@vm0/runner 3.6.2 → 3.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +519 -287
- package/package.json +1 -1
package/index.js
CHANGED
|
@@ -272,14 +272,13 @@ async function subscribeToJobs(server, group, onJob, onConnectionChange) {
|
|
|
272
272
|
}
|
|
273
273
|
|
|
274
274
|
// src/lib/executor.ts
|
|
275
|
-
import
|
|
275
|
+
import path5 from "path";
|
|
276
276
|
|
|
277
277
|
// src/lib/firecracker/vm.ts
|
|
278
|
-
import {
|
|
279
|
-
import
|
|
280
|
-
import
|
|
278
|
+
import { spawn } from "child_process";
|
|
279
|
+
import fs4 from "fs";
|
|
280
|
+
import path3 from "path";
|
|
281
281
|
import readline from "readline";
|
|
282
|
-
import { promisify as promisify3 } from "util";
|
|
283
282
|
|
|
284
283
|
// src/lib/firecracker/client.ts
|
|
285
284
|
import http from "http";
|
|
@@ -291,7 +290,7 @@ var FirecrackerClient = class {
|
|
|
291
290
|
/**
|
|
292
291
|
* Make HTTP request to Firecracker API
|
|
293
292
|
*/
|
|
294
|
-
async request(method,
|
|
293
|
+
async request(method, path8, body) {
|
|
295
294
|
return new Promise((resolve, reject) => {
|
|
296
295
|
const bodyStr = body !== void 0 ? JSON.stringify(body) : void 0;
|
|
297
296
|
const headers = {
|
|
@@ -304,11 +303,11 @@ var FirecrackerClient = class {
|
|
|
304
303
|
headers["Content-Length"] = Buffer.byteLength(bodyStr);
|
|
305
304
|
}
|
|
306
305
|
console.log(
|
|
307
|
-
`[FC API] ${method} ${
|
|
306
|
+
`[FC API] ${method} ${path8}${bodyStr ? ` (${Buffer.byteLength(bodyStr)} bytes)` : ""}`
|
|
308
307
|
);
|
|
309
308
|
const options = {
|
|
310
309
|
socketPath: this.socketPath,
|
|
311
|
-
path:
|
|
310
|
+
path: path8,
|
|
312
311
|
method,
|
|
313
312
|
headers,
|
|
314
313
|
// Disable agent to ensure fresh connection for each request
|
|
@@ -1064,9 +1063,205 @@ async function cleanupOrphanedProxyRules(runnerName) {
|
|
|
1064
1063
|
}
|
|
1065
1064
|
}
|
|
1066
1065
|
|
|
1067
|
-
// src/lib/firecracker/
|
|
1066
|
+
// src/lib/firecracker/overlay-pool.ts
|
|
1067
|
+
import { exec as exec3 } from "child_process";
|
|
1068
|
+
import { randomUUID } from "crypto";
|
|
1069
|
+
import fs3 from "fs";
|
|
1070
|
+
import path2 from "path";
|
|
1071
|
+
import { promisify as promisify3 } from "util";
|
|
1068
1072
|
var execAsync3 = promisify3(exec3);
|
|
1069
|
-
var logger3 = createLogger("
|
|
1073
|
+
var logger3 = createLogger("OverlayPool");
|
|
1074
|
+
var VM0_RUN_DIR2 = "/var/run/vm0";
|
|
1075
|
+
var DEFAULT_POOL_DIR = path2.join(VM0_RUN_DIR2, "overlay-pool");
|
|
1076
|
+
var OVERLAY_SIZE = 2 * 1024 * 1024 * 1024;
|
|
1077
|
+
async function defaultCreateFile(filePath) {
|
|
1078
|
+
const fd = fs3.openSync(filePath, "w");
|
|
1079
|
+
fs3.ftruncateSync(fd, OVERLAY_SIZE);
|
|
1080
|
+
fs3.closeSync(fd);
|
|
1081
|
+
await execAsync3(`mkfs.ext4 -F -q "${filePath}"`);
|
|
1082
|
+
}
|
|
1083
|
+
var OverlayPool = class {
|
|
1084
|
+
initialized = false;
|
|
1085
|
+
queue = [];
|
|
1086
|
+
replenishing = false;
|
|
1087
|
+
config;
|
|
1088
|
+
constructor(config) {
|
|
1089
|
+
this.config = {
|
|
1090
|
+
size: config.size,
|
|
1091
|
+
replenishThreshold: config.replenishThreshold,
|
|
1092
|
+
poolDir: config.poolDir ?? DEFAULT_POOL_DIR,
|
|
1093
|
+
createFile: config.createFile ?? defaultCreateFile
|
|
1094
|
+
};
|
|
1095
|
+
}
|
|
1096
|
+
/**
|
|
1097
|
+
* Generate unique file name using UUID
|
|
1098
|
+
*/
|
|
1099
|
+
generateFileName() {
|
|
1100
|
+
return `overlay-${randomUUID()}.ext4`;
|
|
1101
|
+
}
|
|
1102
|
+
/**
|
|
1103
|
+
* Ensure the pool directory exists
|
|
1104
|
+
*/
|
|
1105
|
+
async ensurePoolDir() {
|
|
1106
|
+
const parentDir = path2.dirname(this.config.poolDir);
|
|
1107
|
+
if (!fs3.existsSync(parentDir)) {
|
|
1108
|
+
await execAsync3(`sudo mkdir -p ${parentDir}`);
|
|
1109
|
+
await execAsync3(`sudo chmod 777 ${parentDir}`);
|
|
1110
|
+
}
|
|
1111
|
+
if (!fs3.existsSync(this.config.poolDir)) {
|
|
1112
|
+
fs3.mkdirSync(this.config.poolDir, { recursive: true });
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
/**
|
|
1116
|
+
* Scan pool directory for overlay files
|
|
1117
|
+
*/
|
|
1118
|
+
scanPoolDir() {
|
|
1119
|
+
if (!fs3.existsSync(this.config.poolDir)) {
|
|
1120
|
+
return [];
|
|
1121
|
+
}
|
|
1122
|
+
return fs3.readdirSync(this.config.poolDir).filter((f) => f.startsWith("overlay-") && f.endsWith(".ext4")).map((f) => path2.join(this.config.poolDir, f));
|
|
1123
|
+
}
|
|
1124
|
+
/**
|
|
1125
|
+
* Replenish the pool in background
|
|
1126
|
+
*/
|
|
1127
|
+
async replenish() {
|
|
1128
|
+
if (this.replenishing || !this.initialized) {
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
const needed = this.config.size - this.queue.length;
|
|
1132
|
+
if (needed <= 0) {
|
|
1133
|
+
return;
|
|
1134
|
+
}
|
|
1135
|
+
this.replenishing = true;
|
|
1136
|
+
logger3.log(`Replenishing pool: creating ${needed} overlay(s)...`);
|
|
1137
|
+
try {
|
|
1138
|
+
const promises = [];
|
|
1139
|
+
for (let i = 0; i < needed; i++) {
|
|
1140
|
+
const filePath = path2.join(
|
|
1141
|
+
this.config.poolDir,
|
|
1142
|
+
this.generateFileName()
|
|
1143
|
+
);
|
|
1144
|
+
promises.push(
|
|
1145
|
+
this.config.createFile(filePath).then(() => {
|
|
1146
|
+
this.queue.push(filePath);
|
|
1147
|
+
})
|
|
1148
|
+
);
|
|
1149
|
+
}
|
|
1150
|
+
await Promise.all(promises);
|
|
1151
|
+
logger3.log(`Pool replenished: ${this.queue.length} available`);
|
|
1152
|
+
} catch (err) {
|
|
1153
|
+
logger3.error(
|
|
1154
|
+
`Replenish failed: ${err instanceof Error ? err.message : "Unknown"}`
|
|
1155
|
+
);
|
|
1156
|
+
} finally {
|
|
1157
|
+
this.replenishing = false;
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
/**
|
|
1161
|
+
* Initialize the overlay pool
|
|
1162
|
+
*/
|
|
1163
|
+
async init() {
|
|
1164
|
+
this.queue = [];
|
|
1165
|
+
logger3.log(
|
|
1166
|
+
`Initializing overlay pool (size=${this.config.size}, threshold=${this.config.replenishThreshold})...`
|
|
1167
|
+
);
|
|
1168
|
+
await this.ensurePoolDir();
|
|
1169
|
+
const existing = this.scanPoolDir();
|
|
1170
|
+
if (existing.length > 0) {
|
|
1171
|
+
logger3.log(`Cleaning up ${existing.length} stale overlay(s)`);
|
|
1172
|
+
for (const file of existing) {
|
|
1173
|
+
fs3.unlinkSync(file);
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
this.initialized = true;
|
|
1177
|
+
await this.replenish();
|
|
1178
|
+
logger3.log("Overlay pool initialized");
|
|
1179
|
+
}
|
|
1180
|
+
/**
|
|
1181
|
+
* Acquire an overlay file from the pool
|
|
1182
|
+
*
|
|
1183
|
+
* Returns the file path. Caller owns the file and must delete it when done.
|
|
1184
|
+
* Falls back to on-demand creation if pool is exhausted.
|
|
1185
|
+
*/
|
|
1186
|
+
async acquire() {
|
|
1187
|
+
if (!this.initialized) {
|
|
1188
|
+
throw new Error("Overlay pool not initialized");
|
|
1189
|
+
}
|
|
1190
|
+
const filePath = this.queue.shift();
|
|
1191
|
+
if (filePath) {
|
|
1192
|
+
logger3.log(`Acquired overlay from pool (${this.queue.length} remaining)`);
|
|
1193
|
+
if (this.queue.length < this.config.replenishThreshold) {
|
|
1194
|
+
this.replenish().catch((err) => {
|
|
1195
|
+
logger3.error(
|
|
1196
|
+
`Background replenish failed: ${err instanceof Error ? err.message : "Unknown"}`
|
|
1197
|
+
);
|
|
1198
|
+
});
|
|
1199
|
+
}
|
|
1200
|
+
return filePath;
|
|
1201
|
+
}
|
|
1202
|
+
logger3.log("Pool exhausted, creating overlay on-demand");
|
|
1203
|
+
const newPath = path2.join(this.config.poolDir, this.generateFileName());
|
|
1204
|
+
await this.config.createFile(newPath);
|
|
1205
|
+
return newPath;
|
|
1206
|
+
}
|
|
1207
|
+
/**
|
|
1208
|
+
* Clean up the overlay pool
|
|
1209
|
+
*/
|
|
1210
|
+
cleanup() {
|
|
1211
|
+
if (!this.initialized) {
|
|
1212
|
+
return;
|
|
1213
|
+
}
|
|
1214
|
+
logger3.log("Cleaning up overlay pool...");
|
|
1215
|
+
for (const file of this.queue) {
|
|
1216
|
+
try {
|
|
1217
|
+
fs3.unlinkSync(file);
|
|
1218
|
+
} catch (err) {
|
|
1219
|
+
logger3.log(
|
|
1220
|
+
`Failed to delete ${file}: ${err instanceof Error ? err.message : "Unknown"}`
|
|
1221
|
+
);
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
this.queue = [];
|
|
1225
|
+
for (const file of this.scanPoolDir()) {
|
|
1226
|
+
try {
|
|
1227
|
+
fs3.unlinkSync(file);
|
|
1228
|
+
} catch (err) {
|
|
1229
|
+
logger3.log(
|
|
1230
|
+
`Failed to delete ${file}: ${err instanceof Error ? err.message : "Unknown"}`
|
|
1231
|
+
);
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
this.initialized = false;
|
|
1235
|
+
this.replenishing = false;
|
|
1236
|
+
logger3.log("Overlay pool cleaned up");
|
|
1237
|
+
}
|
|
1238
|
+
};
|
|
1239
|
+
var overlayPool = null;
|
|
1240
|
+
async function initOverlayPool(config) {
|
|
1241
|
+
if (overlayPool) {
|
|
1242
|
+
overlayPool.cleanup();
|
|
1243
|
+
}
|
|
1244
|
+
overlayPool = new OverlayPool(config);
|
|
1245
|
+
await overlayPool.init();
|
|
1246
|
+
return overlayPool;
|
|
1247
|
+
}
|
|
1248
|
+
function acquireOverlay() {
|
|
1249
|
+
if (!overlayPool) {
|
|
1250
|
+
throw new Error(
|
|
1251
|
+
"Overlay pool not initialized. Call initOverlayPool() first."
|
|
1252
|
+
);
|
|
1253
|
+
}
|
|
1254
|
+
return overlayPool.acquire();
|
|
1255
|
+
}
|
|
1256
|
+
function cleanupOverlayPool() {
|
|
1257
|
+
if (overlayPool) {
|
|
1258
|
+
overlayPool.cleanup();
|
|
1259
|
+
overlayPool = null;
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
// src/lib/firecracker/vm.ts
|
|
1264
|
+
var logger4 = createLogger("VM");
|
|
1070
1265
|
var FirecrackerVM = class {
|
|
1071
1266
|
config;
|
|
1072
1267
|
process = null;
|
|
@@ -1075,16 +1270,15 @@ var FirecrackerVM = class {
|
|
|
1075
1270
|
state = "created";
|
|
1076
1271
|
workDir;
|
|
1077
1272
|
socketPath;
|
|
1078
|
-
vmOverlayPath;
|
|
1079
|
-
//
|
|
1273
|
+
vmOverlayPath = null;
|
|
1274
|
+
// Set during start()
|
|
1080
1275
|
vsockPath;
|
|
1081
1276
|
// Vsock UDS path for host-guest communication
|
|
1082
1277
|
constructor(config) {
|
|
1083
1278
|
this.config = config;
|
|
1084
1279
|
this.workDir = config.workDir || `/tmp/vm0-vm-${config.vmId}`;
|
|
1085
|
-
this.socketPath =
|
|
1086
|
-
this.
|
|
1087
|
-
this.vsockPath = path2.join(this.workDir, "vsock.sock");
|
|
1280
|
+
this.socketPath = path3.join(this.workDir, "firecracker.sock");
|
|
1281
|
+
this.vsockPath = path3.join(this.workDir, "vsock.sock");
|
|
1088
1282
|
}
|
|
1089
1283
|
/**
|
|
1090
1284
|
* Get current VM state
|
|
@@ -1125,25 +1319,23 @@ var FirecrackerVM = class {
|
|
|
1125
1319
|
throw new Error(`Cannot start VM in state: ${this.state}`);
|
|
1126
1320
|
}
|
|
1127
1321
|
try {
|
|
1128
|
-
|
|
1129
|
-
if (
|
|
1130
|
-
|
|
1131
|
-
}
|
|
1132
|
-
|
|
1133
|
-
const
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
await execAsync3(`mkfs.ext4 -F -q "${this.vmOverlayPath}"`);
|
|
1139
|
-
logger3.log(`[VM ${this.config.vmId}] Overlay created`);
|
|
1322
|
+
fs4.mkdirSync(this.workDir, { recursive: true });
|
|
1323
|
+
if (fs4.existsSync(this.socketPath)) {
|
|
1324
|
+
fs4.unlinkSync(this.socketPath);
|
|
1325
|
+
}
|
|
1326
|
+
logger4.log(`[VM ${this.config.vmId}] Setting up overlay and network...`);
|
|
1327
|
+
const setupOverlay = async () => {
|
|
1328
|
+
this.vmOverlayPath = await acquireOverlay();
|
|
1329
|
+
logger4.log(
|
|
1330
|
+
`[VM ${this.config.vmId}] Overlay acquired: ${this.vmOverlayPath}`
|
|
1331
|
+
);
|
|
1140
1332
|
};
|
|
1141
1333
|
const [, networkConfig] = await Promise.all([
|
|
1142
|
-
|
|
1334
|
+
setupOverlay(),
|
|
1143
1335
|
createTapDevice(this.config.vmId)
|
|
1144
1336
|
]);
|
|
1145
1337
|
this.networkConfig = networkConfig;
|
|
1146
|
-
|
|
1338
|
+
logger4.log(`[VM ${this.config.vmId}] Starting Firecracker...`);
|
|
1147
1339
|
this.process = spawn(
|
|
1148
1340
|
this.config.firecrackerBinary,
|
|
1149
1341
|
["--api-sock", this.socketPath],
|
|
@@ -1154,11 +1346,11 @@ var FirecrackerVM = class {
|
|
|
1154
1346
|
}
|
|
1155
1347
|
);
|
|
1156
1348
|
this.process.on("error", (err) => {
|
|
1157
|
-
|
|
1349
|
+
logger4.log(`[VM ${this.config.vmId}] Firecracker error: ${err}`);
|
|
1158
1350
|
this.state = "error";
|
|
1159
1351
|
});
|
|
1160
1352
|
this.process.on("exit", (code, signal) => {
|
|
1161
|
-
|
|
1353
|
+
logger4.log(
|
|
1162
1354
|
`[VM ${this.config.vmId}] Firecracker exited: code=${code}, signal=${signal}`
|
|
1163
1355
|
);
|
|
1164
1356
|
if (this.state !== "stopped") {
|
|
@@ -1171,7 +1363,7 @@ var FirecrackerVM = class {
|
|
|
1171
1363
|
});
|
|
1172
1364
|
stdoutRL.on("line", (line) => {
|
|
1173
1365
|
if (line.trim()) {
|
|
1174
|
-
|
|
1366
|
+
logger4.log(`[VM ${this.config.vmId}] ${line}`);
|
|
1175
1367
|
}
|
|
1176
1368
|
});
|
|
1177
1369
|
}
|
|
@@ -1181,19 +1373,19 @@ var FirecrackerVM = class {
|
|
|
1181
1373
|
});
|
|
1182
1374
|
stderrRL.on("line", (line) => {
|
|
1183
1375
|
if (line.trim()) {
|
|
1184
|
-
|
|
1376
|
+
logger4.log(`[VM ${this.config.vmId}] stderr: ${line}`);
|
|
1185
1377
|
}
|
|
1186
1378
|
});
|
|
1187
1379
|
}
|
|
1188
1380
|
this.client = new FirecrackerClient(this.socketPath);
|
|
1189
|
-
|
|
1381
|
+
logger4.log(`[VM ${this.config.vmId}] Waiting for API...`);
|
|
1190
1382
|
await this.client.waitUntilReady(1e4, 100);
|
|
1191
1383
|
this.state = "configuring";
|
|
1192
1384
|
await this.configure();
|
|
1193
|
-
|
|
1385
|
+
logger4.log(`[VM ${this.config.vmId}] Booting...`);
|
|
1194
1386
|
await this.client.start();
|
|
1195
1387
|
this.state = "running";
|
|
1196
|
-
|
|
1388
|
+
logger4.log(
|
|
1197
1389
|
`[VM ${this.config.vmId}] Running at ${this.networkConfig.guestIp}`
|
|
1198
1390
|
);
|
|
1199
1391
|
} catch (error) {
|
|
@@ -1206,10 +1398,10 @@ var FirecrackerVM = class {
|
|
|
1206
1398
|
* Configure the VM via Firecracker API
|
|
1207
1399
|
*/
|
|
1208
1400
|
async configure() {
|
|
1209
|
-
if (!this.client || !this.networkConfig) {
|
|
1401
|
+
if (!this.client || !this.networkConfig || !this.vmOverlayPath) {
|
|
1210
1402
|
throw new Error("VM not properly initialized");
|
|
1211
1403
|
}
|
|
1212
|
-
|
|
1404
|
+
logger4.log(
|
|
1213
1405
|
`[VM ${this.config.vmId}] Configuring: ${this.config.vcpus} vCPUs, ${this.config.memoryMb}MB RAM`
|
|
1214
1406
|
);
|
|
1215
1407
|
await this.client.setMachineConfig({
|
|
@@ -1218,12 +1410,12 @@ var FirecrackerVM = class {
|
|
|
1218
1410
|
});
|
|
1219
1411
|
const networkBootArgs = generateNetworkBootArgs(this.networkConfig);
|
|
1220
1412
|
const bootArgs = `console=ttyS0 reboot=k panic=1 pci=off nomodules random.trust_cpu=on quiet loglevel=0 nokaslr audit=0 numa=off mitigations=off noresume init=/sbin/vm-init ${networkBootArgs}`;
|
|
1221
|
-
|
|
1413
|
+
logger4.log(`[VM ${this.config.vmId}] Boot args: ${bootArgs}`);
|
|
1222
1414
|
await this.client.setBootSource({
|
|
1223
1415
|
kernel_image_path: this.config.kernelPath,
|
|
1224
1416
|
boot_args: bootArgs
|
|
1225
1417
|
});
|
|
1226
|
-
|
|
1418
|
+
logger4.log(
|
|
1227
1419
|
`[VM ${this.config.vmId}] Base rootfs: ${this.config.rootfsPath}`
|
|
1228
1420
|
);
|
|
1229
1421
|
await this.client.setDrive({
|
|
@@ -1232,14 +1424,14 @@ var FirecrackerVM = class {
|
|
|
1232
1424
|
is_root_device: true,
|
|
1233
1425
|
is_read_only: true
|
|
1234
1426
|
});
|
|
1235
|
-
|
|
1427
|
+
logger4.log(`[VM ${this.config.vmId}] Overlay: ${this.vmOverlayPath}`);
|
|
1236
1428
|
await this.client.setDrive({
|
|
1237
1429
|
drive_id: "overlay",
|
|
1238
1430
|
path_on_host: this.vmOverlayPath,
|
|
1239
1431
|
is_root_device: false,
|
|
1240
1432
|
is_read_only: false
|
|
1241
1433
|
});
|
|
1242
|
-
|
|
1434
|
+
logger4.log(
|
|
1243
1435
|
`[VM ${this.config.vmId}] Network: ${this.networkConfig.tapDevice}`
|
|
1244
1436
|
);
|
|
1245
1437
|
await this.client.setNetworkInterface({
|
|
@@ -1247,7 +1439,7 @@ var FirecrackerVM = class {
|
|
|
1247
1439
|
guest_mac: this.networkConfig.guestMac,
|
|
1248
1440
|
host_dev_name: this.networkConfig.tapDevice
|
|
1249
1441
|
});
|
|
1250
|
-
|
|
1442
|
+
logger4.log(`[VM ${this.config.vmId}] Vsock: ${this.vsockPath}`);
|
|
1251
1443
|
await this.client.setVsock({
|
|
1252
1444
|
vsock_id: "vsock0",
|
|
1253
1445
|
guest_cid: 3,
|
|
@@ -1259,15 +1451,15 @@ var FirecrackerVM = class {
|
|
|
1259
1451
|
*/
|
|
1260
1452
|
async stop() {
|
|
1261
1453
|
if (this.state !== "running") {
|
|
1262
|
-
|
|
1454
|
+
logger4.log(`[VM ${this.config.vmId}] Not running, state: ${this.state}`);
|
|
1263
1455
|
return;
|
|
1264
1456
|
}
|
|
1265
1457
|
this.state = "stopping";
|
|
1266
|
-
|
|
1458
|
+
logger4.log(`[VM ${this.config.vmId}] Stopping...`);
|
|
1267
1459
|
try {
|
|
1268
1460
|
if (this.client) {
|
|
1269
1461
|
await this.client.sendCtrlAltDel().catch((error) => {
|
|
1270
|
-
|
|
1462
|
+
logger4.log(
|
|
1271
1463
|
`[VM ${this.config.vmId}] Graceful shutdown signal failed (VM may already be stopping): ${error instanceof Error ? error.message : error}`
|
|
1272
1464
|
);
|
|
1273
1465
|
});
|
|
@@ -1280,7 +1472,7 @@ var FirecrackerVM = class {
|
|
|
1280
1472
|
* Force kill the VM
|
|
1281
1473
|
*/
|
|
1282
1474
|
async kill() {
|
|
1283
|
-
|
|
1475
|
+
logger4.log(`[VM ${this.config.vmId}] Force killing...`);
|
|
1284
1476
|
await this.cleanup();
|
|
1285
1477
|
}
|
|
1286
1478
|
/**
|
|
@@ -1300,12 +1492,16 @@ var FirecrackerVM = class {
|
|
|
1300
1492
|
);
|
|
1301
1493
|
this.networkConfig = null;
|
|
1302
1494
|
}
|
|
1303
|
-
if (
|
|
1304
|
-
|
|
1495
|
+
if (this.vmOverlayPath && fs4.existsSync(this.vmOverlayPath)) {
|
|
1496
|
+
fs4.unlinkSync(this.vmOverlayPath);
|
|
1497
|
+
this.vmOverlayPath = null;
|
|
1498
|
+
}
|
|
1499
|
+
if (fs4.existsSync(this.workDir)) {
|
|
1500
|
+
fs4.rmSync(this.workDir, { recursive: true, force: true });
|
|
1305
1501
|
}
|
|
1306
1502
|
this.client = null;
|
|
1307
1503
|
this.state = "stopped";
|
|
1308
|
-
|
|
1504
|
+
logger4.log(`[VM ${this.config.vmId}] Stopped`);
|
|
1309
1505
|
}
|
|
1310
1506
|
/**
|
|
1311
1507
|
* Wait for the VM process to exit
|
|
@@ -1336,7 +1532,7 @@ var FirecrackerVM = class {
|
|
|
1336
1532
|
|
|
1337
1533
|
// src/lib/firecracker/vsock.ts
|
|
1338
1534
|
import * as net from "net";
|
|
1339
|
-
import * as
|
|
1535
|
+
import * as fs5 from "fs";
|
|
1340
1536
|
var VSOCK_PORT = 1e3;
|
|
1341
1537
|
var HEADER_SIZE = 4;
|
|
1342
1538
|
var MAX_MESSAGE_SIZE = 16 * 1024 * 1024;
|
|
@@ -1368,8 +1564,8 @@ function encodeExecPayload(command, timeoutMs) {
|
|
|
1368
1564
|
cmdBuf.copy(payload, 8);
|
|
1369
1565
|
return payload;
|
|
1370
1566
|
}
|
|
1371
|
-
function encodeWriteFilePayload(
|
|
1372
|
-
const pathBuf = Buffer.from(
|
|
1567
|
+
function encodeWriteFilePayload(path8, content, sudo) {
|
|
1568
|
+
const pathBuf = Buffer.from(path8, "utf-8");
|
|
1373
1569
|
if (pathBuf.length > 65535) {
|
|
1374
1570
|
throw new Error(`Path too long: ${pathBuf.length} bytes (max 65535)`);
|
|
1375
1571
|
}
|
|
@@ -1676,8 +1872,8 @@ var VsockClient = class {
|
|
|
1676
1872
|
return;
|
|
1677
1873
|
}
|
|
1678
1874
|
const listenerPath = `${this.vsockPath}_${VSOCK_PORT}`;
|
|
1679
|
-
if (
|
|
1680
|
-
|
|
1875
|
+
if (fs5.existsSync(listenerPath)) {
|
|
1876
|
+
fs5.unlinkSync(listenerPath);
|
|
1681
1877
|
}
|
|
1682
1878
|
return new Promise((resolve, reject) => {
|
|
1683
1879
|
const server = net.createServer();
|
|
@@ -1687,8 +1883,8 @@ var VsockClient = class {
|
|
|
1687
1883
|
if (!settled) {
|
|
1688
1884
|
settled = true;
|
|
1689
1885
|
server.close();
|
|
1690
|
-
if (
|
|
1691
|
-
|
|
1886
|
+
if (fs5.existsSync(listenerPath)) {
|
|
1887
|
+
fs5.unlinkSync(listenerPath);
|
|
1692
1888
|
}
|
|
1693
1889
|
reject(new Error(`Guest connection timeout after ${timeoutMs}ms`));
|
|
1694
1890
|
}
|
|
@@ -1698,8 +1894,8 @@ var VsockClient = class {
|
|
|
1698
1894
|
settled = true;
|
|
1699
1895
|
clearTimeout(timeout);
|
|
1700
1896
|
server.close();
|
|
1701
|
-
if (
|
|
1702
|
-
|
|
1897
|
+
if (fs5.existsSync(listenerPath)) {
|
|
1898
|
+
fs5.unlinkSync(listenerPath);
|
|
1703
1899
|
}
|
|
1704
1900
|
reject(err);
|
|
1705
1901
|
}
|
|
@@ -1731,8 +1927,8 @@ var VsockClient = class {
|
|
|
1731
1927
|
}
|
|
1732
1928
|
settled = true;
|
|
1733
1929
|
clearTimeout(timeout);
|
|
1734
|
-
if (
|
|
1735
|
-
|
|
1930
|
+
if (fs5.existsSync(listenerPath)) {
|
|
1931
|
+
fs5.unlinkSync(listenerPath);
|
|
1736
1932
|
}
|
|
1737
1933
|
state = 2 /* Connected */;
|
|
1738
1934
|
this.socket = socket;
|
|
@@ -2226,8 +2422,8 @@ function getErrorMap() {
|
|
|
2226
2422
|
return overrideErrorMap;
|
|
2227
2423
|
}
|
|
2228
2424
|
var makeIssue = (params) => {
|
|
2229
|
-
const { data, path:
|
|
2230
|
-
const fullPath = [...
|
|
2425
|
+
const { data, path: path8, errorMaps, issueData } = params;
|
|
2426
|
+
const fullPath = [...path8, ...issueData.path || []];
|
|
2231
2427
|
const fullIssue = {
|
|
2232
2428
|
...issueData,
|
|
2233
2429
|
path: fullPath
|
|
@@ -2326,11 +2522,11 @@ var errorUtil;
|
|
|
2326
2522
|
errorUtil2.toString = (message) => typeof message === "string" ? message : message === null || message === void 0 ? void 0 : message.message;
|
|
2327
2523
|
})(errorUtil || (errorUtil = {}));
|
|
2328
2524
|
var ParseInputLazyPath = class {
|
|
2329
|
-
constructor(parent, value,
|
|
2525
|
+
constructor(parent, value, path8, key) {
|
|
2330
2526
|
this._cachedPath = [];
|
|
2331
2527
|
this.parent = parent;
|
|
2332
2528
|
this.data = value;
|
|
2333
|
-
this._path =
|
|
2529
|
+
this._path = path8;
|
|
2334
2530
|
this._key = key;
|
|
2335
2531
|
}
|
|
2336
2532
|
get path() {
|
|
@@ -8509,8 +8705,8 @@ var FEATURE_SWITCHES = {
|
|
|
8509
8705
|
var ENV_LOADER_PATH = "/usr/local/bin/vm0-agent/env-loader.mjs";
|
|
8510
8706
|
|
|
8511
8707
|
// src/lib/proxy/vm-registry.ts
|
|
8512
|
-
import
|
|
8513
|
-
var
|
|
8708
|
+
import fs6 from "fs";
|
|
8709
|
+
var logger5 = createLogger("VMRegistry");
|
|
8514
8710
|
var DEFAULT_REGISTRY_PATH = "/tmp/vm0-vm-registry.json";
|
|
8515
8711
|
var VMRegistry = class {
|
|
8516
8712
|
registryPath;
|
|
@@ -8524,8 +8720,8 @@ var VMRegistry = class {
|
|
|
8524
8720
|
*/
|
|
8525
8721
|
load() {
|
|
8526
8722
|
try {
|
|
8527
|
-
if (
|
|
8528
|
-
const content =
|
|
8723
|
+
if (fs6.existsSync(this.registryPath)) {
|
|
8724
|
+
const content = fs6.readFileSync(this.registryPath, "utf-8");
|
|
8529
8725
|
return JSON.parse(content);
|
|
8530
8726
|
}
|
|
8531
8727
|
} catch {
|
|
@@ -8539,8 +8735,8 @@ var VMRegistry = class {
|
|
|
8539
8735
|
this.data.updatedAt = Date.now();
|
|
8540
8736
|
const content = JSON.stringify(this.data, null, 2);
|
|
8541
8737
|
const tempPath = `${this.registryPath}.tmp`;
|
|
8542
|
-
|
|
8543
|
-
|
|
8738
|
+
fs6.writeFileSync(tempPath, content, { mode: 420 });
|
|
8739
|
+
fs6.renameSync(tempPath, this.registryPath);
|
|
8544
8740
|
}
|
|
8545
8741
|
/**
|
|
8546
8742
|
* Register a VM with its IP address
|
|
@@ -8557,7 +8753,7 @@ var VMRegistry = class {
|
|
|
8557
8753
|
this.save();
|
|
8558
8754
|
const firewallInfo = options?.firewallRules ? ` with ${options.firewallRules.length} firewall rules` : "";
|
|
8559
8755
|
const mitmInfo = options?.mitmEnabled ? ", MITM enabled" : "";
|
|
8560
|
-
|
|
8756
|
+
logger5.log(
|
|
8561
8757
|
`Registered VM ${vmIp} for run ${runId}${firewallInfo}${mitmInfo}`
|
|
8562
8758
|
);
|
|
8563
8759
|
}
|
|
@@ -8569,7 +8765,7 @@ var VMRegistry = class {
|
|
|
8569
8765
|
const registration = this.data.vms[vmIp];
|
|
8570
8766
|
delete this.data.vms[vmIp];
|
|
8571
8767
|
this.save();
|
|
8572
|
-
|
|
8768
|
+
logger5.log(`Unregistered VM ${vmIp} (run ${registration.runId})`);
|
|
8573
8769
|
}
|
|
8574
8770
|
}
|
|
8575
8771
|
/**
|
|
@@ -8590,7 +8786,7 @@ var VMRegistry = class {
|
|
|
8590
8786
|
clear() {
|
|
8591
8787
|
this.data.vms = {};
|
|
8592
8788
|
this.save();
|
|
8593
|
-
|
|
8789
|
+
logger5.log("Cleared all registrations");
|
|
8594
8790
|
}
|
|
8595
8791
|
/**
|
|
8596
8792
|
* Get the path to the registry file
|
|
@@ -8613,8 +8809,8 @@ function initVMRegistry(registryPath) {
|
|
|
8613
8809
|
|
|
8614
8810
|
// src/lib/proxy/proxy-manager.ts
|
|
8615
8811
|
import { spawn as spawn2 } from "child_process";
|
|
8616
|
-
import
|
|
8617
|
-
import
|
|
8812
|
+
import fs7 from "fs";
|
|
8813
|
+
import path4 from "path";
|
|
8618
8814
|
|
|
8619
8815
|
// src/lib/proxy/mitm-addon-script.ts
|
|
8620
8816
|
var RUNNER_MITM_ADDON_SCRIPT = `#!/usr/bin/env python3
|
|
@@ -9100,7 +9296,7 @@ addons = [tls_clienthello, request, response]
|
|
|
9100
9296
|
`;
|
|
9101
9297
|
|
|
9102
9298
|
// src/lib/proxy/proxy-manager.ts
|
|
9103
|
-
var
|
|
9299
|
+
var logger6 = createLogger("ProxyManager");
|
|
9104
9300
|
var DEFAULT_PROXY_OPTIONS = {
|
|
9105
9301
|
port: 8080,
|
|
9106
9302
|
registryPath: DEFAULT_REGISTRY_PATH,
|
|
@@ -9111,7 +9307,7 @@ var ProxyManager = class {
|
|
|
9111
9307
|
process = null;
|
|
9112
9308
|
isRunning = false;
|
|
9113
9309
|
constructor(config) {
|
|
9114
|
-
const addonPath =
|
|
9310
|
+
const addonPath = path4.join(config.caDir, "mitm_addon.py");
|
|
9115
9311
|
this.config = {
|
|
9116
9312
|
...DEFAULT_PROXY_OPTIONS,
|
|
9117
9313
|
...config,
|
|
@@ -9138,24 +9334,24 @@ var ProxyManager = class {
|
|
|
9138
9334
|
* Ensure the addon script exists at the configured path
|
|
9139
9335
|
*/
|
|
9140
9336
|
ensureAddonScript() {
|
|
9141
|
-
const addonDir =
|
|
9142
|
-
if (!
|
|
9143
|
-
|
|
9337
|
+
const addonDir = path4.dirname(this.config.addonPath);
|
|
9338
|
+
if (!fs7.existsSync(addonDir)) {
|
|
9339
|
+
fs7.mkdirSync(addonDir, { recursive: true });
|
|
9144
9340
|
}
|
|
9145
|
-
|
|
9341
|
+
fs7.writeFileSync(this.config.addonPath, RUNNER_MITM_ADDON_SCRIPT, {
|
|
9146
9342
|
mode: 493
|
|
9147
9343
|
});
|
|
9148
|
-
|
|
9344
|
+
logger6.log(`Addon script written to ${this.config.addonPath}`);
|
|
9149
9345
|
}
|
|
9150
9346
|
/**
|
|
9151
9347
|
* Validate proxy configuration
|
|
9152
9348
|
*/
|
|
9153
9349
|
validateConfig() {
|
|
9154
|
-
if (!
|
|
9350
|
+
if (!fs7.existsSync(this.config.caDir)) {
|
|
9155
9351
|
throw new Error(`Proxy CA directory not found: ${this.config.caDir}`);
|
|
9156
9352
|
}
|
|
9157
|
-
const caCertPath =
|
|
9158
|
-
if (!
|
|
9353
|
+
const caCertPath = path4.join(this.config.caDir, "mitmproxy-ca.pem");
|
|
9354
|
+
if (!fs7.existsSync(caCertPath)) {
|
|
9159
9355
|
throw new Error(`Proxy CA certificate not found: ${caCertPath}`);
|
|
9160
9356
|
}
|
|
9161
9357
|
this.ensureAddonScript();
|
|
@@ -9165,7 +9361,7 @@ var ProxyManager = class {
|
|
|
9165
9361
|
*/
|
|
9166
9362
|
async start() {
|
|
9167
9363
|
if (this.isRunning) {
|
|
9168
|
-
|
|
9364
|
+
logger6.log("Proxy already running");
|
|
9169
9365
|
return;
|
|
9170
9366
|
}
|
|
9171
9367
|
const mitmproxyInstalled = await this.checkMitmproxyInstalled();
|
|
@@ -9176,11 +9372,11 @@ var ProxyManager = class {
|
|
|
9176
9372
|
}
|
|
9177
9373
|
this.validateConfig();
|
|
9178
9374
|
getVMRegistry();
|
|
9179
|
-
|
|
9180
|
-
|
|
9181
|
-
|
|
9182
|
-
|
|
9183
|
-
|
|
9375
|
+
logger6.log("Starting mitmproxy...");
|
|
9376
|
+
logger6.log(` Port: ${this.config.port}`);
|
|
9377
|
+
logger6.log(` CA Dir: ${this.config.caDir}`);
|
|
9378
|
+
logger6.log(` Addon: ${this.config.addonPath}`);
|
|
9379
|
+
logger6.log(` Registry: ${this.config.registryPath}`);
|
|
9184
9380
|
const args = [
|
|
9185
9381
|
"--mode",
|
|
9186
9382
|
"transparent",
|
|
@@ -9210,18 +9406,23 @@ var ProxyManager = class {
|
|
|
9210
9406
|
mitmLogger.log(data.toString().trim());
|
|
9211
9407
|
});
|
|
9212
9408
|
this.process.on("close", (code) => {
|
|
9213
|
-
|
|
9409
|
+
logger6.log(`mitmproxy exited with code ${code}`);
|
|
9214
9410
|
this.isRunning = false;
|
|
9215
9411
|
this.process = null;
|
|
9216
9412
|
});
|
|
9217
9413
|
this.process.on("error", (err) => {
|
|
9218
|
-
|
|
9414
|
+
logger6.error(`mitmproxy error: ${err.message}`);
|
|
9219
9415
|
this.isRunning = false;
|
|
9220
9416
|
this.process = null;
|
|
9221
9417
|
});
|
|
9222
9418
|
await this.waitForReady();
|
|
9223
9419
|
this.isRunning = true;
|
|
9224
|
-
|
|
9420
|
+
logger6.log("mitmproxy started successfully");
|
|
9421
|
+
process.on("exit", () => {
|
|
9422
|
+
if (this.process && !this.process.killed) {
|
|
9423
|
+
this.process.kill("SIGKILL");
|
|
9424
|
+
}
|
|
9425
|
+
});
|
|
9225
9426
|
}
|
|
9226
9427
|
/**
|
|
9227
9428
|
* Wait for proxy to be ready
|
|
@@ -9247,24 +9448,24 @@ var ProxyManager = class {
|
|
|
9247
9448
|
*/
|
|
9248
9449
|
async stop() {
|
|
9249
9450
|
if (!this.process || !this.isRunning) {
|
|
9250
|
-
|
|
9451
|
+
logger6.log("Proxy not running");
|
|
9251
9452
|
return;
|
|
9252
9453
|
}
|
|
9253
|
-
|
|
9454
|
+
logger6.log("Stopping mitmproxy...");
|
|
9254
9455
|
return new Promise((resolve) => {
|
|
9255
9456
|
if (!this.process) {
|
|
9256
9457
|
resolve();
|
|
9257
9458
|
return;
|
|
9258
9459
|
}
|
|
9259
9460
|
const timeout = setTimeout(() => {
|
|
9260
|
-
|
|
9461
|
+
logger6.log("Force killing mitmproxy...");
|
|
9261
9462
|
this.process?.kill("SIGKILL");
|
|
9262
9463
|
}, 5e3);
|
|
9263
9464
|
this.process.on("close", () => {
|
|
9264
9465
|
clearTimeout(timeout);
|
|
9265
9466
|
this.isRunning = false;
|
|
9266
9467
|
this.process = null;
|
|
9267
|
-
|
|
9468
|
+
logger6.log("mitmproxy stopped");
|
|
9268
9469
|
resolve();
|
|
9269
9470
|
});
|
|
9270
9471
|
this.process.kill("SIGTERM");
|
|
@@ -9409,15 +9610,15 @@ async function withSandboxTiming(actionType, fn) {
|
|
|
9409
9610
|
}
|
|
9410
9611
|
|
|
9411
9612
|
// src/lib/vm-setup/vm-setup.ts
|
|
9412
|
-
var
|
|
9613
|
+
var logger7 = createLogger("VMSetup");
|
|
9413
9614
|
var VM_PROXY_CA_PATH = "/usr/local/share/ca-certificates/vm0-proxy-ca.crt";
|
|
9414
9615
|
async function downloadStorages(guest, manifest) {
|
|
9415
9616
|
const totalArchives = manifest.storages.filter((s) => s.archiveUrl).length + (manifest.artifact?.archiveUrl ? 1 : 0);
|
|
9416
9617
|
if (totalArchives === 0) {
|
|
9417
|
-
|
|
9618
|
+
logger7.log(`No archives to download`);
|
|
9418
9619
|
return;
|
|
9419
9620
|
}
|
|
9420
|
-
|
|
9621
|
+
logger7.log(`Downloading ${totalArchives} archive(s)...`);
|
|
9421
9622
|
const manifestJson = JSON.stringify(manifest);
|
|
9422
9623
|
await guest.writeFile("/tmp/storage-manifest.json", manifestJson);
|
|
9423
9624
|
const result = await guest.exec(
|
|
@@ -9426,23 +9627,23 @@ async function downloadStorages(guest, manifest) {
|
|
|
9426
9627
|
if (result.exitCode !== 0) {
|
|
9427
9628
|
throw new Error(`Storage download failed: ${result.stderr}`);
|
|
9428
9629
|
}
|
|
9429
|
-
|
|
9630
|
+
logger7.log(`Storage download completed`);
|
|
9430
9631
|
}
|
|
9431
9632
|
async function restoreSessionHistory(guest, resumeSession, workingDir, cliAgentType) {
|
|
9432
9633
|
const { sessionId, sessionHistory } = resumeSession;
|
|
9433
9634
|
let sessionPath;
|
|
9434
9635
|
if (cliAgentType === "codex") {
|
|
9435
|
-
|
|
9636
|
+
logger7.log(`Codex resume session will be handled by checkpoint.py`);
|
|
9436
9637
|
return;
|
|
9437
9638
|
} else {
|
|
9438
9639
|
const projectName = workingDir.replace(/^\//, "").replace(/\//g, "-");
|
|
9439
9640
|
sessionPath = `/home/user/.claude/projects/-${projectName}/${sessionId}.jsonl`;
|
|
9440
9641
|
}
|
|
9441
|
-
|
|
9642
|
+
logger7.log(`Restoring session history to ${sessionPath}`);
|
|
9442
9643
|
const dirPath = sessionPath.substring(0, sessionPath.lastIndexOf("/"));
|
|
9443
9644
|
await guest.execOrThrow(`mkdir -p "${dirPath}"`);
|
|
9444
9645
|
await guest.writeFile(sessionPath, sessionHistory);
|
|
9445
|
-
|
|
9646
|
+
logger7.log(
|
|
9446
9647
|
`Session history restored (${sessionHistory.split("\n").length} lines)`
|
|
9447
9648
|
);
|
|
9448
9649
|
}
|
|
@@ -9495,22 +9696,22 @@ function buildEnvironmentVariables(context, apiUrl) {
|
|
|
9495
9696
|
}
|
|
9496
9697
|
|
|
9497
9698
|
// src/lib/network-logs/network-logs.ts
|
|
9498
|
-
import
|
|
9499
|
-
var
|
|
9699
|
+
import fs8 from "fs";
|
|
9700
|
+
var logger8 = createLogger("NetworkLogs");
|
|
9500
9701
|
function getNetworkLogPath(runId) {
|
|
9501
9702
|
return `/tmp/vm0-network-${runId}.jsonl`;
|
|
9502
9703
|
}
|
|
9503
9704
|
function readNetworkLogs(runId) {
|
|
9504
9705
|
const logPath = getNetworkLogPath(runId);
|
|
9505
|
-
if (!
|
|
9706
|
+
if (!fs8.existsSync(logPath)) {
|
|
9506
9707
|
return [];
|
|
9507
9708
|
}
|
|
9508
9709
|
try {
|
|
9509
|
-
const content =
|
|
9710
|
+
const content = fs8.readFileSync(logPath, "utf-8");
|
|
9510
9711
|
const lines = content.split("\n").filter((line) => line.trim());
|
|
9511
9712
|
return lines.map((line) => JSON.parse(line));
|
|
9512
9713
|
} catch (err) {
|
|
9513
|
-
|
|
9714
|
+
logger8.error(
|
|
9514
9715
|
`Failed to read network logs: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9515
9716
|
);
|
|
9516
9717
|
return [];
|
|
@@ -9519,11 +9720,11 @@ function readNetworkLogs(runId) {
|
|
|
9519
9720
|
function cleanupNetworkLogs(runId) {
|
|
9520
9721
|
const logPath = getNetworkLogPath(runId);
|
|
9521
9722
|
try {
|
|
9522
|
-
if (
|
|
9523
|
-
|
|
9723
|
+
if (fs8.existsSync(logPath)) {
|
|
9724
|
+
fs8.unlinkSync(logPath);
|
|
9524
9725
|
}
|
|
9525
9726
|
} catch (err) {
|
|
9526
|
-
|
|
9727
|
+
logger8.error(
|
|
9527
9728
|
`Failed to cleanup network logs: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9528
9729
|
);
|
|
9529
9730
|
}
|
|
@@ -9531,10 +9732,10 @@ function cleanupNetworkLogs(runId) {
|
|
|
9531
9732
|
async function uploadNetworkLogs(apiUrl, sandboxToken, runId) {
|
|
9532
9733
|
const networkLogs = readNetworkLogs(runId);
|
|
9533
9734
|
if (networkLogs.length === 0) {
|
|
9534
|
-
|
|
9735
|
+
logger8.log(`No network logs to upload for ${runId}`);
|
|
9535
9736
|
return;
|
|
9536
9737
|
}
|
|
9537
|
-
|
|
9738
|
+
logger8.log(
|
|
9538
9739
|
`Uploading ${networkLogs.length} network log entries for ${runId}`
|
|
9539
9740
|
);
|
|
9540
9741
|
const headers = {
|
|
@@ -9555,68 +9756,18 @@ async function uploadNetworkLogs(apiUrl, sandboxToken, runId) {
|
|
|
9555
9756
|
});
|
|
9556
9757
|
if (!response.ok) {
|
|
9557
9758
|
const errorText = await response.text();
|
|
9558
|
-
|
|
9759
|
+
logger8.error(`Failed to upload network logs: ${errorText}`);
|
|
9559
9760
|
return;
|
|
9560
9761
|
}
|
|
9561
|
-
|
|
9762
|
+
logger8.log(`Network logs uploaded successfully for ${runId}`);
|
|
9562
9763
|
cleanupNetworkLogs(runId);
|
|
9563
9764
|
}
|
|
9564
9765
|
|
|
9565
9766
|
// src/lib/executor.ts
|
|
9566
|
-
var
|
|
9767
|
+
var logger9 = createLogger("Executor");
|
|
9567
9768
|
function getVmIdFromRunId(runId) {
|
|
9568
9769
|
return runId.split("-")[0] || runId.substring(0, 8);
|
|
9569
9770
|
}
|
|
9570
|
-
var CURL_ERROR_MESSAGES = {
|
|
9571
|
-
6: "DNS resolution failed",
|
|
9572
|
-
7: "Connection refused",
|
|
9573
|
-
28: "Connection timeout",
|
|
9574
|
-
60: "TLS certificate error (proxy CA not trusted)",
|
|
9575
|
-
22: "HTTP error from server"
|
|
9576
|
-
};
|
|
9577
|
-
async function runPreflightCheck(guest, apiUrl, runId, sandboxToken, bypassSecret) {
|
|
9578
|
-
const heartbeatUrl = `${apiUrl}/api/webhooks/agent/heartbeat`;
|
|
9579
|
-
const bypassHeader = bypassSecret ? ` -H "x-vercel-protection-bypass: ${bypassSecret}"` : "";
|
|
9580
|
-
const curlCmd = `curl -sf --connect-timeout 5 --max-time 10 "${heartbeatUrl}" -X POST -H "Content-Type: application/json" -H "Authorization: Bearer ${sandboxToken}"${bypassHeader} -d '{"runId":"${runId}"}'`;
|
|
9581
|
-
const result = await guest.exec(curlCmd, 2e4);
|
|
9582
|
-
if (result.exitCode === 0) {
|
|
9583
|
-
return { success: true };
|
|
9584
|
-
}
|
|
9585
|
-
const errorDetail = CURL_ERROR_MESSAGES[result.exitCode] ?? `curl exit code ${result.exitCode}`;
|
|
9586
|
-
const stderrInfo = result.stderr?.trim() ? ` (${result.stderr.trim()})` : "";
|
|
9587
|
-
return {
|
|
9588
|
-
success: false,
|
|
9589
|
-
error: `Preflight check failed: ${errorDetail}${stderrInfo} - VM cannot reach VM0 API at ${apiUrl}`
|
|
9590
|
-
};
|
|
9591
|
-
}
|
|
9592
|
-
async function reportPreflightFailure(apiUrl, runId, sandboxToken, error, bypassSecret) {
|
|
9593
|
-
const completeUrl = `${apiUrl}/api/webhooks/agent/complete`;
|
|
9594
|
-
const headers = {
|
|
9595
|
-
"Content-Type": "application/json",
|
|
9596
|
-
Authorization: `Bearer ${sandboxToken}`
|
|
9597
|
-
};
|
|
9598
|
-
if (bypassSecret) {
|
|
9599
|
-
headers["x-vercel-protection-bypass"] = bypassSecret;
|
|
9600
|
-
}
|
|
9601
|
-
try {
|
|
9602
|
-
const response = await fetch(completeUrl, {
|
|
9603
|
-
method: "POST",
|
|
9604
|
-
headers,
|
|
9605
|
-
body: JSON.stringify({
|
|
9606
|
-
runId,
|
|
9607
|
-
exitCode: 1,
|
|
9608
|
-
error
|
|
9609
|
-
})
|
|
9610
|
-
});
|
|
9611
|
-
if (!response.ok) {
|
|
9612
|
-
logger8.error(
|
|
9613
|
-
`Failed to report preflight failure: HTTP ${response.status}`
|
|
9614
|
-
);
|
|
9615
|
-
}
|
|
9616
|
-
} catch (err) {
|
|
9617
|
-
logger8.error(`Failed to report preflight failure: ${err}`);
|
|
9618
|
-
}
|
|
9619
|
-
}
|
|
9620
9771
|
async function executeJob(context, config, options = {}) {
|
|
9621
9772
|
setSandboxContext({
|
|
9622
9773
|
apiUrl: config.server.url,
|
|
@@ -9633,9 +9784,9 @@ async function executeJob(context, config, options = {}) {
|
|
|
9633
9784
|
const vmId = getVmIdFromRunId(context.runId);
|
|
9634
9785
|
let vm = null;
|
|
9635
9786
|
let guestIp = null;
|
|
9636
|
-
|
|
9787
|
+
logger9.log(`Starting job ${context.runId} in VM ${vmId}`);
|
|
9637
9788
|
try {
|
|
9638
|
-
const workspacesDir =
|
|
9789
|
+
const workspacesDir = path5.join(process.cwd(), "workspaces");
|
|
9639
9790
|
const vmConfig = {
|
|
9640
9791
|
vmId,
|
|
9641
9792
|
vcpus: config.sandbox.vcpu,
|
|
@@ -9643,30 +9794,30 @@ async function executeJob(context, config, options = {}) {
|
|
|
9643
9794
|
kernelPath: config.firecracker.kernel,
|
|
9644
9795
|
rootfsPath: config.firecracker.rootfs,
|
|
9645
9796
|
firecrackerBinary: config.firecracker.binary,
|
|
9646
|
-
workDir:
|
|
9797
|
+
workDir: path5.join(workspacesDir, `vm0-${vmId}`)
|
|
9647
9798
|
};
|
|
9648
|
-
|
|
9799
|
+
logger9.log(`Creating VM ${vmId}...`);
|
|
9649
9800
|
vm = new FirecrackerVM(vmConfig);
|
|
9650
9801
|
await withSandboxTiming("vm_create", () => vm.start());
|
|
9651
9802
|
guestIp = vm.getGuestIp();
|
|
9652
9803
|
if (!guestIp) {
|
|
9653
9804
|
throw new Error("VM started but no IP address available");
|
|
9654
9805
|
}
|
|
9655
|
-
|
|
9806
|
+
logger9.log(`VM ${vmId} started, guest IP: ${guestIp}`);
|
|
9656
9807
|
const vsockPath = vm.getVsockPath();
|
|
9657
9808
|
const guest = new VsockClient(vsockPath);
|
|
9658
|
-
|
|
9659
|
-
|
|
9809
|
+
logger9.log(`Using vsock for guest communication: ${vsockPath}`);
|
|
9810
|
+
logger9.log(`Waiting for guest connection...`);
|
|
9660
9811
|
await withSandboxTiming(
|
|
9661
9812
|
"guest_wait",
|
|
9662
9813
|
() => guest.waitForGuestConnection(3e4)
|
|
9663
9814
|
);
|
|
9664
|
-
|
|
9815
|
+
logger9.log(`Guest client ready`);
|
|
9665
9816
|
const firewallConfig = context.experimentalFirewall;
|
|
9666
9817
|
if (firewallConfig?.enabled) {
|
|
9667
9818
|
const mitmEnabled = firewallConfig.experimental_mitm ?? false;
|
|
9668
9819
|
const sealSecretsEnabled = firewallConfig.experimental_seal_secrets ?? false;
|
|
9669
|
-
|
|
9820
|
+
logger9.log(
|
|
9670
9821
|
`Setting up network security for VM ${guestIp} (mitm=${mitmEnabled}, sealSecrets=${sealSecretsEnabled})`
|
|
9671
9822
|
);
|
|
9672
9823
|
await withSandboxTiming("network_setup", async () => {
|
|
@@ -9701,52 +9852,23 @@ async function executeJob(context, config, options = {}) {
|
|
|
9701
9852
|
}
|
|
9702
9853
|
const envVars = buildEnvironmentVariables(context, config.server.url);
|
|
9703
9854
|
const envJson = JSON.stringify(envVars);
|
|
9704
|
-
|
|
9855
|
+
logger9.log(
|
|
9705
9856
|
`Writing env JSON (${envJson.length} bytes) to ${ENV_JSON_PATH}`
|
|
9706
9857
|
);
|
|
9707
9858
|
await guest.writeFile(ENV_JSON_PATH, envJson);
|
|
9708
|
-
if (!options.benchmarkMode) {
|
|
9709
|
-
logger8.log(`Running preflight connectivity check...`);
|
|
9710
|
-
const bypassSecret = process.env.VERCEL_AUTOMATION_BYPASS_SECRET;
|
|
9711
|
-
const preflight = await withSandboxTiming(
|
|
9712
|
-
"preflight_check",
|
|
9713
|
-
() => runPreflightCheck(
|
|
9714
|
-
guest,
|
|
9715
|
-
config.server.url,
|
|
9716
|
-
context.runId,
|
|
9717
|
-
context.sandboxToken,
|
|
9718
|
-
bypassSecret
|
|
9719
|
-
)
|
|
9720
|
-
);
|
|
9721
|
-
if (!preflight.success) {
|
|
9722
|
-
logger8.log(`Preflight check failed: ${preflight.error}`);
|
|
9723
|
-
await reportPreflightFailure(
|
|
9724
|
-
config.server.url,
|
|
9725
|
-
context.runId,
|
|
9726
|
-
context.sandboxToken,
|
|
9727
|
-
preflight.error,
|
|
9728
|
-
bypassSecret
|
|
9729
|
-
);
|
|
9730
|
-
return {
|
|
9731
|
-
exitCode: 1,
|
|
9732
|
-
error: preflight.error
|
|
9733
|
-
};
|
|
9734
|
-
}
|
|
9735
|
-
logger8.log(`Preflight check passed`);
|
|
9736
|
-
}
|
|
9737
9859
|
const systemLogFile = `/tmp/vm0-main-${context.runId}.log`;
|
|
9738
9860
|
const startTime = Date.now();
|
|
9739
9861
|
const maxWaitMs = 2 * 60 * 60 * 1e3;
|
|
9740
9862
|
let command;
|
|
9741
9863
|
if (options.benchmarkMode) {
|
|
9742
|
-
|
|
9864
|
+
logger9.log(`Running command directly (benchmark mode)...`);
|
|
9743
9865
|
command = `${context.prompt} > ${systemLogFile} 2>&1`;
|
|
9744
9866
|
} else {
|
|
9745
|
-
|
|
9867
|
+
logger9.log(`Running agent via env-loader...`);
|
|
9746
9868
|
command = `node ${ENV_LOADER_PATH} > ${systemLogFile} 2>&1`;
|
|
9747
9869
|
}
|
|
9748
9870
|
const { pid } = await guest.spawnAndWatch(command, maxWaitMs);
|
|
9749
|
-
|
|
9871
|
+
logger9.log(`Process started with pid=${pid}`);
|
|
9750
9872
|
let exitCode = 1;
|
|
9751
9873
|
let exitEvent;
|
|
9752
9874
|
try {
|
|
@@ -9755,7 +9877,7 @@ async function executeJob(context, config, options = {}) {
|
|
|
9755
9877
|
} catch {
|
|
9756
9878
|
const durationMs2 = Date.now() - startTime;
|
|
9757
9879
|
const duration2 = Math.round(durationMs2 / 1e3);
|
|
9758
|
-
|
|
9880
|
+
logger9.log(`Agent timed out after ${duration2}s`);
|
|
9759
9881
|
recordOperation({
|
|
9760
9882
|
actionType: "agent_execute",
|
|
9761
9883
|
durationMs: durationMs2,
|
|
@@ -9773,7 +9895,7 @@ async function executeJob(context, config, options = {}) {
|
|
|
9773
9895
|
`dmesg | tail -20 | grep -iE "killed|oom" 2>/dev/null`
|
|
9774
9896
|
);
|
|
9775
9897
|
if (dmesgCheck.stdout.toLowerCase().includes("oom") || dmesgCheck.stdout.toLowerCase().includes("killed")) {
|
|
9776
|
-
|
|
9898
|
+
logger9.log(`OOM detected: ${dmesgCheck.stdout}`);
|
|
9777
9899
|
recordOperation({
|
|
9778
9900
|
actionType: "agent_execute",
|
|
9779
9901
|
durationMs,
|
|
@@ -9790,9 +9912,9 @@ async function executeJob(context, config, options = {}) {
|
|
|
9790
9912
|
durationMs,
|
|
9791
9913
|
success: exitCode === 0
|
|
9792
9914
|
});
|
|
9793
|
-
|
|
9915
|
+
logger9.log(`Agent finished in ${duration}s with exit code ${exitCode}`);
|
|
9794
9916
|
if (exitEvent.stderr) {
|
|
9795
|
-
|
|
9917
|
+
logger9.log(
|
|
9796
9918
|
`Stderr (${exitEvent.stderr.length} chars): ${exitEvent.stderr.substring(0, 500)}`
|
|
9797
9919
|
);
|
|
9798
9920
|
}
|
|
@@ -9802,14 +9924,14 @@ async function executeJob(context, config, options = {}) {
|
|
|
9802
9924
|
};
|
|
9803
9925
|
} catch (error) {
|
|
9804
9926
|
const errorMsg = error instanceof Error ? error.message : "Unknown error";
|
|
9805
|
-
|
|
9927
|
+
logger9.error(`Job ${context.runId} failed: ${errorMsg}`);
|
|
9806
9928
|
return {
|
|
9807
9929
|
exitCode: 1,
|
|
9808
9930
|
error: errorMsg
|
|
9809
9931
|
};
|
|
9810
9932
|
} finally {
|
|
9811
9933
|
if (context.experimentalFirewall?.enabled && guestIp) {
|
|
9812
|
-
|
|
9934
|
+
logger9.log(`Cleaning up network security for VM ${guestIp}`);
|
|
9813
9935
|
getVMRegistry().unregister(guestIp);
|
|
9814
9936
|
if (!options.benchmarkMode) {
|
|
9815
9937
|
try {
|
|
@@ -9819,14 +9941,14 @@ async function executeJob(context, config, options = {}) {
|
|
|
9819
9941
|
context.runId
|
|
9820
9942
|
);
|
|
9821
9943
|
} catch (err) {
|
|
9822
|
-
|
|
9944
|
+
logger9.error(
|
|
9823
9945
|
`Failed to upload network logs: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9824
9946
|
);
|
|
9825
9947
|
}
|
|
9826
9948
|
}
|
|
9827
9949
|
}
|
|
9828
9950
|
if (vm) {
|
|
9829
|
-
|
|
9951
|
+
logger9.log(`Cleaning up VM ${vmId}...`);
|
|
9830
9952
|
await withSandboxTiming("cleanup", () => vm.kill());
|
|
9831
9953
|
}
|
|
9832
9954
|
await clearSandboxContext();
|
|
@@ -9835,7 +9957,7 @@ async function executeJob(context, config, options = {}) {
|
|
|
9835
9957
|
|
|
9836
9958
|
// src/lib/runner/status.ts
|
|
9837
9959
|
import { writeFileSync as writeFileSync2 } from "fs";
|
|
9838
|
-
var
|
|
9960
|
+
var logger10 = createLogger("Runner");
|
|
9839
9961
|
function writeStatusFile(statusFilePath, mode, activeRuns, startedAt) {
|
|
9840
9962
|
const status = {
|
|
9841
9963
|
mode,
|
|
@@ -9847,7 +9969,7 @@ function writeStatusFile(statusFilePath, mode, activeRuns, startedAt) {
|
|
|
9847
9969
|
try {
|
|
9848
9970
|
writeFileSync2(statusFilePath, JSON.stringify(status, null, 2));
|
|
9849
9971
|
} catch (err) {
|
|
9850
|
-
|
|
9972
|
+
logger10.error(
|
|
9851
9973
|
`Failed to write status file: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9852
9974
|
);
|
|
9853
9975
|
}
|
|
@@ -9863,27 +9985,97 @@ function createStatusUpdater(statusFilePath, state) {
|
|
|
9863
9985
|
};
|
|
9864
9986
|
}
|
|
9865
9987
|
|
|
9988
|
+
// src/lib/runner/runner-lock.ts
|
|
9989
|
+
import { exec as exec4 } from "child_process";
|
|
9990
|
+
import fs9 from "fs";
|
|
9991
|
+
import path6 from "path";
|
|
9992
|
+
import { promisify as promisify4 } from "util";
|
|
9993
|
+
var execAsync4 = promisify4(exec4);
|
|
9994
|
+
var logger11 = createLogger("RunnerLock");
|
|
9995
|
+
var DEFAULT_RUN_DIR = "/var/run/vm0";
|
|
9996
|
+
var DEFAULT_PID_FILE = `${DEFAULT_RUN_DIR}/runner.pid`;
|
|
9997
|
+
var currentPidFile = null;
|
|
9998
|
+
async function ensureRunDir2(dirPath, skipSudo) {
|
|
9999
|
+
if (!fs9.existsSync(dirPath)) {
|
|
10000
|
+
if (skipSudo) {
|
|
10001
|
+
fs9.mkdirSync(dirPath, { recursive: true });
|
|
10002
|
+
} else {
|
|
10003
|
+
await execAsync4(`sudo mkdir -p ${dirPath}`);
|
|
10004
|
+
await execAsync4(`sudo chmod 777 ${dirPath}`);
|
|
10005
|
+
}
|
|
10006
|
+
}
|
|
10007
|
+
}
|
|
10008
|
+
function isProcessRunning(pid) {
|
|
10009
|
+
try {
|
|
10010
|
+
process.kill(pid, 0);
|
|
10011
|
+
return true;
|
|
10012
|
+
} catch (err) {
|
|
10013
|
+
if (err instanceof Error && "code" in err && err.code === "EPERM") {
|
|
10014
|
+
return true;
|
|
10015
|
+
}
|
|
10016
|
+
return false;
|
|
10017
|
+
}
|
|
10018
|
+
}
|
|
10019
|
+
async function acquireRunnerLock(options = {}) {
|
|
10020
|
+
const pidFile = options.pidFile ?? DEFAULT_PID_FILE;
|
|
10021
|
+
const skipSudo = options.skipSudo ?? false;
|
|
10022
|
+
const runDir = path6.dirname(pidFile);
|
|
10023
|
+
await ensureRunDir2(runDir, skipSudo);
|
|
10024
|
+
if (fs9.existsSync(pidFile)) {
|
|
10025
|
+
const pidStr = fs9.readFileSync(pidFile, "utf-8").trim();
|
|
10026
|
+
const pid = parseInt(pidStr, 10);
|
|
10027
|
+
if (!isNaN(pid) && isProcessRunning(pid)) {
|
|
10028
|
+
logger11.error(`Error: Another runner is already running (PID ${pid})`);
|
|
10029
|
+
logger11.error(`If this is incorrect, remove ${pidFile} and try again.`);
|
|
10030
|
+
process.exit(1);
|
|
10031
|
+
}
|
|
10032
|
+
if (isNaN(pid)) {
|
|
10033
|
+
logger11.log("Cleaning up invalid PID file");
|
|
10034
|
+
} else {
|
|
10035
|
+
logger11.log(`Cleaning up stale PID file (PID ${pid} not running)`);
|
|
10036
|
+
}
|
|
10037
|
+
fs9.unlinkSync(pidFile);
|
|
10038
|
+
}
|
|
10039
|
+
fs9.writeFileSync(pidFile, process.pid.toString());
|
|
10040
|
+
currentPidFile = pidFile;
|
|
10041
|
+
logger11.log(`Runner lock acquired (PID ${process.pid})`);
|
|
10042
|
+
}
|
|
10043
|
+
function releaseRunnerLock() {
|
|
10044
|
+
const pidFile = currentPidFile ?? DEFAULT_PID_FILE;
|
|
10045
|
+
if (fs9.existsSync(pidFile)) {
|
|
10046
|
+
fs9.unlinkSync(pidFile);
|
|
10047
|
+
logger11.log("Runner lock released");
|
|
10048
|
+
}
|
|
10049
|
+
currentPidFile = null;
|
|
10050
|
+
}
|
|
10051
|
+
|
|
9866
10052
|
// src/lib/runner/setup.ts
|
|
9867
|
-
var
|
|
10053
|
+
var logger12 = createLogger("Runner");
|
|
9868
10054
|
async function setupEnvironment(options) {
|
|
9869
10055
|
const { config } = options;
|
|
10056
|
+
await acquireRunnerLock();
|
|
9870
10057
|
const networkCheck = checkNetworkPrerequisites();
|
|
9871
10058
|
if (!networkCheck.ok) {
|
|
9872
|
-
|
|
10059
|
+
logger12.error("Network prerequisites not met:");
|
|
9873
10060
|
for (const error of networkCheck.errors) {
|
|
9874
|
-
|
|
10061
|
+
logger12.error(` - ${error}`);
|
|
9875
10062
|
}
|
|
9876
10063
|
process.exit(1);
|
|
9877
10064
|
}
|
|
9878
|
-
|
|
10065
|
+
logger12.log("Setting up network bridge...");
|
|
9879
10066
|
await setupBridge();
|
|
9880
|
-
|
|
10067
|
+
logger12.log("Flushing bridge ARP cache...");
|
|
9881
10068
|
await flushBridgeArpCache();
|
|
9882
|
-
|
|
10069
|
+
logger12.log("Cleaning up orphaned proxy rules...");
|
|
9883
10070
|
await cleanupOrphanedProxyRules(config.name);
|
|
9884
|
-
|
|
10071
|
+
logger12.log("Cleaning up orphaned IP allocations...");
|
|
9885
10072
|
await cleanupOrphanedAllocations();
|
|
9886
|
-
|
|
10073
|
+
logger12.log("Initializing overlay pool...");
|
|
10074
|
+
await initOverlayPool({
|
|
10075
|
+
size: config.sandbox.max_concurrent + 2,
|
|
10076
|
+
replenishThreshold: config.sandbox.max_concurrent
|
|
10077
|
+
});
|
|
10078
|
+
logger12.log("Initializing network proxy...");
|
|
9887
10079
|
initVMRegistry();
|
|
9888
10080
|
const proxyManager = initProxyManager({
|
|
9889
10081
|
apiUrl: config.server.url,
|
|
@@ -9894,49 +10086,79 @@ async function setupEnvironment(options) {
|
|
|
9894
10086
|
try {
|
|
9895
10087
|
await proxyManager.start();
|
|
9896
10088
|
proxyEnabled = true;
|
|
9897
|
-
|
|
9898
|
-
|
|
10089
|
+
logger12.log("Network proxy initialized successfully");
|
|
10090
|
+
logger12.log("Setting up CIDR proxy rules...");
|
|
9899
10091
|
await setupCIDRProxyRules(config.proxy.port);
|
|
9900
10092
|
} catch (err) {
|
|
9901
|
-
|
|
10093
|
+
logger12.log(
|
|
9902
10094
|
`Network proxy not available: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9903
10095
|
);
|
|
9904
|
-
|
|
10096
|
+
logger12.log(
|
|
9905
10097
|
"Jobs with experimentalFirewall enabled will run without network interception"
|
|
9906
10098
|
);
|
|
9907
10099
|
}
|
|
9908
10100
|
return { proxyEnabled, proxyPort: config.proxy.port };
|
|
9909
10101
|
}
|
|
9910
10102
|
async function cleanupEnvironment(resources) {
|
|
10103
|
+
const errors = [];
|
|
9911
10104
|
if (resources.proxyEnabled) {
|
|
9912
|
-
|
|
9913
|
-
|
|
10105
|
+
try {
|
|
10106
|
+
logger12.log("Cleaning up CIDR proxy rules...");
|
|
10107
|
+
await cleanupCIDRProxyRules(resources.proxyPort);
|
|
10108
|
+
} catch (err) {
|
|
10109
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
10110
|
+
errors.push(error);
|
|
10111
|
+
logger12.error(`Failed to cleanup CIDR proxy rules: ${error.message}`);
|
|
10112
|
+
}
|
|
9914
10113
|
}
|
|
9915
10114
|
if (resources.proxyEnabled) {
|
|
9916
|
-
|
|
9917
|
-
|
|
10115
|
+
try {
|
|
10116
|
+
logger12.log("Stopping network proxy...");
|
|
10117
|
+
await getProxyManager().stop();
|
|
10118
|
+
} catch (err) {
|
|
10119
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
10120
|
+
errors.push(error);
|
|
10121
|
+
logger12.error(`Failed to stop network proxy: ${error.message}`);
|
|
10122
|
+
}
|
|
10123
|
+
}
|
|
10124
|
+
try {
|
|
10125
|
+
cleanupOverlayPool();
|
|
10126
|
+
} catch (err) {
|
|
10127
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
10128
|
+
errors.push(error);
|
|
10129
|
+
logger12.error(`Failed to cleanup overlay pool: ${error.message}`);
|
|
10130
|
+
}
|
|
10131
|
+
try {
|
|
10132
|
+
releaseRunnerLock();
|
|
10133
|
+
} catch (err) {
|
|
10134
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
10135
|
+
errors.push(error);
|
|
10136
|
+
logger12.error(`Failed to release runner lock: ${error.message}`);
|
|
10137
|
+
}
|
|
10138
|
+
if (errors.length > 0) {
|
|
10139
|
+
logger12.error(`Cleanup completed with ${errors.length} error(s)`);
|
|
9918
10140
|
}
|
|
9919
10141
|
}
|
|
9920
10142
|
|
|
9921
10143
|
// src/lib/runner/signals.ts
|
|
9922
|
-
var
|
|
10144
|
+
var logger13 = createLogger("Runner");
|
|
9923
10145
|
function setupSignalHandlers(state, handlers) {
|
|
9924
10146
|
process.on("SIGINT", () => {
|
|
9925
|
-
|
|
9926
|
-
|
|
9927
|
-
state.mode = "stopped";
|
|
10147
|
+
logger13.log("\nShutting down...");
|
|
10148
|
+
state.mode = "stopping";
|
|
9928
10149
|
handlers.updateStatus();
|
|
10150
|
+
handlers.onShutdown();
|
|
9929
10151
|
});
|
|
9930
10152
|
process.on("SIGTERM", () => {
|
|
9931
|
-
|
|
9932
|
-
|
|
9933
|
-
state.mode = "stopped";
|
|
10153
|
+
logger13.log("\nShutting down...");
|
|
10154
|
+
state.mode = "stopping";
|
|
9934
10155
|
handlers.updateStatus();
|
|
10156
|
+
handlers.onShutdown();
|
|
9935
10157
|
});
|
|
9936
10158
|
process.on("SIGUSR1", () => {
|
|
9937
10159
|
if (state.mode === "running") {
|
|
9938
|
-
|
|
9939
|
-
|
|
10160
|
+
logger13.log("\n[Maintenance] Entering drain mode...");
|
|
10161
|
+
logger13.log(
|
|
9940
10162
|
`[Maintenance] Active jobs: ${state.activeRuns.size} (will wait for completion)`
|
|
9941
10163
|
);
|
|
9942
10164
|
state.mode = "draining";
|
|
@@ -9947,7 +10169,7 @@ function setupSignalHandlers(state, handlers) {
|
|
|
9947
10169
|
}
|
|
9948
10170
|
|
|
9949
10171
|
// src/lib/runner/runner.ts
|
|
9950
|
-
var
|
|
10172
|
+
var logger14 = createLogger("Runner");
|
|
9951
10173
|
var Runner = class _Runner {
|
|
9952
10174
|
config;
|
|
9953
10175
|
statusFilePath;
|
|
@@ -9986,41 +10208,43 @@ var Runner = class _Runner {
|
|
|
9986
10208
|
onDrain: () => {
|
|
9987
10209
|
this.pendingJobs.length = 0;
|
|
9988
10210
|
if (this.state.activeRuns.size === 0) {
|
|
9989
|
-
|
|
10211
|
+
logger14.log("[Maintenance] No active jobs, exiting immediately");
|
|
10212
|
+
this.state.mode = "stopping";
|
|
10213
|
+
this.updateStatus();
|
|
9990
10214
|
this.resolveShutdown?.();
|
|
9991
10215
|
}
|
|
9992
10216
|
},
|
|
9993
10217
|
updateStatus: this.updateStatus
|
|
9994
10218
|
});
|
|
9995
|
-
|
|
10219
|
+
logger14.log(
|
|
9996
10220
|
`Starting runner '${this.config.name}' for group '${this.config.group}'...`
|
|
9997
10221
|
);
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
|
|
10222
|
+
logger14.log(`Max concurrent jobs: ${this.config.sandbox.max_concurrent}`);
|
|
10223
|
+
logger14.log(`Status file: ${this.statusFilePath}`);
|
|
10224
|
+
logger14.log("Press Ctrl+C to stop");
|
|
10225
|
+
logger14.log("");
|
|
10002
10226
|
this.updateStatus();
|
|
10003
|
-
|
|
10227
|
+
logger14.log("Checking for pending jobs...");
|
|
10004
10228
|
await this.pollFallback();
|
|
10005
|
-
|
|
10229
|
+
logger14.log("Connecting to realtime job notifications...");
|
|
10006
10230
|
this.subscription = await subscribeToJobs(
|
|
10007
10231
|
this.config.server,
|
|
10008
10232
|
this.config.group,
|
|
10009
10233
|
(notification) => {
|
|
10010
|
-
|
|
10234
|
+
logger14.log(`Ably notification: ${notification.runId}`);
|
|
10011
10235
|
this.processJob(notification.runId).catch(console.error);
|
|
10012
10236
|
},
|
|
10013
10237
|
(connectionState, reason) => {
|
|
10014
|
-
|
|
10238
|
+
logger14.log(
|
|
10015
10239
|
`Ably connection: ${connectionState}${reason ? ` (${reason})` : ""}`
|
|
10016
10240
|
);
|
|
10017
10241
|
}
|
|
10018
10242
|
);
|
|
10019
|
-
|
|
10243
|
+
logger14.log("Connected to realtime job notifications");
|
|
10020
10244
|
this.pollInterval = setInterval(() => {
|
|
10021
10245
|
this.pollFallback().catch(console.error);
|
|
10022
10246
|
}, this.config.sandbox.poll_interval_ms);
|
|
10023
|
-
|
|
10247
|
+
logger14.log(
|
|
10024
10248
|
`Polling fallback enabled (every ${this.config.sandbox.poll_interval_ms / 1e3}s)`
|
|
10025
10249
|
);
|
|
10026
10250
|
await shutdownPromise;
|
|
@@ -10031,7 +10255,7 @@ var Runner = class _Runner {
|
|
|
10031
10255
|
this.subscription.cleanup();
|
|
10032
10256
|
}
|
|
10033
10257
|
if (this.state.jobPromises.size > 0) {
|
|
10034
|
-
|
|
10258
|
+
logger14.log(
|
|
10035
10259
|
`Waiting for ${this.state.jobPromises.size} active job(s) to complete...`
|
|
10036
10260
|
);
|
|
10037
10261
|
await Promise.all(this.state.jobPromises);
|
|
@@ -10039,7 +10263,7 @@ var Runner = class _Runner {
|
|
|
10039
10263
|
await cleanupEnvironment(this.resources);
|
|
10040
10264
|
this.state.mode = "stopped";
|
|
10041
10265
|
this.updateStatus();
|
|
10042
|
-
|
|
10266
|
+
logger14.log("Runner stopped");
|
|
10043
10267
|
process.exit(0);
|
|
10044
10268
|
}
|
|
10045
10269
|
/**
|
|
@@ -10056,11 +10280,11 @@ var Runner = class _Runner {
|
|
|
10056
10280
|
() => pollForJob(this.config.server, this.config.group)
|
|
10057
10281
|
);
|
|
10058
10282
|
if (job) {
|
|
10059
|
-
|
|
10283
|
+
logger14.log(`Poll fallback found job: ${job.runId}`);
|
|
10060
10284
|
await this.processJob(job.runId);
|
|
10061
10285
|
}
|
|
10062
10286
|
} catch (error) {
|
|
10063
|
-
|
|
10287
|
+
logger14.error(
|
|
10064
10288
|
`Poll fallback error: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
10065
10289
|
);
|
|
10066
10290
|
}
|
|
@@ -10070,7 +10294,7 @@ var Runner = class _Runner {
|
|
|
10070
10294
|
*/
|
|
10071
10295
|
async processJob(runId) {
|
|
10072
10296
|
if (this.state.mode !== "running") {
|
|
10073
|
-
|
|
10297
|
+
logger14.log(`Not running (${this.state.mode}), ignoring job ${runId}`);
|
|
10074
10298
|
return;
|
|
10075
10299
|
}
|
|
10076
10300
|
if (this.state.activeRuns.has(runId)) {
|
|
@@ -10078,10 +10302,10 @@ var Runner = class _Runner {
|
|
|
10078
10302
|
}
|
|
10079
10303
|
if (this.state.activeRuns.size >= this.config.sandbox.max_concurrent) {
|
|
10080
10304
|
if (!this.pendingJobs.includes(runId) && this.pendingJobs.length < _Runner.MAX_PENDING_QUEUE_SIZE) {
|
|
10081
|
-
|
|
10305
|
+
logger14.log(`At capacity, queueing job ${runId}`);
|
|
10082
10306
|
this.pendingJobs.push(runId);
|
|
10083
10307
|
} else if (this.pendingJobs.length >= _Runner.MAX_PENDING_QUEUE_SIZE) {
|
|
10084
|
-
|
|
10308
|
+
logger14.log(
|
|
10085
10309
|
`Pending queue full (${_Runner.MAX_PENDING_QUEUE_SIZE}), dropping job ${runId}`
|
|
10086
10310
|
);
|
|
10087
10311
|
}
|
|
@@ -10092,11 +10316,11 @@ var Runner = class _Runner {
|
|
|
10092
10316
|
"claim",
|
|
10093
10317
|
() => claimJob(this.config.server, runId)
|
|
10094
10318
|
);
|
|
10095
|
-
|
|
10319
|
+
logger14.log(`Claimed job: ${context.runId}`);
|
|
10096
10320
|
this.state.activeRuns.add(context.runId);
|
|
10097
10321
|
this.updateStatus();
|
|
10098
10322
|
const jobPromise = this.executeJob(context).catch((error) => {
|
|
10099
|
-
|
|
10323
|
+
logger14.error(
|
|
10100
10324
|
`Job ${context.runId} failed: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
10101
10325
|
);
|
|
10102
10326
|
}).finally(() => {
|
|
@@ -10104,7 +10328,9 @@ var Runner = class _Runner {
|
|
|
10104
10328
|
this.state.jobPromises.delete(jobPromise);
|
|
10105
10329
|
this.updateStatus();
|
|
10106
10330
|
if (this.state.mode === "draining" && this.state.activeRuns.size === 0) {
|
|
10107
|
-
|
|
10331
|
+
logger14.log("[Maintenance] All jobs completed, exiting");
|
|
10332
|
+
this.state.mode = "stopping";
|
|
10333
|
+
this.updateStatus();
|
|
10108
10334
|
this.resolveShutdown?.();
|
|
10109
10335
|
return;
|
|
10110
10336
|
}
|
|
@@ -10117,33 +10343,33 @@ var Runner = class _Runner {
|
|
|
10117
10343
|
});
|
|
10118
10344
|
this.state.jobPromises.add(jobPromise);
|
|
10119
10345
|
} catch (error) {
|
|
10120
|
-
|
|
10346
|
+
logger14.log(
|
|
10121
10347
|
`Could not claim job ${runId}: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
10122
10348
|
);
|
|
10123
10349
|
}
|
|
10124
10350
|
}
|
|
10125
10351
|
async executeJob(context) {
|
|
10126
|
-
|
|
10127
|
-
|
|
10128
|
-
|
|
10352
|
+
logger14.log(` Executing job ${context.runId}...`);
|
|
10353
|
+
logger14.log(` Prompt: ${context.prompt.substring(0, 100)}...`);
|
|
10354
|
+
logger14.log(` Compose version: ${context.agentComposeVersionId}`);
|
|
10129
10355
|
try {
|
|
10130
10356
|
const result = await executeJob(context, this.config);
|
|
10131
|
-
|
|
10357
|
+
logger14.log(
|
|
10132
10358
|
` Job ${context.runId} execution completed with exit code ${result.exitCode}`
|
|
10133
10359
|
);
|
|
10134
10360
|
if (result.exitCode !== 0 && result.error) {
|
|
10135
|
-
|
|
10361
|
+
logger14.error(` Job ${context.runId} failed: ${result.error}`);
|
|
10136
10362
|
}
|
|
10137
10363
|
} catch (err) {
|
|
10138
10364
|
const error = err instanceof Error ? err.message : "Unknown execution error";
|
|
10139
|
-
|
|
10365
|
+
logger14.error(` Job ${context.runId} execution failed: ${error}`);
|
|
10140
10366
|
const result = await completeJob(
|
|
10141
10367
|
this.config.server.url,
|
|
10142
10368
|
context,
|
|
10143
10369
|
1,
|
|
10144
10370
|
error
|
|
10145
10371
|
);
|
|
10146
|
-
|
|
10372
|
+
logger14.log(` Job ${context.runId} reported as ${result.status}`);
|
|
10147
10373
|
}
|
|
10148
10374
|
}
|
|
10149
10375
|
};
|
|
@@ -10174,7 +10400,7 @@ import { dirname as dirname2, join as join3 } from "path";
|
|
|
10174
10400
|
|
|
10175
10401
|
// src/lib/firecracker/process.ts
|
|
10176
10402
|
import { readdirSync, readFileSync as readFileSync2, existsSync as existsSync3 } from "fs";
|
|
10177
|
-
import
|
|
10403
|
+
import path7 from "path";
|
|
10178
10404
|
function parseFirecrackerCmdline(cmdline) {
|
|
10179
10405
|
const args = cmdline.split("\0");
|
|
10180
10406
|
if (!args[0]?.includes("firecracker")) return null;
|
|
@@ -10207,7 +10433,7 @@ function findFirecrackerProcesses() {
|
|
|
10207
10433
|
for (const entry of entries) {
|
|
10208
10434
|
if (!/^\d+$/.test(entry)) continue;
|
|
10209
10435
|
const pid = parseInt(entry, 10);
|
|
10210
|
-
const cmdlinePath =
|
|
10436
|
+
const cmdlinePath = path7.join(procDir, entry, "cmdline");
|
|
10211
10437
|
if (!existsSync3(cmdlinePath)) continue;
|
|
10212
10438
|
try {
|
|
10213
10439
|
const cmdline = readFileSync2(cmdlinePath, "utf-8");
|
|
@@ -10225,7 +10451,7 @@ function findProcessByVmId(vmId) {
|
|
|
10225
10451
|
const processes = findFirecrackerProcesses();
|
|
10226
10452
|
return processes.find((p) => p.vmId === vmId) || null;
|
|
10227
10453
|
}
|
|
10228
|
-
function
|
|
10454
|
+
function isProcessRunning2(pid) {
|
|
10229
10455
|
try {
|
|
10230
10456
|
process.kill(pid, 0);
|
|
10231
10457
|
return true;
|
|
@@ -10234,24 +10460,24 @@ function isProcessRunning(pid) {
|
|
|
10234
10460
|
}
|
|
10235
10461
|
}
|
|
10236
10462
|
async function killProcess(pid, timeoutMs = 5e3) {
|
|
10237
|
-
if (!
|
|
10463
|
+
if (!isProcessRunning2(pid)) return true;
|
|
10238
10464
|
try {
|
|
10239
10465
|
process.kill(pid, "SIGTERM");
|
|
10240
10466
|
} catch {
|
|
10241
|
-
return !
|
|
10467
|
+
return !isProcessRunning2(pid);
|
|
10242
10468
|
}
|
|
10243
10469
|
const startTime = Date.now();
|
|
10244
10470
|
while (Date.now() - startTime < timeoutMs) {
|
|
10245
|
-
if (!
|
|
10471
|
+
if (!isProcessRunning2(pid)) return true;
|
|
10246
10472
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
10247
10473
|
}
|
|
10248
|
-
if (
|
|
10474
|
+
if (isProcessRunning2(pid)) {
|
|
10249
10475
|
try {
|
|
10250
10476
|
process.kill(pid, "SIGKILL");
|
|
10251
10477
|
} catch {
|
|
10252
10478
|
}
|
|
10253
10479
|
}
|
|
10254
|
-
return !
|
|
10480
|
+
return !isProcessRunning2(pid);
|
|
10255
10481
|
}
|
|
10256
10482
|
function findMitmproxyProcess() {
|
|
10257
10483
|
const procDir = "/proc";
|
|
@@ -10264,7 +10490,7 @@ function findMitmproxyProcess() {
|
|
|
10264
10490
|
for (const entry of entries) {
|
|
10265
10491
|
if (!/^\d+$/.test(entry)) continue;
|
|
10266
10492
|
const pid = parseInt(entry, 10);
|
|
10267
|
-
const cmdlinePath =
|
|
10493
|
+
const cmdlinePath = path7.join(procDir, entry, "cmdline");
|
|
10268
10494
|
if (!existsSync3(cmdlinePath)) continue;
|
|
10269
10495
|
try {
|
|
10270
10496
|
const cmdline = readFileSync2(cmdlinePath, "utf-8");
|
|
@@ -10736,6 +10962,12 @@ var benchmarkCommand = new Command4("benchmark").description(
|
|
|
10736
10962
|
}
|
|
10737
10963
|
timer.log("Setting up network bridge...");
|
|
10738
10964
|
await setupBridge();
|
|
10965
|
+
timer.log("Initializing overlay pool...");
|
|
10966
|
+
await initOverlayPool({
|
|
10967
|
+
size: 2,
|
|
10968
|
+
replenishThreshold: 1,
|
|
10969
|
+
poolDir: "/var/run/vm0/overlay-pool-benchmark"
|
|
10970
|
+
});
|
|
10739
10971
|
timer.log(`Executing command: ${prompt}`);
|
|
10740
10972
|
const context = createBenchmarkContext(prompt, options);
|
|
10741
10973
|
const result = await executeJob(context, config, {
|
|
@@ -10756,7 +10988,7 @@ var benchmarkCommand = new Command4("benchmark").description(
|
|
|
10756
10988
|
});
|
|
10757
10989
|
|
|
10758
10990
|
// src/index.ts
|
|
10759
|
-
var version = true ? "3.
|
|
10991
|
+
var version = true ? "3.7.1" : "0.1.0";
|
|
10760
10992
|
program.name("vm0-runner").version(version).description("Self-hosted runner for VM0 agents");
|
|
10761
10993
|
program.addCommand(startCommand);
|
|
10762
10994
|
program.addCommand(doctorCommand);
|