@vm0/runner 3.6.2 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +459 -212
- package/package.json +1 -1
package/index.js
CHANGED
|
@@ -272,14 +272,13 @@ async function subscribeToJobs(server, group, onJob, onConnectionChange) {
|
|
|
272
272
|
}
|
|
273
273
|
|
|
274
274
|
// src/lib/executor.ts
|
|
275
|
-
import
|
|
275
|
+
import path5 from "path";
|
|
276
276
|
|
|
277
277
|
// src/lib/firecracker/vm.ts
|
|
278
|
-
import {
|
|
279
|
-
import
|
|
280
|
-
import
|
|
278
|
+
import { spawn } from "child_process";
|
|
279
|
+
import fs4 from "fs";
|
|
280
|
+
import path3 from "path";
|
|
281
281
|
import readline from "readline";
|
|
282
|
-
import { promisify as promisify3 } from "util";
|
|
283
282
|
|
|
284
283
|
// src/lib/firecracker/client.ts
|
|
285
284
|
import http from "http";
|
|
@@ -291,7 +290,7 @@ var FirecrackerClient = class {
|
|
|
291
290
|
/**
|
|
292
291
|
* Make HTTP request to Firecracker API
|
|
293
292
|
*/
|
|
294
|
-
async request(method,
|
|
293
|
+
async request(method, path8, body) {
|
|
295
294
|
return new Promise((resolve, reject) => {
|
|
296
295
|
const bodyStr = body !== void 0 ? JSON.stringify(body) : void 0;
|
|
297
296
|
const headers = {
|
|
@@ -304,11 +303,11 @@ var FirecrackerClient = class {
|
|
|
304
303
|
headers["Content-Length"] = Buffer.byteLength(bodyStr);
|
|
305
304
|
}
|
|
306
305
|
console.log(
|
|
307
|
-
`[FC API] ${method} ${
|
|
306
|
+
`[FC API] ${method} ${path8}${bodyStr ? ` (${Buffer.byteLength(bodyStr)} bytes)` : ""}`
|
|
308
307
|
);
|
|
309
308
|
const options = {
|
|
310
309
|
socketPath: this.socketPath,
|
|
311
|
-
path:
|
|
310
|
+
path: path8,
|
|
312
311
|
method,
|
|
313
312
|
headers,
|
|
314
313
|
// Disable agent to ensure fresh connection for each request
|
|
@@ -1064,9 +1063,146 @@ async function cleanupOrphanedProxyRules(runnerName) {
|
|
|
1064
1063
|
}
|
|
1065
1064
|
}
|
|
1066
1065
|
|
|
1067
|
-
// src/lib/firecracker/
|
|
1066
|
+
// src/lib/firecracker/overlay-pool.ts
|
|
1067
|
+
import { exec as exec3 } from "child_process";
|
|
1068
|
+
import { randomUUID } from "crypto";
|
|
1069
|
+
import fs3 from "fs";
|
|
1070
|
+
import path2 from "path";
|
|
1071
|
+
import { promisify as promisify3 } from "util";
|
|
1068
1072
|
var execAsync3 = promisify3(exec3);
|
|
1069
|
-
var logger3 = createLogger("
|
|
1073
|
+
var logger3 = createLogger("OverlayPool");
|
|
1074
|
+
var VM0_RUN_DIR2 = "/var/run/vm0";
|
|
1075
|
+
var POOL_DIR = path2.join(VM0_RUN_DIR2, "overlay-pool");
|
|
1076
|
+
var OVERLAY_SIZE = 2 * 1024 * 1024 * 1024;
|
|
1077
|
+
var poolState = {
|
|
1078
|
+
initialized: false,
|
|
1079
|
+
config: null,
|
|
1080
|
+
queue: [],
|
|
1081
|
+
replenishing: false
|
|
1082
|
+
};
|
|
1083
|
+
async function ensurePoolDir() {
|
|
1084
|
+
if (!fs3.existsSync(VM0_RUN_DIR2)) {
|
|
1085
|
+
await execAsync3(`sudo mkdir -p ${VM0_RUN_DIR2}`);
|
|
1086
|
+
await execAsync3(`sudo chmod 777 ${VM0_RUN_DIR2}`);
|
|
1087
|
+
}
|
|
1088
|
+
if (!fs3.existsSync(POOL_DIR)) {
|
|
1089
|
+
fs3.mkdirSync(POOL_DIR, { recursive: true });
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
async function createOverlayFile(filePath) {
|
|
1093
|
+
const fd = fs3.openSync(filePath, "w");
|
|
1094
|
+
fs3.ftruncateSync(fd, OVERLAY_SIZE);
|
|
1095
|
+
fs3.closeSync(fd);
|
|
1096
|
+
await execAsync3(`mkfs.ext4 -F -q "${filePath}"`);
|
|
1097
|
+
}
|
|
1098
|
+
function generateFileName() {
|
|
1099
|
+
return `overlay-${randomUUID()}.ext4`;
|
|
1100
|
+
}
|
|
1101
|
+
function scanPoolDir() {
|
|
1102
|
+
if (!fs3.existsSync(POOL_DIR)) {
|
|
1103
|
+
return [];
|
|
1104
|
+
}
|
|
1105
|
+
return fs3.readdirSync(POOL_DIR).filter((f) => f.startsWith("overlay-") && f.endsWith(".ext4")).map((f) => path2.join(POOL_DIR, f));
|
|
1106
|
+
}
|
|
1107
|
+
async function replenishPool() {
|
|
1108
|
+
if (poolState.replenishing || !poolState.initialized || !poolState.config) {
|
|
1109
|
+
return;
|
|
1110
|
+
}
|
|
1111
|
+
const needed = poolState.config.size - poolState.queue.length;
|
|
1112
|
+
if (needed <= 0) {
|
|
1113
|
+
return;
|
|
1114
|
+
}
|
|
1115
|
+
poolState.replenishing = true;
|
|
1116
|
+
logger3.log(`Replenishing pool: creating ${needed} overlay(s)...`);
|
|
1117
|
+
try {
|
|
1118
|
+
const promises = [];
|
|
1119
|
+
for (let i = 0; i < needed; i++) {
|
|
1120
|
+
const filePath = path2.join(POOL_DIR, generateFileName());
|
|
1121
|
+
promises.push(
|
|
1122
|
+
createOverlayFile(filePath).then(() => {
|
|
1123
|
+
poolState.queue.push(filePath);
|
|
1124
|
+
})
|
|
1125
|
+
);
|
|
1126
|
+
}
|
|
1127
|
+
await Promise.all(promises);
|
|
1128
|
+
logger3.log(`Pool replenished: ${poolState.queue.length} available`);
|
|
1129
|
+
} catch (err) {
|
|
1130
|
+
logger3.error(
|
|
1131
|
+
`Replenish failed: ${err instanceof Error ? err.message : "Unknown"}`
|
|
1132
|
+
);
|
|
1133
|
+
} finally {
|
|
1134
|
+
poolState.replenishing = false;
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
async function initOverlayPool(config) {
|
|
1138
|
+
poolState.config = config;
|
|
1139
|
+
poolState.queue = [];
|
|
1140
|
+
logger3.log(
|
|
1141
|
+
`Initializing overlay pool (size=${config.size}, threshold=${config.replenishThreshold})...`
|
|
1142
|
+
);
|
|
1143
|
+
await ensurePoolDir();
|
|
1144
|
+
const existing = scanPoolDir();
|
|
1145
|
+
if (existing.length > 0) {
|
|
1146
|
+
logger3.log(`Cleaning up ${existing.length} stale overlay(s)`);
|
|
1147
|
+
for (const file of existing) {
|
|
1148
|
+
fs3.unlinkSync(file);
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
poolState.initialized = true;
|
|
1152
|
+
await replenishPool();
|
|
1153
|
+
logger3.log("Overlay pool initialized");
|
|
1154
|
+
}
|
|
1155
|
+
async function acquireOverlay() {
|
|
1156
|
+
const filePath = poolState.queue.shift();
|
|
1157
|
+
if (filePath) {
|
|
1158
|
+
logger3.log(
|
|
1159
|
+
`Acquired overlay from pool (${poolState.queue.length} remaining)`
|
|
1160
|
+
);
|
|
1161
|
+
if (poolState.config && poolState.queue.length < poolState.config.replenishThreshold) {
|
|
1162
|
+
replenishPool().catch((err) => {
|
|
1163
|
+
logger3.error(
|
|
1164
|
+
`Background replenish failed: ${err instanceof Error ? err.message : "Unknown"}`
|
|
1165
|
+
);
|
|
1166
|
+
});
|
|
1167
|
+
}
|
|
1168
|
+
return filePath;
|
|
1169
|
+
}
|
|
1170
|
+
logger3.log("Pool exhausted, creating overlay on-demand");
|
|
1171
|
+
const newPath = path2.join(POOL_DIR, generateFileName());
|
|
1172
|
+
await createOverlayFile(newPath);
|
|
1173
|
+
return newPath;
|
|
1174
|
+
}
|
|
1175
|
+
function cleanupOverlayPool() {
|
|
1176
|
+
if (!poolState.initialized) {
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1179
|
+
logger3.log("Cleaning up overlay pool...");
|
|
1180
|
+
for (const file of poolState.queue) {
|
|
1181
|
+
try {
|
|
1182
|
+
fs3.unlinkSync(file);
|
|
1183
|
+
} catch (err) {
|
|
1184
|
+
logger3.log(
|
|
1185
|
+
`Failed to delete ${file}: ${err instanceof Error ? err.message : "Unknown"}`
|
|
1186
|
+
);
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
poolState.queue = [];
|
|
1190
|
+
for (const file of scanPoolDir()) {
|
|
1191
|
+
try {
|
|
1192
|
+
fs3.unlinkSync(file);
|
|
1193
|
+
} catch (err) {
|
|
1194
|
+
logger3.log(
|
|
1195
|
+
`Failed to delete ${file}: ${err instanceof Error ? err.message : "Unknown"}`
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
poolState.initialized = false;
|
|
1200
|
+
poolState.replenishing = false;
|
|
1201
|
+
logger3.log("Overlay pool cleaned up");
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
// src/lib/firecracker/vm.ts
|
|
1205
|
+
var logger4 = createLogger("VM");
|
|
1070
1206
|
var FirecrackerVM = class {
|
|
1071
1207
|
config;
|
|
1072
1208
|
process = null;
|
|
@@ -1075,16 +1211,15 @@ var FirecrackerVM = class {
|
|
|
1075
1211
|
state = "created";
|
|
1076
1212
|
workDir;
|
|
1077
1213
|
socketPath;
|
|
1078
|
-
vmOverlayPath;
|
|
1079
|
-
//
|
|
1214
|
+
vmOverlayPath = null;
|
|
1215
|
+
// Set by acquireOverlay() during start
|
|
1080
1216
|
vsockPath;
|
|
1081
1217
|
// Vsock UDS path for host-guest communication
|
|
1082
1218
|
constructor(config) {
|
|
1083
1219
|
this.config = config;
|
|
1084
1220
|
this.workDir = config.workDir || `/tmp/vm0-vm-${config.vmId}`;
|
|
1085
|
-
this.socketPath =
|
|
1086
|
-
this.
|
|
1087
|
-
this.vsockPath = path2.join(this.workDir, "vsock.sock");
|
|
1221
|
+
this.socketPath = path3.join(this.workDir, "firecracker.sock");
|
|
1222
|
+
this.vsockPath = path3.join(this.workDir, "vsock.sock");
|
|
1088
1223
|
}
|
|
1089
1224
|
/**
|
|
1090
1225
|
* Get current VM state
|
|
@@ -1125,25 +1260,23 @@ var FirecrackerVM = class {
|
|
|
1125
1260
|
throw new Error(`Cannot start VM in state: ${this.state}`);
|
|
1126
1261
|
}
|
|
1127
1262
|
try {
|
|
1128
|
-
|
|
1129
|
-
if (
|
|
1130
|
-
|
|
1263
|
+
fs4.mkdirSync(this.workDir, { recursive: true });
|
|
1264
|
+
if (fs4.existsSync(this.socketPath)) {
|
|
1265
|
+
fs4.unlinkSync(this.socketPath);
|
|
1131
1266
|
}
|
|
1132
|
-
|
|
1133
|
-
const
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
await execAsync3(`mkfs.ext4 -F -q "${this.vmOverlayPath}"`);
|
|
1139
|
-
logger3.log(`[VM ${this.config.vmId}] Overlay created`);
|
|
1267
|
+
logger4.log(`[VM ${this.config.vmId}] Setting up overlay and network...`);
|
|
1268
|
+
const setupOverlay = async () => {
|
|
1269
|
+
this.vmOverlayPath = await acquireOverlay();
|
|
1270
|
+
logger4.log(
|
|
1271
|
+
`[VM ${this.config.vmId}] Overlay acquired: ${this.vmOverlayPath}`
|
|
1272
|
+
);
|
|
1140
1273
|
};
|
|
1141
1274
|
const [, networkConfig] = await Promise.all([
|
|
1142
|
-
|
|
1275
|
+
setupOverlay(),
|
|
1143
1276
|
createTapDevice(this.config.vmId)
|
|
1144
1277
|
]);
|
|
1145
1278
|
this.networkConfig = networkConfig;
|
|
1146
|
-
|
|
1279
|
+
logger4.log(`[VM ${this.config.vmId}] Starting Firecracker...`);
|
|
1147
1280
|
this.process = spawn(
|
|
1148
1281
|
this.config.firecrackerBinary,
|
|
1149
1282
|
["--api-sock", this.socketPath],
|
|
@@ -1154,11 +1287,11 @@ var FirecrackerVM = class {
|
|
|
1154
1287
|
}
|
|
1155
1288
|
);
|
|
1156
1289
|
this.process.on("error", (err) => {
|
|
1157
|
-
|
|
1290
|
+
logger4.log(`[VM ${this.config.vmId}] Firecracker error: ${err}`);
|
|
1158
1291
|
this.state = "error";
|
|
1159
1292
|
});
|
|
1160
1293
|
this.process.on("exit", (code, signal) => {
|
|
1161
|
-
|
|
1294
|
+
logger4.log(
|
|
1162
1295
|
`[VM ${this.config.vmId}] Firecracker exited: code=${code}, signal=${signal}`
|
|
1163
1296
|
);
|
|
1164
1297
|
if (this.state !== "stopped") {
|
|
@@ -1171,7 +1304,7 @@ var FirecrackerVM = class {
|
|
|
1171
1304
|
});
|
|
1172
1305
|
stdoutRL.on("line", (line) => {
|
|
1173
1306
|
if (line.trim()) {
|
|
1174
|
-
|
|
1307
|
+
logger4.log(`[VM ${this.config.vmId}] ${line}`);
|
|
1175
1308
|
}
|
|
1176
1309
|
});
|
|
1177
1310
|
}
|
|
@@ -1181,19 +1314,19 @@ var FirecrackerVM = class {
|
|
|
1181
1314
|
});
|
|
1182
1315
|
stderrRL.on("line", (line) => {
|
|
1183
1316
|
if (line.trim()) {
|
|
1184
|
-
|
|
1317
|
+
logger4.log(`[VM ${this.config.vmId}] stderr: ${line}`);
|
|
1185
1318
|
}
|
|
1186
1319
|
});
|
|
1187
1320
|
}
|
|
1188
1321
|
this.client = new FirecrackerClient(this.socketPath);
|
|
1189
|
-
|
|
1322
|
+
logger4.log(`[VM ${this.config.vmId}] Waiting for API...`);
|
|
1190
1323
|
await this.client.waitUntilReady(1e4, 100);
|
|
1191
1324
|
this.state = "configuring";
|
|
1192
1325
|
await this.configure();
|
|
1193
|
-
|
|
1326
|
+
logger4.log(`[VM ${this.config.vmId}] Booting...`);
|
|
1194
1327
|
await this.client.start();
|
|
1195
1328
|
this.state = "running";
|
|
1196
|
-
|
|
1329
|
+
logger4.log(
|
|
1197
1330
|
`[VM ${this.config.vmId}] Running at ${this.networkConfig.guestIp}`
|
|
1198
1331
|
);
|
|
1199
1332
|
} catch (error) {
|
|
@@ -1206,10 +1339,10 @@ var FirecrackerVM = class {
|
|
|
1206
1339
|
* Configure the VM via Firecracker API
|
|
1207
1340
|
*/
|
|
1208
1341
|
async configure() {
|
|
1209
|
-
if (!this.client || !this.networkConfig) {
|
|
1342
|
+
if (!this.client || !this.networkConfig || !this.vmOverlayPath) {
|
|
1210
1343
|
throw new Error("VM not properly initialized");
|
|
1211
1344
|
}
|
|
1212
|
-
|
|
1345
|
+
logger4.log(
|
|
1213
1346
|
`[VM ${this.config.vmId}] Configuring: ${this.config.vcpus} vCPUs, ${this.config.memoryMb}MB RAM`
|
|
1214
1347
|
);
|
|
1215
1348
|
await this.client.setMachineConfig({
|
|
@@ -1218,12 +1351,12 @@ var FirecrackerVM = class {
|
|
|
1218
1351
|
});
|
|
1219
1352
|
const networkBootArgs = generateNetworkBootArgs(this.networkConfig);
|
|
1220
1353
|
const bootArgs = `console=ttyS0 reboot=k panic=1 pci=off nomodules random.trust_cpu=on quiet loglevel=0 nokaslr audit=0 numa=off mitigations=off noresume init=/sbin/vm-init ${networkBootArgs}`;
|
|
1221
|
-
|
|
1354
|
+
logger4.log(`[VM ${this.config.vmId}] Boot args: ${bootArgs}`);
|
|
1222
1355
|
await this.client.setBootSource({
|
|
1223
1356
|
kernel_image_path: this.config.kernelPath,
|
|
1224
1357
|
boot_args: bootArgs
|
|
1225
1358
|
});
|
|
1226
|
-
|
|
1359
|
+
logger4.log(
|
|
1227
1360
|
`[VM ${this.config.vmId}] Base rootfs: ${this.config.rootfsPath}`
|
|
1228
1361
|
);
|
|
1229
1362
|
await this.client.setDrive({
|
|
@@ -1232,14 +1365,14 @@ var FirecrackerVM = class {
|
|
|
1232
1365
|
is_root_device: true,
|
|
1233
1366
|
is_read_only: true
|
|
1234
1367
|
});
|
|
1235
|
-
|
|
1368
|
+
logger4.log(`[VM ${this.config.vmId}] Overlay: ${this.vmOverlayPath}`);
|
|
1236
1369
|
await this.client.setDrive({
|
|
1237
1370
|
drive_id: "overlay",
|
|
1238
1371
|
path_on_host: this.vmOverlayPath,
|
|
1239
1372
|
is_root_device: false,
|
|
1240
1373
|
is_read_only: false
|
|
1241
1374
|
});
|
|
1242
|
-
|
|
1375
|
+
logger4.log(
|
|
1243
1376
|
`[VM ${this.config.vmId}] Network: ${this.networkConfig.tapDevice}`
|
|
1244
1377
|
);
|
|
1245
1378
|
await this.client.setNetworkInterface({
|
|
@@ -1247,7 +1380,7 @@ var FirecrackerVM = class {
|
|
|
1247
1380
|
guest_mac: this.networkConfig.guestMac,
|
|
1248
1381
|
host_dev_name: this.networkConfig.tapDevice
|
|
1249
1382
|
});
|
|
1250
|
-
|
|
1383
|
+
logger4.log(`[VM ${this.config.vmId}] Vsock: ${this.vsockPath}`);
|
|
1251
1384
|
await this.client.setVsock({
|
|
1252
1385
|
vsock_id: "vsock0",
|
|
1253
1386
|
guest_cid: 3,
|
|
@@ -1259,15 +1392,15 @@ var FirecrackerVM = class {
|
|
|
1259
1392
|
*/
|
|
1260
1393
|
async stop() {
|
|
1261
1394
|
if (this.state !== "running") {
|
|
1262
|
-
|
|
1395
|
+
logger4.log(`[VM ${this.config.vmId}] Not running, state: ${this.state}`);
|
|
1263
1396
|
return;
|
|
1264
1397
|
}
|
|
1265
1398
|
this.state = "stopping";
|
|
1266
|
-
|
|
1399
|
+
logger4.log(`[VM ${this.config.vmId}] Stopping...`);
|
|
1267
1400
|
try {
|
|
1268
1401
|
if (this.client) {
|
|
1269
1402
|
await this.client.sendCtrlAltDel().catch((error) => {
|
|
1270
|
-
|
|
1403
|
+
logger4.log(
|
|
1271
1404
|
`[VM ${this.config.vmId}] Graceful shutdown signal failed (VM may already be stopping): ${error instanceof Error ? error.message : error}`
|
|
1272
1405
|
);
|
|
1273
1406
|
});
|
|
@@ -1280,7 +1413,7 @@ var FirecrackerVM = class {
|
|
|
1280
1413
|
* Force kill the VM
|
|
1281
1414
|
*/
|
|
1282
1415
|
async kill() {
|
|
1283
|
-
|
|
1416
|
+
logger4.log(`[VM ${this.config.vmId}] Force killing...`);
|
|
1284
1417
|
await this.cleanup();
|
|
1285
1418
|
}
|
|
1286
1419
|
/**
|
|
@@ -1300,12 +1433,16 @@ var FirecrackerVM = class {
|
|
|
1300
1433
|
);
|
|
1301
1434
|
this.networkConfig = null;
|
|
1302
1435
|
}
|
|
1303
|
-
if (
|
|
1304
|
-
|
|
1436
|
+
if (this.vmOverlayPath && fs4.existsSync(this.vmOverlayPath)) {
|
|
1437
|
+
fs4.unlinkSync(this.vmOverlayPath);
|
|
1438
|
+
this.vmOverlayPath = null;
|
|
1439
|
+
}
|
|
1440
|
+
if (fs4.existsSync(this.workDir)) {
|
|
1441
|
+
fs4.rmSync(this.workDir, { recursive: true, force: true });
|
|
1305
1442
|
}
|
|
1306
1443
|
this.client = null;
|
|
1307
1444
|
this.state = "stopped";
|
|
1308
|
-
|
|
1445
|
+
logger4.log(`[VM ${this.config.vmId}] Stopped`);
|
|
1309
1446
|
}
|
|
1310
1447
|
/**
|
|
1311
1448
|
* Wait for the VM process to exit
|
|
@@ -1336,7 +1473,7 @@ var FirecrackerVM = class {
|
|
|
1336
1473
|
|
|
1337
1474
|
// src/lib/firecracker/vsock.ts
|
|
1338
1475
|
import * as net from "net";
|
|
1339
|
-
import * as
|
|
1476
|
+
import * as fs5 from "fs";
|
|
1340
1477
|
var VSOCK_PORT = 1e3;
|
|
1341
1478
|
var HEADER_SIZE = 4;
|
|
1342
1479
|
var MAX_MESSAGE_SIZE = 16 * 1024 * 1024;
|
|
@@ -1368,8 +1505,8 @@ function encodeExecPayload(command, timeoutMs) {
|
|
|
1368
1505
|
cmdBuf.copy(payload, 8);
|
|
1369
1506
|
return payload;
|
|
1370
1507
|
}
|
|
1371
|
-
function encodeWriteFilePayload(
|
|
1372
|
-
const pathBuf = Buffer.from(
|
|
1508
|
+
function encodeWriteFilePayload(path8, content, sudo) {
|
|
1509
|
+
const pathBuf = Buffer.from(path8, "utf-8");
|
|
1373
1510
|
if (pathBuf.length > 65535) {
|
|
1374
1511
|
throw new Error(`Path too long: ${pathBuf.length} bytes (max 65535)`);
|
|
1375
1512
|
}
|
|
@@ -1676,8 +1813,8 @@ var VsockClient = class {
|
|
|
1676
1813
|
return;
|
|
1677
1814
|
}
|
|
1678
1815
|
const listenerPath = `${this.vsockPath}_${VSOCK_PORT}`;
|
|
1679
|
-
if (
|
|
1680
|
-
|
|
1816
|
+
if (fs5.existsSync(listenerPath)) {
|
|
1817
|
+
fs5.unlinkSync(listenerPath);
|
|
1681
1818
|
}
|
|
1682
1819
|
return new Promise((resolve, reject) => {
|
|
1683
1820
|
const server = net.createServer();
|
|
@@ -1687,8 +1824,8 @@ var VsockClient = class {
|
|
|
1687
1824
|
if (!settled) {
|
|
1688
1825
|
settled = true;
|
|
1689
1826
|
server.close();
|
|
1690
|
-
if (
|
|
1691
|
-
|
|
1827
|
+
if (fs5.existsSync(listenerPath)) {
|
|
1828
|
+
fs5.unlinkSync(listenerPath);
|
|
1692
1829
|
}
|
|
1693
1830
|
reject(new Error(`Guest connection timeout after ${timeoutMs}ms`));
|
|
1694
1831
|
}
|
|
@@ -1698,8 +1835,8 @@ var VsockClient = class {
|
|
|
1698
1835
|
settled = true;
|
|
1699
1836
|
clearTimeout(timeout);
|
|
1700
1837
|
server.close();
|
|
1701
|
-
if (
|
|
1702
|
-
|
|
1838
|
+
if (fs5.existsSync(listenerPath)) {
|
|
1839
|
+
fs5.unlinkSync(listenerPath);
|
|
1703
1840
|
}
|
|
1704
1841
|
reject(err);
|
|
1705
1842
|
}
|
|
@@ -1731,8 +1868,8 @@ var VsockClient = class {
|
|
|
1731
1868
|
}
|
|
1732
1869
|
settled = true;
|
|
1733
1870
|
clearTimeout(timeout);
|
|
1734
|
-
if (
|
|
1735
|
-
|
|
1871
|
+
if (fs5.existsSync(listenerPath)) {
|
|
1872
|
+
fs5.unlinkSync(listenerPath);
|
|
1736
1873
|
}
|
|
1737
1874
|
state = 2 /* Connected */;
|
|
1738
1875
|
this.socket = socket;
|
|
@@ -2226,8 +2363,8 @@ function getErrorMap() {
|
|
|
2226
2363
|
return overrideErrorMap;
|
|
2227
2364
|
}
|
|
2228
2365
|
var makeIssue = (params) => {
|
|
2229
|
-
const { data, path:
|
|
2230
|
-
const fullPath = [...
|
|
2366
|
+
const { data, path: path8, errorMaps, issueData } = params;
|
|
2367
|
+
const fullPath = [...path8, ...issueData.path || []];
|
|
2231
2368
|
const fullIssue = {
|
|
2232
2369
|
...issueData,
|
|
2233
2370
|
path: fullPath
|
|
@@ -2326,11 +2463,11 @@ var errorUtil;
|
|
|
2326
2463
|
errorUtil2.toString = (message) => typeof message === "string" ? message : message === null || message === void 0 ? void 0 : message.message;
|
|
2327
2464
|
})(errorUtil || (errorUtil = {}));
|
|
2328
2465
|
var ParseInputLazyPath = class {
|
|
2329
|
-
constructor(parent, value,
|
|
2466
|
+
constructor(parent, value, path8, key) {
|
|
2330
2467
|
this._cachedPath = [];
|
|
2331
2468
|
this.parent = parent;
|
|
2332
2469
|
this.data = value;
|
|
2333
|
-
this._path =
|
|
2470
|
+
this._path = path8;
|
|
2334
2471
|
this._key = key;
|
|
2335
2472
|
}
|
|
2336
2473
|
get path() {
|
|
@@ -8509,8 +8646,8 @@ var FEATURE_SWITCHES = {
|
|
|
8509
8646
|
var ENV_LOADER_PATH = "/usr/local/bin/vm0-agent/env-loader.mjs";
|
|
8510
8647
|
|
|
8511
8648
|
// src/lib/proxy/vm-registry.ts
|
|
8512
|
-
import
|
|
8513
|
-
var
|
|
8649
|
+
import fs6 from "fs";
|
|
8650
|
+
var logger5 = createLogger("VMRegistry");
|
|
8514
8651
|
var DEFAULT_REGISTRY_PATH = "/tmp/vm0-vm-registry.json";
|
|
8515
8652
|
var VMRegistry = class {
|
|
8516
8653
|
registryPath;
|
|
@@ -8524,8 +8661,8 @@ var VMRegistry = class {
|
|
|
8524
8661
|
*/
|
|
8525
8662
|
load() {
|
|
8526
8663
|
try {
|
|
8527
|
-
if (
|
|
8528
|
-
const content =
|
|
8664
|
+
if (fs6.existsSync(this.registryPath)) {
|
|
8665
|
+
const content = fs6.readFileSync(this.registryPath, "utf-8");
|
|
8529
8666
|
return JSON.parse(content);
|
|
8530
8667
|
}
|
|
8531
8668
|
} catch {
|
|
@@ -8539,8 +8676,8 @@ var VMRegistry = class {
|
|
|
8539
8676
|
this.data.updatedAt = Date.now();
|
|
8540
8677
|
const content = JSON.stringify(this.data, null, 2);
|
|
8541
8678
|
const tempPath = `${this.registryPath}.tmp`;
|
|
8542
|
-
|
|
8543
|
-
|
|
8679
|
+
fs6.writeFileSync(tempPath, content, { mode: 420 });
|
|
8680
|
+
fs6.renameSync(tempPath, this.registryPath);
|
|
8544
8681
|
}
|
|
8545
8682
|
/**
|
|
8546
8683
|
* Register a VM with its IP address
|
|
@@ -8557,7 +8694,7 @@ var VMRegistry = class {
|
|
|
8557
8694
|
this.save();
|
|
8558
8695
|
const firewallInfo = options?.firewallRules ? ` with ${options.firewallRules.length} firewall rules` : "";
|
|
8559
8696
|
const mitmInfo = options?.mitmEnabled ? ", MITM enabled" : "";
|
|
8560
|
-
|
|
8697
|
+
logger5.log(
|
|
8561
8698
|
`Registered VM ${vmIp} for run ${runId}${firewallInfo}${mitmInfo}`
|
|
8562
8699
|
);
|
|
8563
8700
|
}
|
|
@@ -8569,7 +8706,7 @@ var VMRegistry = class {
|
|
|
8569
8706
|
const registration = this.data.vms[vmIp];
|
|
8570
8707
|
delete this.data.vms[vmIp];
|
|
8571
8708
|
this.save();
|
|
8572
|
-
|
|
8709
|
+
logger5.log(`Unregistered VM ${vmIp} (run ${registration.runId})`);
|
|
8573
8710
|
}
|
|
8574
8711
|
}
|
|
8575
8712
|
/**
|
|
@@ -8590,7 +8727,7 @@ var VMRegistry = class {
|
|
|
8590
8727
|
clear() {
|
|
8591
8728
|
this.data.vms = {};
|
|
8592
8729
|
this.save();
|
|
8593
|
-
|
|
8730
|
+
logger5.log("Cleared all registrations");
|
|
8594
8731
|
}
|
|
8595
8732
|
/**
|
|
8596
8733
|
* Get the path to the registry file
|
|
@@ -8613,8 +8750,8 @@ function initVMRegistry(registryPath) {
|
|
|
8613
8750
|
|
|
8614
8751
|
// src/lib/proxy/proxy-manager.ts
|
|
8615
8752
|
import { spawn as spawn2 } from "child_process";
|
|
8616
|
-
import
|
|
8617
|
-
import
|
|
8753
|
+
import fs7 from "fs";
|
|
8754
|
+
import path4 from "path";
|
|
8618
8755
|
|
|
8619
8756
|
// src/lib/proxy/mitm-addon-script.ts
|
|
8620
8757
|
var RUNNER_MITM_ADDON_SCRIPT = `#!/usr/bin/env python3
|
|
@@ -9100,7 +9237,7 @@ addons = [tls_clienthello, request, response]
|
|
|
9100
9237
|
`;
|
|
9101
9238
|
|
|
9102
9239
|
// src/lib/proxy/proxy-manager.ts
|
|
9103
|
-
var
|
|
9240
|
+
var logger6 = createLogger("ProxyManager");
|
|
9104
9241
|
var DEFAULT_PROXY_OPTIONS = {
|
|
9105
9242
|
port: 8080,
|
|
9106
9243
|
registryPath: DEFAULT_REGISTRY_PATH,
|
|
@@ -9111,7 +9248,7 @@ var ProxyManager = class {
|
|
|
9111
9248
|
process = null;
|
|
9112
9249
|
isRunning = false;
|
|
9113
9250
|
constructor(config) {
|
|
9114
|
-
const addonPath =
|
|
9251
|
+
const addonPath = path4.join(config.caDir, "mitm_addon.py");
|
|
9115
9252
|
this.config = {
|
|
9116
9253
|
...DEFAULT_PROXY_OPTIONS,
|
|
9117
9254
|
...config,
|
|
@@ -9138,24 +9275,24 @@ var ProxyManager = class {
|
|
|
9138
9275
|
* Ensure the addon script exists at the configured path
|
|
9139
9276
|
*/
|
|
9140
9277
|
ensureAddonScript() {
|
|
9141
|
-
const addonDir =
|
|
9142
|
-
if (!
|
|
9143
|
-
|
|
9278
|
+
const addonDir = path4.dirname(this.config.addonPath);
|
|
9279
|
+
if (!fs7.existsSync(addonDir)) {
|
|
9280
|
+
fs7.mkdirSync(addonDir, { recursive: true });
|
|
9144
9281
|
}
|
|
9145
|
-
|
|
9282
|
+
fs7.writeFileSync(this.config.addonPath, RUNNER_MITM_ADDON_SCRIPT, {
|
|
9146
9283
|
mode: 493
|
|
9147
9284
|
});
|
|
9148
|
-
|
|
9285
|
+
logger6.log(`Addon script written to ${this.config.addonPath}`);
|
|
9149
9286
|
}
|
|
9150
9287
|
/**
|
|
9151
9288
|
* Validate proxy configuration
|
|
9152
9289
|
*/
|
|
9153
9290
|
validateConfig() {
|
|
9154
|
-
if (!
|
|
9291
|
+
if (!fs7.existsSync(this.config.caDir)) {
|
|
9155
9292
|
throw new Error(`Proxy CA directory not found: ${this.config.caDir}`);
|
|
9156
9293
|
}
|
|
9157
|
-
const caCertPath =
|
|
9158
|
-
if (!
|
|
9294
|
+
const caCertPath = path4.join(this.config.caDir, "mitmproxy-ca.pem");
|
|
9295
|
+
if (!fs7.existsSync(caCertPath)) {
|
|
9159
9296
|
throw new Error(`Proxy CA certificate not found: ${caCertPath}`);
|
|
9160
9297
|
}
|
|
9161
9298
|
this.ensureAddonScript();
|
|
@@ -9165,7 +9302,7 @@ var ProxyManager = class {
|
|
|
9165
9302
|
*/
|
|
9166
9303
|
async start() {
|
|
9167
9304
|
if (this.isRunning) {
|
|
9168
|
-
|
|
9305
|
+
logger6.log("Proxy already running");
|
|
9169
9306
|
return;
|
|
9170
9307
|
}
|
|
9171
9308
|
const mitmproxyInstalled = await this.checkMitmproxyInstalled();
|
|
@@ -9176,11 +9313,11 @@ var ProxyManager = class {
|
|
|
9176
9313
|
}
|
|
9177
9314
|
this.validateConfig();
|
|
9178
9315
|
getVMRegistry();
|
|
9179
|
-
|
|
9180
|
-
|
|
9181
|
-
|
|
9182
|
-
|
|
9183
|
-
|
|
9316
|
+
logger6.log("Starting mitmproxy...");
|
|
9317
|
+
logger6.log(` Port: ${this.config.port}`);
|
|
9318
|
+
logger6.log(` CA Dir: ${this.config.caDir}`);
|
|
9319
|
+
logger6.log(` Addon: ${this.config.addonPath}`);
|
|
9320
|
+
logger6.log(` Registry: ${this.config.registryPath}`);
|
|
9184
9321
|
const args = [
|
|
9185
9322
|
"--mode",
|
|
9186
9323
|
"transparent",
|
|
@@ -9210,18 +9347,23 @@ var ProxyManager = class {
|
|
|
9210
9347
|
mitmLogger.log(data.toString().trim());
|
|
9211
9348
|
});
|
|
9212
9349
|
this.process.on("close", (code) => {
|
|
9213
|
-
|
|
9350
|
+
logger6.log(`mitmproxy exited with code ${code}`);
|
|
9214
9351
|
this.isRunning = false;
|
|
9215
9352
|
this.process = null;
|
|
9216
9353
|
});
|
|
9217
9354
|
this.process.on("error", (err) => {
|
|
9218
|
-
|
|
9355
|
+
logger6.error(`mitmproxy error: ${err.message}`);
|
|
9219
9356
|
this.isRunning = false;
|
|
9220
9357
|
this.process = null;
|
|
9221
9358
|
});
|
|
9222
9359
|
await this.waitForReady();
|
|
9223
9360
|
this.isRunning = true;
|
|
9224
|
-
|
|
9361
|
+
logger6.log("mitmproxy started successfully");
|
|
9362
|
+
process.on("exit", () => {
|
|
9363
|
+
if (this.process && !this.process.killed) {
|
|
9364
|
+
this.process.kill("SIGKILL");
|
|
9365
|
+
}
|
|
9366
|
+
});
|
|
9225
9367
|
}
|
|
9226
9368
|
/**
|
|
9227
9369
|
* Wait for proxy to be ready
|
|
@@ -9247,24 +9389,24 @@ var ProxyManager = class {
|
|
|
9247
9389
|
*/
|
|
9248
9390
|
async stop() {
|
|
9249
9391
|
if (!this.process || !this.isRunning) {
|
|
9250
|
-
|
|
9392
|
+
logger6.log("Proxy not running");
|
|
9251
9393
|
return;
|
|
9252
9394
|
}
|
|
9253
|
-
|
|
9395
|
+
logger6.log("Stopping mitmproxy...");
|
|
9254
9396
|
return new Promise((resolve) => {
|
|
9255
9397
|
if (!this.process) {
|
|
9256
9398
|
resolve();
|
|
9257
9399
|
return;
|
|
9258
9400
|
}
|
|
9259
9401
|
const timeout = setTimeout(() => {
|
|
9260
|
-
|
|
9402
|
+
logger6.log("Force killing mitmproxy...");
|
|
9261
9403
|
this.process?.kill("SIGKILL");
|
|
9262
9404
|
}, 5e3);
|
|
9263
9405
|
this.process.on("close", () => {
|
|
9264
9406
|
clearTimeout(timeout);
|
|
9265
9407
|
this.isRunning = false;
|
|
9266
9408
|
this.process = null;
|
|
9267
|
-
|
|
9409
|
+
logger6.log("mitmproxy stopped");
|
|
9268
9410
|
resolve();
|
|
9269
9411
|
});
|
|
9270
9412
|
this.process.kill("SIGTERM");
|
|
@@ -9409,15 +9551,15 @@ async function withSandboxTiming(actionType, fn) {
|
|
|
9409
9551
|
}
|
|
9410
9552
|
|
|
9411
9553
|
// src/lib/vm-setup/vm-setup.ts
|
|
9412
|
-
var
|
|
9554
|
+
var logger7 = createLogger("VMSetup");
|
|
9413
9555
|
var VM_PROXY_CA_PATH = "/usr/local/share/ca-certificates/vm0-proxy-ca.crt";
|
|
9414
9556
|
async function downloadStorages(guest, manifest) {
|
|
9415
9557
|
const totalArchives = manifest.storages.filter((s) => s.archiveUrl).length + (manifest.artifact?.archiveUrl ? 1 : 0);
|
|
9416
9558
|
if (totalArchives === 0) {
|
|
9417
|
-
|
|
9559
|
+
logger7.log(`No archives to download`);
|
|
9418
9560
|
return;
|
|
9419
9561
|
}
|
|
9420
|
-
|
|
9562
|
+
logger7.log(`Downloading ${totalArchives} archive(s)...`);
|
|
9421
9563
|
const manifestJson = JSON.stringify(manifest);
|
|
9422
9564
|
await guest.writeFile("/tmp/storage-manifest.json", manifestJson);
|
|
9423
9565
|
const result = await guest.exec(
|
|
@@ -9426,23 +9568,23 @@ async function downloadStorages(guest, manifest) {
|
|
|
9426
9568
|
if (result.exitCode !== 0) {
|
|
9427
9569
|
throw new Error(`Storage download failed: ${result.stderr}`);
|
|
9428
9570
|
}
|
|
9429
|
-
|
|
9571
|
+
logger7.log(`Storage download completed`);
|
|
9430
9572
|
}
|
|
9431
9573
|
async function restoreSessionHistory(guest, resumeSession, workingDir, cliAgentType) {
|
|
9432
9574
|
const { sessionId, sessionHistory } = resumeSession;
|
|
9433
9575
|
let sessionPath;
|
|
9434
9576
|
if (cliAgentType === "codex") {
|
|
9435
|
-
|
|
9577
|
+
logger7.log(`Codex resume session will be handled by checkpoint.py`);
|
|
9436
9578
|
return;
|
|
9437
9579
|
} else {
|
|
9438
9580
|
const projectName = workingDir.replace(/^\//, "").replace(/\//g, "-");
|
|
9439
9581
|
sessionPath = `/home/user/.claude/projects/-${projectName}/${sessionId}.jsonl`;
|
|
9440
9582
|
}
|
|
9441
|
-
|
|
9583
|
+
logger7.log(`Restoring session history to ${sessionPath}`);
|
|
9442
9584
|
const dirPath = sessionPath.substring(0, sessionPath.lastIndexOf("/"));
|
|
9443
9585
|
await guest.execOrThrow(`mkdir -p "${dirPath}"`);
|
|
9444
9586
|
await guest.writeFile(sessionPath, sessionHistory);
|
|
9445
|
-
|
|
9587
|
+
logger7.log(
|
|
9446
9588
|
`Session history restored (${sessionHistory.split("\n").length} lines)`
|
|
9447
9589
|
);
|
|
9448
9590
|
}
|
|
@@ -9495,22 +9637,22 @@ function buildEnvironmentVariables(context, apiUrl) {
|
|
|
9495
9637
|
}
|
|
9496
9638
|
|
|
9497
9639
|
// src/lib/network-logs/network-logs.ts
|
|
9498
|
-
import
|
|
9499
|
-
var
|
|
9640
|
+
import fs8 from "fs";
|
|
9641
|
+
var logger8 = createLogger("NetworkLogs");
|
|
9500
9642
|
function getNetworkLogPath(runId) {
|
|
9501
9643
|
return `/tmp/vm0-network-${runId}.jsonl`;
|
|
9502
9644
|
}
|
|
9503
9645
|
function readNetworkLogs(runId) {
|
|
9504
9646
|
const logPath = getNetworkLogPath(runId);
|
|
9505
|
-
if (!
|
|
9647
|
+
if (!fs8.existsSync(logPath)) {
|
|
9506
9648
|
return [];
|
|
9507
9649
|
}
|
|
9508
9650
|
try {
|
|
9509
|
-
const content =
|
|
9651
|
+
const content = fs8.readFileSync(logPath, "utf-8");
|
|
9510
9652
|
const lines = content.split("\n").filter((line) => line.trim());
|
|
9511
9653
|
return lines.map((line) => JSON.parse(line));
|
|
9512
9654
|
} catch (err) {
|
|
9513
|
-
|
|
9655
|
+
logger8.error(
|
|
9514
9656
|
`Failed to read network logs: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9515
9657
|
);
|
|
9516
9658
|
return [];
|
|
@@ -9519,11 +9661,11 @@ function readNetworkLogs(runId) {
|
|
|
9519
9661
|
function cleanupNetworkLogs(runId) {
|
|
9520
9662
|
const logPath = getNetworkLogPath(runId);
|
|
9521
9663
|
try {
|
|
9522
|
-
if (
|
|
9523
|
-
|
|
9664
|
+
if (fs8.existsSync(logPath)) {
|
|
9665
|
+
fs8.unlinkSync(logPath);
|
|
9524
9666
|
}
|
|
9525
9667
|
} catch (err) {
|
|
9526
|
-
|
|
9668
|
+
logger8.error(
|
|
9527
9669
|
`Failed to cleanup network logs: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9528
9670
|
);
|
|
9529
9671
|
}
|
|
@@ -9531,10 +9673,10 @@ function cleanupNetworkLogs(runId) {
|
|
|
9531
9673
|
async function uploadNetworkLogs(apiUrl, sandboxToken, runId) {
|
|
9532
9674
|
const networkLogs = readNetworkLogs(runId);
|
|
9533
9675
|
if (networkLogs.length === 0) {
|
|
9534
|
-
|
|
9676
|
+
logger8.log(`No network logs to upload for ${runId}`);
|
|
9535
9677
|
return;
|
|
9536
9678
|
}
|
|
9537
|
-
|
|
9679
|
+
logger8.log(
|
|
9538
9680
|
`Uploading ${networkLogs.length} network log entries for ${runId}`
|
|
9539
9681
|
);
|
|
9540
9682
|
const headers = {
|
|
@@ -9555,15 +9697,15 @@ async function uploadNetworkLogs(apiUrl, sandboxToken, runId) {
|
|
|
9555
9697
|
});
|
|
9556
9698
|
if (!response.ok) {
|
|
9557
9699
|
const errorText = await response.text();
|
|
9558
|
-
|
|
9700
|
+
logger8.error(`Failed to upload network logs: ${errorText}`);
|
|
9559
9701
|
return;
|
|
9560
9702
|
}
|
|
9561
|
-
|
|
9703
|
+
logger8.log(`Network logs uploaded successfully for ${runId}`);
|
|
9562
9704
|
cleanupNetworkLogs(runId);
|
|
9563
9705
|
}
|
|
9564
9706
|
|
|
9565
9707
|
// src/lib/executor.ts
|
|
9566
|
-
var
|
|
9708
|
+
var logger9 = createLogger("Executor");
|
|
9567
9709
|
function getVmIdFromRunId(runId) {
|
|
9568
9710
|
return runId.split("-")[0] || runId.substring(0, 8);
|
|
9569
9711
|
}
|
|
@@ -9609,12 +9751,12 @@ async function reportPreflightFailure(apiUrl, runId, sandboxToken, error, bypass
|
|
|
9609
9751
|
})
|
|
9610
9752
|
});
|
|
9611
9753
|
if (!response.ok) {
|
|
9612
|
-
|
|
9754
|
+
logger9.error(
|
|
9613
9755
|
`Failed to report preflight failure: HTTP ${response.status}`
|
|
9614
9756
|
);
|
|
9615
9757
|
}
|
|
9616
9758
|
} catch (err) {
|
|
9617
|
-
|
|
9759
|
+
logger9.error(`Failed to report preflight failure: ${err}`);
|
|
9618
9760
|
}
|
|
9619
9761
|
}
|
|
9620
9762
|
async function executeJob(context, config, options = {}) {
|
|
@@ -9633,9 +9775,9 @@ async function executeJob(context, config, options = {}) {
|
|
|
9633
9775
|
const vmId = getVmIdFromRunId(context.runId);
|
|
9634
9776
|
let vm = null;
|
|
9635
9777
|
let guestIp = null;
|
|
9636
|
-
|
|
9778
|
+
logger9.log(`Starting job ${context.runId} in VM ${vmId}`);
|
|
9637
9779
|
try {
|
|
9638
|
-
const workspacesDir =
|
|
9780
|
+
const workspacesDir = path5.join(process.cwd(), "workspaces");
|
|
9639
9781
|
const vmConfig = {
|
|
9640
9782
|
vmId,
|
|
9641
9783
|
vcpus: config.sandbox.vcpu,
|
|
@@ -9643,30 +9785,30 @@ async function executeJob(context, config, options = {}) {
|
|
|
9643
9785
|
kernelPath: config.firecracker.kernel,
|
|
9644
9786
|
rootfsPath: config.firecracker.rootfs,
|
|
9645
9787
|
firecrackerBinary: config.firecracker.binary,
|
|
9646
|
-
workDir:
|
|
9788
|
+
workDir: path5.join(workspacesDir, `vm0-${vmId}`)
|
|
9647
9789
|
};
|
|
9648
|
-
|
|
9790
|
+
logger9.log(`Creating VM ${vmId}...`);
|
|
9649
9791
|
vm = new FirecrackerVM(vmConfig);
|
|
9650
9792
|
await withSandboxTiming("vm_create", () => vm.start());
|
|
9651
9793
|
guestIp = vm.getGuestIp();
|
|
9652
9794
|
if (!guestIp) {
|
|
9653
9795
|
throw new Error("VM started but no IP address available");
|
|
9654
9796
|
}
|
|
9655
|
-
|
|
9797
|
+
logger9.log(`VM ${vmId} started, guest IP: ${guestIp}`);
|
|
9656
9798
|
const vsockPath = vm.getVsockPath();
|
|
9657
9799
|
const guest = new VsockClient(vsockPath);
|
|
9658
|
-
|
|
9659
|
-
|
|
9800
|
+
logger9.log(`Using vsock for guest communication: ${vsockPath}`);
|
|
9801
|
+
logger9.log(`Waiting for guest connection...`);
|
|
9660
9802
|
await withSandboxTiming(
|
|
9661
9803
|
"guest_wait",
|
|
9662
9804
|
() => guest.waitForGuestConnection(3e4)
|
|
9663
9805
|
);
|
|
9664
|
-
|
|
9806
|
+
logger9.log(`Guest client ready`);
|
|
9665
9807
|
const firewallConfig = context.experimentalFirewall;
|
|
9666
9808
|
if (firewallConfig?.enabled) {
|
|
9667
9809
|
const mitmEnabled = firewallConfig.experimental_mitm ?? false;
|
|
9668
9810
|
const sealSecretsEnabled = firewallConfig.experimental_seal_secrets ?? false;
|
|
9669
|
-
|
|
9811
|
+
logger9.log(
|
|
9670
9812
|
`Setting up network security for VM ${guestIp} (mitm=${mitmEnabled}, sealSecrets=${sealSecretsEnabled})`
|
|
9671
9813
|
);
|
|
9672
9814
|
await withSandboxTiming("network_setup", async () => {
|
|
@@ -9701,12 +9843,12 @@ async function executeJob(context, config, options = {}) {
|
|
|
9701
9843
|
}
|
|
9702
9844
|
const envVars = buildEnvironmentVariables(context, config.server.url);
|
|
9703
9845
|
const envJson = JSON.stringify(envVars);
|
|
9704
|
-
|
|
9846
|
+
logger9.log(
|
|
9705
9847
|
`Writing env JSON (${envJson.length} bytes) to ${ENV_JSON_PATH}`
|
|
9706
9848
|
);
|
|
9707
9849
|
await guest.writeFile(ENV_JSON_PATH, envJson);
|
|
9708
9850
|
if (!options.benchmarkMode) {
|
|
9709
|
-
|
|
9851
|
+
logger9.log(`Running preflight connectivity check...`);
|
|
9710
9852
|
const bypassSecret = process.env.VERCEL_AUTOMATION_BYPASS_SECRET;
|
|
9711
9853
|
const preflight = await withSandboxTiming(
|
|
9712
9854
|
"preflight_check",
|
|
@@ -9719,7 +9861,7 @@ async function executeJob(context, config, options = {}) {
|
|
|
9719
9861
|
)
|
|
9720
9862
|
);
|
|
9721
9863
|
if (!preflight.success) {
|
|
9722
|
-
|
|
9864
|
+
logger9.log(`Preflight check failed: ${preflight.error}`);
|
|
9723
9865
|
await reportPreflightFailure(
|
|
9724
9866
|
config.server.url,
|
|
9725
9867
|
context.runId,
|
|
@@ -9732,21 +9874,21 @@ async function executeJob(context, config, options = {}) {
|
|
|
9732
9874
|
error: preflight.error
|
|
9733
9875
|
};
|
|
9734
9876
|
}
|
|
9735
|
-
|
|
9877
|
+
logger9.log(`Preflight check passed`);
|
|
9736
9878
|
}
|
|
9737
9879
|
const systemLogFile = `/tmp/vm0-main-${context.runId}.log`;
|
|
9738
9880
|
const startTime = Date.now();
|
|
9739
9881
|
const maxWaitMs = 2 * 60 * 60 * 1e3;
|
|
9740
9882
|
let command;
|
|
9741
9883
|
if (options.benchmarkMode) {
|
|
9742
|
-
|
|
9884
|
+
logger9.log(`Running command directly (benchmark mode)...`);
|
|
9743
9885
|
command = `${context.prompt} > ${systemLogFile} 2>&1`;
|
|
9744
9886
|
} else {
|
|
9745
|
-
|
|
9887
|
+
logger9.log(`Running agent via env-loader...`);
|
|
9746
9888
|
command = `node ${ENV_LOADER_PATH} > ${systemLogFile} 2>&1`;
|
|
9747
9889
|
}
|
|
9748
9890
|
const { pid } = await guest.spawnAndWatch(command, maxWaitMs);
|
|
9749
|
-
|
|
9891
|
+
logger9.log(`Process started with pid=${pid}`);
|
|
9750
9892
|
let exitCode = 1;
|
|
9751
9893
|
let exitEvent;
|
|
9752
9894
|
try {
|
|
@@ -9755,7 +9897,7 @@ async function executeJob(context, config, options = {}) {
|
|
|
9755
9897
|
} catch {
|
|
9756
9898
|
const durationMs2 = Date.now() - startTime;
|
|
9757
9899
|
const duration2 = Math.round(durationMs2 / 1e3);
|
|
9758
|
-
|
|
9900
|
+
logger9.log(`Agent timed out after ${duration2}s`);
|
|
9759
9901
|
recordOperation({
|
|
9760
9902
|
actionType: "agent_execute",
|
|
9761
9903
|
durationMs: durationMs2,
|
|
@@ -9773,7 +9915,7 @@ async function executeJob(context, config, options = {}) {
|
|
|
9773
9915
|
`dmesg | tail -20 | grep -iE "killed|oom" 2>/dev/null`
|
|
9774
9916
|
);
|
|
9775
9917
|
if (dmesgCheck.stdout.toLowerCase().includes("oom") || dmesgCheck.stdout.toLowerCase().includes("killed")) {
|
|
9776
|
-
|
|
9918
|
+
logger9.log(`OOM detected: ${dmesgCheck.stdout}`);
|
|
9777
9919
|
recordOperation({
|
|
9778
9920
|
actionType: "agent_execute",
|
|
9779
9921
|
durationMs,
|
|
@@ -9790,9 +9932,9 @@ async function executeJob(context, config, options = {}) {
|
|
|
9790
9932
|
durationMs,
|
|
9791
9933
|
success: exitCode === 0
|
|
9792
9934
|
});
|
|
9793
|
-
|
|
9935
|
+
logger9.log(`Agent finished in ${duration}s with exit code ${exitCode}`);
|
|
9794
9936
|
if (exitEvent.stderr) {
|
|
9795
|
-
|
|
9937
|
+
logger9.log(
|
|
9796
9938
|
`Stderr (${exitEvent.stderr.length} chars): ${exitEvent.stderr.substring(0, 500)}`
|
|
9797
9939
|
);
|
|
9798
9940
|
}
|
|
@@ -9802,14 +9944,14 @@ async function executeJob(context, config, options = {}) {
|
|
|
9802
9944
|
};
|
|
9803
9945
|
} catch (error) {
|
|
9804
9946
|
const errorMsg = error instanceof Error ? error.message : "Unknown error";
|
|
9805
|
-
|
|
9947
|
+
logger9.error(`Job ${context.runId} failed: ${errorMsg}`);
|
|
9806
9948
|
return {
|
|
9807
9949
|
exitCode: 1,
|
|
9808
9950
|
error: errorMsg
|
|
9809
9951
|
};
|
|
9810
9952
|
} finally {
|
|
9811
9953
|
if (context.experimentalFirewall?.enabled && guestIp) {
|
|
9812
|
-
|
|
9954
|
+
logger9.log(`Cleaning up network security for VM ${guestIp}`);
|
|
9813
9955
|
getVMRegistry().unregister(guestIp);
|
|
9814
9956
|
if (!options.benchmarkMode) {
|
|
9815
9957
|
try {
|
|
@@ -9819,14 +9961,14 @@ async function executeJob(context, config, options = {}) {
|
|
|
9819
9961
|
context.runId
|
|
9820
9962
|
);
|
|
9821
9963
|
} catch (err) {
|
|
9822
|
-
|
|
9964
|
+
logger9.error(
|
|
9823
9965
|
`Failed to upload network logs: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9824
9966
|
);
|
|
9825
9967
|
}
|
|
9826
9968
|
}
|
|
9827
9969
|
}
|
|
9828
9970
|
if (vm) {
|
|
9829
|
-
|
|
9971
|
+
logger9.log(`Cleaning up VM ${vmId}...`);
|
|
9830
9972
|
await withSandboxTiming("cleanup", () => vm.kill());
|
|
9831
9973
|
}
|
|
9832
9974
|
await clearSandboxContext();
|
|
@@ -9835,7 +9977,7 @@ async function executeJob(context, config, options = {}) {
|
|
|
9835
9977
|
|
|
9836
9978
|
// src/lib/runner/status.ts
|
|
9837
9979
|
import { writeFileSync as writeFileSync2 } from "fs";
|
|
9838
|
-
var
|
|
9980
|
+
var logger10 = createLogger("Runner");
|
|
9839
9981
|
function writeStatusFile(statusFilePath, mode, activeRuns, startedAt) {
|
|
9840
9982
|
const status = {
|
|
9841
9983
|
mode,
|
|
@@ -9847,7 +9989,7 @@ function writeStatusFile(statusFilePath, mode, activeRuns, startedAt) {
|
|
|
9847
9989
|
try {
|
|
9848
9990
|
writeFileSync2(statusFilePath, JSON.stringify(status, null, 2));
|
|
9849
9991
|
} catch (err) {
|
|
9850
|
-
|
|
9992
|
+
logger10.error(
|
|
9851
9993
|
`Failed to write status file: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9852
9994
|
);
|
|
9853
9995
|
}
|
|
@@ -9863,27 +10005,98 @@ function createStatusUpdater(statusFilePath, state) {
|
|
|
9863
10005
|
};
|
|
9864
10006
|
}
|
|
9865
10007
|
|
|
10008
|
+
// src/lib/runner/runner-lock.ts
|
|
10009
|
+
import { exec as exec4 } from "child_process";
|
|
10010
|
+
import fs9 from "fs";
|
|
10011
|
+
import path6 from "path";
|
|
10012
|
+
import { promisify as promisify4 } from "util";
|
|
10013
|
+
var execAsync4 = promisify4(exec4);
|
|
10014
|
+
var logger11 = createLogger("RunnerLock");
|
|
10015
|
+
var DEFAULT_RUN_DIR = "/var/run/vm0";
|
|
10016
|
+
var DEFAULT_PID_FILE = `${DEFAULT_RUN_DIR}/runner.pid`;
|
|
10017
|
+
var currentPidFile = null;
|
|
10018
|
+
async function ensureRunDir2(dirPath, skipSudo) {
|
|
10019
|
+
if (!fs9.existsSync(dirPath)) {
|
|
10020
|
+
if (skipSudo) {
|
|
10021
|
+
fs9.mkdirSync(dirPath, { recursive: true });
|
|
10022
|
+
} else {
|
|
10023
|
+
await execAsync4(`sudo mkdir -p ${dirPath}`);
|
|
10024
|
+
await execAsync4(`sudo chmod 777 ${dirPath}`);
|
|
10025
|
+
}
|
|
10026
|
+
}
|
|
10027
|
+
}
|
|
10028
|
+
function isProcessRunning(pid) {
|
|
10029
|
+
try {
|
|
10030
|
+
process.kill(pid, 0);
|
|
10031
|
+
return true;
|
|
10032
|
+
} catch (err) {
|
|
10033
|
+
if (err instanceof Error && "code" in err && err.code === "EPERM") {
|
|
10034
|
+
return true;
|
|
10035
|
+
}
|
|
10036
|
+
return false;
|
|
10037
|
+
}
|
|
10038
|
+
}
|
|
10039
|
+
async function acquireRunnerLock(options = {}) {
|
|
10040
|
+
const pidFile = options.pidFile ?? DEFAULT_PID_FILE;
|
|
10041
|
+
const skipSudo = options.skipSudo ?? false;
|
|
10042
|
+
const runDir = path6.dirname(pidFile);
|
|
10043
|
+
await ensureRunDir2(runDir, skipSudo);
|
|
10044
|
+
if (fs9.existsSync(pidFile)) {
|
|
10045
|
+
const pidStr = fs9.readFileSync(pidFile, "utf-8").trim();
|
|
10046
|
+
const pid = parseInt(pidStr, 10);
|
|
10047
|
+
if (!isNaN(pid) && isProcessRunning(pid)) {
|
|
10048
|
+
logger11.error(`Error: Another runner is already running (PID ${pid})`);
|
|
10049
|
+
logger11.error(`If this is incorrect, remove ${pidFile} and try again.`);
|
|
10050
|
+
process.exit(1);
|
|
10051
|
+
}
|
|
10052
|
+
if (isNaN(pid)) {
|
|
10053
|
+
logger11.log("Cleaning up invalid PID file");
|
|
10054
|
+
} else {
|
|
10055
|
+
logger11.log(`Cleaning up stale PID file (PID ${pid} not running)`);
|
|
10056
|
+
}
|
|
10057
|
+
fs9.unlinkSync(pidFile);
|
|
10058
|
+
}
|
|
10059
|
+
fs9.writeFileSync(pidFile, process.pid.toString());
|
|
10060
|
+
currentPidFile = pidFile;
|
|
10061
|
+
logger11.log(`Runner lock acquired (PID ${process.pid})`);
|
|
10062
|
+
}
|
|
10063
|
+
function releaseRunnerLock() {
|
|
10064
|
+
const pidFile = currentPidFile ?? DEFAULT_PID_FILE;
|
|
10065
|
+
if (fs9.existsSync(pidFile)) {
|
|
10066
|
+
fs9.unlinkSync(pidFile);
|
|
10067
|
+
logger11.log("Runner lock released");
|
|
10068
|
+
}
|
|
10069
|
+
currentPidFile = null;
|
|
10070
|
+
}
|
|
10071
|
+
|
|
9866
10072
|
// src/lib/runner/setup.ts
|
|
9867
|
-
var
|
|
10073
|
+
var logger12 = createLogger("Runner");
|
|
9868
10074
|
async function setupEnvironment(options) {
|
|
9869
10075
|
const { config } = options;
|
|
10076
|
+
await acquireRunnerLock();
|
|
9870
10077
|
const networkCheck = checkNetworkPrerequisites();
|
|
9871
10078
|
if (!networkCheck.ok) {
|
|
9872
|
-
|
|
10079
|
+
logger12.error("Network prerequisites not met:");
|
|
9873
10080
|
for (const error of networkCheck.errors) {
|
|
9874
|
-
|
|
10081
|
+
logger12.error(` - ${error}`);
|
|
9875
10082
|
}
|
|
9876
10083
|
process.exit(1);
|
|
9877
10084
|
}
|
|
9878
|
-
|
|
10085
|
+
logger12.log("Setting up network bridge...");
|
|
9879
10086
|
await setupBridge();
|
|
9880
|
-
|
|
10087
|
+
logger12.log("Flushing bridge ARP cache...");
|
|
9881
10088
|
await flushBridgeArpCache();
|
|
9882
|
-
|
|
10089
|
+
logger12.log("Cleaning up orphaned proxy rules...");
|
|
9883
10090
|
await cleanupOrphanedProxyRules(config.name);
|
|
9884
|
-
|
|
10091
|
+
logger12.log("Cleaning up orphaned IP allocations...");
|
|
9885
10092
|
await cleanupOrphanedAllocations();
|
|
9886
|
-
|
|
10093
|
+
logger12.log("Initializing overlay pool...");
|
|
10094
|
+
await initOverlayPool({
|
|
10095
|
+
size: config.sandbox.max_concurrent + 2,
|
|
10096
|
+
replenishThreshold: config.sandbox.max_concurrent
|
|
10097
|
+
// Start replenishing early to handle bursts
|
|
10098
|
+
});
|
|
10099
|
+
logger12.log("Initializing network proxy...");
|
|
9887
10100
|
initVMRegistry();
|
|
9888
10101
|
const proxyManager = initProxyManager({
|
|
9889
10102
|
apiUrl: config.server.url,
|
|
@@ -9894,49 +10107,79 @@ async function setupEnvironment(options) {
|
|
|
9894
10107
|
try {
|
|
9895
10108
|
await proxyManager.start();
|
|
9896
10109
|
proxyEnabled = true;
|
|
9897
|
-
|
|
9898
|
-
|
|
10110
|
+
logger12.log("Network proxy initialized successfully");
|
|
10111
|
+
logger12.log("Setting up CIDR proxy rules...");
|
|
9899
10112
|
await setupCIDRProxyRules(config.proxy.port);
|
|
9900
10113
|
} catch (err) {
|
|
9901
|
-
|
|
10114
|
+
logger12.log(
|
|
9902
10115
|
`Network proxy not available: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9903
10116
|
);
|
|
9904
|
-
|
|
10117
|
+
logger12.log(
|
|
9905
10118
|
"Jobs with experimentalFirewall enabled will run without network interception"
|
|
9906
10119
|
);
|
|
9907
10120
|
}
|
|
9908
10121
|
return { proxyEnabled, proxyPort: config.proxy.port };
|
|
9909
10122
|
}
|
|
9910
10123
|
async function cleanupEnvironment(resources) {
|
|
10124
|
+
const errors = [];
|
|
9911
10125
|
if (resources.proxyEnabled) {
|
|
9912
|
-
|
|
9913
|
-
|
|
10126
|
+
try {
|
|
10127
|
+
logger12.log("Cleaning up CIDR proxy rules...");
|
|
10128
|
+
await cleanupCIDRProxyRules(resources.proxyPort);
|
|
10129
|
+
} catch (err) {
|
|
10130
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
10131
|
+
errors.push(error);
|
|
10132
|
+
logger12.error(`Failed to cleanup CIDR proxy rules: ${error.message}`);
|
|
10133
|
+
}
|
|
9914
10134
|
}
|
|
9915
10135
|
if (resources.proxyEnabled) {
|
|
9916
|
-
|
|
9917
|
-
|
|
10136
|
+
try {
|
|
10137
|
+
logger12.log("Stopping network proxy...");
|
|
10138
|
+
await getProxyManager().stop();
|
|
10139
|
+
} catch (err) {
|
|
10140
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
10141
|
+
errors.push(error);
|
|
10142
|
+
logger12.error(`Failed to stop network proxy: ${error.message}`);
|
|
10143
|
+
}
|
|
10144
|
+
}
|
|
10145
|
+
try {
|
|
10146
|
+
cleanupOverlayPool();
|
|
10147
|
+
} catch (err) {
|
|
10148
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
10149
|
+
errors.push(error);
|
|
10150
|
+
logger12.error(`Failed to cleanup overlay pool: ${error.message}`);
|
|
10151
|
+
}
|
|
10152
|
+
try {
|
|
10153
|
+
releaseRunnerLock();
|
|
10154
|
+
} catch (err) {
|
|
10155
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
10156
|
+
errors.push(error);
|
|
10157
|
+
logger12.error(`Failed to release runner lock: ${error.message}`);
|
|
10158
|
+
}
|
|
10159
|
+
if (errors.length > 0) {
|
|
10160
|
+
logger12.error(`Cleanup completed with ${errors.length} error(s)`);
|
|
9918
10161
|
}
|
|
9919
10162
|
}
|
|
9920
10163
|
|
|
9921
10164
|
// src/lib/runner/signals.ts
|
|
9922
|
-
var
|
|
10165
|
+
var logger13 = createLogger("Runner");
|
|
9923
10166
|
function setupSignalHandlers(state, handlers) {
|
|
9924
10167
|
process.on("SIGINT", () => {
|
|
9925
|
-
|
|
9926
|
-
|
|
9927
|
-
state.mode = "stopped";
|
|
10168
|
+
logger13.log("\nShutting down...");
|
|
10169
|
+
state.mode = "stopping";
|
|
9928
10170
|
handlers.updateStatus();
|
|
10171
|
+
handlers.onShutdown();
|
|
9929
10172
|
});
|
|
9930
10173
|
process.on("SIGTERM", () => {
|
|
9931
|
-
|
|
9932
|
-
|
|
9933
|
-
state.mode = "stopped";
|
|
10174
|
+
logger13.log("\nShutting down...");
|
|
10175
|
+
state.mode = "stopping";
|
|
9934
10176
|
handlers.updateStatus();
|
|
10177
|
+
handlers.onShutdown();
|
|
9935
10178
|
});
|
|
9936
10179
|
process.on("SIGUSR1", () => {
|
|
9937
10180
|
if (state.mode === "running") {
|
|
9938
|
-
|
|
9939
|
-
|
|
10181
|
+
logger13.log("\n[Maintenance] Entering drain mode...");
|
|
10182
|
+
logger13.log(
|
|
9940
10183
|
`[Maintenance] Active jobs: ${state.activeRuns.size} (will wait for completion)`
|
|
9941
10184
|
);
|
|
9942
10185
|
state.mode = "draining";
|
|
@@ -9947,7 +10190,7 @@ function setupSignalHandlers(state, handlers) {
|
|
|
9947
10190
|
}
|
|
9948
10191
|
|
|
9949
10192
|
// src/lib/runner/runner.ts
|
|
9950
|
-
var
|
|
10193
|
+
var logger14 = createLogger("Runner");
|
|
9951
10194
|
var Runner = class _Runner {
|
|
9952
10195
|
config;
|
|
9953
10196
|
statusFilePath;
|
|
@@ -9986,41 +10229,43 @@ var Runner = class _Runner {
|
|
|
9986
10229
|
onDrain: () => {
|
|
9987
10230
|
this.pendingJobs.length = 0;
|
|
9988
10231
|
if (this.state.activeRuns.size === 0) {
|
|
9989
|
-
|
|
10232
|
+
logger14.log("[Maintenance] No active jobs, exiting immediately");
|
|
10233
|
+
this.state.mode = "stopping";
|
|
10234
|
+
this.updateStatus();
|
|
9990
10235
|
this.resolveShutdown?.();
|
|
9991
10236
|
}
|
|
9992
10237
|
},
|
|
9993
10238
|
updateStatus: this.updateStatus
|
|
9994
10239
|
});
|
|
9995
|
-
|
|
10240
|
+
logger14.log(
|
|
9996
10241
|
`Starting runner '${this.config.name}' for group '${this.config.group}'...`
|
|
9997
10242
|
);
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
|
|
10243
|
+
logger14.log(`Max concurrent jobs: ${this.config.sandbox.max_concurrent}`);
|
|
10244
|
+
logger14.log(`Status file: ${this.statusFilePath}`);
|
|
10245
|
+
logger14.log("Press Ctrl+C to stop");
|
|
10246
|
+
logger14.log("");
|
|
10002
10247
|
this.updateStatus();
|
|
10003
|
-
|
|
10248
|
+
logger14.log("Checking for pending jobs...");
|
|
10004
10249
|
await this.pollFallback();
|
|
10005
|
-
|
|
10250
|
+
logger14.log("Connecting to realtime job notifications...");
|
|
10006
10251
|
this.subscription = await subscribeToJobs(
|
|
10007
10252
|
this.config.server,
|
|
10008
10253
|
this.config.group,
|
|
10009
10254
|
(notification) => {
|
|
10010
|
-
|
|
10255
|
+
logger14.log(`Ably notification: ${notification.runId}`);
|
|
10011
10256
|
this.processJob(notification.runId).catch(console.error);
|
|
10012
10257
|
},
|
|
10013
10258
|
(connectionState, reason) => {
|
|
10014
|
-
|
|
10259
|
+
logger14.log(
|
|
10015
10260
|
`Ably connection: ${connectionState}${reason ? ` (${reason})` : ""}`
|
|
10016
10261
|
);
|
|
10017
10262
|
}
|
|
10018
10263
|
);
|
|
10019
|
-
|
|
10264
|
+
logger14.log("Connected to realtime job notifications");
|
|
10020
10265
|
this.pollInterval = setInterval(() => {
|
|
10021
10266
|
this.pollFallback().catch(console.error);
|
|
10022
10267
|
}, this.config.sandbox.poll_interval_ms);
|
|
10023
|
-
|
|
10268
|
+
logger14.log(
|
|
10024
10269
|
`Polling fallback enabled (every ${this.config.sandbox.poll_interval_ms / 1e3}s)`
|
|
10025
10270
|
);
|
|
10026
10271
|
await shutdownPromise;
|
|
@@ -10031,7 +10276,7 @@ var Runner = class _Runner {
|
|
|
10031
10276
|
this.subscription.cleanup();
|
|
10032
10277
|
}
|
|
10033
10278
|
if (this.state.jobPromises.size > 0) {
|
|
10034
|
-
|
|
10279
|
+
logger14.log(
|
|
10035
10280
|
`Waiting for ${this.state.jobPromises.size} active job(s) to complete...`
|
|
10036
10281
|
);
|
|
10037
10282
|
await Promise.all(this.state.jobPromises);
|
|
@@ -10039,7 +10284,7 @@ var Runner = class _Runner {
|
|
|
10039
10284
|
await cleanupEnvironment(this.resources);
|
|
10040
10285
|
this.state.mode = "stopped";
|
|
10041
10286
|
this.updateStatus();
|
|
10042
|
-
|
|
10287
|
+
logger14.log("Runner stopped");
|
|
10043
10288
|
process.exit(0);
|
|
10044
10289
|
}
|
|
10045
10290
|
/**
|
|
@@ -10056,11 +10301,11 @@ var Runner = class _Runner {
|
|
|
10056
10301
|
() => pollForJob(this.config.server, this.config.group)
|
|
10057
10302
|
);
|
|
10058
10303
|
if (job) {
|
|
10059
|
-
|
|
10304
|
+
logger14.log(`Poll fallback found job: ${job.runId}`);
|
|
10060
10305
|
await this.processJob(job.runId);
|
|
10061
10306
|
}
|
|
10062
10307
|
} catch (error) {
|
|
10063
|
-
|
|
10308
|
+
logger14.error(
|
|
10064
10309
|
`Poll fallback error: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
10065
10310
|
);
|
|
10066
10311
|
}
|
|
@@ -10070,7 +10315,7 @@ var Runner = class _Runner {
|
|
|
10070
10315
|
*/
|
|
10071
10316
|
async processJob(runId) {
|
|
10072
10317
|
if (this.state.mode !== "running") {
|
|
10073
|
-
|
|
10318
|
+
logger14.log(`Not running (${this.state.mode}), ignoring job ${runId}`);
|
|
10074
10319
|
return;
|
|
10075
10320
|
}
|
|
10076
10321
|
if (this.state.activeRuns.has(runId)) {
|
|
@@ -10078,10 +10323,10 @@ var Runner = class _Runner {
|
|
|
10078
10323
|
}
|
|
10079
10324
|
if (this.state.activeRuns.size >= this.config.sandbox.max_concurrent) {
|
|
10080
10325
|
if (!this.pendingJobs.includes(runId) && this.pendingJobs.length < _Runner.MAX_PENDING_QUEUE_SIZE) {
|
|
10081
|
-
|
|
10326
|
+
logger14.log(`At capacity, queueing job ${runId}`);
|
|
10082
10327
|
this.pendingJobs.push(runId);
|
|
10083
10328
|
} else if (this.pendingJobs.length >= _Runner.MAX_PENDING_QUEUE_SIZE) {
|
|
10084
|
-
|
|
10329
|
+
logger14.log(
|
|
10085
10330
|
`Pending queue full (${_Runner.MAX_PENDING_QUEUE_SIZE}), dropping job ${runId}`
|
|
10086
10331
|
);
|
|
10087
10332
|
}
|
|
@@ -10092,11 +10337,11 @@ var Runner = class _Runner {
|
|
|
10092
10337
|
"claim",
|
|
10093
10338
|
() => claimJob(this.config.server, runId)
|
|
10094
10339
|
);
|
|
10095
|
-
|
|
10340
|
+
logger14.log(`Claimed job: ${context.runId}`);
|
|
10096
10341
|
this.state.activeRuns.add(context.runId);
|
|
10097
10342
|
this.updateStatus();
|
|
10098
10343
|
const jobPromise = this.executeJob(context).catch((error) => {
|
|
10099
|
-
|
|
10344
|
+
logger14.error(
|
|
10100
10345
|
`Job ${context.runId} failed: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
10101
10346
|
);
|
|
10102
10347
|
}).finally(() => {
|
|
@@ -10104,7 +10349,9 @@ var Runner = class _Runner {
|
|
|
10104
10349
|
this.state.jobPromises.delete(jobPromise);
|
|
10105
10350
|
this.updateStatus();
|
|
10106
10351
|
if (this.state.mode === "draining" && this.state.activeRuns.size === 0) {
|
|
10107
|
-
|
|
10352
|
+
logger14.log("[Maintenance] All jobs completed, exiting");
|
|
10353
|
+
this.state.mode = "stopping";
|
|
10354
|
+
this.updateStatus();
|
|
10108
10355
|
this.resolveShutdown?.();
|
|
10109
10356
|
return;
|
|
10110
10357
|
}
|
|
@@ -10117,33 +10364,33 @@ var Runner = class _Runner {
|
|
|
10117
10364
|
});
|
|
10118
10365
|
this.state.jobPromises.add(jobPromise);
|
|
10119
10366
|
} catch (error) {
|
|
10120
|
-
|
|
10367
|
+
logger14.log(
|
|
10121
10368
|
`Could not claim job ${runId}: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
10122
10369
|
);
|
|
10123
10370
|
}
|
|
10124
10371
|
}
|
|
10125
10372
|
async executeJob(context) {
|
|
10126
|
-
|
|
10127
|
-
|
|
10128
|
-
|
|
10373
|
+
logger14.log(` Executing job ${context.runId}...`);
|
|
10374
|
+
logger14.log(` Prompt: ${context.prompt.substring(0, 100)}...`);
|
|
10375
|
+
logger14.log(` Compose version: ${context.agentComposeVersionId}`);
|
|
10129
10376
|
try {
|
|
10130
10377
|
const result = await executeJob(context, this.config);
|
|
10131
|
-
|
|
10378
|
+
logger14.log(
|
|
10132
10379
|
` Job ${context.runId} execution completed with exit code ${result.exitCode}`
|
|
10133
10380
|
);
|
|
10134
10381
|
if (result.exitCode !== 0 && result.error) {
|
|
10135
|
-
|
|
10382
|
+
logger14.error(` Job ${context.runId} failed: ${result.error}`);
|
|
10136
10383
|
}
|
|
10137
10384
|
} catch (err) {
|
|
10138
10385
|
const error = err instanceof Error ? err.message : "Unknown execution error";
|
|
10139
|
-
|
|
10386
|
+
logger14.error(` Job ${context.runId} execution failed: ${error}`);
|
|
10140
10387
|
const result = await completeJob(
|
|
10141
10388
|
this.config.server.url,
|
|
10142
10389
|
context,
|
|
10143
10390
|
1,
|
|
10144
10391
|
error
|
|
10145
10392
|
);
|
|
10146
|
-
|
|
10393
|
+
logger14.log(` Job ${context.runId} reported as ${result.status}`);
|
|
10147
10394
|
}
|
|
10148
10395
|
}
|
|
10149
10396
|
};
|
|
@@ -10174,7 +10421,7 @@ import { dirname as dirname2, join as join3 } from "path";
|
|
|
10174
10421
|
|
|
10175
10422
|
// src/lib/firecracker/process.ts
|
|
10176
10423
|
import { readdirSync, readFileSync as readFileSync2, existsSync as existsSync3 } from "fs";
|
|
10177
|
-
import
|
|
10424
|
+
import path7 from "path";
|
|
10178
10425
|
function parseFirecrackerCmdline(cmdline) {
|
|
10179
10426
|
const args = cmdline.split("\0");
|
|
10180
10427
|
if (!args[0]?.includes("firecracker")) return null;
|
|
@@ -10207,7 +10454,7 @@ function findFirecrackerProcesses() {
|
|
|
10207
10454
|
for (const entry of entries) {
|
|
10208
10455
|
if (!/^\d+$/.test(entry)) continue;
|
|
10209
10456
|
const pid = parseInt(entry, 10);
|
|
10210
|
-
const cmdlinePath =
|
|
10457
|
+
const cmdlinePath = path7.join(procDir, entry, "cmdline");
|
|
10211
10458
|
if (!existsSync3(cmdlinePath)) continue;
|
|
10212
10459
|
try {
|
|
10213
10460
|
const cmdline = readFileSync2(cmdlinePath, "utf-8");
|
|
@@ -10225,7 +10472,7 @@ function findProcessByVmId(vmId) {
|
|
|
10225
10472
|
const processes = findFirecrackerProcesses();
|
|
10226
10473
|
return processes.find((p) => p.vmId === vmId) || null;
|
|
10227
10474
|
}
|
|
10228
|
-
function
|
|
10475
|
+
function isProcessRunning2(pid) {
|
|
10229
10476
|
try {
|
|
10230
10477
|
process.kill(pid, 0);
|
|
10231
10478
|
return true;
|
|
@@ -10234,24 +10481,24 @@ function isProcessRunning(pid) {
|
|
|
10234
10481
|
}
|
|
10235
10482
|
}
|
|
10236
10483
|
async function killProcess(pid, timeoutMs = 5e3) {
|
|
10237
|
-
if (!
|
|
10484
|
+
if (!isProcessRunning2(pid)) return true;
|
|
10238
10485
|
try {
|
|
10239
10486
|
process.kill(pid, "SIGTERM");
|
|
10240
10487
|
} catch {
|
|
10241
|
-
return !
|
|
10488
|
+
return !isProcessRunning2(pid);
|
|
10242
10489
|
}
|
|
10243
10490
|
const startTime = Date.now();
|
|
10244
10491
|
while (Date.now() - startTime < timeoutMs) {
|
|
10245
|
-
if (!
|
|
10492
|
+
if (!isProcessRunning2(pid)) return true;
|
|
10246
10493
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
10247
10494
|
}
|
|
10248
|
-
if (
|
|
10495
|
+
if (isProcessRunning2(pid)) {
|
|
10249
10496
|
try {
|
|
10250
10497
|
process.kill(pid, "SIGKILL");
|
|
10251
10498
|
} catch {
|
|
10252
10499
|
}
|
|
10253
10500
|
}
|
|
10254
|
-
return !
|
|
10501
|
+
return !isProcessRunning2(pid);
|
|
10255
10502
|
}
|
|
10256
10503
|
function findMitmproxyProcess() {
|
|
10257
10504
|
const procDir = "/proc";
|
|
@@ -10264,7 +10511,7 @@ function findMitmproxyProcess() {
|
|
|
10264
10511
|
for (const entry of entries) {
|
|
10265
10512
|
if (!/^\d+$/.test(entry)) continue;
|
|
10266
10513
|
const pid = parseInt(entry, 10);
|
|
10267
|
-
const cmdlinePath =
|
|
10514
|
+
const cmdlinePath = path7.join(procDir, entry, "cmdline");
|
|
10268
10515
|
if (!existsSync3(cmdlinePath)) continue;
|
|
10269
10516
|
try {
|
|
10270
10517
|
const cmdline = readFileSync2(cmdlinePath, "utf-8");
|
|
@@ -10756,7 +11003,7 @@ var benchmarkCommand = new Command4("benchmark").description(
|
|
|
10756
11003
|
});
|
|
10757
11004
|
|
|
10758
11005
|
// src/index.ts
|
|
10759
|
-
var version = true ? "3.
|
|
11006
|
+
var version = true ? "3.7.0" : "0.1.0";
|
|
10760
11007
|
program.name("vm0-runner").version(version).description("Self-hosted runner for VM0 agents");
|
|
10761
11008
|
program.addCommand(startCommand);
|
|
10762
11009
|
program.addCommand(doctorCommand);
|