@vm0/runner 2.13.5 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +338 -329
- package/package.json +1 -1
package/index.js
CHANGED
|
@@ -9318,188 +9318,193 @@ async function executeJob2(context, config) {
|
|
|
9318
9318
|
console.log(` Job ${context.runId} reported as ${result.status}`);
|
|
9319
9319
|
}
|
|
9320
9320
|
}
|
|
9321
|
-
var startCommand = new Command("start").description("Start the runner").option("--config <path>", "Config file path", "./runner.yaml").action(
|
|
9322
|
-
|
|
9323
|
-
|
|
9324
|
-
validateFirecrackerPaths(config.firecracker);
|
|
9325
|
-
console.log("Config valid");
|
|
9326
|
-
const datasetSuffix = process.env.AXIOM_DATASET_SUFFIX;
|
|
9327
|
-
if (!datasetSuffix) {
|
|
9328
|
-
throw new Error(
|
|
9329
|
-
"AXIOM_DATASET_SUFFIX is required. Set to 'dev' or 'prod'."
|
|
9330
|
-
);
|
|
9331
|
-
}
|
|
9332
|
-
initMetrics({
|
|
9333
|
-
serviceName: "vm0-runner",
|
|
9334
|
-
runnerLabel: config.name,
|
|
9335
|
-
axiomToken: process.env.AXIOM_TOKEN,
|
|
9336
|
-
environment: datasetSuffix
|
|
9337
|
-
});
|
|
9338
|
-
const networkCheck = checkNetworkPrerequisites();
|
|
9339
|
-
if (!networkCheck.ok) {
|
|
9340
|
-
console.error("Network prerequisites not met:");
|
|
9341
|
-
for (const error of networkCheck.errors) {
|
|
9342
|
-
console.error(` - ${error}`);
|
|
9343
|
-
}
|
|
9344
|
-
process.exit(1);
|
|
9345
|
-
}
|
|
9346
|
-
console.log("Setting up network bridge...");
|
|
9347
|
-
await setupBridge();
|
|
9348
|
-
console.log("Flushing bridge ARP cache...");
|
|
9349
|
-
await flushBridgeArpCache();
|
|
9350
|
-
console.log("Cleaning up orphaned proxy rules...");
|
|
9351
|
-
await cleanupOrphanedProxyRules(config.name);
|
|
9352
|
-
console.log("Cleaning up orphaned IP allocations...");
|
|
9353
|
-
await cleanupOrphanedAllocations();
|
|
9354
|
-
console.log("Initializing network proxy...");
|
|
9355
|
-
initVMRegistry();
|
|
9356
|
-
const proxyManager = initProxyManager({
|
|
9357
|
-
apiUrl: config.server.url,
|
|
9358
|
-
port: config.proxy.port,
|
|
9359
|
-
caDir: config.proxy.ca_dir
|
|
9360
|
-
});
|
|
9361
|
-
let proxyEnabled = false;
|
|
9321
|
+
var startCommand = new Command("start").description("Start the runner").option("--config <path>", "Config file path", "./runner.yaml").action(
|
|
9322
|
+
// eslint-disable-next-line complexity -- TODO: refactor complex function
|
|
9323
|
+
async (options) => {
|
|
9362
9324
|
try {
|
|
9363
|
-
|
|
9364
|
-
|
|
9365
|
-
console.log("
|
|
9366
|
-
|
|
9367
|
-
|
|
9368
|
-
|
|
9369
|
-
|
|
9370
|
-
console.warn(
|
|
9371
|
-
"Jobs with experimentalFirewall enabled will run without network interception"
|
|
9372
|
-
);
|
|
9373
|
-
}
|
|
9374
|
-
const statusFilePath = join2(dirname(options.config), "status.json");
|
|
9375
|
-
const startedAt = /* @__PURE__ */ new Date();
|
|
9376
|
-
const state = { mode: "running" };
|
|
9377
|
-
const updateStatus = () => {
|
|
9378
|
-
writeStatusFile(statusFilePath, state.mode, startedAt);
|
|
9379
|
-
};
|
|
9380
|
-
console.log(
|
|
9381
|
-
`Starting runner '${config.name}' for group '${config.group}'...`
|
|
9382
|
-
);
|
|
9383
|
-
console.log(`Max concurrent jobs: ${config.sandbox.max_concurrent}`);
|
|
9384
|
-
console.log(`Status file: ${statusFilePath}`);
|
|
9385
|
-
console.log("Press Ctrl+C to stop");
|
|
9386
|
-
console.log("");
|
|
9387
|
-
updateStatus();
|
|
9388
|
-
let running = true;
|
|
9389
|
-
process.on("SIGINT", () => {
|
|
9390
|
-
console.log("\nShutting down...");
|
|
9391
|
-
running = false;
|
|
9392
|
-
state.mode = "stopped";
|
|
9393
|
-
updateStatus();
|
|
9394
|
-
});
|
|
9395
|
-
process.on("SIGTERM", () => {
|
|
9396
|
-
console.log("\nShutting down...");
|
|
9397
|
-
running = false;
|
|
9398
|
-
state.mode = "stopped";
|
|
9399
|
-
updateStatus();
|
|
9400
|
-
});
|
|
9401
|
-
process.on("SIGUSR1", () => {
|
|
9402
|
-
if (state.mode === "running") {
|
|
9403
|
-
console.log("\n[Maintenance] Entering drain mode...");
|
|
9404
|
-
console.log(
|
|
9405
|
-
`[Maintenance] Active jobs: ${activeRuns.size} (will wait for completion)`
|
|
9325
|
+
const config = loadConfig(options.config);
|
|
9326
|
+
validateFirecrackerPaths(config.firecracker);
|
|
9327
|
+
console.log("Config valid");
|
|
9328
|
+
const datasetSuffix = process.env.AXIOM_DATASET_SUFFIX;
|
|
9329
|
+
if (!datasetSuffix) {
|
|
9330
|
+
throw new Error(
|
|
9331
|
+
"AXIOM_DATASET_SUFFIX is required. Set to 'dev' or 'prod'."
|
|
9406
9332
|
);
|
|
9407
|
-
state.mode = "draining";
|
|
9408
|
-
updateStatus();
|
|
9409
9333
|
}
|
|
9410
|
-
|
|
9411
|
-
|
|
9412
|
-
|
|
9413
|
-
|
|
9414
|
-
|
|
9415
|
-
|
|
9416
|
-
|
|
9417
|
-
|
|
9418
|
-
|
|
9419
|
-
|
|
9420
|
-
|
|
9421
|
-
updateStatus();
|
|
9422
|
-
}
|
|
9423
|
-
continue;
|
|
9424
|
-
}
|
|
9425
|
-
if (activeRuns.size >= config.sandbox.max_concurrent) {
|
|
9426
|
-
if (jobPromises.size > 0) {
|
|
9427
|
-
await Promise.race(jobPromises);
|
|
9428
|
-
updateStatus();
|
|
9334
|
+
initMetrics({
|
|
9335
|
+
serviceName: "vm0-runner",
|
|
9336
|
+
runnerLabel: config.name,
|
|
9337
|
+
axiomToken: process.env.AXIOM_TOKEN,
|
|
9338
|
+
environment: datasetSuffix
|
|
9339
|
+
});
|
|
9340
|
+
const networkCheck = checkNetworkPrerequisites();
|
|
9341
|
+
if (!networkCheck.ok) {
|
|
9342
|
+
console.error("Network prerequisites not met:");
|
|
9343
|
+
for (const error of networkCheck.errors) {
|
|
9344
|
+
console.error(` - ${error}`);
|
|
9429
9345
|
}
|
|
9430
|
-
|
|
9346
|
+
process.exit(1);
|
|
9431
9347
|
}
|
|
9348
|
+
console.log("Setting up network bridge...");
|
|
9349
|
+
await setupBridge();
|
|
9350
|
+
console.log("Flushing bridge ARP cache...");
|
|
9351
|
+
await flushBridgeArpCache();
|
|
9352
|
+
console.log("Cleaning up orphaned proxy rules...");
|
|
9353
|
+
await cleanupOrphanedProxyRules(config.name);
|
|
9354
|
+
console.log("Cleaning up orphaned IP allocations...");
|
|
9355
|
+
await cleanupOrphanedAllocations();
|
|
9356
|
+
console.log("Initializing network proxy...");
|
|
9357
|
+
initVMRegistry();
|
|
9358
|
+
const proxyManager = initProxyManager({
|
|
9359
|
+
apiUrl: config.server.url,
|
|
9360
|
+
port: config.proxy.port,
|
|
9361
|
+
caDir: config.proxy.ca_dir
|
|
9362
|
+
});
|
|
9363
|
+
let proxyEnabled = false;
|
|
9432
9364
|
try {
|
|
9433
|
-
|
|
9434
|
-
|
|
9435
|
-
|
|
9365
|
+
await proxyManager.start();
|
|
9366
|
+
proxyEnabled = true;
|
|
9367
|
+
console.log("Network proxy initialized successfully");
|
|
9368
|
+
} catch (err) {
|
|
9369
|
+
console.warn(
|
|
9370
|
+
`Network proxy not available: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
9436
9371
|
);
|
|
9437
|
-
|
|
9438
|
-
|
|
9439
|
-
|
|
9372
|
+
console.warn(
|
|
9373
|
+
"Jobs with experimentalFirewall enabled will run without network interception"
|
|
9374
|
+
);
|
|
9375
|
+
}
|
|
9376
|
+
const statusFilePath = join2(dirname(options.config), "status.json");
|
|
9377
|
+
const startedAt = /* @__PURE__ */ new Date();
|
|
9378
|
+
const state = { mode: "running" };
|
|
9379
|
+
const updateStatus = () => {
|
|
9380
|
+
writeStatusFile(statusFilePath, state.mode, startedAt);
|
|
9381
|
+
};
|
|
9382
|
+
console.log(
|
|
9383
|
+
`Starting runner '${config.name}' for group '${config.group}'...`
|
|
9384
|
+
);
|
|
9385
|
+
console.log(`Max concurrent jobs: ${config.sandbox.max_concurrent}`);
|
|
9386
|
+
console.log(`Status file: ${statusFilePath}`);
|
|
9387
|
+
console.log("Press Ctrl+C to stop");
|
|
9388
|
+
console.log("");
|
|
9389
|
+
updateStatus();
|
|
9390
|
+
let running = true;
|
|
9391
|
+
process.on("SIGINT", () => {
|
|
9392
|
+
console.log("\nShutting down...");
|
|
9393
|
+
running = false;
|
|
9394
|
+
state.mode = "stopped";
|
|
9395
|
+
updateStatus();
|
|
9396
|
+
});
|
|
9397
|
+
process.on("SIGTERM", () => {
|
|
9398
|
+
console.log("\nShutting down...");
|
|
9399
|
+
running = false;
|
|
9400
|
+
state.mode = "stopped";
|
|
9401
|
+
updateStatus();
|
|
9402
|
+
});
|
|
9403
|
+
process.on("SIGUSR1", () => {
|
|
9404
|
+
if (state.mode === "running") {
|
|
9405
|
+
console.log("\n[Maintenance] Entering drain mode...");
|
|
9406
|
+
console.log(
|
|
9407
|
+
`[Maintenance] Active jobs: ${activeRuns.size} (will wait for completion)`
|
|
9440
9408
|
);
|
|
9409
|
+
state.mode = "draining";
|
|
9410
|
+
updateStatus();
|
|
9411
|
+
}
|
|
9412
|
+
});
|
|
9413
|
+
const jobPromises = /* @__PURE__ */ new Set();
|
|
9414
|
+
while (running) {
|
|
9415
|
+
if (state.mode === "draining") {
|
|
9416
|
+
if (activeRuns.size === 0) {
|
|
9417
|
+
console.log(
|
|
9418
|
+
"[Maintenance] All jobs completed, exiting drain mode"
|
|
9419
|
+
);
|
|
9420
|
+
running = false;
|
|
9421
|
+
break;
|
|
9422
|
+
}
|
|
9423
|
+
if (jobPromises.size > 0) {
|
|
9424
|
+
await Promise.race(jobPromises);
|
|
9425
|
+
updateStatus();
|
|
9426
|
+
}
|
|
9427
|
+
continue;
|
|
9428
|
+
}
|
|
9429
|
+
if (activeRuns.size >= config.sandbox.max_concurrent) {
|
|
9430
|
+
if (jobPromises.size > 0) {
|
|
9431
|
+
await Promise.race(jobPromises);
|
|
9432
|
+
updateStatus();
|
|
9433
|
+
}
|
|
9441
9434
|
continue;
|
|
9442
9435
|
}
|
|
9443
|
-
console.log(`Found job: ${job.runId}`);
|
|
9444
9436
|
try {
|
|
9445
|
-
const
|
|
9446
|
-
"
|
|
9447
|
-
() =>
|
|
9437
|
+
const job = await withRunnerTiming(
|
|
9438
|
+
"poll",
|
|
9439
|
+
() => pollForJob(config.server, config.group)
|
|
9448
9440
|
);
|
|
9449
|
-
|
|
9450
|
-
|
|
9451
|
-
|
|
9452
|
-
const jobPromise = executeJob2(context, config).catch((error) => {
|
|
9453
|
-
console.error(
|
|
9454
|
-
`Job ${context.runId} failed:`,
|
|
9455
|
-
error instanceof Error ? error.message : "Unknown error"
|
|
9441
|
+
if (!job) {
|
|
9442
|
+
await new Promise(
|
|
9443
|
+
(resolve) => setTimeout(resolve, config.sandbox.poll_interval_ms)
|
|
9456
9444
|
);
|
|
9457
|
-
|
|
9458
|
-
|
|
9459
|
-
|
|
9445
|
+
continue;
|
|
9446
|
+
}
|
|
9447
|
+
console.log(`Found job: ${job.runId}`);
|
|
9448
|
+
try {
|
|
9449
|
+
const context = await withRunnerTiming(
|
|
9450
|
+
"claim",
|
|
9451
|
+
() => claimJob(config.server, job.runId)
|
|
9452
|
+
);
|
|
9453
|
+
console.log(`Claimed job: ${context.runId}`);
|
|
9454
|
+
activeRuns.add(context.runId);
|
|
9460
9455
|
updateStatus();
|
|
9461
|
-
|
|
9462
|
-
|
|
9456
|
+
const jobPromise = executeJob2(context, config).catch((error) => {
|
|
9457
|
+
console.error(
|
|
9458
|
+
`Job ${context.runId} failed:`,
|
|
9459
|
+
error instanceof Error ? error.message : "Unknown error"
|
|
9460
|
+
);
|
|
9461
|
+
}).finally(() => {
|
|
9462
|
+
activeRuns.delete(context.runId);
|
|
9463
|
+
jobPromises.delete(jobPromise);
|
|
9464
|
+
updateStatus();
|
|
9465
|
+
});
|
|
9466
|
+
jobPromises.add(jobPromise);
|
|
9467
|
+
} catch (error) {
|
|
9468
|
+
console.log(
|
|
9469
|
+
`Could not claim job ${job.runId}:`,
|
|
9470
|
+
error instanceof Error ? error.message : "Unknown error"
|
|
9471
|
+
);
|
|
9472
|
+
}
|
|
9463
9473
|
} catch (error) {
|
|
9464
|
-
console.
|
|
9465
|
-
|
|
9474
|
+
console.error(
|
|
9475
|
+
"Polling error:",
|
|
9466
9476
|
error instanceof Error ? error.message : "Unknown error"
|
|
9467
9477
|
);
|
|
9478
|
+
await new Promise((resolve) => setTimeout(resolve, 2e3));
|
|
9468
9479
|
}
|
|
9469
|
-
}
|
|
9470
|
-
|
|
9471
|
-
|
|
9472
|
-
|
|
9480
|
+
}
|
|
9481
|
+
if (jobPromises.size > 0) {
|
|
9482
|
+
console.log(
|
|
9483
|
+
`Waiting for ${jobPromises.size} active job(s) to complete...`
|
|
9473
9484
|
);
|
|
9474
|
-
await
|
|
9485
|
+
await Promise.all(jobPromises);
|
|
9475
9486
|
}
|
|
9487
|
+
if (proxyEnabled) {
|
|
9488
|
+
console.log("Stopping network proxy...");
|
|
9489
|
+
await getProxyManager().stop();
|
|
9490
|
+
}
|
|
9491
|
+
console.log("Flushing metrics...");
|
|
9492
|
+
await flushMetrics();
|
|
9493
|
+
await shutdownMetrics();
|
|
9494
|
+
state.mode = "stopped";
|
|
9495
|
+
updateStatus();
|
|
9496
|
+
console.log("Runner stopped");
|
|
9497
|
+
process.exit(0);
|
|
9498
|
+
} catch (error) {
|
|
9499
|
+
if (error instanceof Error) {
|
|
9500
|
+
console.error(`Error: ${error.message}`);
|
|
9501
|
+
} else {
|
|
9502
|
+
console.error("An unknown error occurred");
|
|
9503
|
+
}
|
|
9504
|
+
process.exit(1);
|
|
9476
9505
|
}
|
|
9477
|
-
if (jobPromises.size > 0) {
|
|
9478
|
-
console.log(
|
|
9479
|
-
`Waiting for ${jobPromises.size} active job(s) to complete...`
|
|
9480
|
-
);
|
|
9481
|
-
await Promise.all(jobPromises);
|
|
9482
|
-
}
|
|
9483
|
-
if (proxyEnabled) {
|
|
9484
|
-
console.log("Stopping network proxy...");
|
|
9485
|
-
await getProxyManager().stop();
|
|
9486
|
-
}
|
|
9487
|
-
console.log("Flushing metrics...");
|
|
9488
|
-
await flushMetrics();
|
|
9489
|
-
await shutdownMetrics();
|
|
9490
|
-
state.mode = "stopped";
|
|
9491
|
-
updateStatus();
|
|
9492
|
-
console.log("Runner stopped");
|
|
9493
|
-
process.exit(0);
|
|
9494
|
-
} catch (error) {
|
|
9495
|
-
if (error instanceof Error) {
|
|
9496
|
-
console.error(`Error: ${error.message}`);
|
|
9497
|
-
} else {
|
|
9498
|
-
console.error("An unknown error occurred");
|
|
9499
|
-
}
|
|
9500
|
-
process.exit(1);
|
|
9501
9506
|
}
|
|
9502
|
-
|
|
9507
|
+
);
|
|
9503
9508
|
|
|
9504
9509
|
// src/commands/doctor.ts
|
|
9505
9510
|
import { Command as Command2 } from "commander";
|
|
@@ -9614,194 +9619,197 @@ function findMitmproxyProcess() {
|
|
|
9614
9619
|
}
|
|
9615
9620
|
|
|
9616
9621
|
// src/commands/doctor.ts
|
|
9617
|
-
var doctorCommand = new Command2("doctor").description("Diagnose runner health, check network, and detect issues").option("--config <path>", "Config file path", "./runner.yaml").action(
|
|
9618
|
-
|
|
9619
|
-
|
|
9620
|
-
|
|
9621
|
-
|
|
9622
|
-
|
|
9623
|
-
|
|
9624
|
-
|
|
9625
|
-
|
|
9626
|
-
|
|
9627
|
-
|
|
9628
|
-
|
|
9629
|
-
|
|
9630
|
-
|
|
9631
|
-
if (status.started_at) {
|
|
9632
|
-
const started = new Date(status.started_at);
|
|
9633
|
-
const uptime = formatUptime(Date.now() - started.getTime());
|
|
9634
|
-
console.log(
|
|
9635
|
-
`Started: ${started.toLocaleString()} (uptime: ${uptime})`
|
|
9622
|
+
var doctorCommand = new Command2("doctor").description("Diagnose runner health, check network, and detect issues").option("--config <path>", "Config file path", "./runner.yaml").action(
|
|
9623
|
+
// eslint-disable-next-line complexity -- TODO: refactor complex function
|
|
9624
|
+
async (options) => {
|
|
9625
|
+
try {
|
|
9626
|
+
const config = loadConfig(options.config);
|
|
9627
|
+
const configDir = dirname2(options.config);
|
|
9628
|
+
const statusFilePath = join3(configDir, "status.json");
|
|
9629
|
+
const workspacesDir = join3(configDir, "workspaces");
|
|
9630
|
+
console.log(`Runner: ${config.name}`);
|
|
9631
|
+
let status = null;
|
|
9632
|
+
if (existsSync3(statusFilePath)) {
|
|
9633
|
+
try {
|
|
9634
|
+
status = JSON.parse(
|
|
9635
|
+
readFileSync3(statusFilePath, "utf-8")
|
|
9636
9636
|
);
|
|
9637
|
+
console.log(`Mode: ${status.mode}`);
|
|
9638
|
+
if (status.started_at) {
|
|
9639
|
+
const started = new Date(status.started_at);
|
|
9640
|
+
const uptime = formatUptime(Date.now() - started.getTime());
|
|
9641
|
+
console.log(
|
|
9642
|
+
`Started: ${started.toLocaleString()} (uptime: ${uptime})`
|
|
9643
|
+
);
|
|
9644
|
+
}
|
|
9645
|
+
} catch {
|
|
9646
|
+
console.log("Mode: unknown (status.json unreadable)");
|
|
9637
9647
|
}
|
|
9638
|
-
}
|
|
9639
|
-
console.log("Mode: unknown (status.json
|
|
9640
|
-
}
|
|
9641
|
-
} else {
|
|
9642
|
-
console.log("Mode: unknown (no status.json)");
|
|
9643
|
-
}
|
|
9644
|
-
console.log("");
|
|
9645
|
-
console.log("API Connectivity:");
|
|
9646
|
-
try {
|
|
9647
|
-
await pollForJob(config.server, config.group);
|
|
9648
|
-
console.log(` \u2713 Connected to ${config.server.url}`);
|
|
9649
|
-
console.log(" \u2713 Authentication: OK");
|
|
9650
|
-
} catch (error) {
|
|
9651
|
-
console.log(` \u2717 Cannot connect to ${config.server.url}`);
|
|
9652
|
-
console.log(
|
|
9653
|
-
` Error: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
9654
|
-
);
|
|
9655
|
-
}
|
|
9656
|
-
console.log("");
|
|
9657
|
-
console.log("Network:");
|
|
9658
|
-
const warnings = [];
|
|
9659
|
-
const bridgeStatus = await checkBridgeStatus();
|
|
9660
|
-
if (bridgeStatus.exists) {
|
|
9661
|
-
console.log(` \u2713 Bridge ${BRIDGE_NAME2} (${bridgeStatus.ip})`);
|
|
9662
|
-
} else {
|
|
9663
|
-
console.log(` \u2717 Bridge ${BRIDGE_NAME2} not found`);
|
|
9664
|
-
warnings.push({
|
|
9665
|
-
message: `Network bridge ${BRIDGE_NAME2} does not exist`
|
|
9666
|
-
});
|
|
9667
|
-
}
|
|
9668
|
-
const proxyPort = config.proxy.port;
|
|
9669
|
-
const mitmProc = findMitmproxyProcess();
|
|
9670
|
-
const portInUse = await isPortInUse(proxyPort);
|
|
9671
|
-
if (mitmProc) {
|
|
9672
|
-
console.log(
|
|
9673
|
-
` \u2713 Proxy mitmproxy (PID ${mitmProc.pid}) on :${proxyPort}`
|
|
9674
|
-
);
|
|
9675
|
-
} else if (portInUse) {
|
|
9676
|
-
console.log(
|
|
9677
|
-
` \u26A0\uFE0F Proxy port :${proxyPort} in use but mitmproxy process not found`
|
|
9678
|
-
);
|
|
9679
|
-
warnings.push({
|
|
9680
|
-
message: `Port ${proxyPort} is in use but mitmproxy process not detected`
|
|
9681
|
-
});
|
|
9682
|
-
} else {
|
|
9683
|
-
console.log(` \u2717 Proxy mitmproxy not running`);
|
|
9684
|
-
warnings.push({ message: "Proxy mitmproxy is not running" });
|
|
9685
|
-
}
|
|
9686
|
-
console.log("");
|
|
9687
|
-
const processes = findFirecrackerProcesses();
|
|
9688
|
-
const tapDevices = await listTapDevices();
|
|
9689
|
-
const workspaces = existsSync3(workspacesDir) ? readdirSync2(workspacesDir).filter((d) => d.startsWith("vm0-")) : [];
|
|
9690
|
-
const jobs = [];
|
|
9691
|
-
const statusVmIds = /* @__PURE__ */ new Set();
|
|
9692
|
-
const allocations = getAllocations();
|
|
9693
|
-
if (status?.active_run_ids) {
|
|
9694
|
-
for (const runId of status.active_run_ids) {
|
|
9695
|
-
const vmId = runId.split("-")[0];
|
|
9696
|
-
if (!vmId) continue;
|
|
9697
|
-
statusVmIds.add(vmId);
|
|
9698
|
-
const proc = processes.find((p) => p.vmId === vmId);
|
|
9699
|
-
const ip = getIPForVm(vmId) ?? "not allocated";
|
|
9700
|
-
jobs.push({
|
|
9701
|
-
runId,
|
|
9702
|
-
vmId,
|
|
9703
|
-
ip,
|
|
9704
|
-
hasProcess: !!proc,
|
|
9705
|
-
pid: proc?.pid
|
|
9706
|
-
});
|
|
9648
|
+
} else {
|
|
9649
|
+
console.log("Mode: unknown (no status.json)");
|
|
9707
9650
|
}
|
|
9708
|
-
|
|
9709
|
-
|
|
9710
|
-
|
|
9711
|
-
|
|
9712
|
-
|
|
9713
|
-
|
|
9714
|
-
|
|
9715
|
-
|
|
9716
|
-
console.log(`Runs (${jobs.length} active, max ${maxConcurrent}):`);
|
|
9717
|
-
if (jobs.length === 0) {
|
|
9718
|
-
console.log(" No active runs");
|
|
9719
|
-
} else {
|
|
9720
|
-
console.log(
|
|
9721
|
-
" Run ID VM ID IP Status"
|
|
9722
|
-
);
|
|
9723
|
-
for (const job of jobs) {
|
|
9724
|
-
const ipConflict = (ipToVmIds.get(job.ip)?.length ?? 0) > 1;
|
|
9725
|
-
let statusText;
|
|
9726
|
-
if (ipConflict) {
|
|
9727
|
-
statusText = "\u26A0\uFE0F IP conflict!";
|
|
9728
|
-
} else if (job.hasProcess) {
|
|
9729
|
-
statusText = `\u2713 Running (PID ${job.pid})`;
|
|
9730
|
-
} else {
|
|
9731
|
-
statusText = "\u26A0\uFE0F No process";
|
|
9732
|
-
}
|
|
9651
|
+
console.log("");
|
|
9652
|
+
console.log("API Connectivity:");
|
|
9653
|
+
try {
|
|
9654
|
+
await pollForJob(config.server, config.group);
|
|
9655
|
+
console.log(` \u2713 Connected to ${config.server.url}`);
|
|
9656
|
+
console.log(" \u2713 Authentication: OK");
|
|
9657
|
+
} catch (error) {
|
|
9658
|
+
console.log(` \u2717 Cannot connect to ${config.server.url}`);
|
|
9733
9659
|
console.log(
|
|
9734
|
-
`
|
|
9660
|
+
` Error: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
9735
9661
|
);
|
|
9736
9662
|
}
|
|
9737
|
-
|
|
9738
|
-
|
|
9739
|
-
|
|
9740
|
-
|
|
9663
|
+
console.log("");
|
|
9664
|
+
console.log("Network:");
|
|
9665
|
+
const warnings = [];
|
|
9666
|
+
const bridgeStatus = await checkBridgeStatus();
|
|
9667
|
+
if (bridgeStatus.exists) {
|
|
9668
|
+
console.log(` \u2713 Bridge ${BRIDGE_NAME2} (${bridgeStatus.ip})`);
|
|
9669
|
+
} else {
|
|
9670
|
+
console.log(` \u2717 Bridge ${BRIDGE_NAME2} not found`);
|
|
9741
9671
|
warnings.push({
|
|
9742
|
-
message: `
|
|
9672
|
+
message: `Network bridge ${BRIDGE_NAME2} does not exist`
|
|
9743
9673
|
});
|
|
9744
9674
|
}
|
|
9745
|
-
|
|
9746
|
-
|
|
9747
|
-
|
|
9675
|
+
const proxyPort = config.proxy.port;
|
|
9676
|
+
const mitmProc = findMitmproxyProcess();
|
|
9677
|
+
const portInUse = await isPortInUse(proxyPort);
|
|
9678
|
+
if (mitmProc) {
|
|
9679
|
+
console.log(
|
|
9680
|
+
` \u2713 Proxy mitmproxy (PID ${mitmProc.pid}) on :${proxyPort}`
|
|
9681
|
+
);
|
|
9682
|
+
} else if (portInUse) {
|
|
9683
|
+
console.log(
|
|
9684
|
+
` \u26A0\uFE0F Proxy port :${proxyPort} in use but mitmproxy process not found`
|
|
9685
|
+
);
|
|
9748
9686
|
warnings.push({
|
|
9749
|
-
message: `
|
|
9687
|
+
message: `Port ${proxyPort} is in use but mitmproxy process not detected`
|
|
9750
9688
|
});
|
|
9689
|
+
} else {
|
|
9690
|
+
console.log(` \u2717 Proxy mitmproxy not running`);
|
|
9691
|
+
warnings.push({ message: "Proxy mitmproxy is not running" });
|
|
9751
9692
|
}
|
|
9752
|
-
|
|
9753
|
-
|
|
9754
|
-
|
|
9755
|
-
|
|
9756
|
-
|
|
9757
|
-
|
|
9758
|
-
|
|
9693
|
+
console.log("");
|
|
9694
|
+
const processes = findFirecrackerProcesses();
|
|
9695
|
+
const tapDevices = await listTapDevices();
|
|
9696
|
+
const workspaces = existsSync3(workspacesDir) ? readdirSync2(workspacesDir).filter((d) => d.startsWith("vm0-")) : [];
|
|
9697
|
+
const jobs = [];
|
|
9698
|
+
const statusVmIds = /* @__PURE__ */ new Set();
|
|
9699
|
+
const allocations = getAllocations();
|
|
9700
|
+
if (status?.active_run_ids) {
|
|
9701
|
+
for (const runId of status.active_run_ids) {
|
|
9702
|
+
const vmId = runId.split("-")[0];
|
|
9703
|
+
if (!vmId) continue;
|
|
9704
|
+
statusVmIds.add(vmId);
|
|
9705
|
+
const proc = processes.find((p) => p.vmId === vmId);
|
|
9706
|
+
const ip = getIPForVm(vmId) ?? "not allocated";
|
|
9707
|
+
jobs.push({
|
|
9708
|
+
runId,
|
|
9709
|
+
vmId,
|
|
9710
|
+
ip,
|
|
9711
|
+
hasProcess: !!proc,
|
|
9712
|
+
pid: proc?.pid
|
|
9713
|
+
});
|
|
9714
|
+
}
|
|
9759
9715
|
}
|
|
9760
|
-
|
|
9761
|
-
|
|
9762
|
-
|
|
9763
|
-
|
|
9764
|
-
|
|
9765
|
-
message: `Orphan TAP device: ${tap} (no matching job or process)`
|
|
9766
|
-
});
|
|
9716
|
+
const ipToVmIds = /* @__PURE__ */ new Map();
|
|
9717
|
+
for (const [ip, allocation] of allocations) {
|
|
9718
|
+
const existing = ipToVmIds.get(ip) ?? [];
|
|
9719
|
+
existing.push(allocation.vmId);
|
|
9720
|
+
ipToVmIds.set(ip, existing);
|
|
9767
9721
|
}
|
|
9768
|
-
|
|
9769
|
-
|
|
9770
|
-
|
|
9771
|
-
|
|
9722
|
+
const maxConcurrent = config.sandbox.max_concurrent;
|
|
9723
|
+
console.log(`Runs (${jobs.length} active, max ${maxConcurrent}):`);
|
|
9724
|
+
if (jobs.length === 0) {
|
|
9725
|
+
console.log(" No active runs");
|
|
9726
|
+
} else {
|
|
9727
|
+
console.log(
|
|
9728
|
+
" Run ID VM ID IP Status"
|
|
9729
|
+
);
|
|
9730
|
+
for (const job of jobs) {
|
|
9731
|
+
const ipConflict = (ipToVmIds.get(job.ip)?.length ?? 0) > 1;
|
|
9732
|
+
let statusText;
|
|
9733
|
+
if (ipConflict) {
|
|
9734
|
+
statusText = "\u26A0\uFE0F IP conflict!";
|
|
9735
|
+
} else if (job.hasProcess) {
|
|
9736
|
+
statusText = `\u2713 Running (PID ${job.pid})`;
|
|
9737
|
+
} else {
|
|
9738
|
+
statusText = "\u26A0\uFE0F No process";
|
|
9739
|
+
}
|
|
9740
|
+
console.log(
|
|
9741
|
+
` ${job.runId} ${job.vmId} ${job.ip.padEnd(15)} ${statusText}`
|
|
9742
|
+
);
|
|
9743
|
+
}
|
|
9744
|
+
}
|
|
9745
|
+
console.log("");
|
|
9746
|
+
for (const job of jobs) {
|
|
9747
|
+
if (!job.hasProcess) {
|
|
9748
|
+
warnings.push({
|
|
9749
|
+
message: `Run ${job.vmId} in status.json but no Firecracker process running`
|
|
9750
|
+
});
|
|
9751
|
+
}
|
|
9752
|
+
}
|
|
9753
|
+
for (const [ip, vmIds] of ipToVmIds) {
|
|
9754
|
+
if (vmIds.length > 1) {
|
|
9755
|
+
warnings.push({
|
|
9756
|
+
message: `IP conflict: ${ip} assigned to ${vmIds.join(", ")}`
|
|
9757
|
+
});
|
|
9758
|
+
}
|
|
9759
|
+
}
|
|
9760
|
+
const processVmIds = new Set(processes.map((p) => p.vmId));
|
|
9761
|
+
for (const proc of processes) {
|
|
9762
|
+
if (!statusVmIds.has(proc.vmId)) {
|
|
9763
|
+
warnings.push({
|
|
9764
|
+
message: `Orphan process: PID ${proc.pid} (vmId ${proc.vmId}) not in status.json`
|
|
9765
|
+
});
|
|
9766
|
+
}
|
|
9767
|
+
}
|
|
9768
|
+
for (const tap of tapDevices) {
|
|
9769
|
+
const vmId = tap.replace("tap", "");
|
|
9770
|
+
if (!processVmIds.has(vmId) && !statusVmIds.has(vmId)) {
|
|
9771
|
+
warnings.push({
|
|
9772
|
+
message: `Orphan TAP device: ${tap} (no matching job or process)`
|
|
9773
|
+
});
|
|
9774
|
+
}
|
|
9775
|
+
}
|
|
9776
|
+
for (const ws of workspaces) {
|
|
9777
|
+
const vmId = ws.replace("vm0-", "");
|
|
9778
|
+
if (!processVmIds.has(vmId) && !statusVmIds.has(vmId)) {
|
|
9779
|
+
warnings.push({
|
|
9780
|
+
message: `Orphan workspace: ${ws} (no matching job or process)`
|
|
9781
|
+
});
|
|
9782
|
+
}
|
|
9783
|
+
}
|
|
9784
|
+
const activeVmIps = new Set(jobs.map((j) => j.ip));
|
|
9785
|
+
const iptablesRules = await listIptablesNatRules();
|
|
9786
|
+
const orphanedIptables = await findOrphanedIptablesRules(
|
|
9787
|
+
iptablesRules,
|
|
9788
|
+
activeVmIps,
|
|
9789
|
+
proxyPort
|
|
9790
|
+
);
|
|
9791
|
+
for (const rule of orphanedIptables) {
|
|
9772
9792
|
warnings.push({
|
|
9773
|
-
message: `Orphan
|
|
9793
|
+
message: `Orphan iptables rule: redirect ${rule.sourceIp}:${rule.destPort} -> :${rule.redirectPort}`
|
|
9774
9794
|
});
|
|
9775
9795
|
}
|
|
9776
|
-
|
|
9777
|
-
|
|
9778
|
-
|
|
9779
|
-
|
|
9780
|
-
|
|
9781
|
-
|
|
9782
|
-
|
|
9783
|
-
);
|
|
9784
|
-
for (const rule of orphanedIptables) {
|
|
9785
|
-
warnings.push({
|
|
9786
|
-
message: `Orphan iptables rule: redirect ${rule.sourceIp}:${rule.destPort} -> :${rule.redirectPort}`
|
|
9787
|
-
});
|
|
9788
|
-
}
|
|
9789
|
-
console.log("Warnings:");
|
|
9790
|
-
if (warnings.length === 0) {
|
|
9791
|
-
console.log(" None");
|
|
9792
|
-
} else {
|
|
9793
|
-
for (const w of warnings) {
|
|
9794
|
-
console.log(` - ${w.message}`);
|
|
9796
|
+
console.log("Warnings:");
|
|
9797
|
+
if (warnings.length === 0) {
|
|
9798
|
+
console.log(" None");
|
|
9799
|
+
} else {
|
|
9800
|
+
for (const w of warnings) {
|
|
9801
|
+
console.log(` - ${w.message}`);
|
|
9802
|
+
}
|
|
9795
9803
|
}
|
|
9804
|
+
process.exit(warnings.length > 0 ? 1 : 0);
|
|
9805
|
+
} catch (error) {
|
|
9806
|
+
console.error(
|
|
9807
|
+
`Error: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
9808
|
+
);
|
|
9809
|
+
process.exit(1);
|
|
9796
9810
|
}
|
|
9797
|
-
process.exit(warnings.length > 0 ? 1 : 0);
|
|
9798
|
-
} catch (error) {
|
|
9799
|
-
console.error(
|
|
9800
|
-
`Error: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
9801
|
-
);
|
|
9802
|
-
process.exit(1);
|
|
9803
9811
|
}
|
|
9804
|
-
|
|
9812
|
+
);
|
|
9805
9813
|
function formatUptime(ms) {
|
|
9806
9814
|
const seconds = Math.floor(ms / 1e3);
|
|
9807
9815
|
const minutes = Math.floor(seconds / 60);
|
|
@@ -9819,6 +9827,7 @@ import { existsSync as existsSync4, readFileSync as readFileSync4, writeFileSync
|
|
|
9819
9827
|
import { dirname as dirname3, join as join4 } from "path";
|
|
9820
9828
|
import * as readline2 from "readline";
|
|
9821
9829
|
var killCommand = new Command3("kill").description("Force terminate a run and clean up all resources").argument("<run-id>", "Run ID (full UUID or short 8-char vmId)").option("--config <path>", "Config file path", "./runner.yaml").option("--force", "Skip confirmation prompt").action(
|
|
9830
|
+
// eslint-disable-next-line complexity -- TODO: refactor complex function
|
|
9822
9831
|
async (runIdArg, options) => {
|
|
9823
9832
|
try {
|
|
9824
9833
|
loadConfig(options.config);
|
|
@@ -10081,7 +10090,7 @@ var benchmarkCommand = new Command4("benchmark").description(
|
|
|
10081
10090
|
});
|
|
10082
10091
|
|
|
10083
10092
|
// src/index.ts
|
|
10084
|
-
var version = true ? "2.
|
|
10093
|
+
var version = true ? "2.14.0" : "0.1.0";
|
|
10085
10094
|
program.name("vm0-runner").version(version).description("Self-hosted runner for VM0 agents");
|
|
10086
10095
|
program.addCommand(startCommand);
|
|
10087
10096
|
program.addCommand(doctorCommand);
|