@calltelemetry/ct-lab-mcp 0.8.3 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/slot/manager.d.ts +12 -1
- package/dist/src/slot/manager.d.ts.map +1 -1
- package/dist/src/slot/manager.js +280 -8
- package/dist/src/slot/manager.js.map +1 -1
- package/dist/src/slot/types.d.ts +36 -0
- package/dist/src/slot/types.d.ts.map +1 -1
- package/dist/src/slot/types.js.map +1 -1
- package/dist/src/tools/index.d.ts +3 -0
- package/dist/src/tools/index.d.ts.map +1 -1
- package/dist/src/tools/index.js +9 -1
- package/dist/src/tools/index.js.map +1 -1
- package/dist/src/tools/prune-untracked-vms.d.ts +48 -0
- package/dist/src/tools/prune-untracked-vms.d.ts.map +1 -0
- package/dist/src/tools/prune-untracked-vms.js +88 -0
- package/dist/src/tools/prune-untracked-vms.js.map +1 -0
- package/dist/src/transports/sse.d.ts.map +1 -1
- package/dist/src/transports/sse.js +78 -0
- package/dist/src/transports/sse.js.map +1 -1
- package/dist/src/utils/formatters.d.ts +4 -0
- package/dist/src/utils/formatters.d.ts.map +1 -1
- package/dist/src/utils/formatters.js +39 -0
- package/dist/src/utils/formatters.js.map +1 -1
- package/package.json +1 -1
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* atomic concurrency locking (AsyncMutex), TTL lease management, candidate slot ranking,
|
|
5
5
|
* structured metadata labeling/indexing, and automated expired slot purge & reclamation.
|
|
6
6
|
*/
|
|
7
|
-
import { HostId, SlotDefinition, HostCapacityStatus, GlobalCapacitySummary, SlotCapacityReport, SlotClaimRequest, SlotClaimResult, ReleaseSlotOptions, ReleaseSlotParams, ReleaseSlotResult, LeaseExtensionResult, ListSlotsOptions, PurgeSlotsOptions, PurgeSlotsResult, UpdateSlotMetadataOptions, UpdateSlotMetadataResult, ISlotManager, CubeDefinition, ClaimCubeParams, ReleaseCubeParams, CubeClaimResult, FindSlotParams, FindSlotResult, PurgeAndRedeployParams, PurgeAndRedeployResult } from "./types.js";
|
|
7
|
+
import { HostId, SlotDefinition, HostCapacityStatus, GlobalCapacitySummary, SlotCapacityReport, SlotClaimRequest, SlotClaimResult, ReleaseSlotOptions, ReleaseSlotParams, ReleaseSlotResult, LeaseExtensionResult, ListSlotsOptions, PurgeSlotsOptions, PurgeSlotsResult, UpdateSlotMetadataOptions, UpdateSlotMetadataResult, ISlotManager, CubeDefinition, ClaimCubeParams, ReleaseCubeParams, CubeClaimResult, FindSlotParams, FindSlotResult, PurgeAndRedeployParams, PurgeAndRedeployResult, PruneUntrackedVmsParams, PruneUntrackedVmsResult } from "./types.js";
|
|
8
8
|
import { MultiHostProxmoxManager } from "../proxmox/index.js";
|
|
9
9
|
import { MultiHypervisorManager } from "../hypervisors/manager.js";
|
|
10
10
|
import { SafetyGuardrailEngine } from "../guardrails/safety.js";
|
|
@@ -133,6 +133,17 @@ export declare class SlotManager implements ISlotManager {
|
|
|
133
133
|
* One-shot purge of old/expired disposable test VM and replacement with fresh slot allocation.
|
|
134
134
|
*/
|
|
135
135
|
purgeAndRedeploy(params: PurgeAndRedeployParams): Promise<PurgeAndRedeployResult>;
|
|
136
|
+
/**
|
|
137
|
+
* Safe Prune of Abandoned / Untracked Disposable Test VMs across Hypervisors.
|
|
138
|
+
* Scans inventory for stopped disposable VMs not tied to any active slot lease,
|
|
139
|
+
* enforces strict non-bypassable guardrails, and frees hypervisor datastore and memory.
|
|
140
|
+
*/
|
|
141
|
+
pruneUntrackedVms(params?: PruneUntrackedVmsParams): Promise<PruneUntrackedVmsResult>;
|
|
142
|
+
/**
|
|
143
|
+
* Heartbeat-driven lease auto-extension for active sessions.
|
|
144
|
+
* If lease is active and expiring within 45 mins, extends by extendMinutes (default 30m) up to 8h cap.
|
|
145
|
+
*/
|
|
146
|
+
touchSlotLease(slotIdOrClaimOrLinear: string, extendMinutes?: number): Promise<SlotDefinition | null>;
|
|
136
147
|
/**
|
|
137
148
|
* Attach a created VMID or VM name to a claimed slot.
|
|
138
149
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../../../src/slot/manager.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,MAAM,EACN,cAAc,EAEd,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,EAClB,gBAAgB,EAEhB,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAGhB,yBAAyB,EACzB,wBAAwB,EAExB,YAAY,EASZ,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,EAKf,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAgC,MAAM,yBAAyB,CAAC;AAC9F,OAAO,EAAa,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../../../src/slot/manager.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,MAAM,EACN,cAAc,EAEd,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,EAClB,gBAAgB,EAEhB,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAGhB,yBAAyB,EACzB,wBAAwB,EAExB,YAAY,EASZ,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,EAKf,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,uBAAuB,EAExB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAgC,MAAM,yBAAyB,CAAC;AAC9F,OAAO,EAAa,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAGrE,cAAc,YAAY,CAAC;AAE3B,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IACpC,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE;IAC1C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IACpC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,GAAG,MAAM,CAeT;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,qBAAqB,EAAE,CAAC;IAChC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,cAAc,CAAC,EAAE,uBAAuB,CAAC;IACzC,iBAAiB,CAAC,EAAE,sBAAsB,CAAC;IAC3C,eAAe,CAAC,EAAE,qBAAqB,CAAC;IACxC,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AA2BD,eAAO,MAAM,wBAAwB,EAAE,qBAWtC,CAAC;AAEF,eAAO,MAAM,aAAa,EAAE,cAAc,EAuCzC,CAAC;AAEF,qBAAa,WAAY,YAAW,YAAY;IAC9C,OAAO,CAAC,KAAK,CAAiD;IAC9D,OAAO,CAAC,KAAK,CAA0C;IACvD,OAAO,CAAC,KAAK,CAA0C;IACvD,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,cAAc,CAAC,CAA0B;IACjD,OAAO,CAAC,iBAAiB,CAAC,CAAyB;IACnD,OAAO,CAAC,eAAe,CAAwB;IAC/C,OAAO,CAAC,SAAS,CAAoB;IACrC,OAAO,CAAC,cAAc,CAAU;gBAEpB,OAAO,GAAE,kBAAuB;IAyB5C;;OAEG;IACH,oBAAoB,CAAC,iBAAiB,EAAE,sBAAsB,GAAG,IAAI;IAIrE;;OAEG;IACH,kBAAkB,CAAC,eAAe,EAAE,qBAAqB,GAAG,IAAI;IAIhE;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,qBAAqB,GAAG,IAAI;IAwD/C;;OAEG;IACH,OAAO,CAAC,eAAe;IAKvB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,SAAS;IAuBjB;;OAEG;IACH,OAAO,CAAC,yBAAyB;IA2CjC;;OAEG;IACG,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;YAItD,iBAAiB;IAgW/B;;OAEG;IACG,WAAW,CACf,gBAAgB,EAAE,MAAM,GAAG,kBAAkB,GAAG,iBAAiB,EACjE,aAAa,CAAC,EAAE,OAAO,GACtB,OAAO,CAAC,iBAAiB,CAAC;IA0I7B;;OAEG;IACG,WAAW,CACf,OAAO,EAAE,MAAM,EACf,iBAAiB,EAAE,MAAM,EACzB,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,CAAC;IA+EhC;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC;IAmF/F;;OAEG;IACG,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IA+F1E;;OAEG;IACG,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC,qBAAqB,GAAG,kBAAkB,CAAC;IAuE7F;;OAEG;IACG,iBAAiB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,qBAAqB,GAAG,kBAAkB,CAAC;IAgBpH;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;IAMvC;;;OAGG;IACG,UAAU,CAAC,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAI9D,kBAAkB;IA6ThC;;;;OAIG;IACG,QAAQ,CAAC,MAAM,GAAE,cAAmB,GAAG,OAAO,CAAC,cAAc,CAAC;IA4SpE;;OAEG;IACG,gBAAgB,CAAC,MAAM,EAAE,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAgIvF;;;;OAIG;IACG,iBAAiB,CAAC,MAAM,GAAE,uBAA4B,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAuK/F;;;OAGG;IACG,cAAc,CAAC,qBAAqB,EAAE,MAAM,EAAE,aAAa,SAAK,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAsCvG;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAmBhG;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAQ7D;;OAEG;IACG,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAUvE;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAqC/B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAiBjC;;OAEG;IACG,SAAS,CAAC,OAAO,UAAQ,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAkE3D;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAiB7D;;OAEG;IACG,SAAS,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAwFlE;;OAEG;IACG,WAAW,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CA6D7F"}
|
package/dist/src/slot/manager.js
CHANGED
|
@@ -9,6 +9,7 @@ import { CapacityExceededError, GlobalCapacityExceededError, SlotNotFoundError,
|
|
|
9
9
|
import { AsyncMutex } from "../utils/mutex.js";
|
|
10
10
|
import { defaultSafetyGuardrailEngine } from "../guardrails/safety.js";
|
|
11
11
|
import { telemetry } from "../telemetry/index.js";
|
|
12
|
+
import { logger } from "../utils/logger.js";
|
|
12
13
|
export * from "./types.js";
|
|
13
14
|
/**
|
|
14
15
|
* Calculates pure hardware capacity in standard slots (default 8GB).
|
|
@@ -1377,13 +1378,33 @@ export class SlotManager {
|
|
|
1377
1378
|
const totalMem = hostConfig.totalMemoryMb;
|
|
1378
1379
|
const reserved = hostConfig.reservedHeadroomMb || 0;
|
|
1379
1380
|
const freeMem = Math.max(0, totalMem - reserved - (hostSlots.length - availCount) * (hostConfig.slotSizeMb || 8192));
|
|
1380
|
-
//
|
|
1381
|
-
|
|
1381
|
+
// Host tiering: proxmox-mini (Tier 1 Primary), proxmox-lab (Tier 2 Secondary), ESXi (Tier 3 Fallback)
|
|
1382
|
+
let tier = "secondary";
|
|
1383
|
+
let tierBoost = 10000;
|
|
1384
|
+
let recommendationReason = "Secondary placement: Proxmox Lab appliance cluster";
|
|
1385
|
+
if (hid === "proxmox-mini") {
|
|
1386
|
+
tier = "primary";
|
|
1387
|
+
tierBoost = 20000;
|
|
1388
|
+
recommendationReason = "Primary placement: Minisforum high-IOPS NVMe appliance host";
|
|
1389
|
+
}
|
|
1390
|
+
else if (hid === "proxmox-lab") {
|
|
1391
|
+
tier = "secondary";
|
|
1392
|
+
tierBoost = 10000;
|
|
1393
|
+
recommendationReason = "Secondary placement: Proxmox Lab appliance cluster";
|
|
1394
|
+
}
|
|
1395
|
+
else if (hostConfig.hypervisorType === "esxi" || hid.startsWith("esxi")) {
|
|
1396
|
+
tier = "fallback";
|
|
1397
|
+
tierBoost = 0; // Least preferred host — contains CUCM/CUBE and testbed fixtures
|
|
1398
|
+
recommendationReason = "Fallback placement: ESXi Intel host (reserved for fixtures / heavy spillover)";
|
|
1399
|
+
}
|
|
1400
|
+
// Available slots score with tier priority: Proxmox Mini always outranks Proxmox Lab, and Proxmox always outranks ESXi
|
|
1401
|
+
const score = (availCount > 0 ? tierBoost : 0) + availCount * 100 + Math.floor(freeMem / 1024) * 10 + Math.floor(freeDisk / 100);
|
|
1382
1402
|
allCandidates.push({
|
|
1383
1403
|
hostId: hid,
|
|
1384
1404
|
name: hostConfig.name,
|
|
1385
1405
|
hypervisorType: hostConfig.hypervisorType || (hid.startsWith("esxi") ? "esxi" : "proxmox"),
|
|
1386
1406
|
status: "online",
|
|
1407
|
+
tier,
|
|
1387
1408
|
availableSlots: availCount,
|
|
1388
1409
|
totalPureSlots: pureCapacity,
|
|
1389
1410
|
freeMemoryMb: freeMem,
|
|
@@ -1391,15 +1412,56 @@ export class SlotManager {
|
|
|
1391
1412
|
freeDiskGb: freeDisk,
|
|
1392
1413
|
cpuUsedPercent: 0,
|
|
1393
1414
|
runningVms: hostSlots.length - availCount,
|
|
1394
|
-
score
|
|
1415
|
+
score,
|
|
1416
|
+
recommendationReason
|
|
1395
1417
|
});
|
|
1396
1418
|
}
|
|
1397
1419
|
// Sort candidates by score descending
|
|
1398
1420
|
allCandidates.sort((a, b) => b.score - a.score);
|
|
1399
1421
|
const availableSlotsTotal = allCandidates.reduce((sum, c) => sum + c.availableSlots, 0);
|
|
1422
|
+
// Check if Proxmox hosts have expired leases or disposable VMs
|
|
1423
|
+
const proxmoxReclaimableSlots = Array.from(this.slots.values()).filter((s) => (s.hostId === "proxmox-mini" || s.hostId === "proxmox-lab") &&
|
|
1424
|
+
(s.status === "EXPIRED" || (s.expiresAt && new Date(s.expiresAt).getTime() <= Date.now())));
|
|
1400
1425
|
// If capacity is available
|
|
1401
1426
|
if (availableSlotsTotal > 0) {
|
|
1402
|
-
|
|
1427
|
+
let winningCandidate = allCandidates[0];
|
|
1428
|
+
// Strict ESXi Fallback Rule: If winning host is ESXi, but Proxmox Mini/Lab has reclaimable expired slots
|
|
1429
|
+
// AND purgeIfFull is requested, auto-purge Proxmox instead of spilling over to ESXi!
|
|
1430
|
+
if (winningCandidate.hypervisorType === "esxi" && proxmoxReclaimableSlots.length > 0 && params.purgeIfFull) {
|
|
1431
|
+
const targetPurgedHost = proxmoxReclaimableSlots.some((s) => s.hostId === "proxmox-mini")
|
|
1432
|
+
? "proxmox-mini"
|
|
1433
|
+
: "proxmox-lab";
|
|
1434
|
+
const purgeRes = await this.purgeSlotsInternal({
|
|
1435
|
+
host: targetPurgedHost,
|
|
1436
|
+
destroyVms: true,
|
|
1437
|
+
includeOrphans: true,
|
|
1438
|
+
dryRun: false
|
|
1439
|
+
});
|
|
1440
|
+
if (purgeRes.purgedSlotsCount > 0 || purgeRes.destroyedVmsCount > 0) {
|
|
1441
|
+
const tag = params.tag || params.linearIssue || params.requester || "find-slot-reclaim";
|
|
1442
|
+
const claimResult = await this.claimSlotInternal({
|
|
1443
|
+
tag,
|
|
1444
|
+
requester: params.requester || tag,
|
|
1445
|
+
linearIssue: params.linearIssue,
|
|
1446
|
+
label: params.label,
|
|
1447
|
+
workloadType: params.workloadType,
|
|
1448
|
+
targetVersion: params.targetVersion,
|
|
1449
|
+
branch: params.branch,
|
|
1450
|
+
host: targetPurgedHost,
|
|
1451
|
+
durationMinutes: params.durationMinutes,
|
|
1452
|
+
allowEsxiSlots: allowEsxi,
|
|
1453
|
+
purpose: `Auto-reclaimed Proxmox slot via find_slot to avoid ESXi fallback`
|
|
1454
|
+
});
|
|
1455
|
+
return {
|
|
1456
|
+
status: "auto_reclaimed_and_claimed",
|
|
1457
|
+
candidateHost: allCandidates.find((c) => c.hostId === targetPurgedHost) || allCandidates[0],
|
|
1458
|
+
allCandidates,
|
|
1459
|
+
availableSlotsTotal,
|
|
1460
|
+
claim: claimResult,
|
|
1461
|
+
message: `Proxmox hosts were at capacity, but auto-purged expired lease(s) on '${targetPurgedHost}' to keep ESXi fixture host free. Successfully claimed '${claimResult.slotId}' on '${targetPurgedHost}'.`
|
|
1462
|
+
};
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1403
1465
|
let claimResult;
|
|
1404
1466
|
if (params.autoClaim) {
|
|
1405
1467
|
const tag = params.tag || params.linearIssue || params.requester || "find-slot-claim";
|
|
@@ -1425,12 +1487,16 @@ export class SlotManager {
|
|
|
1425
1487
|
message: `Found available slot and successfully auto-claimed '${claimResult.slotId}' on host '${winningCandidate.hostId}'.`
|
|
1426
1488
|
};
|
|
1427
1489
|
}
|
|
1490
|
+
let msg = `Found ${availableSlotsTotal} available 8GB slot(s) across ${allCandidates.length} eligible host(s). Recommended placement: '${winningCandidate.hostId}' (${winningCandidate.availableSlots} slots available).`;
|
|
1491
|
+
if (winningCandidate.hypervisorType === "esxi" && proxmoxReclaimableSlots.length > 0) {
|
|
1492
|
+
msg += ` (Note: Proxmox Mini has ${proxmoxReclaimableSlots.length} expired lease(s) that can be reclaimed with purgeIfFull=true to avoid placing on ESXi).`;
|
|
1493
|
+
}
|
|
1428
1494
|
return {
|
|
1429
1495
|
status: "available",
|
|
1430
1496
|
candidateHost: winningCandidate,
|
|
1431
1497
|
allCandidates,
|
|
1432
1498
|
availableSlotsTotal,
|
|
1433
|
-
message:
|
|
1499
|
+
message: msg
|
|
1434
1500
|
};
|
|
1435
1501
|
}
|
|
1436
1502
|
// If capacity is NOT available (availableSlotsTotal === 0)
|
|
@@ -1602,18 +1668,31 @@ export class SlotManager {
|
|
|
1602
1668
|
}
|
|
1603
1669
|
// 3. Find associated slot if any
|
|
1604
1670
|
const slotToRelease = matchingSlot || Array.from(this.slots.values()).find((s) => (s.vmId !== null && String(s.vmId) === String(targetVm.id)) || (linearIssue && s.linearIssue === linearIssue));
|
|
1605
|
-
// 4.
|
|
1671
|
+
// 4. Pre-destruction snapshot (if requested)
|
|
1672
|
+
if (params.snapshotBeforeDestroy) {
|
|
1673
|
+
try {
|
|
1674
|
+
const drivers = this.hypervisorManager.getAllDrivers();
|
|
1675
|
+
const driver = drivers.find((d) => d.hostName === targetVm.hostName);
|
|
1676
|
+
if (driver && driver.createSnapshot) {
|
|
1677
|
+
await driver.createSnapshot(targetVm.id, `pre-redeploy-${Date.now()}`, "Pre-purge snapshot before redeploy", false);
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
catch (snapErr) {
|
|
1681
|
+
logger.warn(`Failed to create pre-redeploy snapshot for VM ${targetVm.name}:`, snapErr);
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
// 5. Destroy old VM
|
|
1606
1685
|
await this.hypervisorManager.vmAction({
|
|
1607
1686
|
vmIdOrName: String(targetVm.id),
|
|
1608
1687
|
targetHost: targetVm.hostName,
|
|
1609
1688
|
action: "destroy",
|
|
1610
1689
|
force: true
|
|
1611
1690
|
});
|
|
1612
|
-
//
|
|
1691
|
+
// 6. Release existing slot if present
|
|
1613
1692
|
if (slotToRelease) {
|
|
1614
1693
|
this.resetSlot(slotToRelease);
|
|
1615
1694
|
}
|
|
1616
|
-
//
|
|
1695
|
+
// 7. Claim new slot on the same host (or auto)
|
|
1617
1696
|
const targetHost = params.host || targetVm.hostName;
|
|
1618
1697
|
const tag = linearIssue || targetVm.name;
|
|
1619
1698
|
const newSlot = await this.claimSlotInternal({
|
|
@@ -1642,6 +1721,199 @@ export class SlotManager {
|
|
|
1642
1721
|
};
|
|
1643
1722
|
});
|
|
1644
1723
|
}
|
|
1724
|
+
/**
|
|
1725
|
+
* Safe Prune of Abandoned / Untracked Disposable Test VMs across Hypervisors.
|
|
1726
|
+
* Scans inventory for stopped disposable VMs not tied to any active slot lease,
|
|
1727
|
+
* enforces strict non-bypassable guardrails, and frees hypervisor datastore and memory.
|
|
1728
|
+
*/
|
|
1729
|
+
async pruneUntrackedVms(params = {}) {
|
|
1730
|
+
return this.mutex.runExclusive(async () => {
|
|
1731
|
+
if (!this.hypervisorManager) {
|
|
1732
|
+
throw new InvalidSlotOperationError("HypervisorManager is required to prune untracked VMs.");
|
|
1733
|
+
}
|
|
1734
|
+
this.sweepExpiredSlotsInternal();
|
|
1735
|
+
const dryRun = params.dryRun ?? true;
|
|
1736
|
+
const targetHost = params.host && params.host !== "all" && params.host !== "auto"
|
|
1737
|
+
? this.normalizeHostId(params.host)
|
|
1738
|
+
: undefined;
|
|
1739
|
+
const linearFilter = params.linearIssue;
|
|
1740
|
+
const inventory = await this.hypervisorManager.listUnifiedInventory({
|
|
1741
|
+
includeTemplates: false,
|
|
1742
|
+
host: targetHost
|
|
1743
|
+
});
|
|
1744
|
+
// Active slot VM IDs that must NEVER be touched
|
|
1745
|
+
const activeSlotVmIds = new Set(Array.from(this.slots.values())
|
|
1746
|
+
.filter((s) => s.status === "CLAIMED" && s.vmId !== null && s.vmId !== undefined)
|
|
1747
|
+
.map((s) => String(s.vmId)));
|
|
1748
|
+
const prunedVms = [];
|
|
1749
|
+
let skippedProtectedCount = 0;
|
|
1750
|
+
let reclaimedMemoryMb = 0;
|
|
1751
|
+
for (const vm of inventory) {
|
|
1752
|
+
if (vm.isTemplate)
|
|
1753
|
+
continue;
|
|
1754
|
+
if (activeSlotVmIds.has(String(vm.id)))
|
|
1755
|
+
continue;
|
|
1756
|
+
// Check if VM matches disposable patterns or specific linear filter
|
|
1757
|
+
const nameLower = vm.name.toLowerCase();
|
|
1758
|
+
const isDisposablePattern = nameLower.startsWith("ct-") ||
|
|
1759
|
+
nameLower.includes("uat") ||
|
|
1760
|
+
nameLower.includes("qa-") ||
|
|
1761
|
+
nameLower.includes("demo") ||
|
|
1762
|
+
nameLower.includes("test") ||
|
|
1763
|
+
Boolean(vm.linearIssue);
|
|
1764
|
+
if (linearFilter) {
|
|
1765
|
+
const matchesLinear = vm.linearIssue === linearFilter || nameLower.includes(linearFilter.toLowerCase());
|
|
1766
|
+
if (!matchesLinear)
|
|
1767
|
+
continue;
|
|
1768
|
+
}
|
|
1769
|
+
else if (!isDisposablePattern && !params.force) {
|
|
1770
|
+
continue;
|
|
1771
|
+
}
|
|
1772
|
+
// Strict Safety Guardrail Check
|
|
1773
|
+
const protCheck = this.guardrailEngine.isProtected({
|
|
1774
|
+
vmid: vm.id,
|
|
1775
|
+
name: vm.name,
|
|
1776
|
+
isProtected: vm.isProtected,
|
|
1777
|
+
tags: vm.tags
|
|
1778
|
+
});
|
|
1779
|
+
const memMb = Math.round((vm.memoryBytes || 0) / (1024 * 1024));
|
|
1780
|
+
if (protCheck.protected) {
|
|
1781
|
+
skippedProtectedCount++;
|
|
1782
|
+
prunedVms.push({
|
|
1783
|
+
vmId: vm.id,
|
|
1784
|
+
name: vm.name,
|
|
1785
|
+
hostName: vm.hostName,
|
|
1786
|
+
hypervisor: vm.hypervisor,
|
|
1787
|
+
status: vm.status,
|
|
1788
|
+
memoryMb: memMb,
|
|
1789
|
+
linearIssue: vm.linearIssue,
|
|
1790
|
+
statusAction: "skipped_protected",
|
|
1791
|
+
reason: `Protected infrastructure preserved: ${protCheck.reason}`
|
|
1792
|
+
});
|
|
1793
|
+
continue;
|
|
1794
|
+
}
|
|
1795
|
+
if (dryRun) {
|
|
1796
|
+
reclaimedMemoryMb += memMb;
|
|
1797
|
+
prunedVms.push({
|
|
1798
|
+
vmId: vm.id,
|
|
1799
|
+
name: vm.name,
|
|
1800
|
+
hostName: vm.hostName,
|
|
1801
|
+
hypervisor: vm.hypervisor,
|
|
1802
|
+
status: vm.status,
|
|
1803
|
+
memoryMb: memMb,
|
|
1804
|
+
linearIssue: vm.linearIssue,
|
|
1805
|
+
statusAction: "dry_run",
|
|
1806
|
+
reason: "Would be destroyed (untracked disposable test VM)"
|
|
1807
|
+
});
|
|
1808
|
+
}
|
|
1809
|
+
else {
|
|
1810
|
+
try {
|
|
1811
|
+
if (params.snapshotBeforeDestroy) {
|
|
1812
|
+
try {
|
|
1813
|
+
const drivers = this.hypervisorManager.getAllDrivers();
|
|
1814
|
+
const driver = drivers.find((d) => d.hostName === vm.hostName);
|
|
1815
|
+
if (driver && driver.createSnapshot) {
|
|
1816
|
+
await driver.createSnapshot(vm.id, `pre-prune-${Date.now()}`, "Pre-prune snapshot", false);
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
catch (snapErr) {
|
|
1820
|
+
logger.warn(`Failed to create pre-prune snapshot for VM ${vm.name}:`, snapErr);
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
await this.hypervisorManager.vmAction({
|
|
1824
|
+
vmIdOrName: String(vm.id),
|
|
1825
|
+
targetHost: vm.hostName,
|
|
1826
|
+
action: "destroy",
|
|
1827
|
+
force: true
|
|
1828
|
+
});
|
|
1829
|
+
reclaimedMemoryMb += memMb;
|
|
1830
|
+
prunedVms.push({
|
|
1831
|
+
vmId: vm.id,
|
|
1832
|
+
name: vm.name,
|
|
1833
|
+
hostName: vm.hostName,
|
|
1834
|
+
hypervisor: vm.hypervisor,
|
|
1835
|
+
status: vm.status,
|
|
1836
|
+
memoryMb: memMb,
|
|
1837
|
+
linearIssue: vm.linearIssue,
|
|
1838
|
+
statusAction: "pruned",
|
|
1839
|
+
reason: "Destroyed untracked disposable test VM"
|
|
1840
|
+
});
|
|
1841
|
+
this.telemetry.emit({
|
|
1842
|
+
event: "vm.purged",
|
|
1843
|
+
actor: "system",
|
|
1844
|
+
status: "success",
|
|
1845
|
+
vmId: vm.id,
|
|
1846
|
+
vmName: vm.name,
|
|
1847
|
+
hostId: vm.hostName,
|
|
1848
|
+
hypervisorType: vm.hypervisor,
|
|
1849
|
+
metadata: { action: "prune_untracked_vms", linearIssue: vm.linearIssue }
|
|
1850
|
+
});
|
|
1851
|
+
}
|
|
1852
|
+
catch (err) {
|
|
1853
|
+
prunedVms.push({
|
|
1854
|
+
vmId: vm.id,
|
|
1855
|
+
name: vm.name,
|
|
1856
|
+
hostName: vm.hostName,
|
|
1857
|
+
hypervisor: vm.hypervisor,
|
|
1858
|
+
status: vm.status,
|
|
1859
|
+
memoryMb: memMb,
|
|
1860
|
+
linearIssue: vm.linearIssue,
|
|
1861
|
+
statusAction: "failed",
|
|
1862
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1863
|
+
});
|
|
1864
|
+
}
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
const eligibleCount = prunedVms.filter((v) => v.statusAction !== "skipped_protected").length;
|
|
1868
|
+
const prunedCount = prunedVms.filter((v) => v.statusAction === "pruned").length;
|
|
1869
|
+
return {
|
|
1870
|
+
dryRun,
|
|
1871
|
+
scannedVmsCount: inventory.length,
|
|
1872
|
+
eligibleVmsCount: eligibleCount,
|
|
1873
|
+
prunedVmsCount: dryRun ? eligibleCount : prunedCount,
|
|
1874
|
+
skippedProtectedCount,
|
|
1875
|
+
reclaimedMemoryMb,
|
|
1876
|
+
prunedVms,
|
|
1877
|
+
message: dryRun
|
|
1878
|
+
? `[Dry-run] Identified ${eligibleCount} untracked disposable VM(s) (~${Math.round(reclaimedMemoryMb / 1024)} GB RAM) eligible for pruning (${skippedProtectedCount} protected VMs preserved).`
|
|
1879
|
+
: `Successfully pruned ${prunedCount} untracked disposable VM(s), reclaiming ~${Math.round(reclaimedMemoryMb / 1024)} GB RAM (${skippedProtectedCount} protected VMs preserved).`
|
|
1880
|
+
};
|
|
1881
|
+
});
|
|
1882
|
+
}
|
|
1883
|
+
/**
|
|
1884
|
+
* Heartbeat-driven lease auto-extension for active sessions.
|
|
1885
|
+
* If lease is active and expiring within 45 mins, extends by extendMinutes (default 30m) up to 8h cap.
|
|
1886
|
+
*/
|
|
1887
|
+
async touchSlotLease(slotIdOrClaimOrLinear, extendMinutes = 30) {
|
|
1888
|
+
return this.mutex.runExclusive(async () => {
|
|
1889
|
+
this.sweepExpiredSlotsInternal();
|
|
1890
|
+
const slot = Array.from(this.slots.values()).find((s) => s.slotId === slotIdOrClaimOrLinear ||
|
|
1891
|
+
s.claimId === slotIdOrClaimOrLinear ||
|
|
1892
|
+
s.linearIssue === slotIdOrClaimOrLinear ||
|
|
1893
|
+
(s.vmId !== null && String(s.vmId) === slotIdOrClaimOrLinear));
|
|
1894
|
+
if (!slot || (slot.status !== "CLAIMED" && slot.status !== "active") || !slot.expiresAt) {
|
|
1895
|
+
return null;
|
|
1896
|
+
}
|
|
1897
|
+
const now = Date.now();
|
|
1898
|
+
const currentExpiry = new Date(slot.expiresAt).getTime();
|
|
1899
|
+
const remainingMs = currentExpiry - now;
|
|
1900
|
+
// Only extend if within 45 minutes of expiry
|
|
1901
|
+
if (remainingMs <= 45 * 60 * 1000) {
|
|
1902
|
+
const claimedAtMs = slot.claimedAt ? new Date(slot.claimedAt).getTime() : now;
|
|
1903
|
+
const currentTotalMinutes = Math.round((currentExpiry - claimedAtMs) / (60 * 1000));
|
|
1904
|
+
const maxTotalMinutes = 480; // 8 hours max lease cap
|
|
1905
|
+
if (currentTotalMinutes < maxTotalMinutes) {
|
|
1906
|
+
const extensionMs = Math.min(extendMinutes, maxTotalMinutes - currentTotalMinutes) * 60 * 1000;
|
|
1907
|
+
const newExpiresAt = new Date(currentExpiry + extensionMs).toISOString();
|
|
1908
|
+
slot.expiresAt = newExpiresAt;
|
|
1909
|
+
slot.durationMinutes = (slot.durationMinutes || 120) + Math.round(extensionMs / 60000);
|
|
1910
|
+
slot.ttlMinutes = Math.round((new Date(newExpiresAt).getTime() - now) / 60000);
|
|
1911
|
+
slot.ttlSeconds = Math.round((new Date(newExpiresAt).getTime() - now) / 1000);
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
return { ...slot };
|
|
1915
|
+
});
|
|
1916
|
+
}
|
|
1645
1917
|
/**
|
|
1646
1918
|
* Attach a created VMID or VM name to a claimed slot.
|
|
1647
1919
|
*/
|