@calltelemetry/ct-lab-mcp 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/slot/manager.d.ts +12 -1
- package/dist/src/slot/manager.d.ts.map +1 -1
- package/dist/src/slot/manager.js +302 -8
- package/dist/src/slot/manager.js.map +1 -1
- package/dist/src/slot/types.d.ts +48 -0
- package/dist/src/slot/types.d.ts.map +1 -1
- package/dist/src/slot/types.js.map +1 -1
- package/dist/src/tools/find-slot.d.ts +8 -0
- package/dist/src/tools/find-slot.d.ts.map +1 -1
- package/dist/src/tools/find-slot.js +13 -2
- package/dist/src/tools/find-slot.js.map +1 -1
- package/dist/src/tools/index.d.ts +3 -0
- package/dist/src/tools/index.d.ts.map +1 -1
- package/dist/src/tools/index.js +9 -1
- package/dist/src/tools/index.js.map +1 -1
- package/dist/src/tools/prune-untracked-vms.d.ts +56 -0
- package/dist/src/tools/prune-untracked-vms.d.ts.map +1 -0
- package/dist/src/tools/prune-untracked-vms.js +100 -0
- package/dist/src/tools/prune-untracked-vms.js.map +1 -0
- package/dist/src/tools/purge-and-redeploy.js +2 -2
- package/dist/src/tools/purge-and-redeploy.js.map +1 -1
- package/dist/src/transports/sse.d.ts.map +1 -1
- package/dist/src/transports/sse.js +78 -0
- package/dist/src/transports/sse.js.map +1 -1
- package/dist/src/utils/formatters.d.ts +4 -0
- package/dist/src/utils/formatters.d.ts.map +1 -1
- package/dist/src/utils/formatters.js +39 -0
- package/dist/src/utils/formatters.js.map +1 -1
- package/package.json +1 -1
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* atomic concurrency locking (AsyncMutex), TTL lease management, candidate slot ranking,
|
|
5
5
|
* structured metadata labeling/indexing, and automated expired slot purge & reclamation.
|
|
6
6
|
*/
|
|
7
|
-
import { HostId, SlotDefinition, HostCapacityStatus, GlobalCapacitySummary, SlotCapacityReport, SlotClaimRequest, SlotClaimResult, ReleaseSlotOptions, ReleaseSlotParams, ReleaseSlotResult, LeaseExtensionResult, ListSlotsOptions, PurgeSlotsOptions, PurgeSlotsResult, UpdateSlotMetadataOptions, UpdateSlotMetadataResult, ISlotManager, CubeDefinition, ClaimCubeParams, ReleaseCubeParams, CubeClaimResult, FindSlotParams, FindSlotResult, PurgeAndRedeployParams, PurgeAndRedeployResult } from "./types.js";
|
|
7
|
+
import { HostId, SlotDefinition, HostCapacityStatus, GlobalCapacitySummary, SlotCapacityReport, SlotClaimRequest, SlotClaimResult, ReleaseSlotOptions, ReleaseSlotParams, ReleaseSlotResult, LeaseExtensionResult, ListSlotsOptions, PurgeSlotsOptions, PurgeSlotsResult, UpdateSlotMetadataOptions, UpdateSlotMetadataResult, ISlotManager, CubeDefinition, ClaimCubeParams, ReleaseCubeParams, CubeClaimResult, FindSlotParams, FindSlotResult, PurgeAndRedeployParams, PurgeAndRedeployResult, PruneUntrackedVmsParams, PruneUntrackedVmsResult } from "./types.js";
|
|
8
8
|
import { MultiHostProxmoxManager } from "../proxmox/index.js";
|
|
9
9
|
import { MultiHypervisorManager } from "../hypervisors/manager.js";
|
|
10
10
|
import { SafetyGuardrailEngine } from "../guardrails/safety.js";
|
|
@@ -133,6 +133,17 @@ export declare class SlotManager implements ISlotManager {
|
|
|
133
133
|
* One-shot purge of old/expired disposable test VM and replacement with fresh slot allocation.
|
|
134
134
|
*/
|
|
135
135
|
purgeAndRedeploy(params: PurgeAndRedeployParams): Promise<PurgeAndRedeployResult>;
|
|
136
|
+
/**
|
|
137
|
+
* Safe Prune of Abandoned / Untracked Disposable Test VMs across Hypervisors.
|
|
138
|
+
* Scans inventory for stopped disposable VMs not tied to any active slot lease,
|
|
139
|
+
* enforces strict non-bypassable guardrails, and frees hypervisor datastore and memory.
|
|
140
|
+
*/
|
|
141
|
+
pruneUntrackedVms(params?: PruneUntrackedVmsParams): Promise<PruneUntrackedVmsResult>;
|
|
142
|
+
/**
|
|
143
|
+
* Heartbeat-driven lease auto-extension for active sessions.
|
|
144
|
+
* If lease is active and expiring within 45 mins, extends by extendMinutes (default 30m) up to 8h cap.
|
|
145
|
+
*/
|
|
146
|
+
touchSlotLease(slotIdOrClaimOrLinear: string, extendMinutes?: number): Promise<SlotDefinition | null>;
|
|
136
147
|
/**
|
|
137
148
|
* Attach a created VMID or VM name to a claimed slot.
|
|
138
149
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../../../src/slot/manager.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,MAAM,EACN,cAAc,EAEd,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,EAClB,gBAAgB,EAEhB,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAGhB,yBAAyB,EACzB,wBAAwB,EAExB,YAAY,EASZ,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,EAKf,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAgC,MAAM,yBAAyB,CAAC;AAC9F,OAAO,EAAa,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../../../src/slot/manager.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,MAAM,EACN,cAAc,EAEd,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,EAClB,gBAAgB,EAEhB,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAGhB,yBAAyB,EACzB,wBAAwB,EAExB,YAAY,EASZ,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,EAKf,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,uBAAuB,EAExB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAgC,MAAM,yBAAyB,CAAC;AAC9F,OAAO,EAAa,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAGrE,cAAc,YAAY,CAAC;AAE3B,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IACpC,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE;IAC1C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IACpC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,GAAG,MAAM,CAeT;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,qBAAqB,EAAE,CAAC;IAChC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,cAAc,CAAC,EAAE,uBAAuB,CAAC;IACzC,iBAAiB,CAAC,EAAE,sBAAsB,CAAC;IAC3C,eAAe,CAAC,EAAE,qBAAqB,CAAC;IACxC,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AA2BD,eAAO,MAAM,wBAAwB,EAAE,qBAWtC,CAAC;AAEF,eAAO,MAAM,aAAa,EAAE,cAAc,EAuCzC,CAAC;AAEF,qBAAa,WAAY,YAAW,YAAY;IAC9C,OAAO,CAAC,KAAK,CAAiD;IAC9D,OAAO,CAAC,KAAK,CAA0C;IACvD,OAAO,CAAC,KAAK,CAA0C;IACvD,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,cAAc,CAAC,CAA0B;IACjD,OAAO,CAAC,iBAAiB,CAAC,CAAyB;IACnD,OAAO,CAAC,eAAe,CAAwB;IAC/C,OAAO,CAAC,SAAS,CAAoB;IACrC,OAAO,CAAC,cAAc,CAAU;gBAEpB,OAAO,GAAE,kBAAuB;IAyB5C;;OAEG;IACH,oBAAoB,CAAC,iBAAiB,EAAE,sBAAsB,GAAG,IAAI;IAIrE;;OAEG;IACH,kBAAkB,CAAC,eAAe,EAAE,qBAAqB,GAAG,IAAI;IAIhE;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,qBAAqB,GAAG,IAAI;IAwD/C;;OAEG;IACH,OAAO,CAAC,eAAe;IAKvB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,SAAS;IAuBjB;;OAEG;IACH,OAAO,CAAC,yBAAyB;IA2CjC;;OAEG;IACG,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;YAItD,iBAAiB;IAgW/B;;OAEG;IACG,WAAW,CACf,gBAAgB,EAAE,MAAM,GAAG,kBAAkB,GAAG,iBAAiB,EACjE,aAAa,CAAC,EAAE,OAAO,GACtB,OAAO,CAAC,iBAAiB,CAAC;IA0I7B;;OAEG;IACG,WAAW,CACf,OAAO,EAAE,MAAM,EACf,iBAAiB,EAAE,MAAM,EACzB,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,CAAC;IA+EhC;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC;IAmF/F;;OAEG;IACG,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IA+F1E;;OAEG;IACG,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC,qBAAqB,GAAG,kBAAkB,CAAC;IAuE7F;;OAEG;IACG,iBAAiB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,qBAAqB,GAAG,kBAAkB,CAAC;IAgBpH;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;IAMvC;;;OAGG;IACG,UAAU,CAAC,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAI9D,kBAAkB;IA6ThC;;;;OAIG;IACG,QAAQ,CAAC,MAAM,GAAE,cAAmB,GAAG,OAAO,CAAC,cAAc,CAAC;IA8SpE;;OAEG;IACG,gBAAgB,CAAC,MAAM,EAAE,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAgIvF;;;;OAIG;IACG,iBAAiB,CAAC,MAAM,GAAE,uBAA4B,GAAG,OAAO,CAAC,uBAAuB,CAAC;IA2L/F;;;OAGG;IACG,cAAc,CAAC,qBAAqB,EAAE,MAAM,EAAE,aAAa,SAAK,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAsCvG;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAmBhG;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAQ7D;;OAEG;IACG,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAUvE;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAqC/B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAiBjC;;OAEG;IACG,SAAS,CAAC,OAAO,UAAQ,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAkE3D;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAiB7D;;OAEG;IACG,SAAS,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAwFlE;;OAEG;IACG,WAAW,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CA6D7F"}
|
package/dist/src/slot/manager.js
CHANGED
|
@@ -9,6 +9,7 @@ import { CapacityExceededError, GlobalCapacityExceededError, SlotNotFoundError,
|
|
|
9
9
|
import { AsyncMutex } from "../utils/mutex.js";
|
|
10
10
|
import { defaultSafetyGuardrailEngine } from "../guardrails/safety.js";
|
|
11
11
|
import { telemetry } from "../telemetry/index.js";
|
|
12
|
+
import { logger } from "../utils/logger.js";
|
|
12
13
|
export * from "./types.js";
|
|
13
14
|
/**
|
|
14
15
|
* Calculates pure hardware capacity in standard slots (default 8GB).
|
|
@@ -1370,6 +1371,10 @@ export class SlotManager {
|
|
|
1370
1371
|
continue;
|
|
1371
1372
|
if (hostConfig.hypervisorType === "esxi" && !allowEsxi)
|
|
1372
1373
|
continue;
|
|
1374
|
+
if (params.excludeHosts && params.excludeHosts.some((ex) => this.normalizeHostId(ex) === hid))
|
|
1375
|
+
continue;
|
|
1376
|
+
if (params.minFreeDiskGb && (hostConfig.freeDiskGb ?? 0) < params.minFreeDiskGb)
|
|
1377
|
+
continue;
|
|
1373
1378
|
const hostSlots = Array.from(this.slots.values()).filter((s) => s.hostId === hid);
|
|
1374
1379
|
const availCount = hostSlots.filter((s) => s.status === "AVAILABLE").length;
|
|
1375
1380
|
const pureCapacity = calculatePureCapacity({ ...hostConfig, allowEsxiSlots: allowEsxi });
|
|
@@ -1377,13 +1382,33 @@ export class SlotManager {
|
|
|
1377
1382
|
const totalMem = hostConfig.totalMemoryMb;
|
|
1378
1383
|
const reserved = hostConfig.reservedHeadroomMb || 0;
|
|
1379
1384
|
const freeMem = Math.max(0, totalMem - reserved - (hostSlots.length - availCount) * (hostConfig.slotSizeMb || 8192));
|
|
1380
|
-
//
|
|
1381
|
-
|
|
1385
|
+
// Host tiering: proxmox-mini (Tier 1 Primary), proxmox-lab (Tier 2 Secondary), ESXi (Tier 3 Fallback)
|
|
1386
|
+
let tier = "secondary";
|
|
1387
|
+
let tierBoost = 10000;
|
|
1388
|
+
let recommendationReason = "Secondary placement: Proxmox Lab appliance cluster";
|
|
1389
|
+
if (hid === "proxmox-mini") {
|
|
1390
|
+
tier = "primary";
|
|
1391
|
+
tierBoost = 20000;
|
|
1392
|
+
recommendationReason = "Primary placement: Minisforum high-IOPS NVMe appliance host";
|
|
1393
|
+
}
|
|
1394
|
+
else if (hid === "proxmox-lab") {
|
|
1395
|
+
tier = "secondary";
|
|
1396
|
+
tierBoost = 10000;
|
|
1397
|
+
recommendationReason = "Secondary placement: Proxmox Lab appliance cluster";
|
|
1398
|
+
}
|
|
1399
|
+
else if (hostConfig.hypervisorType === "esxi" || hid.startsWith("esxi")) {
|
|
1400
|
+
tier = "fallback";
|
|
1401
|
+
tierBoost = 0; // Least preferred host — contains CUCM/CUBE and testbed fixtures
|
|
1402
|
+
recommendationReason = "Fallback placement: ESXi Intel host (reserved for fixtures / heavy spillover)";
|
|
1403
|
+
}
|
|
1404
|
+
// Available slots score with tier priority: Proxmox Mini always outranks Proxmox Lab, and Proxmox always outranks ESXi
|
|
1405
|
+
const score = (availCount > 0 ? tierBoost : 0) + availCount * 100 + Math.floor(freeMem / 1024) * 10 + Math.floor(freeDisk / 100);
|
|
1382
1406
|
allCandidates.push({
|
|
1383
1407
|
hostId: hid,
|
|
1384
1408
|
name: hostConfig.name,
|
|
1385
1409
|
hypervisorType: hostConfig.hypervisorType || (hid.startsWith("esxi") ? "esxi" : "proxmox"),
|
|
1386
1410
|
status: "online",
|
|
1411
|
+
tier,
|
|
1387
1412
|
availableSlots: availCount,
|
|
1388
1413
|
totalPureSlots: pureCapacity,
|
|
1389
1414
|
freeMemoryMb: freeMem,
|
|
@@ -1391,15 +1416,56 @@ export class SlotManager {
|
|
|
1391
1416
|
freeDiskGb: freeDisk,
|
|
1392
1417
|
cpuUsedPercent: 0,
|
|
1393
1418
|
runningVms: hostSlots.length - availCount,
|
|
1394
|
-
score
|
|
1419
|
+
score,
|
|
1420
|
+
recommendationReason
|
|
1395
1421
|
});
|
|
1396
1422
|
}
|
|
1397
1423
|
// Sort candidates by score descending
|
|
1398
1424
|
allCandidates.sort((a, b) => b.score - a.score);
|
|
1399
1425
|
const availableSlotsTotal = allCandidates.reduce((sum, c) => sum + c.availableSlots, 0);
|
|
1426
|
+
// Check if Proxmox hosts have expired leases or disposable VMs
|
|
1427
|
+
const proxmoxReclaimableSlots = Array.from(this.slots.values()).filter((s) => (s.hostId === "proxmox-mini" || s.hostId === "proxmox-lab") &&
|
|
1428
|
+
(s.status === "EXPIRED" || (s.expiresAt && new Date(s.expiresAt).getTime() <= Date.now())));
|
|
1400
1429
|
// If capacity is available
|
|
1401
1430
|
if (availableSlotsTotal > 0) {
|
|
1402
|
-
|
|
1431
|
+
let winningCandidate = allCandidates[0];
|
|
1432
|
+
// Strict ESXi Fallback Rule: If winning host is ESXi, but Proxmox Mini/Lab has reclaimable expired slots
|
|
1433
|
+
// AND purgeIfFull is requested, auto-purge Proxmox instead of spilling over to ESXi!
|
|
1434
|
+
if (winningCandidate.hypervisorType === "esxi" && proxmoxReclaimableSlots.length > 0 && params.purgeIfFull) {
|
|
1435
|
+
const targetPurgedHost = proxmoxReclaimableSlots.some((s) => s.hostId === "proxmox-mini")
|
|
1436
|
+
? "proxmox-mini"
|
|
1437
|
+
: "proxmox-lab";
|
|
1438
|
+
const purgeRes = await this.purgeSlotsInternal({
|
|
1439
|
+
host: targetPurgedHost,
|
|
1440
|
+
destroyVms: true,
|
|
1441
|
+
includeOrphans: true,
|
|
1442
|
+
dryRun: false
|
|
1443
|
+
});
|
|
1444
|
+
if (purgeRes.purgedSlotsCount > 0 || purgeRes.destroyedVmsCount > 0) {
|
|
1445
|
+
const tag = params.tag || params.linearIssue || params.requester || "find-slot-reclaim";
|
|
1446
|
+
const claimResult = await this.claimSlotInternal({
|
|
1447
|
+
tag,
|
|
1448
|
+
requester: params.requester || tag,
|
|
1449
|
+
linearIssue: params.linearIssue,
|
|
1450
|
+
label: params.label,
|
|
1451
|
+
workloadType: params.workloadType,
|
|
1452
|
+
targetVersion: params.targetVersion,
|
|
1453
|
+
branch: params.branch,
|
|
1454
|
+
host: targetPurgedHost,
|
|
1455
|
+
durationMinutes: params.durationMinutes,
|
|
1456
|
+
allowEsxiSlots: allowEsxi,
|
|
1457
|
+
purpose: `Auto-reclaimed Proxmox slot via find_slot to avoid ESXi fallback`
|
|
1458
|
+
});
|
|
1459
|
+
return {
|
|
1460
|
+
status: "auto_reclaimed_and_claimed",
|
|
1461
|
+
candidateHost: allCandidates.find((c) => c.hostId === targetPurgedHost) || allCandidates[0],
|
|
1462
|
+
allCandidates,
|
|
1463
|
+
availableSlotsTotal,
|
|
1464
|
+
claim: claimResult,
|
|
1465
|
+
message: `Proxmox hosts were at capacity, but auto-purged expired lease(s) on '${targetPurgedHost}' to keep ESXi fixture host free. Successfully claimed '${claimResult.slotId}' on '${targetPurgedHost}'.`
|
|
1466
|
+
};
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1403
1469
|
let claimResult;
|
|
1404
1470
|
if (params.autoClaim) {
|
|
1405
1471
|
const tag = params.tag || params.linearIssue || params.requester || "find-slot-claim";
|
|
@@ -1425,12 +1491,16 @@ export class SlotManager {
|
|
|
1425
1491
|
message: `Found available slot and successfully auto-claimed '${claimResult.slotId}' on host '${winningCandidate.hostId}'.`
|
|
1426
1492
|
};
|
|
1427
1493
|
}
|
|
1494
|
+
let msg = `Found ${availableSlotsTotal} available 8GB slot(s) across ${allCandidates.length} eligible host(s). Recommended placement: '${winningCandidate.hostId}' (${winningCandidate.availableSlots} slots available).`;
|
|
1495
|
+
if (winningCandidate.hypervisorType === "esxi" && proxmoxReclaimableSlots.length > 0) {
|
|
1496
|
+
msg += ` (Note: Proxmox Mini has ${proxmoxReclaimableSlots.length} expired lease(s) that can be reclaimed with purgeIfFull=true to avoid placing on ESXi).`;
|
|
1497
|
+
}
|
|
1428
1498
|
return {
|
|
1429
1499
|
status: "available",
|
|
1430
1500
|
candidateHost: winningCandidate,
|
|
1431
1501
|
allCandidates,
|
|
1432
1502
|
availableSlotsTotal,
|
|
1433
|
-
message:
|
|
1503
|
+
message: msg
|
|
1434
1504
|
};
|
|
1435
1505
|
}
|
|
1436
1506
|
// If capacity is NOT available (availableSlotsTotal === 0)
|
|
@@ -1602,18 +1672,31 @@ export class SlotManager {
|
|
|
1602
1672
|
}
|
|
1603
1673
|
// 3. Find associated slot if any
|
|
1604
1674
|
const slotToRelease = matchingSlot || Array.from(this.slots.values()).find((s) => (s.vmId !== null && String(s.vmId) === String(targetVm.id)) || (linearIssue && s.linearIssue === linearIssue));
|
|
1605
|
-
// 4.
|
|
1675
|
+
// 4. Pre-destruction snapshot (if requested)
|
|
1676
|
+
if (params.snapshotBeforeDestroy) {
|
|
1677
|
+
try {
|
|
1678
|
+
const drivers = this.hypervisorManager.getAllDrivers();
|
|
1679
|
+
const driver = drivers.find((d) => d.hostName === targetVm.hostName);
|
|
1680
|
+
if (driver && driver.createSnapshot) {
|
|
1681
|
+
await driver.createSnapshot(targetVm.id, `pre-redeploy-${Date.now()}`, "Pre-purge snapshot before redeploy", false);
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
catch (snapErr) {
|
|
1685
|
+
logger.warn(`Failed to create pre-redeploy snapshot for VM ${targetVm.name}:`, snapErr);
|
|
1686
|
+
}
|
|
1687
|
+
}
|
|
1688
|
+
// 5. Destroy old VM
|
|
1606
1689
|
await this.hypervisorManager.vmAction({
|
|
1607
1690
|
vmIdOrName: String(targetVm.id),
|
|
1608
1691
|
targetHost: targetVm.hostName,
|
|
1609
1692
|
action: "destroy",
|
|
1610
1693
|
force: true
|
|
1611
1694
|
});
|
|
1612
|
-
//
|
|
1695
|
+
// 6. Release existing slot if present
|
|
1613
1696
|
if (slotToRelease) {
|
|
1614
1697
|
this.resetSlot(slotToRelease);
|
|
1615
1698
|
}
|
|
1616
|
-
//
|
|
1699
|
+
// 7. Claim new slot on the same host (or auto)
|
|
1617
1700
|
const targetHost = params.host || targetVm.hostName;
|
|
1618
1701
|
const tag = linearIssue || targetVm.name;
|
|
1619
1702
|
const newSlot = await this.claimSlotInternal({
|
|
@@ -1642,6 +1725,217 @@ export class SlotManager {
|
|
|
1642
1725
|
};
|
|
1643
1726
|
});
|
|
1644
1727
|
}
|
|
1728
|
+
/**
|
|
1729
|
+
* Safe Prune of Abandoned / Untracked Disposable Test VMs across Hypervisors.
|
|
1730
|
+
* Scans inventory for stopped disposable VMs not tied to any active slot lease,
|
|
1731
|
+
* enforces strict non-bypassable guardrails, and frees hypervisor datastore and memory.
|
|
1732
|
+
*/
|
|
1733
|
+
async pruneUntrackedVms(params = {}) {
|
|
1734
|
+
return this.mutex.runExclusive(async () => {
|
|
1735
|
+
if (!this.hypervisorManager) {
|
|
1736
|
+
throw new InvalidSlotOperationError("HypervisorManager is required to prune untracked VMs.");
|
|
1737
|
+
}
|
|
1738
|
+
this.sweepExpiredSlotsInternal();
|
|
1739
|
+
const dryRun = params.dryRun ?? true;
|
|
1740
|
+
const targetHost = params.host && params.host !== "all" && params.host !== "auto"
|
|
1741
|
+
? this.normalizeHostId(params.host)
|
|
1742
|
+
: undefined;
|
|
1743
|
+
const linearFilter = params.linearIssue;
|
|
1744
|
+
const inventory = await this.hypervisorManager.listUnifiedInventory({
|
|
1745
|
+
includeTemplates: false,
|
|
1746
|
+
host: targetHost
|
|
1747
|
+
});
|
|
1748
|
+
// Active slot VM IDs that must NEVER be touched
|
|
1749
|
+
const activeSlotVmIds = new Set(Array.from(this.slots.values())
|
|
1750
|
+
.filter((s) => s.status === "CLAIMED" && s.vmId !== null && s.vmId !== undefined)
|
|
1751
|
+
.map((s) => String(s.vmId)));
|
|
1752
|
+
const prunedVms = [];
|
|
1753
|
+
let skippedProtectedCount = 0;
|
|
1754
|
+
let reclaimedMemoryMb = 0;
|
|
1755
|
+
for (const vm of inventory) {
|
|
1756
|
+
if (vm.isTemplate)
|
|
1757
|
+
continue;
|
|
1758
|
+
if (activeSlotVmIds.has(String(vm.id)))
|
|
1759
|
+
continue;
|
|
1760
|
+
if (params.vmIds && params.vmIds.length > 0) {
|
|
1761
|
+
const targetVmIdSet = new Set(params.vmIds.map((id) => String(id)));
|
|
1762
|
+
if (!targetVmIdSet.has(String(vm.id)) && !targetVmIdSet.has(vm.name)) {
|
|
1763
|
+
continue;
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
const processedCount = prunedVms.filter((v) => v.statusAction === "pruned" || v.statusAction === "dry_run").length;
|
|
1767
|
+
if (params.limit && processedCount >= params.limit) {
|
|
1768
|
+
break;
|
|
1769
|
+
}
|
|
1770
|
+
// Check if VM matches disposable patterns or specific linear filter
|
|
1771
|
+
const nameLower = vm.name.toLowerCase();
|
|
1772
|
+
const isDisposablePattern = nameLower.startsWith("ct-") ||
|
|
1773
|
+
nameLower.includes("uat") ||
|
|
1774
|
+
nameLower.includes("qa-") ||
|
|
1775
|
+
nameLower.includes("demo") ||
|
|
1776
|
+
nameLower.includes("test") ||
|
|
1777
|
+
Boolean(vm.linearIssue);
|
|
1778
|
+
if (linearFilter) {
|
|
1779
|
+
const matchesLinear = vm.linearIssue === linearFilter || nameLower.includes(linearFilter.toLowerCase());
|
|
1780
|
+
if (!matchesLinear)
|
|
1781
|
+
continue;
|
|
1782
|
+
}
|
|
1783
|
+
else if (!isDisposablePattern && !params.force) {
|
|
1784
|
+
continue;
|
|
1785
|
+
}
|
|
1786
|
+
// Strict Safety Guardrail Check
|
|
1787
|
+
const protCheck = this.guardrailEngine.isProtected({
|
|
1788
|
+
vmid: vm.id,
|
|
1789
|
+
name: vm.name,
|
|
1790
|
+
isProtected: vm.isProtected,
|
|
1791
|
+
tags: vm.tags
|
|
1792
|
+
});
|
|
1793
|
+
const memMb = Math.round((vm.memoryBytes || 0) / (1024 * 1024));
|
|
1794
|
+
if (protCheck.protected) {
|
|
1795
|
+
skippedProtectedCount++;
|
|
1796
|
+
prunedVms.push({
|
|
1797
|
+
vmId: vm.id,
|
|
1798
|
+
name: vm.name,
|
|
1799
|
+
hostName: vm.hostName,
|
|
1800
|
+
hypervisor: vm.hypervisor,
|
|
1801
|
+
status: vm.status,
|
|
1802
|
+
memoryMb: memMb,
|
|
1803
|
+
linearIssue: vm.linearIssue,
|
|
1804
|
+
statusAction: "skipped_protected",
|
|
1805
|
+
reason: `Protected infrastructure preserved: ${protCheck.reason}`
|
|
1806
|
+
});
|
|
1807
|
+
continue;
|
|
1808
|
+
}
|
|
1809
|
+
if (dryRun) {
|
|
1810
|
+
reclaimedMemoryMb += memMb;
|
|
1811
|
+
prunedVms.push({
|
|
1812
|
+
vmId: vm.id,
|
|
1813
|
+
name: vm.name,
|
|
1814
|
+
hostName: vm.hostName,
|
|
1815
|
+
hypervisor: vm.hypervisor,
|
|
1816
|
+
status: vm.status,
|
|
1817
|
+
memoryMb: memMb,
|
|
1818
|
+
linearIssue: vm.linearIssue,
|
|
1819
|
+
statusAction: "dry_run",
|
|
1820
|
+
reason: "Would be destroyed (untracked disposable test VM)"
|
|
1821
|
+
});
|
|
1822
|
+
}
|
|
1823
|
+
else {
|
|
1824
|
+
try {
|
|
1825
|
+
if (params.snapshotBeforeDestroy) {
|
|
1826
|
+
try {
|
|
1827
|
+
const drivers = this.hypervisorManager.getAllDrivers();
|
|
1828
|
+
const driver = drivers.find((d) => d.hostName === vm.hostName);
|
|
1829
|
+
if (driver && driver.createSnapshot) {
|
|
1830
|
+
await driver.createSnapshot(vm.id, `pre-prune-${Date.now()}`, "Pre-prune snapshot", false);
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
catch (snapErr) {
|
|
1834
|
+
logger.warn(`Failed to create pre-prune snapshot for VM ${vm.name}:`, snapErr);
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
await this.hypervisorManager.vmAction({
|
|
1838
|
+
vmIdOrName: String(vm.id),
|
|
1839
|
+
targetHost: vm.hostName,
|
|
1840
|
+
action: "destroy",
|
|
1841
|
+
force: true
|
|
1842
|
+
});
|
|
1843
|
+
reclaimedMemoryMb += memMb;
|
|
1844
|
+
prunedVms.push({
|
|
1845
|
+
vmId: vm.id,
|
|
1846
|
+
name: vm.name,
|
|
1847
|
+
hostName: vm.hostName,
|
|
1848
|
+
hypervisor: vm.hypervisor,
|
|
1849
|
+
status: vm.status,
|
|
1850
|
+
memoryMb: memMb,
|
|
1851
|
+
linearIssue: vm.linearIssue,
|
|
1852
|
+
statusAction: "pruned",
|
|
1853
|
+
reason: "Destroyed untracked disposable test VM"
|
|
1854
|
+
});
|
|
1855
|
+
this.telemetry.emit({
|
|
1856
|
+
event: "vm.purged",
|
|
1857
|
+
actor: "system",
|
|
1858
|
+
status: "success",
|
|
1859
|
+
vmId: vm.id,
|
|
1860
|
+
vmName: vm.name,
|
|
1861
|
+
hostId: vm.hostName,
|
|
1862
|
+
hypervisorType: vm.hypervisor,
|
|
1863
|
+
metadata: { action: "prune_untracked_vms", linearIssue: vm.linearIssue }
|
|
1864
|
+
});
|
|
1865
|
+
}
|
|
1866
|
+
catch (err) {
|
|
1867
|
+
prunedVms.push({
|
|
1868
|
+
vmId: vm.id,
|
|
1869
|
+
name: vm.name,
|
|
1870
|
+
hostName: vm.hostName,
|
|
1871
|
+
hypervisor: vm.hypervisor,
|
|
1872
|
+
status: vm.status,
|
|
1873
|
+
memoryMb: memMb,
|
|
1874
|
+
linearIssue: vm.linearIssue,
|
|
1875
|
+
statusAction: "failed",
|
|
1876
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1877
|
+
});
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
const eligibleCount = prunedVms.filter((v) => v.statusAction !== "skipped_protected").length;
|
|
1882
|
+
const prunedCount = prunedVms.filter((v) => v.statusAction === "pruned").length;
|
|
1883
|
+
return {
|
|
1884
|
+
dryRun,
|
|
1885
|
+
scannedVmsCount: inventory.length,
|
|
1886
|
+
eligibleVmsCount: eligibleCount,
|
|
1887
|
+
prunedVmsCount: dryRun ? eligibleCount : prunedCount,
|
|
1888
|
+
skippedProtectedCount,
|
|
1889
|
+
reclaimedMemoryMb,
|
|
1890
|
+
summary: {
|
|
1891
|
+
scannedVms: inventory.length,
|
|
1892
|
+
eligibleVms: eligibleCount,
|
|
1893
|
+
prunedVms: dryRun ? 0 : prunedCount,
|
|
1894
|
+
skippedProtected: skippedProtectedCount,
|
|
1895
|
+
reclaimedMemoryMb,
|
|
1896
|
+
reclaimedMemoryGb: Number((reclaimedMemoryMb / 1024).toFixed(1)),
|
|
1897
|
+
},
|
|
1898
|
+
prunedVms,
|
|
1899
|
+
message: dryRun
|
|
1900
|
+
? `[Dry-run] Identified ${eligibleCount} untracked disposable VM(s) (~${Math.round(reclaimedMemoryMb / 1024)} GB RAM) eligible for pruning (${skippedProtectedCount} protected VMs preserved).`
|
|
1901
|
+
: `Successfully pruned ${prunedCount} untracked disposable VM(s), reclaiming ~${Math.round(reclaimedMemoryMb / 1024)} GB RAM (${skippedProtectedCount} protected VMs preserved).`
|
|
1902
|
+
};
|
|
1903
|
+
});
|
|
1904
|
+
}
|
|
1905
|
+
/**
|
|
1906
|
+
* Heartbeat-driven lease auto-extension for active sessions.
|
|
1907
|
+
* If lease is active and expiring within 45 mins, extends by extendMinutes (default 30m) up to 8h cap.
|
|
1908
|
+
*/
|
|
1909
|
+
async touchSlotLease(slotIdOrClaimOrLinear, extendMinutes = 30) {
|
|
1910
|
+
return this.mutex.runExclusive(async () => {
|
|
1911
|
+
this.sweepExpiredSlotsInternal();
|
|
1912
|
+
const slot = Array.from(this.slots.values()).find((s) => s.slotId === slotIdOrClaimOrLinear ||
|
|
1913
|
+
s.claimId === slotIdOrClaimOrLinear ||
|
|
1914
|
+
s.linearIssue === slotIdOrClaimOrLinear ||
|
|
1915
|
+
(s.vmId !== null && String(s.vmId) === slotIdOrClaimOrLinear));
|
|
1916
|
+
if (!slot || (slot.status !== "CLAIMED" && slot.status !== "active") || !slot.expiresAt) {
|
|
1917
|
+
return null;
|
|
1918
|
+
}
|
|
1919
|
+
const now = Date.now();
|
|
1920
|
+
const currentExpiry = new Date(slot.expiresAt).getTime();
|
|
1921
|
+
const remainingMs = currentExpiry - now;
|
|
1922
|
+
// Only extend if within 45 minutes of expiry
|
|
1923
|
+
if (remainingMs <= 45 * 60 * 1000) {
|
|
1924
|
+
const claimedAtMs = slot.claimedAt ? new Date(slot.claimedAt).getTime() : now;
|
|
1925
|
+
const currentTotalMinutes = Math.round((currentExpiry - claimedAtMs) / (60 * 1000));
|
|
1926
|
+
const maxTotalMinutes = 480; // 8 hours max lease cap
|
|
1927
|
+
if (currentTotalMinutes < maxTotalMinutes) {
|
|
1928
|
+
const extensionMs = Math.min(extendMinutes, maxTotalMinutes - currentTotalMinutes) * 60 * 1000;
|
|
1929
|
+
const newExpiresAt = new Date(currentExpiry + extensionMs).toISOString();
|
|
1930
|
+
slot.expiresAt = newExpiresAt;
|
|
1931
|
+
slot.durationMinutes = (slot.durationMinutes || 120) + Math.round(extensionMs / 60000);
|
|
1932
|
+
slot.ttlMinutes = Math.round((new Date(newExpiresAt).getTime() - now) / 60000);
|
|
1933
|
+
slot.ttlSeconds = Math.round((new Date(newExpiresAt).getTime() - now) / 1000);
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
return { ...slot };
|
|
1937
|
+
});
|
|
1938
|
+
}
|
|
1645
1939
|
/**
|
|
1646
1940
|
* Attach a created VMID or VM name to a claimed slot.
|
|
1647
1941
|
*/
|