@calltelemetry/ct-lab-mcp 0.8.2 → 0.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/src/slot/manager.d.ts +27 -1
  2. package/dist/src/slot/manager.d.ts.map +1 -1
  3. package/dist/src/slot/manager.js +1080 -429
  4. package/dist/src/slot/manager.js.map +1 -1
  5. package/dist/src/slot/types.d.ts +123 -0
  6. package/dist/src/slot/types.d.ts.map +1 -1
  7. package/dist/src/slot/types.js.map +1 -1
  8. package/dist/src/tools/appliance-exec.d.ts +4 -4
  9. package/dist/src/tools/find-slot.d.ts +80 -0
  10. package/dist/src/tools/find-slot.d.ts.map +1 -0
  11. package/dist/src/tools/find-slot.js +134 -0
  12. package/dist/src/tools/find-slot.js.map +1 -0
  13. package/dist/src/tools/fleet-tools.d.ts +6 -6
  14. package/dist/src/tools/index.d.ts +3 -0
  15. package/dist/src/tools/index.d.ts.map +1 -1
  16. package/dist/src/tools/index.js +19 -1
  17. package/dist/src/tools/index.js.map +1 -1
  18. package/dist/src/tools/list-hosts.d.ts +2 -2
  19. package/dist/src/tools/list-inventory.d.ts +2 -2
  20. package/dist/src/tools/provision-tools.d.ts +16 -8
  21. package/dist/src/tools/provision-tools.d.ts.map +1 -1
  22. package/dist/src/tools/provision-tools.js +11 -0
  23. package/dist/src/tools/provision-tools.js.map +1 -1
  24. package/dist/src/tools/prune-untracked-vms.d.ts +48 -0
  25. package/dist/src/tools/prune-untracked-vms.d.ts.map +1 -0
  26. package/dist/src/tools/prune-untracked-vms.js +88 -0
  27. package/dist/src/tools/prune-untracked-vms.js.map +1 -0
  28. package/dist/src/tools/purge-and-redeploy.d.ts +64 -0
  29. package/dist/src/tools/purge-and-redeploy.d.ts.map +1 -0
  30. package/dist/src/tools/purge-and-redeploy.js +115 -0
  31. package/dist/src/tools/purge-and-redeploy.js.map +1 -0
  32. package/dist/src/tools/purge-slots.d.ts +6 -6
  33. package/dist/src/tools/slot-tools.d.ts +30 -18
  34. package/dist/src/tools/slot-tools.d.ts.map +1 -1
  35. package/dist/src/tools/slot-tools.js +11 -0
  36. package/dist/src/tools/slot-tools.js.map +1 -1
  37. package/dist/src/tools/vm-action.d.ts +8 -8
  38. package/dist/src/tools/vm-cleanup.d.ts +4 -4
  39. package/dist/src/transports/sse.d.ts.map +1 -1
  40. package/dist/src/transports/sse.js +78 -0
  41. package/dist/src/transports/sse.js.map +1 -1
  42. package/dist/src/utils/formatters.d.ts +12 -0
  43. package/dist/src/utils/formatters.d.ts.map +1 -1
  44. package/dist/src/utils/formatters.js +152 -0
  45. package/dist/src/utils/formatters.js.map +1 -1
  46. package/package.json +1 -1
@@ -9,6 +9,7 @@ import { CapacityExceededError, GlobalCapacityExceededError, SlotNotFoundError,
9
9
  import { AsyncMutex } from "../utils/mutex.js";
10
10
  import { defaultSafetyGuardrailEngine } from "../guardrails/safety.js";
11
11
  import { telemetry } from "../telemetry/index.js";
12
+ import { logger } from "../utils/logger.js";
12
13
  export * from "./types.js";
13
14
  /**
14
15
  * Calculates pure hardware capacity in standard slots (default 8GB).
@@ -16,7 +17,12 @@ export * from "./types.js";
16
17
  */
17
18
  export function calculatePureCapacity(host) {
18
19
  if (host.maxSlots !== undefined) {
19
- return host.maxSlots;
20
+ if (host.hypervisorType === "esxi" && host.maxSlots === 0 && host.allowEsxiSlots) {
21
+ // Allow dynamic calculation below for ESXi when unlocked
22
+ }
23
+ else {
24
+ return host.maxSlots;
25
+ }
20
26
  }
21
27
  const slotSize = host.slotSizeMb || 8192;
22
28
  if (host.totalMemoryMb !== undefined) {
@@ -112,12 +118,14 @@ export class SlotManager {
112
118
  hypervisorManager;
113
119
  guardrailEngine;
114
120
  telemetry;
121
+ allowEsxiSlots;
115
122
  constructor(options = {}) {
116
123
  this.defaultDurationMinutes = options.defaultDurationMinutes ?? 120;
117
124
  this.proxmoxManager = options.proxmoxManager;
118
125
  this.hypervisorManager = options.hypervisorManager;
119
126
  this.guardrailEngine = options.guardrailEngine || defaultSafetyGuardrailEngine;
120
127
  this.telemetry = options.telemetryRegistry || telemetry;
128
+ this.allowEsxiSlots = options.allowEsxiSlots ?? (process.env.ALLOW_ESXI_SLOTS === "true");
121
129
  const hostConfigs = options.hosts && options.hosts.length > 0
122
130
  ? options.hosts
123
131
  : [...DEFAULT_HOST_CONFIGS];
@@ -148,7 +156,14 @@ export class SlotManager {
148
156
  */
149
157
  registerHost(host) {
150
158
  const normalizedId = this.normalizeHostId(host.hostId);
151
- const pureSlots = calculatePureCapacity(host);
159
+ let pureSlots = calculatePureCapacity({
160
+ ...host,
161
+ allowEsxiSlots: this.allowEsxiSlots
162
+ });
163
+ if (normalizedId.startsWith("esxi") && this.allowEsxiSlots && host.maxSlots === 0) {
164
+ const usableMb = Math.max(0, host.totalMemoryMb - (host.reservedHeadroomMb || 0));
165
+ pureSlots = Math.floor(usableMb / (host.slotSizeMb || 8192));
166
+ }
152
167
  const normalizedHost = {
153
168
  ...host,
154
169
  hostId: normalizedId,
@@ -282,20 +297,24 @@ export class SlotManager {
282
297
  * Atomically claims an 8GB appliance slot.
283
298
  */
284
299
  async claimSlot(request) {
285
- return this.mutex.runExclusive(async () => {
286
- // 1. Reclaim any expired leases first
287
- this.sweepExpiredSlotsInternal();
288
- const tagInput = request.tag || request.linearIssue || request.requester;
289
- if (!tagInput || typeof tagInput !== "string" || tagInput.trim() === "") {
290
- throw new InvalidSlotOperationError("A valid non-empty 'tag', 'linearIssue', or 'requester' is required to claim a slot.");
291
- }
292
- const durationMinutes = request.durationMinutes ?? request.ttlMinutes ?? (request.ttlSeconds ? Math.ceil(request.ttlSeconds / 60) : this.defaultDurationMinutes);
293
- if (durationMinutes < 1 || durationMinutes > 1440) {
294
- throw new InvalidSlotOperationError(`Claim duration must be between 1 and 1440 minutes (requested: ${durationMinutes}).`);
295
- }
296
- const rawRequestedHost = request.host || request.targetHost;
297
- const requestedHost = rawRequestedHost ? this.normalizeHostId(rawRequestedHost) : undefined;
298
- const hypervisorTypePref = request.hypervisorType || "any";
300
+ return this.mutex.runExclusive(async () => this.claimSlotInternal(request));
301
+ }
302
+ async claimSlotInternal(request) {
303
+ // 1. Reclaim any expired leases first
304
+ this.sweepExpiredSlotsInternal();
305
+ const tagInput = request.tag || request.linearIssue || request.requester;
306
+ if (!tagInput || typeof tagInput !== "string" || tagInput.trim() === "") {
307
+ throw new InvalidSlotOperationError("A valid non-empty 'tag', 'linearIssue', or 'requester' is required to claim a slot.");
308
+ }
309
+ const durationMinutes = request.durationMinutes ?? request.ttlMinutes ?? (request.ttlSeconds ? Math.ceil(request.ttlSeconds / 60) : this.defaultDurationMinutes);
310
+ if (durationMinutes < 1 || durationMinutes > 1440) {
311
+ throw new InvalidSlotOperationError(`Claim duration must be between 1 and 1440 minutes (requested: ${durationMinutes}).`);
312
+ }
313
+ const rawRequestedHost = request.host || request.targetHost;
314
+ const requestedHost = rawRequestedHost ? this.normalizeHostId(rawRequestedHost) : undefined;
315
+ const hypervisorTypePref = request.hypervisorType || "any";
316
+ const allowEsxi = request.allowEsxiSlots ?? this.allowEsxiSlots;
317
+ if (!allowEsxi) {
299
318
  // 1. Explicit ESXi Host Rejection
300
319
  if (requestedHost &&
301
320
  (requestedHost === "esxi-intel-192-168-123-176" ||
@@ -307,213 +326,246 @@ export class SlotManager {
307
326
  if (hypervisorTypePref === "esxi") {
308
327
  throw new HypervisorSlotUnsupportedError("Slot leasing is not supported on ESXi hypervisors. Appliance slots are exclusively managed on Proxmox VE (proxmox-lab and proxmox-mini).", requestedHost, "esxi");
309
328
  }
310
- let targetSlot;
311
- if (requestedHost && requestedHost !== "auto") {
312
- // Specific host claim
313
- const hostConfig = this.hosts.get(requestedHost);
314
- if (!hostConfig) {
315
- const validHosts = Array.from(this.hosts.keys()).join(", ");
316
- throw new InvalidSlotOperationError(`Unknown host '${requestedHost}' for slot allocation. Available: [${validHosts}]`);
329
+ }
330
+ let targetSlot;
331
+ if (requestedHost && requestedHost !== "auto") {
332
+ // Specific host claim
333
+ const hostConfig = this.hosts.get(requestedHost);
334
+ if (!hostConfig) {
335
+ const validHosts = Array.from(this.hosts.keys()).join(", ");
336
+ throw new InvalidSlotOperationError(`Unknown host '${requestedHost}' for slot allocation. Available: [${validHosts}]`);
337
+ }
338
+ // ESXi Datastore check (>= 10GB free required)
339
+ if (hostConfig.hypervisorType === "esxi") {
340
+ const freeDisk = hostConfig.freeDiskGb ?? 100;
341
+ if (freeDisk < 10) {
342
+ throw new DatastoreCapacityExceededError(requestedHost, 10, freeDisk);
317
343
  }
318
- // ESXi Datastore check (>= 10GB free required)
319
- if (hostConfig.hypervisorType === "esxi") {
320
- const freeDisk = hostConfig.freeDiskGb ?? 100;
321
- if (freeDisk < 10) {
322
- throw new DatastoreCapacityExceededError(requestedHost, 10, freeDisk);
323
- }
344
+ }
345
+ const hostSlots = Array.from(this.slots.values())
346
+ .filter((s) => s.hostId === requestedHost)
347
+ .sort((a, b) => a.slotIndex - b.slotIndex);
348
+ let availableSlot = hostSlots.find((s) => s.status === "AVAILABLE");
349
+ // Dynamic slot expansion when box pure capacity exceeds pre-allocated slots
350
+ if (!availableSlot) {
351
+ const pureSlots = calculatePureCapacity(hostConfig);
352
+ if (hostSlots.length < pureSlots) {
353
+ const nextIndex = hostSlots.length + 1;
354
+ const slotId = `slot-${requestedHost}-${nextIndex}`;
355
+ availableSlot = {
356
+ slotId,
357
+ hostId: requestedHost,
358
+ hostName: hostConfig.name,
359
+ hypervisorType: hostConfig.hypervisorType,
360
+ slotIndex: nextIndex,
361
+ memoryMb: hostConfig.slotSizeMb || 8192,
362
+ allocatedMemoryMb: hostConfig.slotSizeMb || 8192,
363
+ allocatedMemoryBytes: (hostConfig.slotSizeMb || 8192) * 1024 * 1024,
364
+ status: "AVAILABLE",
365
+ claimId: null,
366
+ claimedAt: null,
367
+ expiresAt: null,
368
+ durationMinutes: null,
369
+ ttlMinutes: null,
370
+ ttlSeconds: null,
371
+ tag: null,
372
+ label: null,
373
+ workloadType: null,
374
+ targetVersion: null,
375
+ branch: null,
376
+ customTags: [],
377
+ requester: null,
378
+ linearIssue: null,
379
+ sessionId: null,
380
+ purpose: null,
381
+ vmId: null,
382
+ vmName: null,
383
+ metadata: {}
384
+ };
385
+ this.slots.set(slotId, availableSlot);
386
+ hostSlots.push(availableSlot);
324
387
  }
325
- const hostSlots = Array.from(this.slots.values())
326
- .filter((s) => s.hostId === requestedHost)
327
- .sort((a, b) => a.slotIndex - b.slotIndex);
328
- let availableSlot = hostSlots.find((s) => s.status === "AVAILABLE");
329
- // Dynamic slot expansion when box pure capacity exceeds pre-allocated slots
330
- if (!availableSlot) {
331
- const pureSlots = calculatePureCapacity(hostConfig);
332
- if (hostSlots.length < pureSlots) {
333
- const nextIndex = hostSlots.length + 1;
334
- const slotId = `slot-${requestedHost}-${nextIndex}`;
335
- availableSlot = {
336
- slotId,
337
- hostId: requestedHost,
338
- hostName: hostConfig.name,
339
- hypervisorType: hostConfig.hypervisorType,
340
- slotIndex: nextIndex,
341
- memoryMb: hostConfig.slotSizeMb || 8192,
342
- allocatedMemoryMb: hostConfig.slotSizeMb || 8192,
343
- allocatedMemoryBytes: (hostConfig.slotSizeMb || 8192) * 1024 * 1024,
344
- status: "AVAILABLE",
345
- claimId: null,
346
- claimedAt: null,
347
- expiresAt: null,
348
- durationMinutes: null,
349
- ttlMinutes: null,
350
- ttlSeconds: null,
351
- tag: null,
352
- label: null,
353
- workloadType: null,
354
- targetVersion: null,
355
- branch: null,
356
- customTags: [],
357
- requester: null,
358
- linearIssue: null,
359
- sessionId: null,
360
- purpose: null,
361
- vmId: null,
362
- vmName: null,
363
- metadata: {}
364
- };
365
- this.slots.set(slotId, availableSlot);
366
- hostSlots.push(availableSlot);
367
- }
388
+ }
389
+ if (!availableSlot && request.purgeIfFull) {
390
+ const purgeRes = await this.purgeSlotsInternal({
391
+ host: requestedHost,
392
+ destroyVms: true,
393
+ includeOrphans: true,
394
+ dryRun: false
395
+ });
396
+ if (purgeRes.purgedSlotsCount > 0 || purgeRes.destroyedVmsCount > 0) {
397
+ const recheckedSlots = Array.from(this.slots.values())
398
+ .filter((s) => s.hostId === requestedHost)
399
+ .sort((a, b) => a.slotIndex - b.slotIndex);
400
+ availableSlot = recheckedSlots.find((s) => s.status === "AVAILABLE");
368
401
  }
369
- if (!availableSlot) {
370
- const allocatedCount = hostSlots.filter((s) => s.status === "CLAIMED" || s.status === "active").length;
371
- const hostLimit = hostConfig.maxSlots ?? calculatePureCapacity(hostConfig);
372
- if (requestedHost === "proxmox-lab") {
373
- throw new CapacityExceededError(`Host 'proxmox-lab' has reached its hard quota limit of 2 concurrent 8GB slots (16GB allocated). 16GB is strictly reserved for CUCM and core lab infrastructure.`, "proxmox-lab", hostLimit, allocatedCount);
374
- }
375
- else if (requestedHost === "proxmox-mini") {
376
- throw new CapacityExceededError(`Host 'proxmox-mini' has reached its hard quota limit of 4 concurrent 8GB slots (32GB allocated).`, "proxmox-mini", hostLimit, allocatedCount);
377
- }
378
- else {
379
- throw new CapacityExceededError(`Host '${requestedHost}' has reached its hard quota limit of ${hostLimit} concurrent slots.`, requestedHost, hostLimit, allocatedCount);
380
- }
402
+ }
403
+ if (!availableSlot) {
404
+ const allocatedCount = hostSlots.filter((s) => s.status === "CLAIMED" || s.status === "active").length;
405
+ const hostLimit = hostConfig.maxSlots ?? calculatePureCapacity(hostConfig);
406
+ if (requestedHost === "proxmox-lab") {
407
+ throw new CapacityExceededError(`Host 'proxmox-lab' has reached its hard quota limit of 2 concurrent 8GB slots (16GB allocated). 16GB is strictly reserved for CUCM and core lab infrastructure.`, "proxmox-lab", hostLimit, allocatedCount);
408
+ }
409
+ else if (requestedHost === "proxmox-mini") {
410
+ throw new CapacityExceededError(`Host 'proxmox-mini' has reached its hard quota limit of 4 concurrent 8GB slots (32GB allocated).`, "proxmox-mini", hostLimit, allocatedCount);
411
+ }
412
+ else {
413
+ throw new CapacityExceededError(`Host '${requestedHost}' has reached its hard quota limit of ${hostLimit} concurrent slots.`, requestedHost, hostLimit, allocatedCount);
381
414
  }
382
- targetSlot = availableSlot;
383
415
  }
384
- else {
385
- // Homogeneous-fleet auto-placement. Every eligible Proxmox host is a
386
- // peer at the request boundary; production uses system randomness,
387
- // while tests may supply an explicit seed for reproducible placement.
388
- // Filter candidate hosts based on hypervisor preference (excluding ESXi and 0-capacity hosts)
389
- const candidateHostConfigs = Array.from(this.hosts.values()).filter((h) => {
390
- const pureSlots = calculatePureCapacity(h);
391
- if (h.hypervisorType === "esxi" || pureSlots === 0)
392
- return false;
393
- if (hypervisorTypePref === "proxmox" && h.hypervisorType !== "proxmox")
394
- return false;
395
- return true;
396
- });
397
- // Evaluate available slots and datastore health per host
398
- const hostStats = candidateHostConfigs.map((h) => {
399
- const hostSlots = Array.from(this.slots.values()).filter((s) => s.hostId === h.hostId);
400
- const availableSlots = hostSlots.filter((s) => s.status === "AVAILABLE").length;
401
- const freeDiskGb = h.freeDiskGb ?? 100;
402
- const datastoreOk = h.hypervisorType !== "esxi" || freeDiskGb >= 10;
403
- return { host: h, availableSlots, hostSlots, datastoreOk };
416
+ targetSlot = availableSlot;
417
+ }
418
+ else {
419
+ // Homogeneous-fleet auto-placement. Every eligible host is a
420
+ // peer at the request boundary; production uses system randomness,
421
+ // while tests may supply an explicit seed for reproducible placement.
422
+ // Filter candidate hosts based on hypervisor preference (excluding ESXi unless allowEsxi)
423
+ const candidateHostConfigs = Array.from(this.hosts.values()).filter((h) => {
424
+ const pureSlots = calculatePureCapacity({ ...h, allowEsxiSlots: allowEsxi });
425
+ if (h.hypervisorType === "esxi" && !allowEsxi)
426
+ return false;
427
+ if (pureSlots === 0 && (!allowEsxi || h.hypervisorType !== "esxi"))
428
+ return false;
429
+ if (hypervisorTypePref === "proxmox" && h.hypervisorType !== "proxmox")
430
+ return false;
431
+ if (hypervisorTypePref === "esxi" && h.hypervisorType !== "esxi")
432
+ return false;
433
+ return true;
434
+ });
435
+ // Evaluate available slots and datastore health per host
436
+ const hostStats = candidateHostConfigs.map((h) => {
437
+ const hostSlots = Array.from(this.slots.values()).filter((s) => s.hostId === h.hostId);
438
+ const availableSlots = hostSlots.filter((s) => s.status === "AVAILABLE").length;
439
+ const freeDiskGb = h.freeDiskGb ?? 100;
440
+ const datastoreOk = h.hypervisorType !== "esxi" || freeDiskGb >= 10;
441
+ return { host: h, availableSlots, hostSlots, datastoreOk };
442
+ });
443
+ let totalAvailable = hostStats.reduce((sum, h) => sum + (h.datastoreOk ? h.availableSlots : 0), 0);
444
+ if (totalAvailable <= 0 && request.purgeIfFull) {
445
+ const purgeRes = await this.purgeSlotsInternal({
446
+ destroyVms: true,
447
+ includeOrphans: true,
448
+ dryRun: false
404
449
  });
405
- const totalAvailable = hostStats.reduce((sum, h) => sum + (h.datastoreOk ? h.availableSlots : 0), 0);
406
- if (totalAvailable <= 0) {
407
- if (hypervisorTypePref === "proxmox") {
408
- throw new CapacityExceededError("All Proxmox cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", "proxmox-cluster", 6, 6);
450
+ if (purgeRes.purgedSlotsCount > 0 || purgeRes.destroyedVmsCount > 0) {
451
+ for (const h of hostStats) {
452
+ const hSlots = Array.from(this.slots.values()).filter((s) => s.hostId === h.host.hostId);
453
+ h.availableSlots = hSlots.filter((s) => s.status === "AVAILABLE").length;
454
+ h.hostSlots = hSlots;
409
455
  }
410
- throw new GlobalCapacityExceededError("All 6 cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", 6, 6);
456
+ totalAvailable = hostStats.reduce((sum, h) => sum + (h.datastoreOk ? h.availableSlots : 0), 0);
411
457
  }
412
- const validCandidates = hostStats
413
- .filter((h) => h.datastoreOk && h.availableSlots > 0)
414
- .sort((a, b) => a.host.hostId.localeCompare(b.host.hostId));
415
- const candidateIndex = request.placementSeed
416
- ? createHash("sha256")
417
- .update(request.placementSeed)
418
- .digest()
419
- .readUInt32BE(0) % validCandidates.length
420
- : randomInt(validCandidates.length);
421
- const winningHostStat = validCandidates[candidateIndex];
422
- if (!winningHostStat) {
423
- throw new GlobalCapacityExceededError("All 6 cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", 6, 6);
458
+ }
459
+ if (totalAvailable <= 0) {
460
+ if (hypervisorTypePref === "proxmox") {
461
+ throw new CapacityExceededError("All Proxmox cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", "proxmox-cluster", 6, 6);
424
462
  }
425
- const availableSlotsOnWinner = winningHostStat.hostSlots
426
- .filter((s) => s.status === "AVAILABLE")
427
- .sort((a, b) => a.slotIndex - b.slotIndex);
428
- targetSlot = availableSlotsOnWinner[0];
463
+ throw new GlobalCapacityExceededError("All 6 cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", 6, 6);
429
464
  }
430
- if (!targetSlot) {
465
+ const validCandidates = hostStats
466
+ .filter((h) => h.datastoreOk && h.availableSlots > 0)
467
+ .sort((a, b) => a.host.hostId.localeCompare(b.host.hostId));
468
+ const candidateIndex = request.placementSeed
469
+ ? createHash("sha256")
470
+ .update(request.placementSeed)
471
+ .digest()
472
+ .readUInt32BE(0) % validCandidates.length
473
+ : randomInt(validCandidates.length);
474
+ const winningHostStat = validCandidates[candidateIndex];
475
+ if (!winningHostStat) {
431
476
  throw new GlobalCapacityExceededError("All 6 cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", 6, 6);
432
477
  }
433
- // Assign claim details
434
- const claimId = `claim-${randomUUID().substring(0, 8)}`;
435
- const now = new Date();
436
- const claimedAt = now.toISOString();
437
- const expiresAt = new Date(now.getTime() + durationMinutes * 60 * 1000).toISOString();
438
- const linearIssue = request.linearIssue || (request.tag && request.tag.startsWith("CTUAT-") ? request.tag : null);
439
- // Normalize custom tags
440
- const customTags = [
441
- ...this.normalizeTagsArray(request.customTags),
442
- ...this.normalizeTagsArray(request.tags)
443
- ];
444
- // Deduplicate tags
445
- const uniqueTags = Array.from(new Set(customTags));
446
- // Auto-assign human readable label if not provided
447
- const label = request.label || (linearIssue
448
- ? `[${linearIssue}] ${request.workloadType || "workload"}: Slot ${targetSlot.slotIndex}`
449
- : request.purpose
450
- ? `${request.purpose} (Slot ${targetSlot.slotIndex})`
451
- : `${request.tag || tagInput} Lease`);
452
- targetSlot.status = "CLAIMED";
453
- targetSlot.claimId = claimId;
454
- targetSlot.claimedAt = claimedAt;
455
- targetSlot.expiresAt = expiresAt;
456
- targetSlot.durationMinutes = durationMinutes;
457
- targetSlot.ttlMinutes = durationMinutes;
458
- targetSlot.ttlSeconds = durationMinutes * 60;
459
- targetSlot.tag = (request.tag || tagInput).trim();
460
- targetSlot.label = label;
461
- targetSlot.workloadType = request.workloadType || null;
462
- targetSlot.targetVersion = request.targetVersion || null;
463
- targetSlot.branch = request.branch || null;
464
- targetSlot.customTags = uniqueTags;
465
- targetSlot.requester = request.requester || request.tag || null;
466
- targetSlot.linearIssue = linearIssue;
467
- targetSlot.sessionId = request.sessionId || null;
468
- targetSlot.purpose = request.purpose || null;
469
- targetSlot.metadata = request.metadata || {};
470
- // Emit audit telemetry
471
- this.telemetry.emit({
472
- event: "slot.claim",
473
- actor: targetSlot.requester || targetSlot.sessionId || "unknown",
474
- status: "success",
475
- linearIssue: linearIssue || undefined,
476
- slotId: targetSlot.slotId,
477
- claimId,
478
- hostId: targetSlot.hostId,
479
- hypervisorType: targetSlot.hypervisorType,
480
- metadata: {
481
- label: targetSlot.label,
482
- workloadType: targetSlot.workloadType,
483
- targetVersion: targetSlot.targetVersion,
484
- branch: targetSlot.branch,
485
- durationMinutes,
486
- expiresAt
487
- }
488
- });
489
- return {
490
- claimId,
491
- slotId: targetSlot.slotId,
492
- hostId: targetSlot.hostId,
493
- hostName: targetSlot.hostName || targetSlot.hostId,
494
- hypervisorType: targetSlot.hypervisorType || (targetSlot.hostId.startsWith("esxi") ? "esxi" : "proxmox"),
495
- slotIndex: targetSlot.slotIndex,
496
- memoryMb: targetSlot.memoryMb,
497
- allocatedMemoryMb: targetSlot.memoryMb,
498
- allocatedMemoryBytes: targetSlot.memoryMb * 1024 * 1024,
499
- tag: targetSlot.tag,
500
- label: targetSlot.label || undefined,
501
- workloadType: targetSlot.workloadType || undefined,
502
- targetVersion: targetSlot.targetVersion || undefined,
503
- branch: targetSlot.branch || undefined,
504
- customTags: targetSlot.customTags.length > 0 ? targetSlot.customTags : undefined,
505
- requester: targetSlot.requester || undefined,
506
- linearIssue: targetSlot.linearIssue || undefined,
507
- claimedAt,
508
- expiresAt,
478
+ const availableSlotsOnWinner = winningHostStat.hostSlots
479
+ .filter((s) => s.status === "AVAILABLE")
480
+ .sort((a, b) => a.slotIndex - b.slotIndex);
481
+ targetSlot = availableSlotsOnWinner[0];
482
+ }
483
+ if (!targetSlot) {
484
+ throw new GlobalCapacityExceededError("All 6 cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", 6, 6);
485
+ }
486
+ // Assign claim details
487
+ const claimId = `claim-${randomUUID().substring(0, 8)}`;
488
+ const now = new Date();
489
+ const claimedAt = now.toISOString();
490
+ const expiresAt = new Date(now.getTime() + durationMinutes * 60 * 1000).toISOString();
491
+ const linearIssue = request.linearIssue || (request.tag && request.tag.startsWith("CTUAT-") ? request.tag : null);
492
+ // Normalize custom tags
493
+ const customTags = [
494
+ ...this.normalizeTagsArray(request.customTags),
495
+ ...this.normalizeTagsArray(request.tags)
496
+ ];
497
+ // Deduplicate tags
498
+ const uniqueTags = Array.from(new Set(customTags));
499
+ // Auto-assign human readable label if not provided
500
+ const label = request.label || (linearIssue
501
+ ? `[${linearIssue}] ${request.workloadType || "workload"}: Slot ${targetSlot.slotIndex}`
502
+ : request.purpose
503
+ ? `${request.purpose} (Slot ${targetSlot.slotIndex})`
504
+ : `${request.tag || tagInput} Lease`);
505
+ targetSlot.status = "CLAIMED";
506
+ targetSlot.claimId = claimId;
507
+ targetSlot.claimedAt = claimedAt;
508
+ targetSlot.expiresAt = expiresAt;
509
+ targetSlot.durationMinutes = durationMinutes;
510
+ targetSlot.ttlMinutes = durationMinutes;
511
+ targetSlot.ttlSeconds = durationMinutes * 60;
512
+ targetSlot.tag = (request.tag || tagInput).trim();
513
+ targetSlot.label = label;
514
+ targetSlot.workloadType = request.workloadType || null;
515
+ targetSlot.targetVersion = request.targetVersion || null;
516
+ targetSlot.branch = request.branch || null;
517
+ targetSlot.customTags = uniqueTags;
518
+ targetSlot.requester = request.requester || request.tag || null;
519
+ targetSlot.linearIssue = linearIssue;
520
+ targetSlot.sessionId = request.sessionId || null;
521
+ targetSlot.purpose = request.purpose || null;
522
+ targetSlot.metadata = request.metadata || {};
523
+ // Emit audit telemetry
524
+ this.telemetry.emit({
525
+ event: "slot.claim",
526
+ actor: targetSlot.requester || targetSlot.sessionId || "unknown",
527
+ status: "success",
528
+ linearIssue: linearIssue || undefined,
529
+ slotId: targetSlot.slotId,
530
+ claimId,
531
+ hostId: targetSlot.hostId,
532
+ hypervisorType: targetSlot.hypervisorType,
533
+ metadata: {
534
+ label: targetSlot.label,
535
+ workloadType: targetSlot.workloadType,
536
+ targetVersion: targetSlot.targetVersion,
537
+ branch: targetSlot.branch,
509
538
  durationMinutes,
510
- ttlMinutes: durationMinutes,
511
- ttlSeconds: durationMinutes * 60,
512
- status: "active",
513
- purpose: targetSlot.purpose || undefined,
514
- sessionId: targetSlot.sessionId || undefined
515
- };
539
+ expiresAt
540
+ }
516
541
  });
542
+ return {
543
+ claimId,
544
+ slotId: targetSlot.slotId,
545
+ hostId: targetSlot.hostId,
546
+ hostName: targetSlot.hostName || targetSlot.hostId,
547
+ hypervisorType: targetSlot.hypervisorType || (targetSlot.hostId.startsWith("esxi") ? "esxi" : "proxmox"),
548
+ slotIndex: targetSlot.slotIndex,
549
+ memoryMb: targetSlot.memoryMb,
550
+ allocatedMemoryMb: targetSlot.memoryMb,
551
+ allocatedMemoryBytes: targetSlot.memoryMb * 1024 * 1024,
552
+ tag: targetSlot.tag,
553
+ label: targetSlot.label || undefined,
554
+ workloadType: targetSlot.workloadType || undefined,
555
+ targetVersion: targetSlot.targetVersion || undefined,
556
+ branch: targetSlot.branch || undefined,
557
+ customTags: targetSlot.customTags.length > 0 ? targetSlot.customTags : undefined,
558
+ requester: targetSlot.requester || undefined,
559
+ linearIssue: targetSlot.linearIssue || undefined,
560
+ claimedAt,
561
+ expiresAt,
562
+ durationMinutes,
563
+ ttlMinutes: durationMinutes,
564
+ ttlSeconds: durationMinutes * 60,
565
+ status: "active",
566
+ purpose: targetSlot.purpose || undefined,
567
+ sessionId: targetSlot.sessionId || undefined
568
+ };
517
569
  }
518
570
  /**
519
571
  * Releases one or more slots matching claimId, tag, slotId, requester, or linearIssue.
@@ -971,149 +1023,164 @@ export class SlotManager {
971
1023
  * Gated by SafetyGuardrailEngine non-bypassable protections.
972
1024
  */
973
1025
  async purgeSlots(options = {}) {
974
- return this.mutex.runExclusive(async () => {
975
- const now = Date.now();
976
- const dryRun = options.dryRun ?? false;
977
- const destroyVms = options.destroyVms ?? true;
978
- const force = options.force ?? false;
979
- const includeOrphans = options.includeOrphans ?? false;
980
- const targetHost = options.host && options.host !== "all" ? this.normalizeHostId(options.host) : undefined;
981
- const targetHypervisor = options.hypervisorType && options.hypervisorType !== "all" && options.hypervisorType !== "any"
982
- ? options.hypervisorType
983
- : undefined;
984
- const matchingSlots = [];
985
- const reasons = new Map();
986
- for (const slot of this.slots.values()) {
987
- if (slot.status !== "CLAIMED" && slot.status !== "active" && slot.status !== "EXPIRED")
988
- continue;
989
- if (targetHost && slot.hostId !== targetHost)
990
- continue;
991
- if (targetHypervisor && slot.hypervisorType !== targetHypervisor)
992
- continue;
993
- let shouldPurge = false;
994
- let reason = "manual_purge";
995
- const expiresTime = slot.expiresAt ? new Date(slot.expiresAt).getTime() : 0;
996
- const claimedTime = slot.claimedAt ? new Date(slot.claimedAt).getTime() : 0;
997
- const isExpired = expiresTime > 0 && expiresTime <= now;
998
- if (options.linearIssue) {
999
- const matchesLinear = slot.linearIssue === options.linearIssue || slot.tag === options.linearIssue;
1000
- if (matchesLinear) {
1001
- shouldPurge = true;
1002
- reason = isExpired ? "expired" : "linear_match";
1003
- }
1004
- }
1005
- else if (options.olderThanMinutes !== undefined) {
1006
- const ageMinutes = (now - claimedTime) / (60 * 1000);
1007
- if (ageMinutes >= options.olderThanMinutes || isExpired) {
1008
- shouldPurge = true;
1009
- reason = isExpired ? "expired" : "older_than_threshold";
1010
- }
1011
- }
1012
- else if (isExpired) {
1026
+ return this.mutex.runExclusive(async () => this.purgeSlotsInternal(options));
1027
+ }
1028
+ async purgeSlotsInternal(options = {}) {
1029
+ const now = Date.now();
1030
+ const dryRun = options.dryRun ?? false;
1031
+ const destroyVms = options.destroyVms ?? true;
1032
+ const force = options.force ?? false;
1033
+ const includeOrphans = options.includeOrphans ?? false;
1034
+ const targetHost = options.host && options.host !== "all" ? this.normalizeHostId(options.host) : undefined;
1035
+ const targetHypervisor = options.hypervisorType && options.hypervisorType !== "all" && options.hypervisorType !== "any"
1036
+ ? options.hypervisorType
1037
+ : undefined;
1038
+ const matchingSlots = [];
1039
+ const reasons = new Map();
1040
+ for (const slot of this.slots.values()) {
1041
+ if (slot.status !== "CLAIMED" && slot.status !== "active" && slot.status !== "EXPIRED")
1042
+ continue;
1043
+ if (targetHost && slot.hostId !== targetHost)
1044
+ continue;
1045
+ if (targetHypervisor && slot.hypervisorType !== targetHypervisor)
1046
+ continue;
1047
+ let shouldPurge = false;
1048
+ let reason = "manual_purge";
1049
+ const expiresTime = slot.expiresAt ? new Date(slot.expiresAt).getTime() : 0;
1050
+ const claimedTime = slot.claimedAt ? new Date(slot.claimedAt).getTime() : 0;
1051
+ const isExpired = expiresTime > 0 && expiresTime <= now;
1052
+ if (options.linearIssue) {
1053
+ const matchesLinear = slot.linearIssue === options.linearIssue || slot.tag === options.linearIssue;
1054
+ if (matchesLinear) {
1013
1055
  shouldPurge = true;
1014
- reason = "expired";
1056
+ reason = isExpired ? "expired" : "linear_match";
1015
1057
  }
1016
- else if (force) {
1058
+ }
1059
+ else if (options.olderThanMinutes !== undefined) {
1060
+ const ageMinutes = (now - claimedTime) / (60 * 1000);
1061
+ if (ageMinutes >= options.olderThanMinutes || isExpired) {
1017
1062
  shouldPurge = true;
1018
- reason = "forced";
1019
- }
1020
- if (shouldPurge) {
1021
- matchingSlots.push(slot);
1022
- reasons.set(slot.slotId, reason);
1063
+ reason = isExpired ? "expired" : "older_than_threshold";
1023
1064
  }
1024
1065
  }
1025
- const purgedSlots = [];
1026
- const destroyedVms = [];
1027
- let skippedProtectedCount = 0;
1028
- let reclaimedMemoryMb = 0;
1029
- for (const slot of matchingSlots) {
1030
- const reason = reasons.get(slot.slotId) || "manual_purge";
1031
- const purgedSlot = {
1032
- slotId: slot.slotId,
1033
- hostId: slot.hostId,
1034
- claimId: slot.claimId,
1035
- claimedAt: slot.claimedAt,
1036
- expiresAt: slot.expiresAt,
1037
- linearIssue: slot.linearIssue,
1038
- label: slot.label,
1039
- workloadType: slot.workloadType,
1040
- vmId: slot.vmId,
1041
- vmName: slot.vmName,
1042
- reason
1066
+ else if (isExpired) {
1067
+ shouldPurge = true;
1068
+ reason = "expired";
1069
+ }
1070
+ else if (force) {
1071
+ shouldPurge = true;
1072
+ reason = "forced";
1073
+ }
1074
+ if (shouldPurge) {
1075
+ matchingSlots.push(slot);
1076
+ reasons.set(slot.slotId, reason);
1077
+ }
1078
+ }
1079
+ const purgedSlots = [];
1080
+ const destroyedVms = [];
1081
+ let skippedProtectedCount = 0;
1082
+ let reclaimedMemoryMb = 0;
1083
+ for (const slot of matchingSlots) {
1084
+ const reason = reasons.get(slot.slotId) || "manual_purge";
1085
+ const purgedSlot = {
1086
+ slotId: slot.slotId,
1087
+ hostId: slot.hostId,
1088
+ claimId: slot.claimId,
1089
+ claimedAt: slot.claimedAt,
1090
+ expiresAt: slot.expiresAt,
1091
+ linearIssue: slot.linearIssue,
1092
+ label: slot.label,
1093
+ workloadType: slot.workloadType,
1094
+ vmId: slot.vmId,
1095
+ vmName: slot.vmName,
1096
+ reason
1097
+ };
1098
+ let canReclaimSlot = true;
1099
+ // Process attached VM cleanup if present
1100
+ if (slot.vmId !== null && slot.vmId !== undefined) {
1101
+ const vmTarget = {
1102
+ vmid: slot.vmId,
1103
+ name: slot.vmName || undefined,
1104
+ isProtected: false
1043
1105
  };
1044
- let canReclaimSlot = true;
1045
- // Process attached VM cleanup if present
1046
- if (slot.vmId !== null && slot.vmId !== undefined) {
1047
- const vmTarget = {
1048
- vmid: slot.vmId,
1049
- name: slot.vmName || undefined,
1050
- isProtected: false
1051
- };
1052
- // Guardrail safety check
1053
- const protCheck = this.guardrailEngine.isProtected(vmTarget);
1054
- if (protCheck.protected) {
1055
- canReclaimSlot = false;
1056
- skippedProtectedCount++;
1106
+ // Guardrail safety check
1107
+ const protCheck = this.guardrailEngine.isProtected(vmTarget);
1108
+ if (protCheck.protected) {
1109
+ canReclaimSlot = false;
1110
+ skippedProtectedCount++;
1111
+ destroyedVms.push({
1112
+ vmId: slot.vmId,
1113
+ vmName: slot.vmName || undefined,
1114
+ hostId: slot.hostId,
1115
+ hypervisorType: slot.hypervisorType,
1116
+ status: "skipped_protected",
1117
+ reason: `Protected infrastructure preserved: ${protCheck.reason}`
1118
+ });
1119
+ this.telemetry.emit({
1120
+ event: "vm.guardrail_blocked",
1121
+ actor: slot.requester || "system",
1122
+ status: "blocked",
1123
+ linearIssue: slot.linearIssue || undefined,
1124
+ vmId: slot.vmId,
1125
+ vmName: slot.vmName || undefined,
1126
+ hostId: slot.hostId,
1127
+ hypervisorType: slot.hypervisorType,
1128
+ error: `Guardrail blocked purge destruction: ${protCheck.reason}`
1129
+ });
1130
+ }
1131
+ else {
1132
+ if (dryRun) {
1057
1133
  destroyedVms.push({
1058
1134
  vmId: slot.vmId,
1059
1135
  vmName: slot.vmName || undefined,
1060
1136
  hostId: slot.hostId,
1061
1137
  hypervisorType: slot.hypervisorType,
1062
- status: "skipped_protected",
1063
- reason: `Protected infrastructure preserved: ${protCheck.reason}`
1064
- });
1065
- this.telemetry.emit({
1066
- event: "vm.guardrail_blocked",
1067
- actor: slot.requester || "system",
1068
- status: "blocked",
1069
- linearIssue: slot.linearIssue || undefined,
1070
- vmId: slot.vmId,
1071
- vmName: slot.vmName || undefined,
1072
- hostId: slot.hostId,
1073
- hypervisorType: slot.hypervisorType,
1074
- error: `Guardrail blocked purge destruction: ${protCheck.reason}`
1138
+ status: "dry_run",
1139
+ reason: `Would be destroyed during purge (${reason})`
1075
1140
  });
1076
1141
  }
1077
- else {
1078
- if (dryRun) {
1142
+ else if (destroyVms && this.hypervisorManager) {
1143
+ try {
1144
+ await this.hypervisorManager.vmAction({
1145
+ vmIdOrName: String(slot.vmId),
1146
+ targetHost: slot.hostId,
1147
+ action: "destroy",
1148
+ force: true
1149
+ });
1079
1150
  destroyedVms.push({
1080
1151
  vmId: slot.vmId,
1081
1152
  vmName: slot.vmName || undefined,
1082
1153
  hostId: slot.hostId,
1083
1154
  hypervisorType: slot.hypervisorType,
1084
- status: "dry_run",
1085
- reason: `Would be destroyed during purge (${reason})`
1155
+ status: "destroyed",
1156
+ reason: `Purged and destroyed attached VM (${reason})`
1157
+ });
1158
+ this.telemetry.emit({
1159
+ event: "vm.purged",
1160
+ actor: slot.requester || "system",
1161
+ status: "success",
1162
+ linearIssue: slot.linearIssue || undefined,
1163
+ vmId: slot.vmId,
1164
+ vmName: slot.vmName || undefined,
1165
+ hostId: slot.hostId,
1166
+ hypervisorType: slot.hypervisorType,
1167
+ metadata: { slotId: slot.slotId, reason }
1086
1168
  });
1087
1169
  }
1088
- else if (destroyVms && this.hypervisorManager) {
1089
- try {
1090
- await this.hypervisorManager.vmAction({
1091
- vmIdOrName: String(slot.vmId),
1092
- targetHost: slot.hostId,
1093
- action: "destroy",
1094
- force: true
1095
- });
1170
+ catch (err) {
1171
+ const errMsg = err instanceof Error ? err.message : String(err);
1172
+ if (errMsg.toLowerCase().includes("not found")) {
1173
+ canReclaimSlot = true;
1096
1174
  destroyedVms.push({
1097
1175
  vmId: slot.vmId,
1098
1176
  vmName: slot.vmName || undefined,
1099
1177
  hostId: slot.hostId,
1100
1178
  hypervisorType: slot.hypervisorType,
1101
1179
  status: "destroyed",
1102
- reason: `Purged and destroyed attached VM (${reason})`
1103
- });
1104
- this.telemetry.emit({
1105
- event: "vm.purged",
1106
- actor: slot.requester || "system",
1107
- status: "success",
1108
- linearIssue: slot.linearIssue || undefined,
1109
- vmId: slot.vmId,
1110
- vmName: slot.vmName || undefined,
1111
- hostId: slot.hostId,
1112
- hypervisorType: slot.hypervisorType,
1113
- metadata: { slotId: slot.slotId, reason }
1180
+ reason: `Attached VM already absent from hypervisor (${reason})`
1114
1181
  });
1115
1182
  }
1116
- catch (err) {
1183
+ else {
1117
1184
  canReclaimSlot = false;
1118
1185
  destroyedVms.push({
1119
1186
  vmId: slot.vmId,
@@ -1121,148 +1188,732 @@ export class SlotManager {
1121
1188
  hostId: slot.hostId,
1122
1189
  hypervisorType: slot.hypervisorType,
1123
1190
  status: "failed",
1124
- error: err instanceof Error ? err.message : String(err)
1191
+ error: errMsg
1125
1192
  });
1126
1193
  }
1127
1194
  }
1128
- else if (!dryRun) {
1129
- canReclaimSlot = false;
1195
+ }
1196
+ else if (!dryRun) {
1197
+ canReclaimSlot = false;
1198
+ destroyedVms.push({
1199
+ vmId: slot.vmId,
1200
+ vmName: slot.vmName || undefined,
1201
+ hostId: slot.hostId,
1202
+ hypervisorType: slot.hypervisorType,
1203
+ status: "failed",
1204
+ error: destroyVms
1205
+ ? "No hypervisor manager is available to verify VM deletion"
1206
+ : "VM destruction was disabled; occupied capacity was retained"
1207
+ });
1208
+ }
1209
+ }
1210
+ }
1211
+ // Dry-run reports the candidate. A live purge only releases scheduler
1212
+ // capacity after the attached VM is absent or destruction succeeded.
1213
+ if (dryRun || canReclaimSlot) {
1214
+ purgedSlots.push(purgedSlot);
1215
+ reclaimedMemoryMb += slot.memoryMb;
1216
+ }
1217
+ if (!dryRun && canReclaimSlot) {
1218
+ this.telemetry.emit({
1219
+ event: "slot.purge",
1220
+ actor: slot.requester || "system",
1221
+ status: "success",
1222
+ linearIssue: slot.linearIssue || undefined,
1223
+ slotId: slot.slotId,
1224
+ claimId: slot.claimId || undefined,
1225
+ vmId: slot.vmId || undefined,
1226
+ vmName: slot.vmName || undefined,
1227
+ hostId: slot.hostId,
1228
+ hypervisorType: slot.hypervisorType,
1229
+ metadata: {
1230
+ reason,
1231
+ freedMemoryMb: slot.memoryMb
1232
+ }
1233
+ });
1234
+ this.resetSlot(slot);
1235
+ }
1236
+ }
1237
+ // Check orphan VMs if requested
1238
+ if (includeOrphans && this.hypervisorManager) {
1239
+ try {
1240
+ const allVms = await this.hypervisorManager.listUnifiedInventory({
1241
+ includeTemplates: false,
1242
+ host: targetHost,
1243
+ hypervisor: targetHypervisor
1244
+ });
1245
+ const activeVmIds = new Set(Array.from(this.slots.values())
1246
+ .map((s) => s.vmId)
1247
+ .filter((id) => id !== null && id !== undefined)
1248
+ .map((id) => String(id)));
1249
+ for (const vm of allVms) {
1250
+ // Check if VM is an untracked disposable VM without active lease
1251
+ if (!vm.isTemplate && !vm.isProtected && !activeVmIds.has(String(vm.id))) {
1252
+ const protCheck = this.guardrailEngine.isProtected({
1253
+ vmid: vm.id,
1254
+ name: vm.name,
1255
+ isProtected: vm.isProtected,
1256
+ tags: vm.tags
1257
+ });
1258
+ if (protCheck.protected) {
1259
+ skippedProtectedCount++;
1130
1260
  destroyedVms.push({
1131
- vmId: slot.vmId,
1132
- vmName: slot.vmName || undefined,
1133
- hostId: slot.hostId,
1134
- hypervisorType: slot.hypervisorType,
1135
- status: "failed",
1136
- error: destroyVms
1137
- ? "No hypervisor manager is available to verify VM deletion"
1138
- : "VM destruction was disabled; occupied capacity was retained"
1261
+ vmId: vm.id,
1262
+ vmName: vm.name,
1263
+ hostId: vm.hostName,
1264
+ hypervisorType: vm.hypervisor,
1265
+ status: "skipped_protected",
1266
+ reason: `Protected orphan preserved: ${protCheck.reason}`
1139
1267
  });
1140
1268
  }
1269
+ else {
1270
+ if (dryRun) {
1271
+ destroyedVms.push({
1272
+ vmId: vm.id,
1273
+ vmName: vm.name,
1274
+ hostId: vm.hostName,
1275
+ hypervisorType: vm.hypervisor,
1276
+ status: "dry_run",
1277
+ reason: "Orphaned disposable VM with no active slot lease"
1278
+ });
1279
+ }
1280
+ else if (destroyVms) {
1281
+ try {
1282
+ await this.hypervisorManager.vmAction({
1283
+ vmIdOrName: String(vm.id),
1284
+ targetHost: vm.hostName,
1285
+ action: "destroy",
1286
+ force: true
1287
+ });
1288
+ destroyedVms.push({
1289
+ vmId: vm.id,
1290
+ vmName: vm.name,
1291
+ hostId: vm.hostName,
1292
+ hypervisorType: vm.hypervisor,
1293
+ status: "destroyed",
1294
+ reason: "Destroyed orphaned disposable VM with no active slot lease"
1295
+ });
1296
+ }
1297
+ catch (orphanErr) {
1298
+ destroyedVms.push({
1299
+ vmId: vm.id,
1300
+ vmName: vm.name,
1301
+ hostId: vm.hostName,
1302
+ hypervisorType: vm.hypervisor,
1303
+ status: "failed",
1304
+ error: orphanErr instanceof Error ? orphanErr.message : String(orphanErr)
1305
+ });
1306
+ }
1307
+ }
1308
+ }
1141
1309
  }
1142
1310
  }
1143
- // Dry-run reports the candidate. A live purge only releases scheduler
1144
- // capacity after the attached VM is absent or destruction succeeded.
1145
- if (dryRun || canReclaimSlot) {
1146
- purgedSlots.push(purgedSlot);
1147
- reclaimedMemoryMb += slot.memoryMb;
1148
- }
1149
- if (!dryRun && canReclaimSlot) {
1150
- this.telemetry.emit({
1151
- event: "slot.purge",
1152
- actor: slot.requester || "system",
1153
- status: "success",
1154
- linearIssue: slot.linearIssue || undefined,
1155
- slotId: slot.slotId,
1156
- claimId: slot.claimId || undefined,
1157
- vmId: slot.vmId || undefined,
1158
- vmName: slot.vmName || undefined,
1159
- hostId: slot.hostId,
1160
- hypervisorType: slot.hypervisorType,
1161
- metadata: {
1162
- reason,
1163
- freedMemoryMb: slot.memoryMb
1311
+ }
1312
+ catch (invErr) {
1313
+ console.error("[purgeSlots] Orphan inventory inspection error:", invErr);
1314
+ }
1315
+ }
1316
+ const destroyedCount = destroyedVms.filter((v) => v.status === "destroyed").length;
1317
+ const reclaimedMemoryBytes = reclaimedMemoryMb * 1024 * 1024;
1318
+ const prefix = dryRun ? "[DRY RUN] " : "";
1319
+ const message = `${prefix}Purged ${purgedSlots.length} slot lease(s), reclaimed ${reclaimedMemoryMb / 1024} GB RAM, processed ${destroyedVms.length} VM(s) (${destroyedCount} destroyed, ${skippedProtectedCount} protected skipped).`;
1320
+ return {
1321
+ purgedSlotsCount: purgedSlots.length,
1322
+ purgedSlots,
1323
+ destroyedVmsCount: destroyedCount,
1324
+ destroyedVms,
1325
+ skippedProtectedVmsCount: skippedProtectedCount,
1326
+ reclaimedMemoryBytes,
1327
+ reclaimedMemoryMb,
1328
+ dryRun,
1329
+ timestamp: new Date(now).toISOString(),
1330
+ message
1331
+ };
1332
+ }
1333
+ /**
1334
+ * Intelligently discovers available slot capacity across all hypervisors,
1335
+ * ranks candidate hosts, and reports reclaimable expired/disposable VMs if fleet is full.
1336
+ * Optionally auto-claims when autoClaim=true, or auto-reclaims and claims when purgeIfFull=true.
1337
+ */
1338
+ async findSlot(params = {}) {
1339
+ return this.mutex.runExclusive(async () => {
1340
+ this.sweepExpiredSlotsInternal();
1341
+ const allowEsxi = params.allowEsxiSlots ?? this.allowEsxiSlots;
1342
+ const targetHost = params.host && params.host !== "all" && params.host !== "auto"
1343
+ ? this.normalizeHostId(params.host)
1344
+ : undefined;
1345
+ const targetHyp = params.hypervisorType && params.hypervisorType !== "all" && params.hypervisorType !== "any"
1346
+ ? params.hypervisorType
1347
+ : undefined;
1348
+ // Sync with live capacity if hypervisorManager is present
1349
+ if (this.hypervisorManager) {
1350
+ try {
1351
+ const liveCaps = await this.hypervisorManager.listAllHostsCapacity();
1352
+ for (const cap of liveCaps) {
1353
+ const hid = this.normalizeHostId(cap.name);
1354
+ const cfg = this.hosts.get(hid);
1355
+ if (cfg) {
1356
+ cfg.totalMemoryMb = Math.round(cap.memoryTotalBytes / (1024 * 1024));
1357
+ cfg.freeDiskGb = Math.round(cap.storageFreeBytes / (1024 * 1024 * 1024));
1164
1358
  }
1359
+ }
1360
+ }
1361
+ catch {
1362
+ // fallback to registered configs
1363
+ }
1364
+ }
1365
+ // Gather candidate hosts
1366
+ const allCandidates = [];
1367
+ for (const [hid, hostConfig] of this.hosts.entries()) {
1368
+ if (targetHost && hid !== targetHost)
1369
+ continue;
1370
+ if (targetHyp && hostConfig.hypervisorType !== targetHyp)
1371
+ continue;
1372
+ if (hostConfig.hypervisorType === "esxi" && !allowEsxi)
1373
+ continue;
1374
+ const hostSlots = Array.from(this.slots.values()).filter((s) => s.hostId === hid);
1375
+ const availCount = hostSlots.filter((s) => s.status === "AVAILABLE").length;
1376
+ const pureCapacity = calculatePureCapacity({ ...hostConfig, allowEsxiSlots: allowEsxi });
1377
+ const freeDisk = hostConfig.freeDiskGb ?? 100;
1378
+ const totalMem = hostConfig.totalMemoryMb;
1379
+ const reserved = hostConfig.reservedHeadroomMb || 0;
1380
+ const freeMem = Math.max(0, totalMem - reserved - (hostSlots.length - availCount) * (hostConfig.slotSizeMb || 8192));
1381
+ // Host tiering: proxmox-mini (Tier 1 Primary), proxmox-lab (Tier 2 Secondary), ESXi (Tier 3 Fallback)
1382
+ let tier = "secondary";
1383
+ let tierBoost = 10000;
1384
+ let recommendationReason = "Secondary placement: Proxmox Lab appliance cluster";
1385
+ if (hid === "proxmox-mini") {
1386
+ tier = "primary";
1387
+ tierBoost = 20000;
1388
+ recommendationReason = "Primary placement: Minisforum high-IOPS NVMe appliance host";
1389
+ }
1390
+ else if (hid === "proxmox-lab") {
1391
+ tier = "secondary";
1392
+ tierBoost = 10000;
1393
+ recommendationReason = "Secondary placement: Proxmox Lab appliance cluster";
1394
+ }
1395
+ else if (hostConfig.hypervisorType === "esxi" || hid.startsWith("esxi")) {
1396
+ tier = "fallback";
1397
+ tierBoost = 0; // Least preferred host — contains CUCM/CUBE and testbed fixtures
1398
+ recommendationReason = "Fallback placement: ESXi Intel host (reserved for fixtures / heavy spillover)";
1399
+ }
1400
+ // Available slots score with tier priority: Proxmox Mini always outranks Proxmox Lab, and Proxmox always outranks ESXi
1401
+ const score = (availCount > 0 ? tierBoost : 0) + availCount * 100 + Math.floor(freeMem / 1024) * 10 + Math.floor(freeDisk / 100);
1402
+ allCandidates.push({
1403
+ hostId: hid,
1404
+ name: hostConfig.name,
1405
+ hypervisorType: hostConfig.hypervisorType || (hid.startsWith("esxi") ? "esxi" : "proxmox"),
1406
+ status: "online",
1407
+ tier,
1408
+ availableSlots: availCount,
1409
+ totalPureSlots: pureCapacity,
1410
+ freeMemoryMb: freeMem,
1411
+ totalMemoryMb: totalMem,
1412
+ freeDiskGb: freeDisk,
1413
+ cpuUsedPercent: 0,
1414
+ runningVms: hostSlots.length - availCount,
1415
+ score,
1416
+ recommendationReason
1417
+ });
1418
+ }
1419
+ // Sort candidates by score descending
1420
+ allCandidates.sort((a, b) => b.score - a.score);
1421
+ const availableSlotsTotal = allCandidates.reduce((sum, c) => sum + c.availableSlots, 0);
1422
+ // Check if Proxmox hosts have expired leases or disposable VMs
1423
+ const proxmoxReclaimableSlots = Array.from(this.slots.values()).filter((s) => (s.hostId === "proxmox-mini" || s.hostId === "proxmox-lab") &&
1424
+ (s.status === "EXPIRED" || (s.expiresAt && new Date(s.expiresAt).getTime() <= Date.now())));
1425
+ // If capacity is available
1426
+ if (availableSlotsTotal > 0) {
1427
+ let winningCandidate = allCandidates[0];
1428
+ // Strict ESXi Fallback Rule: If winning host is ESXi, but Proxmox Mini/Lab has reclaimable expired slots
1429
+ // AND purgeIfFull is requested, auto-purge Proxmox instead of spilling over to ESXi!
1430
+ if (winningCandidate.hypervisorType === "esxi" && proxmoxReclaimableSlots.length > 0 && params.purgeIfFull) {
1431
+ const targetPurgedHost = proxmoxReclaimableSlots.some((s) => s.hostId === "proxmox-mini")
1432
+ ? "proxmox-mini"
1433
+ : "proxmox-lab";
1434
+ const purgeRes = await this.purgeSlotsInternal({
1435
+ host: targetPurgedHost,
1436
+ destroyVms: true,
1437
+ includeOrphans: true,
1438
+ dryRun: false
1165
1439
  });
1166
- this.resetSlot(slot);
1440
+ if (purgeRes.purgedSlotsCount > 0 || purgeRes.destroyedVmsCount > 0) {
1441
+ const tag = params.tag || params.linearIssue || params.requester || "find-slot-reclaim";
1442
+ const claimResult = await this.claimSlotInternal({
1443
+ tag,
1444
+ requester: params.requester || tag,
1445
+ linearIssue: params.linearIssue,
1446
+ label: params.label,
1447
+ workloadType: params.workloadType,
1448
+ targetVersion: params.targetVersion,
1449
+ branch: params.branch,
1450
+ host: targetPurgedHost,
1451
+ durationMinutes: params.durationMinutes,
1452
+ allowEsxiSlots: allowEsxi,
1453
+ purpose: `Auto-reclaimed Proxmox slot via find_slot to avoid ESXi fallback`
1454
+ });
1455
+ return {
1456
+ status: "auto_reclaimed_and_claimed",
1457
+ candidateHost: allCandidates.find((c) => c.hostId === targetPurgedHost) || allCandidates[0],
1458
+ allCandidates,
1459
+ availableSlotsTotal,
1460
+ claim: claimResult,
1461
+ message: `Proxmox hosts were at capacity, but auto-purged expired lease(s) on '${targetPurgedHost}' to keep ESXi fixture host free. Successfully claimed '${claimResult.slotId}' on '${targetPurgedHost}'.`
1462
+ };
1463
+ }
1464
+ }
1465
+ let claimResult;
1466
+ if (params.autoClaim) {
1467
+ const tag = params.tag || params.linearIssue || params.requester || "find-slot-claim";
1468
+ claimResult = await this.claimSlotInternal({
1469
+ tag,
1470
+ requester: params.requester || tag,
1471
+ linearIssue: params.linearIssue,
1472
+ label: params.label,
1473
+ workloadType: params.workloadType,
1474
+ targetVersion: params.targetVersion,
1475
+ branch: params.branch,
1476
+ host: winningCandidate.hostId,
1477
+ durationMinutes: params.durationMinutes,
1478
+ allowEsxiSlots: allowEsxi,
1479
+ purpose: `Auto-claimed via find_slot on ${winningCandidate.hostId}`
1480
+ });
1481
+ return {
1482
+ status: "auto_claimed",
1483
+ candidateHost: winningCandidate,
1484
+ allCandidates,
1485
+ availableSlotsTotal: availableSlotsTotal - 1,
1486
+ claim: claimResult,
1487
+ message: `Found available slot and successfully auto-claimed '${claimResult.slotId}' on host '${winningCandidate.hostId}'.`
1488
+ };
1167
1489
  }
1490
+ let msg = `Found ${availableSlotsTotal} available 8GB slot(s) across ${allCandidates.length} eligible host(s). Recommended placement: '${winningCandidate.hostId}' (${winningCandidate.availableSlots} slots available).`;
1491
+ if (winningCandidate.hypervisorType === "esxi" && proxmoxReclaimableSlots.length > 0) {
1492
+ msg += ` (Note: Proxmox Mini has ${proxmoxReclaimableSlots.length} expired lease(s) that can be reclaimed with purgeIfFull=true to avoid placing on ESXi).`;
1493
+ }
1494
+ return {
1495
+ status: "available",
1496
+ candidateHost: winningCandidate,
1497
+ allCandidates,
1498
+ availableSlotsTotal,
1499
+ message: msg
1500
+ };
1168
1501
  }
1169
- // Check orphan VMs if requested
1170
- if (includeOrphans && this.hypervisorManager) {
1502
+ // If capacity is NOT available (availableSlotsTotal === 0)
1503
+ // Inspect reclaimable resources (expired leases + untracked disposable VMs)
1504
+ const expiredSlots = Array.from(this.slots.values())
1505
+ .filter((s) => s.status === "EXPIRED" || (s.expiresAt && new Date(s.expiresAt).getTime() <= Date.now()))
1506
+ .map((s) => ({
1507
+ slotId: s.slotId,
1508
+ hostId: s.hostId,
1509
+ claimId: s.claimId,
1510
+ claimedAt: s.claimedAt,
1511
+ expiresAt: s.expiresAt,
1512
+ linearIssue: s.linearIssue,
1513
+ label: s.label,
1514
+ workloadType: s.workloadType,
1515
+ vmId: s.vmId,
1516
+ vmName: s.vmName,
1517
+ reason: "expired"
1518
+ }));
1519
+ const disposableVms = [];
1520
+ if (this.hypervisorManager) {
1171
1521
  try {
1172
- const allVms = await this.hypervisorManager.listUnifiedInventory({
1522
+ const inv = await this.hypervisorManager.listUnifiedInventory({
1173
1523
  includeTemplates: false,
1174
1524
  host: targetHost,
1175
- hypervisor: targetHypervisor
1525
+ hypervisor: targetHyp
1176
1526
  });
1177
1527
  const activeVmIds = new Set(Array.from(this.slots.values())
1178
- .map((s) => s.vmId)
1179
- .filter((id) => id !== null && id !== undefined)
1180
- .map((id) => String(id)));
1181
- for (const vm of allVms) {
1182
- // Check if VM is an untracked disposable VM without active lease
1528
+ .filter((s) => s.status === "CLAIMED" && s.vmId !== null && s.vmId !== undefined)
1529
+ .map((s) => String(s.vmId)));
1530
+ for (const vm of inv) {
1183
1531
  if (!vm.isTemplate && !vm.isProtected && !activeVmIds.has(String(vm.id))) {
1184
- const protCheck = this.guardrailEngine.isProtected({
1532
+ const prot = this.guardrailEngine.isProtected({
1185
1533
  vmid: vm.id,
1186
1534
  name: vm.name,
1187
1535
  isProtected: vm.isProtected,
1188
1536
  tags: vm.tags
1189
1537
  });
1190
- if (protCheck.protected) {
1191
- skippedProtectedCount++;
1192
- destroyedVms.push({
1538
+ if (!prot.protected) {
1539
+ disposableVms.push({
1193
1540
  vmId: vm.id,
1194
- vmName: vm.name,
1195
- hostId: vm.hostName,
1196
- hypervisorType: vm.hypervisor,
1197
- status: "skipped_protected",
1198
- reason: `Protected orphan preserved: ${protCheck.reason}`
1541
+ name: vm.name,
1542
+ hostName: vm.hostName,
1543
+ hypervisor: vm.hypervisor,
1544
+ status: vm.status,
1545
+ memoryMb: Math.round(vm.memoryBytes / (1024 * 1024)),
1546
+ linearIssue: vm.linearIssue
1199
1547
  });
1200
1548
  }
1201
- else {
1202
- if (dryRun) {
1203
- destroyedVms.push({
1204
- vmId: vm.id,
1205
- vmName: vm.name,
1206
- hostId: vm.hostName,
1207
- hypervisorType: vm.hypervisor,
1208
- status: "dry_run",
1209
- reason: "Orphaned disposable VM with no active slot lease"
1210
- });
1211
- }
1212
- else if (destroyVms) {
1213
- try {
1214
- await this.hypervisorManager.vmAction({
1215
- vmIdOrName: String(vm.id),
1216
- targetHost: vm.hostName,
1217
- action: "destroy",
1218
- force: true
1219
- });
1220
- destroyedVms.push({
1221
- vmId: vm.id,
1222
- vmName: vm.name,
1223
- hostId: vm.hostName,
1224
- hypervisorType: vm.hypervisor,
1225
- status: "destroyed",
1226
- reason: "Destroyed orphaned disposable VM with no active slot lease"
1227
- });
1228
- }
1229
- catch (orphanErr) {
1230
- destroyedVms.push({
1231
- vmId: vm.id,
1232
- vmName: vm.name,
1233
- hostId: vm.hostName,
1234
- hypervisorType: vm.hypervisor,
1235
- status: "failed",
1236
- error: orphanErr instanceof Error ? orphanErr.message : String(orphanErr)
1237
- });
1238
- }
1549
+ }
1550
+ }
1551
+ }
1552
+ catch {
1553
+ // ignore inventory errors
1554
+ }
1555
+ }
1556
+ const reclaimableSlotsEst = expiredSlots.length + Math.floor(disposableVms.reduce((acc, v) => acc + v.memoryMb, 0) / 8192);
1557
+ const reclaimableMemoryMb = expiredSlots.length * 8192 + disposableVms.reduce((acc, v) => acc + v.memoryMb, 0);
1558
+ const reclaimable = {
1559
+ expiredSlotsCount: expiredSlots.length,
1560
+ expiredSlots,
1561
+ disposableVmsCount: disposableVms.length,
1562
+ disposableVms,
1563
+ reclaimableMemoryMb,
1564
+ reclaimableSlotsEst
1565
+ };
1566
+ // If purgeIfFull is requested and we have reclaimable resources, auto-purge and claim!
1567
+ if (params.purgeIfFull && (expiredSlots.length > 0 || disposableVms.length > 0)) {
1568
+ const purgeRes = await this.purgeSlotsInternal({
1569
+ host: targetHost,
1570
+ hypervisorType: targetHyp,
1571
+ destroyVms: true,
1572
+ includeOrphans: true,
1573
+ dryRun: false
1574
+ });
1575
+ if (purgeRes.purgedSlotsCount > 0 || purgeRes.destroyedVmsCount > 0) {
1576
+ const tag = params.tag || params.linearIssue || params.requester || "find-slot-reclaim";
1577
+ const claimResult = await this.claimSlotInternal({
1578
+ tag,
1579
+ requester: params.requester || tag,
1580
+ linearIssue: params.linearIssue,
1581
+ label: params.label,
1582
+ workloadType: params.workloadType,
1583
+ targetVersion: params.targetVersion,
1584
+ branch: params.branch,
1585
+ host: targetHost || "auto",
1586
+ durationMinutes: params.durationMinutes,
1587
+ allowEsxiSlots: allowEsxi,
1588
+ purpose: `Auto-reclaimed and claimed via find_slot`
1589
+ });
1590
+ return {
1591
+ status: "auto_reclaimed_and_claimed",
1592
+ candidateHost: allCandidates.find((c) => c.hostId === claimResult.hostId) || allCandidates[0],
1593
+ allCandidates,
1594
+ availableSlotsTotal: 0,
1595
+ claim: claimResult,
1596
+ reclaimable,
1597
+ message: `Purged ${purgeRes.purgedSlotsCount} expired slot(s) and ${purgeRes.destroyedVmsCount} disposable VM(s), then successfully claimed '${claimResult.slotId}' on host '${claimResult.hostId}'.`
1598
+ };
1599
+ }
1600
+ }
1601
+ return {
1602
+ status: "full",
1603
+ allCandidates,
1604
+ availableSlotsTotal: 0,
1605
+ reclaimable,
1606
+ message: `No slots currently available across ${allCandidates.length} host(s). Found ${expiredSlots.length} expired lease(s) and ${disposableVms.length} disposable VM(s) (~${Math.round(reclaimableMemoryMb / 1024)}GB reclaimable). Use purge_slots or call find_slot with purgeIfFull=true to reclaim capacity.`
1607
+ };
1608
+ });
1609
+ }
1610
+ /**
1611
+ * One-shot purge of old/expired disposable test VM and replacement with fresh slot allocation.
1612
+ */
1613
+ async purgeAndRedeploy(params) {
1614
+ return this.mutex.runExclusive(async () => {
1615
+ if (!this.hypervisorManager) {
1616
+ throw new InvalidSlotOperationError("HypervisorManager is required for purge_and_redeploy operations.");
1617
+ }
1618
+ const linearIssue = params.linearIssue;
1619
+ const targetVmIdOrName = params.targetVmIdOrName;
1620
+ if (!linearIssue && !targetVmIdOrName) {
1621
+ throw new InvalidSlotOperationError("Either 'linearIssue' or 'targetVmIdOrName' is required for purge_and_redeploy.");
1622
+ }
1623
+ // 1. Locate target VM
1624
+ let targetVmidStr = targetVmIdOrName;
1625
+ const matchingSlot = Array.from(this.slots.values()).find((s) => (targetVmIdOrName && (String(s.vmId) === targetVmIdOrName || s.vmName === targetVmIdOrName)) ||
1626
+ (linearIssue && s.linearIssue === linearIssue));
1627
+ if (!targetVmidStr && matchingSlot && matchingSlot.vmId != null) {
1628
+ targetVmidStr = String(matchingSlot.vmId);
1629
+ }
1630
+ const inventory = await this.hypervisorManager.listUnifiedInventory({
1631
+ includeTemplates: false,
1632
+ host: params.host
1633
+ });
1634
+ const targetVm = inventory.find((vm) => {
1635
+ if (targetVmidStr && (String(vm.id) === targetVmidStr || vm.name === targetVmidStr)) {
1636
+ return true;
1637
+ }
1638
+ if (linearIssue && (vm.linearIssue === linearIssue || vm.name.toLowerCase().includes(linearIssue.toLowerCase()))) {
1639
+ return true;
1640
+ }
1641
+ return false;
1642
+ });
1643
+ if (!targetVm) {
1644
+ throw new SlotNotFoundError(`No existing VM found matching '${targetVmIdOrName || linearIssue}'.`);
1645
+ }
1646
+ // 2. Safety guardrail check
1647
+ const protCheck = this.guardrailEngine.isProtected({
1648
+ vmid: targetVm.id,
1649
+ name: targetVm.name,
1650
+ isProtected: targetVm.isProtected,
1651
+ tags: targetVm.tags
1652
+ });
1653
+ if (protCheck.protected) {
1654
+ throw new InvalidSlotOperationError(`Refusing to destroy protected VM '${targetVm.name}' (${targetVm.id}): ${protCheck.reason}`);
1655
+ }
1656
+ if (params.dryRun) {
1657
+ return {
1658
+ success: true,
1659
+ dryRun: true,
1660
+ purgedVm: {
1661
+ vmId: targetVm.id,
1662
+ name: targetVm.name,
1663
+ host: targetVm.hostName,
1664
+ hypervisor: targetVm.hypervisor
1665
+ },
1666
+ message: `[Dry-run] Would purge VM '${targetVm.name}' (${targetVm.id}) on '${targetVm.hostName}' and claim fresh slot for redeployment.`
1667
+ };
1668
+ }
1669
+ // 3. Find associated slot if any
1670
+ const slotToRelease = matchingSlot || Array.from(this.slots.values()).find((s) => (s.vmId !== null && String(s.vmId) === String(targetVm.id)) || (linearIssue && s.linearIssue === linearIssue));
1671
+ // 4. Pre-destruction snapshot (if requested)
1672
+ if (params.snapshotBeforeDestroy) {
1673
+ try {
1674
+ const drivers = this.hypervisorManager.getAllDrivers();
1675
+ const driver = drivers.find((d) => d.hostName === targetVm.hostName);
1676
+ if (driver && driver.createSnapshot) {
1677
+ await driver.createSnapshot(targetVm.id, `pre-redeploy-${Date.now()}`, "Pre-purge snapshot before redeploy", false);
1678
+ }
1679
+ }
1680
+ catch (snapErr) {
1681
+ logger.warn(`Failed to create pre-redeploy snapshot for VM ${targetVm.name}:`, snapErr);
1682
+ }
1683
+ }
1684
+ // 5. Destroy old VM
1685
+ await this.hypervisorManager.vmAction({
1686
+ vmIdOrName: String(targetVm.id),
1687
+ targetHost: targetVm.hostName,
1688
+ action: "destroy",
1689
+ force: true
1690
+ });
1691
+ // 6. Release existing slot if present
1692
+ if (slotToRelease) {
1693
+ this.resetSlot(slotToRelease);
1694
+ }
1695
+ // 7. Claim new slot on the same host (or auto)
1696
+ const targetHost = params.host || targetVm.hostName;
1697
+ const tag = linearIssue || targetVm.name;
1698
+ const newSlot = await this.claimSlotInternal({
1699
+ tag,
1700
+ requester: linearIssue || "purge-and-redeploy",
1701
+ linearIssue,
1702
+ label: `Redeployed ${linearIssue || targetVm.name}`,
1703
+ workloadType: params.workloadType || "uat",
1704
+ targetVersion: params.targetVersion || params.version,
1705
+ branch: params.branch,
1706
+ host: targetHost,
1707
+ durationMinutes: params.durationMinutes || 120,
1708
+ purpose: `Purged old VM ${targetVm.name} and claimed capacity for fresh deployment`
1709
+ });
1710
+ return {
1711
+ success: true,
1712
+ purgedVm: {
1713
+ vmId: targetVm.id,
1714
+ name: targetVm.name,
1715
+ host: targetVm.hostName,
1716
+ hypervisor: targetVm.hypervisor
1717
+ },
1718
+ reclaimedSlotId: matchingSlot?.slotId,
1719
+ newSlot,
1720
+ message: `Successfully purged old VM '${targetVm.name}' (${targetVm.id}) and claimed fresh slot '${newSlot.slotId}' on host '${newSlot.hostId}'.`
1721
+ };
1722
+ });
1723
+ }
1724
+ /**
1725
+ * Safe Prune of Abandoned / Untracked Disposable Test VMs across Hypervisors.
1726
+ * Scans inventory for stopped disposable VMs not tied to any active slot lease,
1727
+ * enforces strict non-bypassable guardrails, and frees hypervisor datastore and memory.
1728
+ */
1729
+ async pruneUntrackedVms(params = {}) {
1730
+ return this.mutex.runExclusive(async () => {
1731
+ if (!this.hypervisorManager) {
1732
+ throw new InvalidSlotOperationError("HypervisorManager is required to prune untracked VMs.");
1733
+ }
1734
+ this.sweepExpiredSlotsInternal();
1735
+ const dryRun = params.dryRun ?? true;
1736
+ const targetHost = params.host && params.host !== "all" && params.host !== "auto"
1737
+ ? this.normalizeHostId(params.host)
1738
+ : undefined;
1739
+ const linearFilter = params.linearIssue;
1740
+ const inventory = await this.hypervisorManager.listUnifiedInventory({
1741
+ includeTemplates: false,
1742
+ host: targetHost
1743
+ });
1744
+ // Active slot VM IDs that must NEVER be touched
1745
+ const activeSlotVmIds = new Set(Array.from(this.slots.values())
1746
+ .filter((s) => s.status === "CLAIMED" && s.vmId !== null && s.vmId !== undefined)
1747
+ .map((s) => String(s.vmId)));
1748
+ const prunedVms = [];
1749
+ let skippedProtectedCount = 0;
1750
+ let reclaimedMemoryMb = 0;
1751
+ for (const vm of inventory) {
1752
+ if (vm.isTemplate)
1753
+ continue;
1754
+ if (activeSlotVmIds.has(String(vm.id)))
1755
+ continue;
1756
+ // Check if VM matches disposable patterns or specific linear filter
1757
+ const nameLower = vm.name.toLowerCase();
1758
+ const isDisposablePattern = nameLower.startsWith("ct-") ||
1759
+ nameLower.includes("uat") ||
1760
+ nameLower.includes("qa-") ||
1761
+ nameLower.includes("demo") ||
1762
+ nameLower.includes("test") ||
1763
+ Boolean(vm.linearIssue);
1764
+ if (linearFilter) {
1765
+ const matchesLinear = vm.linearIssue === linearFilter || nameLower.includes(linearFilter.toLowerCase());
1766
+ if (!matchesLinear)
1767
+ continue;
1768
+ }
1769
+ else if (!isDisposablePattern && !params.force) {
1770
+ continue;
1771
+ }
1772
+ // Strict Safety Guardrail Check
1773
+ const protCheck = this.guardrailEngine.isProtected({
1774
+ vmid: vm.id,
1775
+ name: vm.name,
1776
+ isProtected: vm.isProtected,
1777
+ tags: vm.tags
1778
+ });
1779
+ const memMb = Math.round((vm.memoryBytes || 0) / (1024 * 1024));
1780
+ if (protCheck.protected) {
1781
+ skippedProtectedCount++;
1782
+ prunedVms.push({
1783
+ vmId: vm.id,
1784
+ name: vm.name,
1785
+ hostName: vm.hostName,
1786
+ hypervisor: vm.hypervisor,
1787
+ status: vm.status,
1788
+ memoryMb: memMb,
1789
+ linearIssue: vm.linearIssue,
1790
+ statusAction: "skipped_protected",
1791
+ reason: `Protected infrastructure preserved: ${protCheck.reason}`
1792
+ });
1793
+ continue;
1794
+ }
1795
+ if (dryRun) {
1796
+ reclaimedMemoryMb += memMb;
1797
+ prunedVms.push({
1798
+ vmId: vm.id,
1799
+ name: vm.name,
1800
+ hostName: vm.hostName,
1801
+ hypervisor: vm.hypervisor,
1802
+ status: vm.status,
1803
+ memoryMb: memMb,
1804
+ linearIssue: vm.linearIssue,
1805
+ statusAction: "dry_run",
1806
+ reason: "Would be destroyed (untracked disposable test VM)"
1807
+ });
1808
+ }
1809
+ else {
1810
+ try {
1811
+ if (params.snapshotBeforeDestroy) {
1812
+ try {
1813
+ const drivers = this.hypervisorManager.getAllDrivers();
1814
+ const driver = drivers.find((d) => d.hostName === vm.hostName);
1815
+ if (driver && driver.createSnapshot) {
1816
+ await driver.createSnapshot(vm.id, `pre-prune-${Date.now()}`, "Pre-prune snapshot", false);
1239
1817
  }
1240
1818
  }
1819
+ catch (snapErr) {
1820
+ logger.warn(`Failed to create pre-prune snapshot for VM ${vm.name}:`, snapErr);
1821
+ }
1241
1822
  }
1823
+ await this.hypervisorManager.vmAction({
1824
+ vmIdOrName: String(vm.id),
1825
+ targetHost: vm.hostName,
1826
+ action: "destroy",
1827
+ force: true
1828
+ });
1829
+ reclaimedMemoryMb += memMb;
1830
+ prunedVms.push({
1831
+ vmId: vm.id,
1832
+ name: vm.name,
1833
+ hostName: vm.hostName,
1834
+ hypervisor: vm.hypervisor,
1835
+ status: vm.status,
1836
+ memoryMb: memMb,
1837
+ linearIssue: vm.linearIssue,
1838
+ statusAction: "pruned",
1839
+ reason: "Destroyed untracked disposable test VM"
1840
+ });
1841
+ this.telemetry.emit({
1842
+ event: "vm.purged",
1843
+ actor: "system",
1844
+ status: "success",
1845
+ vmId: vm.id,
1846
+ vmName: vm.name,
1847
+ hostId: vm.hostName,
1848
+ hypervisorType: vm.hypervisor,
1849
+ metadata: { action: "prune_untracked_vms", linearIssue: vm.linearIssue }
1850
+ });
1851
+ }
1852
+ catch (err) {
1853
+ prunedVms.push({
1854
+ vmId: vm.id,
1855
+ name: vm.name,
1856
+ hostName: vm.hostName,
1857
+ hypervisor: vm.hypervisor,
1858
+ status: vm.status,
1859
+ memoryMb: memMb,
1860
+ linearIssue: vm.linearIssue,
1861
+ statusAction: "failed",
1862
+ error: err instanceof Error ? err.message : String(err)
1863
+ });
1242
1864
  }
1243
- }
1244
- catch (invErr) {
1245
- console.error("[purgeSlots] Orphan inventory inspection error:", invErr);
1246
1865
  }
1247
1866
  }
1248
- const destroyedCount = destroyedVms.filter((v) => v.status === "destroyed").length;
1249
- const reclaimedMemoryBytes = reclaimedMemoryMb * 1024 * 1024;
1250
- const prefix = dryRun ? "[DRY RUN] " : "";
1251
- const message = `${prefix}Purged ${purgedSlots.length} slot lease(s), reclaimed ${reclaimedMemoryMb / 1024} GB RAM, processed ${destroyedVms.length} VM(s) (${destroyedCount} destroyed, ${skippedProtectedCount} protected skipped).`;
1867
+ const eligibleCount = prunedVms.filter((v) => v.statusAction !== "skipped_protected").length;
1868
+ const prunedCount = prunedVms.filter((v) => v.statusAction === "pruned").length;
1252
1869
  return {
1253
- purgedSlotsCount: purgedSlots.length,
1254
- purgedSlots,
1255
- destroyedVmsCount: destroyedCount,
1256
- destroyedVms,
1257
- skippedProtectedVmsCount: skippedProtectedCount,
1258
- reclaimedMemoryBytes,
1259
- reclaimedMemoryMb,
1260
1870
  dryRun,
1261
- timestamp: new Date(now).toISOString(),
1262
- message
1871
+ scannedVmsCount: inventory.length,
1872
+ eligibleVmsCount: eligibleCount,
1873
+ prunedVmsCount: dryRun ? eligibleCount : prunedCount,
1874
+ skippedProtectedCount,
1875
+ reclaimedMemoryMb,
1876
+ prunedVms,
1877
+ message: dryRun
1878
+ ? `[Dry-run] Identified ${eligibleCount} untracked disposable VM(s) (~${Math.round(reclaimedMemoryMb / 1024)} GB RAM) eligible for pruning (${skippedProtectedCount} protected VMs preserved).`
1879
+ : `Successfully pruned ${prunedCount} untracked disposable VM(s), reclaiming ~${Math.round(reclaimedMemoryMb / 1024)} GB RAM (${skippedProtectedCount} protected VMs preserved).`
1263
1880
  };
1264
1881
  });
1265
1882
  }
1883
+ /**
1884
+ * Heartbeat-driven lease auto-extension for active sessions.
1885
+ * If lease is active and expiring within 45 mins, extends by extendMinutes (default 30m) up to 8h cap.
1886
+ */
1887
+ async touchSlotLease(slotIdOrClaimOrLinear, extendMinutes = 30) {
1888
+ return this.mutex.runExclusive(async () => {
1889
+ this.sweepExpiredSlotsInternal();
1890
+ const slot = Array.from(this.slots.values()).find((s) => s.slotId === slotIdOrClaimOrLinear ||
1891
+ s.claimId === slotIdOrClaimOrLinear ||
1892
+ s.linearIssue === slotIdOrClaimOrLinear ||
1893
+ (s.vmId !== null && String(s.vmId) === slotIdOrClaimOrLinear));
1894
+ if (!slot || (slot.status !== "CLAIMED" && slot.status !== "active") || !slot.expiresAt) {
1895
+ return null;
1896
+ }
1897
+ const now = Date.now();
1898
+ const currentExpiry = new Date(slot.expiresAt).getTime();
1899
+ const remainingMs = currentExpiry - now;
1900
+ // Only extend if within 45 minutes of expiry
1901
+ if (remainingMs <= 45 * 60 * 1000) {
1902
+ const claimedAtMs = slot.claimedAt ? new Date(slot.claimedAt).getTime() : now;
1903
+ const currentTotalMinutes = Math.round((currentExpiry - claimedAtMs) / (60 * 1000));
1904
+ const maxTotalMinutes = 480; // 8 hours max lease cap
1905
+ if (currentTotalMinutes < maxTotalMinutes) {
1906
+ const extensionMs = Math.min(extendMinutes, maxTotalMinutes - currentTotalMinutes) * 60 * 1000;
1907
+ const newExpiresAt = new Date(currentExpiry + extensionMs).toISOString();
1908
+ slot.expiresAt = newExpiresAt;
1909
+ slot.durationMinutes = (slot.durationMinutes || 120) + Math.round(extensionMs / 60000);
1910
+ slot.ttlMinutes = Math.round((new Date(newExpiresAt).getTime() - now) / 60000);
1911
+ slot.ttlSeconds = Math.round((new Date(newExpiresAt).getTime() - now) / 1000);
1912
+ }
1913
+ }
1914
+ return { ...slot };
1915
+ });
1916
+ }
1266
1917
  /**
1267
1918
  * Attach a created VMID or VM name to a claimed slot.
1268
1919
  */