@calltelemetry/ct-lab-mcp 0.8.2 → 0.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/src/slot/manager.d.ts +16 -1
  2. package/dist/src/slot/manager.d.ts.map +1 -1
  3. package/dist/src/slot/manager.js +810 -431
  4. package/dist/src/slot/manager.js.map +1 -1
  5. package/dist/src/slot/types.d.ts +87 -0
  6. package/dist/src/slot/types.d.ts.map +1 -1
  7. package/dist/src/slot/types.js.map +1 -1
  8. package/dist/src/tools/appliance-exec.d.ts +4 -4
  9. package/dist/src/tools/find-slot.d.ts +80 -0
  10. package/dist/src/tools/find-slot.d.ts.map +1 -0
  11. package/dist/src/tools/find-slot.js +134 -0
  12. package/dist/src/tools/find-slot.js.map +1 -0
  13. package/dist/src/tools/fleet-tools.d.ts +6 -6
  14. package/dist/src/tools/index.d.ts.map +1 -1
  15. package/dist/src/tools/index.js +11 -1
  16. package/dist/src/tools/index.js.map +1 -1
  17. package/dist/src/tools/list-hosts.d.ts +2 -2
  18. package/dist/src/tools/list-inventory.d.ts +2 -2
  19. package/dist/src/tools/provision-tools.d.ts +16 -8
  20. package/dist/src/tools/provision-tools.d.ts.map +1 -1
  21. package/dist/src/tools/provision-tools.js +11 -0
  22. package/dist/src/tools/provision-tools.js.map +1 -1
  23. package/dist/src/tools/purge-and-redeploy.d.ts +64 -0
  24. package/dist/src/tools/purge-and-redeploy.d.ts.map +1 -0
  25. package/dist/src/tools/purge-and-redeploy.js +115 -0
  26. package/dist/src/tools/purge-and-redeploy.js.map +1 -0
  27. package/dist/src/tools/purge-slots.d.ts +6 -6
  28. package/dist/src/tools/slot-tools.d.ts +30 -18
  29. package/dist/src/tools/slot-tools.d.ts.map +1 -1
  30. package/dist/src/tools/slot-tools.js +11 -0
  31. package/dist/src/tools/slot-tools.js.map +1 -1
  32. package/dist/src/tools/vm-action.d.ts +8 -8
  33. package/dist/src/tools/vm-cleanup.d.ts +4 -4
  34. package/dist/src/utils/formatters.d.ts +8 -0
  35. package/dist/src/utils/formatters.d.ts.map +1 -1
  36. package/dist/src/utils/formatters.js +113 -0
  37. package/dist/src/utils/formatters.js.map +1 -1
  38. package/package.json +1 -1
@@ -16,7 +16,12 @@ export * from "./types.js";
16
16
  */
17
17
  export function calculatePureCapacity(host) {
18
18
  if (host.maxSlots !== undefined) {
19
- return host.maxSlots;
19
+ if (host.hypervisorType === "esxi" && host.maxSlots === 0 && host.allowEsxiSlots) {
20
+ // Allow dynamic calculation below for ESXi when unlocked
21
+ }
22
+ else {
23
+ return host.maxSlots;
24
+ }
20
25
  }
21
26
  const slotSize = host.slotSizeMb || 8192;
22
27
  if (host.totalMemoryMb !== undefined) {
@@ -112,12 +117,14 @@ export class SlotManager {
112
117
  hypervisorManager;
113
118
  guardrailEngine;
114
119
  telemetry;
120
+ allowEsxiSlots;
115
121
  constructor(options = {}) {
116
122
  this.defaultDurationMinutes = options.defaultDurationMinutes ?? 120;
117
123
  this.proxmoxManager = options.proxmoxManager;
118
124
  this.hypervisorManager = options.hypervisorManager;
119
125
  this.guardrailEngine = options.guardrailEngine || defaultSafetyGuardrailEngine;
120
126
  this.telemetry = options.telemetryRegistry || telemetry;
127
+ this.allowEsxiSlots = options.allowEsxiSlots ?? (process.env.ALLOW_ESXI_SLOTS === "true");
121
128
  const hostConfigs = options.hosts && options.hosts.length > 0
122
129
  ? options.hosts
123
130
  : [...DEFAULT_HOST_CONFIGS];
@@ -148,7 +155,14 @@ export class SlotManager {
148
155
  */
149
156
  registerHost(host) {
150
157
  const normalizedId = this.normalizeHostId(host.hostId);
151
- const pureSlots = calculatePureCapacity(host);
158
+ let pureSlots = calculatePureCapacity({
159
+ ...host,
160
+ allowEsxiSlots: this.allowEsxiSlots
161
+ });
162
+ if (normalizedId.startsWith("esxi") && this.allowEsxiSlots && host.maxSlots === 0) {
163
+ const usableMb = Math.max(0, host.totalMemoryMb - (host.reservedHeadroomMb || 0));
164
+ pureSlots = Math.floor(usableMb / (host.slotSizeMb || 8192));
165
+ }
152
166
  const normalizedHost = {
153
167
  ...host,
154
168
  hostId: normalizedId,
@@ -282,20 +296,24 @@ export class SlotManager {
282
296
  * Atomically claims an 8GB appliance slot.
283
297
  */
284
298
  async claimSlot(request) {
285
- return this.mutex.runExclusive(async () => {
286
- // 1. Reclaim any expired leases first
287
- this.sweepExpiredSlotsInternal();
288
- const tagInput = request.tag || request.linearIssue || request.requester;
289
- if (!tagInput || typeof tagInput !== "string" || tagInput.trim() === "") {
290
- throw new InvalidSlotOperationError("A valid non-empty 'tag', 'linearIssue', or 'requester' is required to claim a slot.");
291
- }
292
- const durationMinutes = request.durationMinutes ?? request.ttlMinutes ?? (request.ttlSeconds ? Math.ceil(request.ttlSeconds / 60) : this.defaultDurationMinutes);
293
- if (durationMinutes < 1 || durationMinutes > 1440) {
294
- throw new InvalidSlotOperationError(`Claim duration must be between 1 and 1440 minutes (requested: ${durationMinutes}).`);
295
- }
296
- const rawRequestedHost = request.host || request.targetHost;
297
- const requestedHost = rawRequestedHost ? this.normalizeHostId(rawRequestedHost) : undefined;
298
- const hypervisorTypePref = request.hypervisorType || "any";
299
+ return this.mutex.runExclusive(async () => this.claimSlotInternal(request));
300
+ }
301
+ async claimSlotInternal(request) {
302
+ // 1. Reclaim any expired leases first
303
+ this.sweepExpiredSlotsInternal();
304
+ const tagInput = request.tag || request.linearIssue || request.requester;
305
+ if (!tagInput || typeof tagInput !== "string" || tagInput.trim() === "") {
306
+ throw new InvalidSlotOperationError("A valid non-empty 'tag', 'linearIssue', or 'requester' is required to claim a slot.");
307
+ }
308
+ const durationMinutes = request.durationMinutes ?? request.ttlMinutes ?? (request.ttlSeconds ? Math.ceil(request.ttlSeconds / 60) : this.defaultDurationMinutes);
309
+ if (durationMinutes < 1 || durationMinutes > 1440) {
310
+ throw new InvalidSlotOperationError(`Claim duration must be between 1 and 1440 minutes (requested: ${durationMinutes}).`);
311
+ }
312
+ const rawRequestedHost = request.host || request.targetHost;
313
+ const requestedHost = rawRequestedHost ? this.normalizeHostId(rawRequestedHost) : undefined;
314
+ const hypervisorTypePref = request.hypervisorType || "any";
315
+ const allowEsxi = request.allowEsxiSlots ?? this.allowEsxiSlots;
316
+ if (!allowEsxi) {
299
317
  // 1. Explicit ESXi Host Rejection
300
318
  if (requestedHost &&
301
319
  (requestedHost === "esxi-intel-192-168-123-176" ||
@@ -307,213 +325,246 @@ export class SlotManager {
307
325
  if (hypervisorTypePref === "esxi") {
308
326
  throw new HypervisorSlotUnsupportedError("Slot leasing is not supported on ESXi hypervisors. Appliance slots are exclusively managed on Proxmox VE (proxmox-lab and proxmox-mini).", requestedHost, "esxi");
309
327
  }
310
- let targetSlot;
311
- if (requestedHost && requestedHost !== "auto") {
312
- // Specific host claim
313
- const hostConfig = this.hosts.get(requestedHost);
314
- if (!hostConfig) {
315
- const validHosts = Array.from(this.hosts.keys()).join(", ");
316
- throw new InvalidSlotOperationError(`Unknown host '${requestedHost}' for slot allocation. Available: [${validHosts}]`);
328
+ }
329
+ let targetSlot;
330
+ if (requestedHost && requestedHost !== "auto") {
331
+ // Specific host claim
332
+ const hostConfig = this.hosts.get(requestedHost);
333
+ if (!hostConfig) {
334
+ const validHosts = Array.from(this.hosts.keys()).join(", ");
335
+ throw new InvalidSlotOperationError(`Unknown host '${requestedHost}' for slot allocation. Available: [${validHosts}]`);
336
+ }
337
+ // ESXi Datastore check (>= 10GB free required)
338
+ if (hostConfig.hypervisorType === "esxi") {
339
+ const freeDisk = hostConfig.freeDiskGb ?? 100;
340
+ if (freeDisk < 10) {
341
+ throw new DatastoreCapacityExceededError(requestedHost, 10, freeDisk);
317
342
  }
318
- // ESXi Datastore check (>= 10GB free required)
319
- if (hostConfig.hypervisorType === "esxi") {
320
- const freeDisk = hostConfig.freeDiskGb ?? 100;
321
- if (freeDisk < 10) {
322
- throw new DatastoreCapacityExceededError(requestedHost, 10, freeDisk);
323
- }
343
+ }
344
+ const hostSlots = Array.from(this.slots.values())
345
+ .filter((s) => s.hostId === requestedHost)
346
+ .sort((a, b) => a.slotIndex - b.slotIndex);
347
+ let availableSlot = hostSlots.find((s) => s.status === "AVAILABLE");
348
+ // Dynamic slot expansion when box pure capacity exceeds pre-allocated slots
349
+ if (!availableSlot) {
350
+ const pureSlots = calculatePureCapacity(hostConfig);
351
+ if (hostSlots.length < pureSlots) {
352
+ const nextIndex = hostSlots.length + 1;
353
+ const slotId = `slot-${requestedHost}-${nextIndex}`;
354
+ availableSlot = {
355
+ slotId,
356
+ hostId: requestedHost,
357
+ hostName: hostConfig.name,
358
+ hypervisorType: hostConfig.hypervisorType,
359
+ slotIndex: nextIndex,
360
+ memoryMb: hostConfig.slotSizeMb || 8192,
361
+ allocatedMemoryMb: hostConfig.slotSizeMb || 8192,
362
+ allocatedMemoryBytes: (hostConfig.slotSizeMb || 8192) * 1024 * 1024,
363
+ status: "AVAILABLE",
364
+ claimId: null,
365
+ claimedAt: null,
366
+ expiresAt: null,
367
+ durationMinutes: null,
368
+ ttlMinutes: null,
369
+ ttlSeconds: null,
370
+ tag: null,
371
+ label: null,
372
+ workloadType: null,
373
+ targetVersion: null,
374
+ branch: null,
375
+ customTags: [],
376
+ requester: null,
377
+ linearIssue: null,
378
+ sessionId: null,
379
+ purpose: null,
380
+ vmId: null,
381
+ vmName: null,
382
+ metadata: {}
383
+ };
384
+ this.slots.set(slotId, availableSlot);
385
+ hostSlots.push(availableSlot);
324
386
  }
325
- const hostSlots = Array.from(this.slots.values())
326
- .filter((s) => s.hostId === requestedHost)
327
- .sort((a, b) => a.slotIndex - b.slotIndex);
328
- let availableSlot = hostSlots.find((s) => s.status === "AVAILABLE");
329
- // Dynamic slot expansion when box pure capacity exceeds pre-allocated slots
330
- if (!availableSlot) {
331
- const pureSlots = calculatePureCapacity(hostConfig);
332
- if (hostSlots.length < pureSlots) {
333
- const nextIndex = hostSlots.length + 1;
334
- const slotId = `slot-${requestedHost}-${nextIndex}`;
335
- availableSlot = {
336
- slotId,
337
- hostId: requestedHost,
338
- hostName: hostConfig.name,
339
- hypervisorType: hostConfig.hypervisorType,
340
- slotIndex: nextIndex,
341
- memoryMb: hostConfig.slotSizeMb || 8192,
342
- allocatedMemoryMb: hostConfig.slotSizeMb || 8192,
343
- allocatedMemoryBytes: (hostConfig.slotSizeMb || 8192) * 1024 * 1024,
344
- status: "AVAILABLE",
345
- claimId: null,
346
- claimedAt: null,
347
- expiresAt: null,
348
- durationMinutes: null,
349
- ttlMinutes: null,
350
- ttlSeconds: null,
351
- tag: null,
352
- label: null,
353
- workloadType: null,
354
- targetVersion: null,
355
- branch: null,
356
- customTags: [],
357
- requester: null,
358
- linearIssue: null,
359
- sessionId: null,
360
- purpose: null,
361
- vmId: null,
362
- vmName: null,
363
- metadata: {}
364
- };
365
- this.slots.set(slotId, availableSlot);
366
- hostSlots.push(availableSlot);
367
- }
387
+ }
388
+ if (!availableSlot && request.purgeIfFull) {
389
+ const purgeRes = await this.purgeSlotsInternal({
390
+ host: requestedHost,
391
+ destroyVms: true,
392
+ includeOrphans: true,
393
+ dryRun: false
394
+ });
395
+ if (purgeRes.purgedSlotsCount > 0 || purgeRes.destroyedVmsCount > 0) {
396
+ const recheckedSlots = Array.from(this.slots.values())
397
+ .filter((s) => s.hostId === requestedHost)
398
+ .sort((a, b) => a.slotIndex - b.slotIndex);
399
+ availableSlot = recheckedSlots.find((s) => s.status === "AVAILABLE");
368
400
  }
369
- if (!availableSlot) {
370
- const allocatedCount = hostSlots.filter((s) => s.status === "CLAIMED" || s.status === "active").length;
371
- const hostLimit = hostConfig.maxSlots ?? calculatePureCapacity(hostConfig);
372
- if (requestedHost === "proxmox-lab") {
373
- throw new CapacityExceededError(`Host 'proxmox-lab' has reached its hard quota limit of 2 concurrent 8GB slots (16GB allocated). 16GB is strictly reserved for CUCM and core lab infrastructure.`, "proxmox-lab", hostLimit, allocatedCount);
374
- }
375
- else if (requestedHost === "proxmox-mini") {
376
- throw new CapacityExceededError(`Host 'proxmox-mini' has reached its hard quota limit of 4 concurrent 8GB slots (32GB allocated).`, "proxmox-mini", hostLimit, allocatedCount);
377
- }
378
- else {
379
- throw new CapacityExceededError(`Host '${requestedHost}' has reached its hard quota limit of ${hostLimit} concurrent slots.`, requestedHost, hostLimit, allocatedCount);
380
- }
401
+ }
402
+ if (!availableSlot) {
403
+ const allocatedCount = hostSlots.filter((s) => s.status === "CLAIMED" || s.status === "active").length;
404
+ const hostLimit = hostConfig.maxSlots ?? calculatePureCapacity(hostConfig);
405
+ if (requestedHost === "proxmox-lab") {
406
+ throw new CapacityExceededError(`Host 'proxmox-lab' has reached its hard quota limit of 2 concurrent 8GB slots (16GB allocated). 16GB is strictly reserved for CUCM and core lab infrastructure.`, "proxmox-lab", hostLimit, allocatedCount);
407
+ }
408
+ else if (requestedHost === "proxmox-mini") {
409
+ throw new CapacityExceededError(`Host 'proxmox-mini' has reached its hard quota limit of 4 concurrent 8GB slots (32GB allocated).`, "proxmox-mini", hostLimit, allocatedCount);
410
+ }
411
+ else {
412
+ throw new CapacityExceededError(`Host '${requestedHost}' has reached its hard quota limit of ${hostLimit} concurrent slots.`, requestedHost, hostLimit, allocatedCount);
381
413
  }
382
- targetSlot = availableSlot;
383
414
  }
384
- else {
385
- // Homogeneous-fleet auto-placement. Every eligible Proxmox host is a
386
- // peer at the request boundary; production uses system randomness,
387
- // while tests may supply an explicit seed for reproducible placement.
388
- // Filter candidate hosts based on hypervisor preference (excluding ESXi and 0-capacity hosts)
389
- const candidateHostConfigs = Array.from(this.hosts.values()).filter((h) => {
390
- const pureSlots = calculatePureCapacity(h);
391
- if (h.hypervisorType === "esxi" || pureSlots === 0)
392
- return false;
393
- if (hypervisorTypePref === "proxmox" && h.hypervisorType !== "proxmox")
394
- return false;
395
- return true;
396
- });
397
- // Evaluate available slots and datastore health per host
398
- const hostStats = candidateHostConfigs.map((h) => {
399
- const hostSlots = Array.from(this.slots.values()).filter((s) => s.hostId === h.hostId);
400
- const availableSlots = hostSlots.filter((s) => s.status === "AVAILABLE").length;
401
- const freeDiskGb = h.freeDiskGb ?? 100;
402
- const datastoreOk = h.hypervisorType !== "esxi" || freeDiskGb >= 10;
403
- return { host: h, availableSlots, hostSlots, datastoreOk };
415
+ targetSlot = availableSlot;
416
+ }
417
+ else {
418
+ // Homogeneous-fleet auto-placement. Every eligible host is a
419
+ // peer at the request boundary; production uses system randomness,
420
+ // while tests may supply an explicit seed for reproducible placement.
421
+ // Filter candidate hosts based on hypervisor preference (excluding ESXi unless allowEsxi)
422
+ const candidateHostConfigs = Array.from(this.hosts.values()).filter((h) => {
423
+ const pureSlots = calculatePureCapacity({ ...h, allowEsxiSlots: allowEsxi });
424
+ if (h.hypervisorType === "esxi" && !allowEsxi)
425
+ return false;
426
+ if (pureSlots === 0 && (!allowEsxi || h.hypervisorType !== "esxi"))
427
+ return false;
428
+ if (hypervisorTypePref === "proxmox" && h.hypervisorType !== "proxmox")
429
+ return false;
430
+ if (hypervisorTypePref === "esxi" && h.hypervisorType !== "esxi")
431
+ return false;
432
+ return true;
433
+ });
434
+ // Evaluate available slots and datastore health per host
435
+ const hostStats = candidateHostConfigs.map((h) => {
436
+ const hostSlots = Array.from(this.slots.values()).filter((s) => s.hostId === h.hostId);
437
+ const availableSlots = hostSlots.filter((s) => s.status === "AVAILABLE").length;
438
+ const freeDiskGb = h.freeDiskGb ?? 100;
439
+ const datastoreOk = h.hypervisorType !== "esxi" || freeDiskGb >= 10;
440
+ return { host: h, availableSlots, hostSlots, datastoreOk };
441
+ });
442
+ let totalAvailable = hostStats.reduce((sum, h) => sum + (h.datastoreOk ? h.availableSlots : 0), 0);
443
+ if (totalAvailable <= 0 && request.purgeIfFull) {
444
+ const purgeRes = await this.purgeSlotsInternal({
445
+ destroyVms: true,
446
+ includeOrphans: true,
447
+ dryRun: false
404
448
  });
405
- const totalAvailable = hostStats.reduce((sum, h) => sum + (h.datastoreOk ? h.availableSlots : 0), 0);
406
- if (totalAvailable <= 0) {
407
- if (hypervisorTypePref === "proxmox") {
408
- throw new CapacityExceededError("All Proxmox cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", "proxmox-cluster", 6, 6);
449
+ if (purgeRes.purgedSlotsCount > 0 || purgeRes.destroyedVmsCount > 0) {
450
+ for (const h of hostStats) {
451
+ const hSlots = Array.from(this.slots.values()).filter((s) => s.hostId === h.host.hostId);
452
+ h.availableSlots = hSlots.filter((s) => s.status === "AVAILABLE").length;
453
+ h.hostSlots = hSlots;
409
454
  }
410
- throw new GlobalCapacityExceededError("All 6 cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", 6, 6);
455
+ totalAvailable = hostStats.reduce((sum, h) => sum + (h.datastoreOk ? h.availableSlots : 0), 0);
411
456
  }
412
- const validCandidates = hostStats
413
- .filter((h) => h.datastoreOk && h.availableSlots > 0)
414
- .sort((a, b) => a.host.hostId.localeCompare(b.host.hostId));
415
- const candidateIndex = request.placementSeed
416
- ? createHash("sha256")
417
- .update(request.placementSeed)
418
- .digest()
419
- .readUInt32BE(0) % validCandidates.length
420
- : randomInt(validCandidates.length);
421
- const winningHostStat = validCandidates[candidateIndex];
422
- if (!winningHostStat) {
423
- throw new GlobalCapacityExceededError("All 6 cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", 6, 6);
457
+ }
458
+ if (totalAvailable <= 0) {
459
+ if (hypervisorTypePref === "proxmox") {
460
+ throw new CapacityExceededError("All Proxmox cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", "proxmox-cluster", 6, 6);
424
461
  }
425
- const availableSlotsOnWinner = winningHostStat.hostSlots
426
- .filter((s) => s.status === "AVAILABLE")
427
- .sort((a, b) => a.slotIndex - b.slotIndex);
428
- targetSlot = availableSlotsOnWinner[0];
462
+ throw new GlobalCapacityExceededError("All 6 cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", 6, 6);
429
463
  }
430
- if (!targetSlot) {
464
+ const validCandidates = hostStats
465
+ .filter((h) => h.datastoreOk && h.availableSlots > 0)
466
+ .sort((a, b) => a.host.hostId.localeCompare(b.host.hostId));
467
+ const candidateIndex = request.placementSeed
468
+ ? createHash("sha256")
469
+ .update(request.placementSeed)
470
+ .digest()
471
+ .readUInt32BE(0) % validCandidates.length
472
+ : randomInt(validCandidates.length);
473
+ const winningHostStat = validCandidates[candidateIndex];
474
+ if (!winningHostStat) {
431
475
  throw new GlobalCapacityExceededError("All 6 cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", 6, 6);
432
476
  }
433
- // Assign claim details
434
- const claimId = `claim-${randomUUID().substring(0, 8)}`;
435
- const now = new Date();
436
- const claimedAt = now.toISOString();
437
- const expiresAt = new Date(now.getTime() + durationMinutes * 60 * 1000).toISOString();
438
- const linearIssue = request.linearIssue || (request.tag && request.tag.startsWith("CTUAT-") ? request.tag : null);
439
- // Normalize custom tags
440
- const customTags = [
441
- ...this.normalizeTagsArray(request.customTags),
442
- ...this.normalizeTagsArray(request.tags)
443
- ];
444
- // Deduplicate tags
445
- const uniqueTags = Array.from(new Set(customTags));
446
- // Auto-assign human readable label if not provided
447
- const label = request.label || (linearIssue
448
- ? `[${linearIssue}] ${request.workloadType || "workload"}: Slot ${targetSlot.slotIndex}`
449
- : request.purpose
450
- ? `${request.purpose} (Slot ${targetSlot.slotIndex})`
451
- : `${request.tag || tagInput} Lease`);
452
- targetSlot.status = "CLAIMED";
453
- targetSlot.claimId = claimId;
454
- targetSlot.claimedAt = claimedAt;
455
- targetSlot.expiresAt = expiresAt;
456
- targetSlot.durationMinutes = durationMinutes;
457
- targetSlot.ttlMinutes = durationMinutes;
458
- targetSlot.ttlSeconds = durationMinutes * 60;
459
- targetSlot.tag = (request.tag || tagInput).trim();
460
- targetSlot.label = label;
461
- targetSlot.workloadType = request.workloadType || null;
462
- targetSlot.targetVersion = request.targetVersion || null;
463
- targetSlot.branch = request.branch || null;
464
- targetSlot.customTags = uniqueTags;
465
- targetSlot.requester = request.requester || request.tag || null;
466
- targetSlot.linearIssue = linearIssue;
467
- targetSlot.sessionId = request.sessionId || null;
468
- targetSlot.purpose = request.purpose || null;
469
- targetSlot.metadata = request.metadata || {};
470
- // Emit audit telemetry
471
- this.telemetry.emit({
472
- event: "slot.claim",
473
- actor: targetSlot.requester || targetSlot.sessionId || "unknown",
474
- status: "success",
475
- linearIssue: linearIssue || undefined,
476
- slotId: targetSlot.slotId,
477
- claimId,
478
- hostId: targetSlot.hostId,
479
- hypervisorType: targetSlot.hypervisorType,
480
- metadata: {
481
- label: targetSlot.label,
482
- workloadType: targetSlot.workloadType,
483
- targetVersion: targetSlot.targetVersion,
484
- branch: targetSlot.branch,
485
- durationMinutes,
486
- expiresAt
487
- }
488
- });
489
- return {
490
- claimId,
491
- slotId: targetSlot.slotId,
492
- hostId: targetSlot.hostId,
493
- hostName: targetSlot.hostName || targetSlot.hostId,
494
- hypervisorType: targetSlot.hypervisorType || (targetSlot.hostId.startsWith("esxi") ? "esxi" : "proxmox"),
495
- slotIndex: targetSlot.slotIndex,
496
- memoryMb: targetSlot.memoryMb,
497
- allocatedMemoryMb: targetSlot.memoryMb,
498
- allocatedMemoryBytes: targetSlot.memoryMb * 1024 * 1024,
499
- tag: targetSlot.tag,
500
- label: targetSlot.label || undefined,
501
- workloadType: targetSlot.workloadType || undefined,
502
- targetVersion: targetSlot.targetVersion || undefined,
503
- branch: targetSlot.branch || undefined,
504
- customTags: targetSlot.customTags.length > 0 ? targetSlot.customTags : undefined,
505
- requester: targetSlot.requester || undefined,
506
- linearIssue: targetSlot.linearIssue || undefined,
507
- claimedAt,
508
- expiresAt,
477
+ const availableSlotsOnWinner = winningHostStat.hostSlots
478
+ .filter((s) => s.status === "AVAILABLE")
479
+ .sort((a, b) => a.slotIndex - b.slotIndex);
480
+ targetSlot = availableSlotsOnWinner[0];
481
+ }
482
+ if (!targetSlot) {
483
+ throw new GlobalCapacityExceededError("All 6 cluster slots are currently allocated (2/2 on proxmox-lab, 4/4 on proxmox-mini).", 6, 6);
484
+ }
485
+ // Assign claim details
486
+ const claimId = `claim-${randomUUID().substring(0, 8)}`;
487
+ const now = new Date();
488
+ const claimedAt = now.toISOString();
489
+ const expiresAt = new Date(now.getTime() + durationMinutes * 60 * 1000).toISOString();
490
+ const linearIssue = request.linearIssue || (request.tag && request.tag.startsWith("CTUAT-") ? request.tag : null);
491
+ // Normalize custom tags
492
+ const customTags = [
493
+ ...this.normalizeTagsArray(request.customTags),
494
+ ...this.normalizeTagsArray(request.tags)
495
+ ];
496
+ // Deduplicate tags
497
+ const uniqueTags = Array.from(new Set(customTags));
498
+ // Auto-assign human readable label if not provided
499
+ const label = request.label || (linearIssue
500
+ ? `[${linearIssue}] ${request.workloadType || "workload"}: Slot ${targetSlot.slotIndex}`
501
+ : request.purpose
502
+ ? `${request.purpose} (Slot ${targetSlot.slotIndex})`
503
+ : `${request.tag || tagInput} Lease`);
504
+ targetSlot.status = "CLAIMED";
505
+ targetSlot.claimId = claimId;
506
+ targetSlot.claimedAt = claimedAt;
507
+ targetSlot.expiresAt = expiresAt;
508
+ targetSlot.durationMinutes = durationMinutes;
509
+ targetSlot.ttlMinutes = durationMinutes;
510
+ targetSlot.ttlSeconds = durationMinutes * 60;
511
+ targetSlot.tag = (request.tag || tagInput).trim();
512
+ targetSlot.label = label;
513
+ targetSlot.workloadType = request.workloadType || null;
514
+ targetSlot.targetVersion = request.targetVersion || null;
515
+ targetSlot.branch = request.branch || null;
516
+ targetSlot.customTags = uniqueTags;
517
+ targetSlot.requester = request.requester || request.tag || null;
518
+ targetSlot.linearIssue = linearIssue;
519
+ targetSlot.sessionId = request.sessionId || null;
520
+ targetSlot.purpose = request.purpose || null;
521
+ targetSlot.metadata = request.metadata || {};
522
+ // Emit audit telemetry
523
+ this.telemetry.emit({
524
+ event: "slot.claim",
525
+ actor: targetSlot.requester || targetSlot.sessionId || "unknown",
526
+ status: "success",
527
+ linearIssue: linearIssue || undefined,
528
+ slotId: targetSlot.slotId,
529
+ claimId,
530
+ hostId: targetSlot.hostId,
531
+ hypervisorType: targetSlot.hypervisorType,
532
+ metadata: {
533
+ label: targetSlot.label,
534
+ workloadType: targetSlot.workloadType,
535
+ targetVersion: targetSlot.targetVersion,
536
+ branch: targetSlot.branch,
509
537
  durationMinutes,
510
- ttlMinutes: durationMinutes,
511
- ttlSeconds: durationMinutes * 60,
512
- status: "active",
513
- purpose: targetSlot.purpose || undefined,
514
- sessionId: targetSlot.sessionId || undefined
515
- };
538
+ expiresAt
539
+ }
516
540
  });
541
+ return {
542
+ claimId,
543
+ slotId: targetSlot.slotId,
544
+ hostId: targetSlot.hostId,
545
+ hostName: targetSlot.hostName || targetSlot.hostId,
546
+ hypervisorType: targetSlot.hypervisorType || (targetSlot.hostId.startsWith("esxi") ? "esxi" : "proxmox"),
547
+ slotIndex: targetSlot.slotIndex,
548
+ memoryMb: targetSlot.memoryMb,
549
+ allocatedMemoryMb: targetSlot.memoryMb,
550
+ allocatedMemoryBytes: targetSlot.memoryMb * 1024 * 1024,
551
+ tag: targetSlot.tag,
552
+ label: targetSlot.label || undefined,
553
+ workloadType: targetSlot.workloadType || undefined,
554
+ targetVersion: targetSlot.targetVersion || undefined,
555
+ branch: targetSlot.branch || undefined,
556
+ customTags: targetSlot.customTags.length > 0 ? targetSlot.customTags : undefined,
557
+ requester: targetSlot.requester || undefined,
558
+ linearIssue: targetSlot.linearIssue || undefined,
559
+ claimedAt,
560
+ expiresAt,
561
+ durationMinutes,
562
+ ttlMinutes: durationMinutes,
563
+ ttlSeconds: durationMinutes * 60,
564
+ status: "active",
565
+ purpose: targetSlot.purpose || undefined,
566
+ sessionId: targetSlot.sessionId || undefined
567
+ };
517
568
  }
518
569
  /**
519
570
  * Releases one or more slots matching claimId, tag, slotId, requester, or linearIssue.
@@ -971,149 +1022,164 @@ export class SlotManager {
971
1022
  * Gated by SafetyGuardrailEngine non-bypassable protections.
972
1023
  */
973
1024
  async purgeSlots(options = {}) {
974
- return this.mutex.runExclusive(async () => {
975
- const now = Date.now();
976
- const dryRun = options.dryRun ?? false;
977
- const destroyVms = options.destroyVms ?? true;
978
- const force = options.force ?? false;
979
- const includeOrphans = options.includeOrphans ?? false;
980
- const targetHost = options.host && options.host !== "all" ? this.normalizeHostId(options.host) : undefined;
981
- const targetHypervisor = options.hypervisorType && options.hypervisorType !== "all" && options.hypervisorType !== "any"
982
- ? options.hypervisorType
983
- : undefined;
984
- const matchingSlots = [];
985
- const reasons = new Map();
986
- for (const slot of this.slots.values()) {
987
- if (slot.status !== "CLAIMED" && slot.status !== "active" && slot.status !== "EXPIRED")
988
- continue;
989
- if (targetHost && slot.hostId !== targetHost)
990
- continue;
991
- if (targetHypervisor && slot.hypervisorType !== targetHypervisor)
992
- continue;
993
- let shouldPurge = false;
994
- let reason = "manual_purge";
995
- const expiresTime = slot.expiresAt ? new Date(slot.expiresAt).getTime() : 0;
996
- const claimedTime = slot.claimedAt ? new Date(slot.claimedAt).getTime() : 0;
997
- const isExpired = expiresTime > 0 && expiresTime <= now;
998
- if (options.linearIssue) {
999
- const matchesLinear = slot.linearIssue === options.linearIssue || slot.tag === options.linearIssue;
1000
- if (matchesLinear) {
1001
- shouldPurge = true;
1002
- reason = isExpired ? "expired" : "linear_match";
1003
- }
1004
- }
1005
- else if (options.olderThanMinutes !== undefined) {
1006
- const ageMinutes = (now - claimedTime) / (60 * 1000);
1007
- if (ageMinutes >= options.olderThanMinutes || isExpired) {
1008
- shouldPurge = true;
1009
- reason = isExpired ? "expired" : "older_than_threshold";
1010
- }
1011
- }
1012
- else if (isExpired) {
1025
+ return this.mutex.runExclusive(async () => this.purgeSlotsInternal(options));
1026
+ }
1027
+ async purgeSlotsInternal(options = {}) {
1028
+ const now = Date.now();
1029
+ const dryRun = options.dryRun ?? false;
1030
+ const destroyVms = options.destroyVms ?? true;
1031
+ const force = options.force ?? false;
1032
+ const includeOrphans = options.includeOrphans ?? false;
1033
+ const targetHost = options.host && options.host !== "all" ? this.normalizeHostId(options.host) : undefined;
1034
+ const targetHypervisor = options.hypervisorType && options.hypervisorType !== "all" && options.hypervisorType !== "any"
1035
+ ? options.hypervisorType
1036
+ : undefined;
1037
+ const matchingSlots = [];
1038
+ const reasons = new Map();
1039
+ for (const slot of this.slots.values()) {
1040
+ if (slot.status !== "CLAIMED" && slot.status !== "active" && slot.status !== "EXPIRED")
1041
+ continue;
1042
+ if (targetHost && slot.hostId !== targetHost)
1043
+ continue;
1044
+ if (targetHypervisor && slot.hypervisorType !== targetHypervisor)
1045
+ continue;
1046
+ let shouldPurge = false;
1047
+ let reason = "manual_purge";
1048
+ const expiresTime = slot.expiresAt ? new Date(slot.expiresAt).getTime() : 0;
1049
+ const claimedTime = slot.claimedAt ? new Date(slot.claimedAt).getTime() : 0;
1050
+ const isExpired = expiresTime > 0 && expiresTime <= now;
1051
+ if (options.linearIssue) {
1052
+ const matchesLinear = slot.linearIssue === options.linearIssue || slot.tag === options.linearIssue;
1053
+ if (matchesLinear) {
1013
1054
  shouldPurge = true;
1014
- reason = "expired";
1055
+ reason = isExpired ? "expired" : "linear_match";
1015
1056
  }
1016
- else if (force) {
1057
+ }
1058
+ else if (options.olderThanMinutes !== undefined) {
1059
+ const ageMinutes = (now - claimedTime) / (60 * 1000);
1060
+ if (ageMinutes >= options.olderThanMinutes || isExpired) {
1017
1061
  shouldPurge = true;
1018
- reason = "forced";
1019
- }
1020
- if (shouldPurge) {
1021
- matchingSlots.push(slot);
1022
- reasons.set(slot.slotId, reason);
1062
+ reason = isExpired ? "expired" : "older_than_threshold";
1023
1063
  }
1024
1064
  }
1025
- const purgedSlots = [];
1026
- const destroyedVms = [];
1027
- let skippedProtectedCount = 0;
1028
- let reclaimedMemoryMb = 0;
1029
- for (const slot of matchingSlots) {
1030
- const reason = reasons.get(slot.slotId) || "manual_purge";
1031
- const purgedSlot = {
1032
- slotId: slot.slotId,
1033
- hostId: slot.hostId,
1034
- claimId: slot.claimId,
1035
- claimedAt: slot.claimedAt,
1036
- expiresAt: slot.expiresAt,
1037
- linearIssue: slot.linearIssue,
1038
- label: slot.label,
1039
- workloadType: slot.workloadType,
1040
- vmId: slot.vmId,
1041
- vmName: slot.vmName,
1042
- reason
1065
+ else if (isExpired) {
1066
+ shouldPurge = true;
1067
+ reason = "expired";
1068
+ }
1069
+ else if (force) {
1070
+ shouldPurge = true;
1071
+ reason = "forced";
1072
+ }
1073
+ if (shouldPurge) {
1074
+ matchingSlots.push(slot);
1075
+ reasons.set(slot.slotId, reason);
1076
+ }
1077
+ }
1078
+ const purgedSlots = [];
1079
+ const destroyedVms = [];
1080
+ let skippedProtectedCount = 0;
1081
+ let reclaimedMemoryMb = 0;
1082
+ for (const slot of matchingSlots) {
1083
+ const reason = reasons.get(slot.slotId) || "manual_purge";
1084
+ const purgedSlot = {
1085
+ slotId: slot.slotId,
1086
+ hostId: slot.hostId,
1087
+ claimId: slot.claimId,
1088
+ claimedAt: slot.claimedAt,
1089
+ expiresAt: slot.expiresAt,
1090
+ linearIssue: slot.linearIssue,
1091
+ label: slot.label,
1092
+ workloadType: slot.workloadType,
1093
+ vmId: slot.vmId,
1094
+ vmName: slot.vmName,
1095
+ reason
1096
+ };
1097
+ let canReclaimSlot = true;
1098
+ // Process attached VM cleanup if present
1099
+ if (slot.vmId !== null && slot.vmId !== undefined) {
1100
+ const vmTarget = {
1101
+ vmid: slot.vmId,
1102
+ name: slot.vmName || undefined,
1103
+ isProtected: false
1043
1104
  };
1044
- let canReclaimSlot = true;
1045
- // Process attached VM cleanup if present
1046
- if (slot.vmId !== null && slot.vmId !== undefined) {
1047
- const vmTarget = {
1048
- vmid: slot.vmId,
1049
- name: slot.vmName || undefined,
1050
- isProtected: false
1051
- };
1052
- // Guardrail safety check
1053
- const protCheck = this.guardrailEngine.isProtected(vmTarget);
1054
- if (protCheck.protected) {
1055
- canReclaimSlot = false;
1056
- skippedProtectedCount++;
1105
+ // Guardrail safety check
1106
+ const protCheck = this.guardrailEngine.isProtected(vmTarget);
1107
+ if (protCheck.protected) {
1108
+ canReclaimSlot = false;
1109
+ skippedProtectedCount++;
1110
+ destroyedVms.push({
1111
+ vmId: slot.vmId,
1112
+ vmName: slot.vmName || undefined,
1113
+ hostId: slot.hostId,
1114
+ hypervisorType: slot.hypervisorType,
1115
+ status: "skipped_protected",
1116
+ reason: `Protected infrastructure preserved: ${protCheck.reason}`
1117
+ });
1118
+ this.telemetry.emit({
1119
+ event: "vm.guardrail_blocked",
1120
+ actor: slot.requester || "system",
1121
+ status: "blocked",
1122
+ linearIssue: slot.linearIssue || undefined,
1123
+ vmId: slot.vmId,
1124
+ vmName: slot.vmName || undefined,
1125
+ hostId: slot.hostId,
1126
+ hypervisorType: slot.hypervisorType,
1127
+ error: `Guardrail blocked purge destruction: ${protCheck.reason}`
1128
+ });
1129
+ }
1130
+ else {
1131
+ if (dryRun) {
1057
1132
  destroyedVms.push({
1058
1133
  vmId: slot.vmId,
1059
1134
  vmName: slot.vmName || undefined,
1060
1135
  hostId: slot.hostId,
1061
1136
  hypervisorType: slot.hypervisorType,
1062
- status: "skipped_protected",
1063
- reason: `Protected infrastructure preserved: ${protCheck.reason}`
1064
- });
1065
- this.telemetry.emit({
1066
- event: "vm.guardrail_blocked",
1067
- actor: slot.requester || "system",
1068
- status: "blocked",
1069
- linearIssue: slot.linearIssue || undefined,
1070
- vmId: slot.vmId,
1071
- vmName: slot.vmName || undefined,
1072
- hostId: slot.hostId,
1073
- hypervisorType: slot.hypervisorType,
1074
- error: `Guardrail blocked purge destruction: ${protCheck.reason}`
1137
+ status: "dry_run",
1138
+ reason: `Would be destroyed during purge (${reason})`
1075
1139
  });
1076
1140
  }
1077
- else {
1078
- if (dryRun) {
1141
+ else if (destroyVms && this.hypervisorManager) {
1142
+ try {
1143
+ await this.hypervisorManager.vmAction({
1144
+ vmIdOrName: String(slot.vmId),
1145
+ targetHost: slot.hostId,
1146
+ action: "destroy",
1147
+ force: true
1148
+ });
1079
1149
  destroyedVms.push({
1080
1150
  vmId: slot.vmId,
1081
1151
  vmName: slot.vmName || undefined,
1082
1152
  hostId: slot.hostId,
1083
1153
  hypervisorType: slot.hypervisorType,
1084
- status: "dry_run",
1085
- reason: `Would be destroyed during purge (${reason})`
1154
+ status: "destroyed",
1155
+ reason: `Purged and destroyed attached VM (${reason})`
1156
+ });
1157
+ this.telemetry.emit({
1158
+ event: "vm.purged",
1159
+ actor: slot.requester || "system",
1160
+ status: "success",
1161
+ linearIssue: slot.linearIssue || undefined,
1162
+ vmId: slot.vmId,
1163
+ vmName: slot.vmName || undefined,
1164
+ hostId: slot.hostId,
1165
+ hypervisorType: slot.hypervisorType,
1166
+ metadata: { slotId: slot.slotId, reason }
1086
1167
  });
1087
1168
  }
1088
- else if (destroyVms && this.hypervisorManager) {
1089
- try {
1090
- await this.hypervisorManager.vmAction({
1091
- vmIdOrName: String(slot.vmId),
1092
- targetHost: slot.hostId,
1093
- action: "destroy",
1094
- force: true
1095
- });
1169
+ catch (err) {
1170
+ const errMsg = err instanceof Error ? err.message : String(err);
1171
+ if (errMsg.toLowerCase().includes("not found")) {
1172
+ canReclaimSlot = true;
1096
1173
  destroyedVms.push({
1097
1174
  vmId: slot.vmId,
1098
1175
  vmName: slot.vmName || undefined,
1099
1176
  hostId: slot.hostId,
1100
1177
  hypervisorType: slot.hypervisorType,
1101
1178
  status: "destroyed",
1102
- reason: `Purged and destroyed attached VM (${reason})`
1103
- });
1104
- this.telemetry.emit({
1105
- event: "vm.purged",
1106
- actor: slot.requester || "system",
1107
- status: "success",
1108
- linearIssue: slot.linearIssue || undefined,
1109
- vmId: slot.vmId,
1110
- vmName: slot.vmName || undefined,
1111
- hostId: slot.hostId,
1112
- hypervisorType: slot.hypervisorType,
1113
- metadata: { slotId: slot.slotId, reason }
1179
+ reason: `Attached VM already absent from hypervisor (${reason})`
1114
1180
  });
1115
1181
  }
1116
- catch (err) {
1182
+ else {
1117
1183
  canReclaimSlot = false;
1118
1184
  destroyedVms.push({
1119
1185
  vmId: slot.vmId,
@@ -1121,145 +1187,458 @@ export class SlotManager {
1121
1187
  hostId: slot.hostId,
1122
1188
  hypervisorType: slot.hypervisorType,
1123
1189
  status: "failed",
1124
- error: err instanceof Error ? err.message : String(err)
1190
+ error: errMsg
1125
1191
  });
1126
1192
  }
1127
1193
  }
1128
- else if (!dryRun) {
1129
- canReclaimSlot = false;
1194
+ }
1195
+ else if (!dryRun) {
1196
+ canReclaimSlot = false;
1197
+ destroyedVms.push({
1198
+ vmId: slot.vmId,
1199
+ vmName: slot.vmName || undefined,
1200
+ hostId: slot.hostId,
1201
+ hypervisorType: slot.hypervisorType,
1202
+ status: "failed",
1203
+ error: destroyVms
1204
+ ? "No hypervisor manager is available to verify VM deletion"
1205
+ : "VM destruction was disabled; occupied capacity was retained"
1206
+ });
1207
+ }
1208
+ }
1209
+ }
1210
+ // Dry-run reports the candidate. A live purge only releases scheduler
1211
+ // capacity after the attached VM is absent or destruction succeeded.
1212
+ if (dryRun || canReclaimSlot) {
1213
+ purgedSlots.push(purgedSlot);
1214
+ reclaimedMemoryMb += slot.memoryMb;
1215
+ }
1216
+ if (!dryRun && canReclaimSlot) {
1217
+ this.telemetry.emit({
1218
+ event: "slot.purge",
1219
+ actor: slot.requester || "system",
1220
+ status: "success",
1221
+ linearIssue: slot.linearIssue || undefined,
1222
+ slotId: slot.slotId,
1223
+ claimId: slot.claimId || undefined,
1224
+ vmId: slot.vmId || undefined,
1225
+ vmName: slot.vmName || undefined,
1226
+ hostId: slot.hostId,
1227
+ hypervisorType: slot.hypervisorType,
1228
+ metadata: {
1229
+ reason,
1230
+ freedMemoryMb: slot.memoryMb
1231
+ }
1232
+ });
1233
+ this.resetSlot(slot);
1234
+ }
1235
+ }
1236
+ // Check orphan VMs if requested
1237
+ if (includeOrphans && this.hypervisorManager) {
1238
+ try {
1239
+ const allVms = await this.hypervisorManager.listUnifiedInventory({
1240
+ includeTemplates: false,
1241
+ host: targetHost,
1242
+ hypervisor: targetHypervisor
1243
+ });
1244
+ const activeVmIds = new Set(Array.from(this.slots.values())
1245
+ .map((s) => s.vmId)
1246
+ .filter((id) => id !== null && id !== undefined)
1247
+ .map((id) => String(id)));
1248
+ for (const vm of allVms) {
1249
+ // Check if VM is an untracked disposable VM without active lease
1250
+ if (!vm.isTemplate && !vm.isProtected && !activeVmIds.has(String(vm.id))) {
1251
+ const protCheck = this.guardrailEngine.isProtected({
1252
+ vmid: vm.id,
1253
+ name: vm.name,
1254
+ isProtected: vm.isProtected,
1255
+ tags: vm.tags
1256
+ });
1257
+ if (protCheck.protected) {
1258
+ skippedProtectedCount++;
1130
1259
  destroyedVms.push({
1131
- vmId: slot.vmId,
1132
- vmName: slot.vmName || undefined,
1133
- hostId: slot.hostId,
1134
- hypervisorType: slot.hypervisorType,
1135
- status: "failed",
1136
- error: destroyVms
1137
- ? "No hypervisor manager is available to verify VM deletion"
1138
- : "VM destruction was disabled; occupied capacity was retained"
1260
+ vmId: vm.id,
1261
+ vmName: vm.name,
1262
+ hostId: vm.hostName,
1263
+ hypervisorType: vm.hypervisor,
1264
+ status: "skipped_protected",
1265
+ reason: `Protected orphan preserved: ${protCheck.reason}`
1139
1266
  });
1140
1267
  }
1268
+ else {
1269
+ if (dryRun) {
1270
+ destroyedVms.push({
1271
+ vmId: vm.id,
1272
+ vmName: vm.name,
1273
+ hostId: vm.hostName,
1274
+ hypervisorType: vm.hypervisor,
1275
+ status: "dry_run",
1276
+ reason: "Orphaned disposable VM with no active slot lease"
1277
+ });
1278
+ }
1279
+ else if (destroyVms) {
1280
+ try {
1281
+ await this.hypervisorManager.vmAction({
1282
+ vmIdOrName: String(vm.id),
1283
+ targetHost: vm.hostName,
1284
+ action: "destroy",
1285
+ force: true
1286
+ });
1287
+ destroyedVms.push({
1288
+ vmId: vm.id,
1289
+ vmName: vm.name,
1290
+ hostId: vm.hostName,
1291
+ hypervisorType: vm.hypervisor,
1292
+ status: "destroyed",
1293
+ reason: "Destroyed orphaned disposable VM with no active slot lease"
1294
+ });
1295
+ }
1296
+ catch (orphanErr) {
1297
+ destroyedVms.push({
1298
+ vmId: vm.id,
1299
+ vmName: vm.name,
1300
+ hostId: vm.hostName,
1301
+ hypervisorType: vm.hypervisor,
1302
+ status: "failed",
1303
+ error: orphanErr instanceof Error ? orphanErr.message : String(orphanErr)
1304
+ });
1305
+ }
1306
+ }
1307
+ }
1141
1308
  }
1142
1309
  }
1143
- // Dry-run reports the candidate. A live purge only releases scheduler
1144
- // capacity after the attached VM is absent or destruction succeeded.
1145
- if (dryRun || canReclaimSlot) {
1146
- purgedSlots.push(purgedSlot);
1147
- reclaimedMemoryMb += slot.memoryMb;
1148
- }
1149
- if (!dryRun && canReclaimSlot) {
1150
- this.telemetry.emit({
1151
- event: "slot.purge",
1152
- actor: slot.requester || "system",
1153
- status: "success",
1154
- linearIssue: slot.linearIssue || undefined,
1155
- slotId: slot.slotId,
1156
- claimId: slot.claimId || undefined,
1157
- vmId: slot.vmId || undefined,
1158
- vmName: slot.vmName || undefined,
1159
- hostId: slot.hostId,
1160
- hypervisorType: slot.hypervisorType,
1161
- metadata: {
1162
- reason,
1163
- freedMemoryMb: slot.memoryMb
1310
+ }
1311
+ catch (invErr) {
1312
+ console.error("[purgeSlots] Orphan inventory inspection error:", invErr);
1313
+ }
1314
+ }
1315
+ const destroyedCount = destroyedVms.filter((v) => v.status === "destroyed").length;
1316
+ const reclaimedMemoryBytes = reclaimedMemoryMb * 1024 * 1024;
1317
+ const prefix = dryRun ? "[DRY RUN] " : "";
1318
+ const message = `${prefix}Purged ${purgedSlots.length} slot lease(s), reclaimed ${reclaimedMemoryMb / 1024} GB RAM, processed ${destroyedVms.length} VM(s) (${destroyedCount} destroyed, ${skippedProtectedCount} protected skipped).`;
1319
+ return {
1320
+ purgedSlotsCount: purgedSlots.length,
1321
+ purgedSlots,
1322
+ destroyedVmsCount: destroyedCount,
1323
+ destroyedVms,
1324
+ skippedProtectedVmsCount: skippedProtectedCount,
1325
+ reclaimedMemoryBytes,
1326
+ reclaimedMemoryMb,
1327
+ dryRun,
1328
+ timestamp: new Date(now).toISOString(),
1329
+ message
1330
+ };
1331
+ }
1332
+ /**
1333
+ * Intelligently discovers available slot capacity across all hypervisors,
1334
+ * ranks candidate hosts, and reports reclaimable expired/disposable VMs if fleet is full.
1335
+ * Optionally auto-claims when autoClaim=true, or auto-reclaims and claims when purgeIfFull=true.
1336
+ */
1337
+ async findSlot(params = {}) {
1338
+ return this.mutex.runExclusive(async () => {
1339
+ this.sweepExpiredSlotsInternal();
1340
+ const allowEsxi = params.allowEsxiSlots ?? this.allowEsxiSlots;
1341
+ const targetHost = params.host && params.host !== "all" && params.host !== "auto"
1342
+ ? this.normalizeHostId(params.host)
1343
+ : undefined;
1344
+ const targetHyp = params.hypervisorType && params.hypervisorType !== "all" && params.hypervisorType !== "any"
1345
+ ? params.hypervisorType
1346
+ : undefined;
1347
+ // Sync with live capacity if hypervisorManager is present
1348
+ if (this.hypervisorManager) {
1349
+ try {
1350
+ const liveCaps = await this.hypervisorManager.listAllHostsCapacity();
1351
+ for (const cap of liveCaps) {
1352
+ const hid = this.normalizeHostId(cap.name);
1353
+ const cfg = this.hosts.get(hid);
1354
+ if (cfg) {
1355
+ cfg.totalMemoryMb = Math.round(cap.memoryTotalBytes / (1024 * 1024));
1356
+ cfg.freeDiskGb = Math.round(cap.storageFreeBytes / (1024 * 1024 * 1024));
1164
1357
  }
1358
+ }
1359
+ }
1360
+ catch {
1361
+ // fallback to registered configs
1362
+ }
1363
+ }
1364
+ // Gather candidate hosts
1365
+ const allCandidates = [];
1366
+ for (const [hid, hostConfig] of this.hosts.entries()) {
1367
+ if (targetHost && hid !== targetHost)
1368
+ continue;
1369
+ if (targetHyp && hostConfig.hypervisorType !== targetHyp)
1370
+ continue;
1371
+ if (hostConfig.hypervisorType === "esxi" && !allowEsxi)
1372
+ continue;
1373
+ const hostSlots = Array.from(this.slots.values()).filter((s) => s.hostId === hid);
1374
+ const availCount = hostSlots.filter((s) => s.status === "AVAILABLE").length;
1375
+ const pureCapacity = calculatePureCapacity({ ...hostConfig, allowEsxiSlots: allowEsxi });
1376
+ const freeDisk = hostConfig.freeDiskGb ?? 100;
1377
+ const totalMem = hostConfig.totalMemoryMb;
1378
+ const reserved = hostConfig.reservedHeadroomMb || 0;
1379
+ const freeMem = Math.max(0, totalMem - reserved - (hostSlots.length - availCount) * (hostConfig.slotSizeMb || 8192));
1380
+ // Score based on available slots, free memory, and datastore space
1381
+ const score = availCount * 100 + Math.floor(freeMem / 1024) * 10 + Math.floor(freeDisk / 100);
1382
+ allCandidates.push({
1383
+ hostId: hid,
1384
+ name: hostConfig.name,
1385
+ hypervisorType: hostConfig.hypervisorType || (hid.startsWith("esxi") ? "esxi" : "proxmox"),
1386
+ status: "online",
1387
+ availableSlots: availCount,
1388
+ totalPureSlots: pureCapacity,
1389
+ freeMemoryMb: freeMem,
1390
+ totalMemoryMb: totalMem,
1391
+ freeDiskGb: freeDisk,
1392
+ cpuUsedPercent: 0,
1393
+ runningVms: hostSlots.length - availCount,
1394
+ score
1395
+ });
1396
+ }
1397
+ // Sort candidates by score descending
1398
+ allCandidates.sort((a, b) => b.score - a.score);
1399
+ const availableSlotsTotal = allCandidates.reduce((sum, c) => sum + c.availableSlots, 0);
1400
+ // If capacity is available
1401
+ if (availableSlotsTotal > 0) {
1402
+ const winningCandidate = allCandidates[0];
1403
+ let claimResult;
1404
+ if (params.autoClaim) {
1405
+ const tag = params.tag || params.linearIssue || params.requester || "find-slot-claim";
1406
+ claimResult = await this.claimSlotInternal({
1407
+ tag,
1408
+ requester: params.requester || tag,
1409
+ linearIssue: params.linearIssue,
1410
+ label: params.label,
1411
+ workloadType: params.workloadType,
1412
+ targetVersion: params.targetVersion,
1413
+ branch: params.branch,
1414
+ host: winningCandidate.hostId,
1415
+ durationMinutes: params.durationMinutes,
1416
+ allowEsxiSlots: allowEsxi,
1417
+ purpose: `Auto-claimed via find_slot on ${winningCandidate.hostId}`
1165
1418
  });
1166
- this.resetSlot(slot);
1419
+ return {
1420
+ status: "auto_claimed",
1421
+ candidateHost: winningCandidate,
1422
+ allCandidates,
1423
+ availableSlotsTotal: availableSlotsTotal - 1,
1424
+ claim: claimResult,
1425
+ message: `Found available slot and successfully auto-claimed '${claimResult.slotId}' on host '${winningCandidate.hostId}'.`
1426
+ };
1167
1427
  }
1428
+ return {
1429
+ status: "available",
1430
+ candidateHost: winningCandidate,
1431
+ allCandidates,
1432
+ availableSlotsTotal,
1433
+ message: `Found ${availableSlotsTotal} available 8GB slot(s) across ${allCandidates.length} eligible host(s). Recommended placement: '${winningCandidate.hostId}' (${winningCandidate.availableSlots} slots available).`
1434
+ };
1168
1435
  }
1169
- // Check orphan VMs if requested
1170
- if (includeOrphans && this.hypervisorManager) {
1436
+ // If capacity is NOT available (availableSlotsTotal === 0)
1437
+ // Inspect reclaimable resources (expired leases + untracked disposable VMs)
1438
+ const expiredSlots = Array.from(this.slots.values())
1439
+ .filter((s) => s.status === "EXPIRED" || (s.expiresAt && new Date(s.expiresAt).getTime() <= Date.now()))
1440
+ .map((s) => ({
1441
+ slotId: s.slotId,
1442
+ hostId: s.hostId,
1443
+ claimId: s.claimId,
1444
+ claimedAt: s.claimedAt,
1445
+ expiresAt: s.expiresAt,
1446
+ linearIssue: s.linearIssue,
1447
+ label: s.label,
1448
+ workloadType: s.workloadType,
1449
+ vmId: s.vmId,
1450
+ vmName: s.vmName,
1451
+ reason: "expired"
1452
+ }));
1453
+ const disposableVms = [];
1454
+ if (this.hypervisorManager) {
1171
1455
  try {
1172
- const allVms = await this.hypervisorManager.listUnifiedInventory({
1456
+ const inv = await this.hypervisorManager.listUnifiedInventory({
1173
1457
  includeTemplates: false,
1174
1458
  host: targetHost,
1175
- hypervisor: targetHypervisor
1459
+ hypervisor: targetHyp
1176
1460
  });
1177
1461
  const activeVmIds = new Set(Array.from(this.slots.values())
1178
- .map((s) => s.vmId)
1179
- .filter((id) => id !== null && id !== undefined)
1180
- .map((id) => String(id)));
1181
- for (const vm of allVms) {
1182
- // Check if VM is an untracked disposable VM without active lease
1462
+ .filter((s) => s.status === "CLAIMED" && s.vmId !== null && s.vmId !== undefined)
1463
+ .map((s) => String(s.vmId)));
1464
+ for (const vm of inv) {
1183
1465
  if (!vm.isTemplate && !vm.isProtected && !activeVmIds.has(String(vm.id))) {
1184
- const protCheck = this.guardrailEngine.isProtected({
1466
+ const prot = this.guardrailEngine.isProtected({
1185
1467
  vmid: vm.id,
1186
1468
  name: vm.name,
1187
1469
  isProtected: vm.isProtected,
1188
1470
  tags: vm.tags
1189
1471
  });
1190
- if (protCheck.protected) {
1191
- skippedProtectedCount++;
1192
- destroyedVms.push({
1472
+ if (!prot.protected) {
1473
+ disposableVms.push({
1193
1474
  vmId: vm.id,
1194
- vmName: vm.name,
1195
- hostId: vm.hostName,
1196
- hypervisorType: vm.hypervisor,
1197
- status: "skipped_protected",
1198
- reason: `Protected orphan preserved: ${protCheck.reason}`
1475
+ name: vm.name,
1476
+ hostName: vm.hostName,
1477
+ hypervisor: vm.hypervisor,
1478
+ status: vm.status,
1479
+ memoryMb: Math.round(vm.memoryBytes / (1024 * 1024)),
1480
+ linearIssue: vm.linearIssue
1199
1481
  });
1200
1482
  }
1201
- else {
1202
- if (dryRun) {
1203
- destroyedVms.push({
1204
- vmId: vm.id,
1205
- vmName: vm.name,
1206
- hostId: vm.hostName,
1207
- hypervisorType: vm.hypervisor,
1208
- status: "dry_run",
1209
- reason: "Orphaned disposable VM with no active slot lease"
1210
- });
1211
- }
1212
- else if (destroyVms) {
1213
- try {
1214
- await this.hypervisorManager.vmAction({
1215
- vmIdOrName: String(vm.id),
1216
- targetHost: vm.hostName,
1217
- action: "destroy",
1218
- force: true
1219
- });
1220
- destroyedVms.push({
1221
- vmId: vm.id,
1222
- vmName: vm.name,
1223
- hostId: vm.hostName,
1224
- hypervisorType: vm.hypervisor,
1225
- status: "destroyed",
1226
- reason: "Destroyed orphaned disposable VM with no active slot lease"
1227
- });
1228
- }
1229
- catch (orphanErr) {
1230
- destroyedVms.push({
1231
- vmId: vm.id,
1232
- vmName: vm.name,
1233
- hostId: vm.hostName,
1234
- hypervisorType: vm.hypervisor,
1235
- status: "failed",
1236
- error: orphanErr instanceof Error ? orphanErr.message : String(orphanErr)
1237
- });
1238
- }
1239
- }
1240
- }
1241
1483
  }
1242
1484
  }
1243
1485
  }
1244
- catch (invErr) {
1245
- console.error("[purgeSlots] Orphan inventory inspection error:", invErr);
1486
+ catch {
1487
+ // ignore inventory errors
1488
+ }
1489
+ }
1490
+ const reclaimableSlotsEst = expiredSlots.length + Math.floor(disposableVms.reduce((acc, v) => acc + v.memoryMb, 0) / 8192);
1491
+ const reclaimableMemoryMb = expiredSlots.length * 8192 + disposableVms.reduce((acc, v) => acc + v.memoryMb, 0);
1492
+ const reclaimable = {
1493
+ expiredSlotsCount: expiredSlots.length,
1494
+ expiredSlots,
1495
+ disposableVmsCount: disposableVms.length,
1496
+ disposableVms,
1497
+ reclaimableMemoryMb,
1498
+ reclaimableSlotsEst
1499
+ };
1500
+ // If purgeIfFull is requested and we have reclaimable resources, auto-purge and claim!
1501
+ if (params.purgeIfFull && (expiredSlots.length > 0 || disposableVms.length > 0)) {
1502
+ const purgeRes = await this.purgeSlotsInternal({
1503
+ host: targetHost,
1504
+ hypervisorType: targetHyp,
1505
+ destroyVms: true,
1506
+ includeOrphans: true,
1507
+ dryRun: false
1508
+ });
1509
+ if (purgeRes.purgedSlotsCount > 0 || purgeRes.destroyedVmsCount > 0) {
1510
+ const tag = params.tag || params.linearIssue || params.requester || "find-slot-reclaim";
1511
+ const claimResult = await this.claimSlotInternal({
1512
+ tag,
1513
+ requester: params.requester || tag,
1514
+ linearIssue: params.linearIssue,
1515
+ label: params.label,
1516
+ workloadType: params.workloadType,
1517
+ targetVersion: params.targetVersion,
1518
+ branch: params.branch,
1519
+ host: targetHost || "auto",
1520
+ durationMinutes: params.durationMinutes,
1521
+ allowEsxiSlots: allowEsxi,
1522
+ purpose: `Auto-reclaimed and claimed via find_slot`
1523
+ });
1524
+ return {
1525
+ status: "auto_reclaimed_and_claimed",
1526
+ candidateHost: allCandidates.find((c) => c.hostId === claimResult.hostId) || allCandidates[0],
1527
+ allCandidates,
1528
+ availableSlotsTotal: 0,
1529
+ claim: claimResult,
1530
+ reclaimable,
1531
+ message: `Purged ${purgeRes.purgedSlotsCount} expired slot(s) and ${purgeRes.destroyedVmsCount} disposable VM(s), then successfully claimed '${claimResult.slotId}' on host '${claimResult.hostId}'.`
1532
+ };
1246
1533
  }
1247
1534
  }
1248
- const destroyedCount = destroyedVms.filter((v) => v.status === "destroyed").length;
1249
- const reclaimedMemoryBytes = reclaimedMemoryMb * 1024 * 1024;
1250
- const prefix = dryRun ? "[DRY RUN] " : "";
1251
- const message = `${prefix}Purged ${purgedSlots.length} slot lease(s), reclaimed ${reclaimedMemoryMb / 1024} GB RAM, processed ${destroyedVms.length} VM(s) (${destroyedCount} destroyed, ${skippedProtectedCount} protected skipped).`;
1252
1535
  return {
1253
- purgedSlotsCount: purgedSlots.length,
1254
- purgedSlots,
1255
- destroyedVmsCount: destroyedCount,
1256
- destroyedVms,
1257
- skippedProtectedVmsCount: skippedProtectedCount,
1258
- reclaimedMemoryBytes,
1259
- reclaimedMemoryMb,
1260
- dryRun,
1261
- timestamp: new Date(now).toISOString(),
1262
- message
1536
+ status: "full",
1537
+ allCandidates,
1538
+ availableSlotsTotal: 0,
1539
+ reclaimable,
1540
+ message: `No slots currently available across ${allCandidates.length} host(s). Found ${expiredSlots.length} expired lease(s) and ${disposableVms.length} disposable VM(s) (~${Math.round(reclaimableMemoryMb / 1024)}GB reclaimable). Use purge_slots or call find_slot with purgeIfFull=true to reclaim capacity.`
1541
+ };
1542
+ });
1543
+ }
1544
+ /**
1545
+ * One-shot purge of old/expired disposable test VM and replacement with fresh slot allocation.
1546
+ */
1547
+ async purgeAndRedeploy(params) {
1548
+ return this.mutex.runExclusive(async () => {
1549
+ if (!this.hypervisorManager) {
1550
+ throw new InvalidSlotOperationError("HypervisorManager is required for purge_and_redeploy operations.");
1551
+ }
1552
+ const linearIssue = params.linearIssue;
1553
+ const targetVmIdOrName = params.targetVmIdOrName;
1554
+ if (!linearIssue && !targetVmIdOrName) {
1555
+ throw new InvalidSlotOperationError("Either 'linearIssue' or 'targetVmIdOrName' is required for purge_and_redeploy.");
1556
+ }
1557
+ // 1. Locate target VM
1558
+ let targetVmidStr = targetVmIdOrName;
1559
+ const matchingSlot = Array.from(this.slots.values()).find((s) => (targetVmIdOrName && (String(s.vmId) === targetVmIdOrName || s.vmName === targetVmIdOrName)) ||
1560
+ (linearIssue && s.linearIssue === linearIssue));
1561
+ if (!targetVmidStr && matchingSlot && matchingSlot.vmId != null) {
1562
+ targetVmidStr = String(matchingSlot.vmId);
1563
+ }
1564
+ const inventory = await this.hypervisorManager.listUnifiedInventory({
1565
+ includeTemplates: false,
1566
+ host: params.host
1567
+ });
1568
+ const targetVm = inventory.find((vm) => {
1569
+ if (targetVmidStr && (String(vm.id) === targetVmidStr || vm.name === targetVmidStr)) {
1570
+ return true;
1571
+ }
1572
+ if (linearIssue && (vm.linearIssue === linearIssue || vm.name.toLowerCase().includes(linearIssue.toLowerCase()))) {
1573
+ return true;
1574
+ }
1575
+ return false;
1576
+ });
1577
+ if (!targetVm) {
1578
+ throw new SlotNotFoundError(`No existing VM found matching '${targetVmIdOrName || linearIssue}'.`);
1579
+ }
1580
+ // 2. Safety guardrail check
1581
+ const protCheck = this.guardrailEngine.isProtected({
1582
+ vmid: targetVm.id,
1583
+ name: targetVm.name,
1584
+ isProtected: targetVm.isProtected,
1585
+ tags: targetVm.tags
1586
+ });
1587
+ if (protCheck.protected) {
1588
+ throw new InvalidSlotOperationError(`Refusing to destroy protected VM '${targetVm.name}' (${targetVm.id}): ${protCheck.reason}`);
1589
+ }
1590
+ if (params.dryRun) {
1591
+ return {
1592
+ success: true,
1593
+ dryRun: true,
1594
+ purgedVm: {
1595
+ vmId: targetVm.id,
1596
+ name: targetVm.name,
1597
+ host: targetVm.hostName,
1598
+ hypervisor: targetVm.hypervisor
1599
+ },
1600
+ message: `[Dry-run] Would purge VM '${targetVm.name}' (${targetVm.id}) on '${targetVm.hostName}' and claim fresh slot for redeployment.`
1601
+ };
1602
+ }
1603
+ // 3. Find associated slot if any
1604
+ const slotToRelease = matchingSlot || Array.from(this.slots.values()).find((s) => (s.vmId !== null && String(s.vmId) === String(targetVm.id)) || (linearIssue && s.linearIssue === linearIssue));
1605
+ // 4. Destroy old VM
1606
+ await this.hypervisorManager.vmAction({
1607
+ vmIdOrName: String(targetVm.id),
1608
+ targetHost: targetVm.hostName,
1609
+ action: "destroy",
1610
+ force: true
1611
+ });
1612
+ // 5. Release existing slot if present
1613
+ if (slotToRelease) {
1614
+ this.resetSlot(slotToRelease);
1615
+ }
1616
+ // 6. Claim new slot on the same host (or auto)
1617
+ const targetHost = params.host || targetVm.hostName;
1618
+ const tag = linearIssue || targetVm.name;
1619
+ const newSlot = await this.claimSlotInternal({
1620
+ tag,
1621
+ requester: linearIssue || "purge-and-redeploy",
1622
+ linearIssue,
1623
+ label: `Redeployed ${linearIssue || targetVm.name}`,
1624
+ workloadType: params.workloadType || "uat",
1625
+ targetVersion: params.targetVersion || params.version,
1626
+ branch: params.branch,
1627
+ host: targetHost,
1628
+ durationMinutes: params.durationMinutes || 120,
1629
+ purpose: `Purged old VM ${targetVm.name} and claimed capacity for fresh deployment`
1630
+ });
1631
+ return {
1632
+ success: true,
1633
+ purgedVm: {
1634
+ vmId: targetVm.id,
1635
+ name: targetVm.name,
1636
+ host: targetVm.hostName,
1637
+ hypervisor: targetVm.hypervisor
1638
+ },
1639
+ reclaimedSlotId: matchingSlot?.slotId,
1640
+ newSlot,
1641
+ message: `Successfully purged old VM '${targetVm.name}' (${targetVm.id}) and claimed fresh slot '${newSlot.slotId}' on host '${newSlot.hostId}'.`
1263
1642
  };
1264
1643
  });
1265
1644
  }