@appkit/llamacpp-cli 1.5.0 → 1.7.0

package/src/lib/history-manager.ts

@@ -0,0 +1,172 @@
+import { mkdir, readFile, writeFile, access, rename } from 'fs/promises';
+import { join } from 'path';
+import { homedir } from 'os';
+import { ServerMetrics, SystemMetrics } from '../types/monitor-types.js';
+import { HistoryData, HistorySnapshot, TIME_WINDOW_HOURS, TimeWindow } from '../types/history-types.js';
+
+export class HistoryManager {
+  private serverId: string;
+  private historyDir: string;
+  private historyPath: string;
+  private readonly MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
+
+  constructor(serverId: string) {
+    this.serverId = serverId;
+    this.historyDir = join(homedir(), '.llamacpp', 'history');
+    this.historyPath = join(this.historyDir, `${serverId}.json`);
+  }
+
+  /**
+   * Append a new snapshot to history (with auto-pruning)
+   */
+  async appendSnapshot(serverMetrics: ServerMetrics, systemMetrics?: SystemMetrics): Promise<void> {
+    try {
+      // Ensure history directory exists
+      await mkdir(this.historyDir, { recursive: true });
+
+      // Load existing history
+      const historyData = await this.loadHistoryData();
+
+      // Create new snapshot
+      const snapshot: HistorySnapshot = {
+        timestamp: Date.now(),
+        server: {
+          healthy: serverMetrics.healthy,
+          uptime: serverMetrics.uptime,
+          activeSlots: serverMetrics.activeSlots,
+          idleSlots: serverMetrics.idleSlots,
+          totalSlots: serverMetrics.totalSlots,
+          avgPromptSpeed: serverMetrics.avgPromptSpeed,
+          avgGenerateSpeed: serverMetrics.avgGenerateSpeed,
+          processMemory: serverMetrics.processMemory,
+          processCpuUsage: serverMetrics.processCpuUsage,
+        },
+        system: systemMetrics ? {
+          gpuUsage: systemMetrics.gpuUsage,
+          cpuUsage: systemMetrics.cpuUsage,
+          aneUsage: systemMetrics.aneUsage,
+          temperature: systemMetrics.temperature,
+          memoryUsed: systemMetrics.memoryUsed,
+          memoryTotal: systemMetrics.memoryTotal,
+        } : undefined,
+      };
+
+      // Append new snapshot
+      historyData.snapshots.push(snapshot);
+
+      // Prune old snapshots (keep only last 24h)
+      historyData.snapshots = this.pruneOldSnapshots(historyData.snapshots, this.MAX_AGE_MS);
+
+      // Atomic write: write to temp file in same directory, then rename
+      // This prevents read collisions during concurrent access
+      // IMPORTANT: temp file MUST be in same directory as destination for rename to work across filesystems
+      const tempPath = join(this.historyDir, `.${this.serverId}-${Date.now()}.tmp`);
+      await writeFile(tempPath, JSON.stringify(historyData, null, 2), 'utf-8');
+      await rename(tempPath, this.historyPath);
+    } catch (error) {
+      // Silent failure - don't interrupt monitoring
+      // Don't throw - just return silently to avoid polluting console
+      return;
+    }
+  }
+
+  /**
+   * Load all snapshots within specified time window
+   */
+  async loadHistory(windowHours: number): Promise<HistorySnapshot[]> {
+    // Retry logic for file I/O collisions during concurrent read/write
+    const maxRetries = 3;
+    let lastError: Error | null = null;
+
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      try {
+        const historyData = await this.loadHistoryData();
+        return this.filterByTimeWindow(historyData.snapshots, windowHours);
+      } catch (error) {
+        lastError = error as Error;
+        // Wait briefly before retry (exponential backoff)
+        if (attempt < maxRetries - 1) {
+          await new Promise(resolve => setTimeout(resolve, 50 * Math.pow(2, attempt)));
+        }
+      }
+    }
+
+    // All retries failed - throw error so it can be handled upstream
+    throw new Error(`Failed to load history after ${maxRetries} attempts: ${lastError?.message || 'Unknown error'}`);
+  }
+
+  /**
+   * Load history for specific time window type
+   */
+  async loadHistoryByWindow(window: TimeWindow): Promise<HistorySnapshot[]> {
+    return this.loadHistory(TIME_WINDOW_HOURS[window]);
+  }
+
+  /**
+   * Get file path for server history
+   */
+  getHistoryPath(): string {
+    return this.historyPath;
+  }
+
+  /**
+   * Check if history file exists
+   */
+  async hasHistory(): Promise<boolean> {
+    try {
+      await access(this.historyPath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Clear all history for server
+   */
+  async clearHistory(): Promise<void> {
+    const emptyHistory: HistoryData = {
+      serverId: this.serverId,
+      snapshots: [],
+    };
+
+    await mkdir(this.historyDir, { recursive: true });
+
+    // Atomic write - temp file in same directory as destination
+    const tempPath = join(this.historyDir, `.${this.serverId}-${Date.now()}.tmp`);
+    await writeFile(tempPath, JSON.stringify(emptyHistory, null, 2), 'utf-8');
+    await rename(tempPath, this.historyPath);
+  }
+
+  /**
+   * Load full history data from file
+   */
+  private async loadHistoryData(): Promise<HistoryData> {
+    try {
+      const content = await readFile(this.historyPath, 'utf-8');
+      return JSON.parse(content) as HistoryData;
+    } catch (error) {
+      // File doesn't exist or is corrupted, return empty history
+      return {
+        serverId: this.serverId,
+        snapshots: [],
+      };
+    }
+  }
+
+  /**
+   * Prune snapshots older than maxAge
+   */
+  private pruneOldSnapshots(snapshots: HistorySnapshot[], maxAgeMs: number): HistorySnapshot[] {
+    const cutoff = Date.now() - maxAgeMs;
+    return snapshots.filter(s => s.timestamp >= cutoff);
+  }
+
+  /**
+   * Filter snapshots by time window
+   */
+  private filterByTimeWindow(snapshots: HistorySnapshot[], windowHours: number): HistorySnapshot[] {
+    const cutoff = Date.now() - (windowHours * 60 * 60 * 1000);
+    return snapshots.filter(s => s.timestamp >= cutoff);
+  }
+}

package/src/lib/metrics-aggregator.ts

@@ -2,7 +2,7 @@ import { ServerConfig } from '../types/server-config.js';
 import { ServerMetrics, SlotInfo, MonitorData } from '../types/monitor-types.js';
 import { statusChecker } from './status-checker.js';
 import { systemCollector } from './system-collector.js';
-import { getProcessMemory } from '../utils/process-utils.js';
+import { getProcessMemory, getProcessCpu } from '../utils/process-utils.js';
 
 /**
  * Aggregates metrics from llama.cpp server API endpoints
@@ -128,10 +128,12 @@ export class MetricsAggregator {
    * Aggregate all server metrics
    * @param server - Server configuration
    * @param processMemory - Optional pre-fetched process memory (for batch collection)
+   * @param processCpuUsage - Optional pre-fetched process CPU usage (for batch collection)
    */
   async collectServerMetrics(
     server: ServerConfig,
-    processMemory?: number | null
+    processMemory?: number | null,
+    processCpuUsage?: number | null
   ): Promise<ServerMetrics> {
     const now = Date.now();
 
@@ -167,14 +169,17 @@ export class MetricsAggregator {
     }
 
     // Fetch detailed metrics in parallel
-    // If processMemory was pre-fetched (batch mode), use it; otherwise fetch individually
-    const [healthy, props, slots, fetchedMemory] = await Promise.all([
+    // If processMemory/CPU were pre-fetched (batch mode), use them; otherwise fetch individually
+    const [healthy, props, slots, fetchedMemory, fetchedCpu] = await Promise.all([
       this.getHealth(),
       this.getProps(),
       this.getSlots(),
       processMemory !== undefined
         ? Promise.resolve(processMemory)
         : (server.pid ? getProcessMemory(server.pid) : Promise.resolve(null)),
+      processCpuUsage !== undefined
+        ? Promise.resolve(processCpuUsage)
+        : (server.pid ? getProcessCpu(server.pid) : Promise.resolve(null)),
     ]);
 
     // Calculate slot statistics
@@ -201,6 +206,13 @@ export class MetricsAggregator {
           ) / processingSlots.length
         : undefined;
 
+    // Calculate total memory (CPU + Metal GPU memory if available)
+    let totalMemory = fetchedMemory ?? undefined;
+    if (totalMemory !== undefined && server.metalMemoryMB) {
+      // Add Metal memory (convert MB to bytes)
+      totalMemory += server.metalMemoryMB * 1024 * 1024;
+    }
+
     return {
       server,
       healthy,
@@ -214,7 +226,8 @@ export class MetricsAggregator {
       slots,
       avgPromptSpeed,
       avgGenerateSpeed,
-      processMemory: fetchedMemory ?? undefined,
+      processMemory: totalMemory,
+      processCpuUsage: fetchedCpu ?? undefined,
       timestamp: now,
       stale: false,
     };

package/src/lib/system-collector.ts

@@ -178,16 +178,12 @@ export class SystemCollector {
    */
   private parseVmStatOutput(output: string): {
     memoryUsed: number;
-    memoryTotal: number;
   } {
     const lines = output.split('\n');
     const pageSize = 16384; // 16KB on Apple Silicon
     let pagesActive = 0;
     let pagesWired = 0;
     let pagesCompressed = 0;
-    let pagesFree = 0;
-    let pagesInactive = 0;
-    let pagesSpeculative = 0;
 
     for (const line of lines) {
       const match = line.match(/Pages (.*?):\s+(\d+)\./);
@@ -198,49 +194,56 @@ export class SystemCollector {
         if (name === 'active') pagesActive = value;
         else if (name === 'wired down') pagesWired = value;
         else if (name === 'compressed') pagesCompressed = value;
-        else if (name === 'free') pagesFree = value;
-        else if (name === 'inactive') pagesInactive = value;
-        else if (name === 'speculative') pagesSpeculative = value;
       }
     }
 
     // Calculate used memory (active + wired + compressed)
+    // This matches what Activity Monitor and macmon report as "used"
     const usedPages = pagesActive + pagesWired + pagesCompressed;
     const memoryUsed = usedPages * pageSize;
 
-    // Calculate total memory (used + free + inactive + speculative)
-    const totalPages =
-      pagesActive +
-      pagesWired +
-      pagesCompressed +
-      pagesFree +
-      pagesInactive +
-      pagesSpeculative;
-    const memoryTotal = totalPages * pageSize;
-
-    return { memoryUsed, memoryTotal };
+    return { memoryUsed };
+  }
+
+  /**
+   * Get total system memory from sysctl
+   * Returns installed RAM size in bytes
+   */
+  private async getTotalMemory(): Promise<number> {
+    try {
+      const output = await execCommand('sysctl -n hw.memsize 2>/dev/null');
+      return parseInt(output.trim(), 10) || 0;
+    } catch {
+      return 0;
+    }
   }
 
   /**
-   * Collect vm_stat memory metrics
+   * Collect vm_stat memory metrics + total system memory from sysctl
    */
-  private async getVmStatMetrics(): Promise<{
+  private async getMemoryMetrics(): Promise<{
     memoryUsed: number;
     memoryTotal: number;
   }> {
     try {
-      const output = await execCommand('vm_stat 2>/dev/null');
-      return this.parseVmStatOutput(output);
+      // Get used memory from vm_stat
+      const vmStatOutput = await execCommand('vm_stat 2>/dev/null');
+      const { memoryUsed } = this.parseVmStatOutput(vmStatOutput);
+
+      // Get total installed RAM from sysctl (this is accurate)
+      const memoryTotal = await this.getTotalMemory();
+
+      return { memoryUsed, memoryTotal };
     } catch {
-      // Fallback to zeros if vm_stat fails
+      // Fallback to zeros if commands fail
       return { memoryUsed: 0, memoryTotal: 0 };
     }
   }
 
   /**
    * Collect all system metrics
-   * Attempts macmon first (GPU/CPU/ANE + memory), falls back to vm_stat (memory only)
-   * Caches results for 1.5s to prevent spawning multiple macmon processes
+   * Attempts macmon first (GPU/CPU/ANE), always gets memory from vm_stat + sysctl
+   * Caches results for 4s to prevent spawning multiple macmon processes
    */
   async collectSystemMetrics(): Promise<SystemMetrics> {
     const now = Date.now();
@@ -276,11 +279,11 @@ export class SystemCollector {
     const warnings: string[] = [];
     const now = Date.now();
 
-    // Try macmon first
+    // Try macmon first for GPU/CPU/ANE
     const macmonMetrics = await this.getMacmonMetrics();
 
-    // Always get memory from vm_stat (more reliable than macmon)
-    const memoryMetrics = await this.getVmStatMetrics();
+    // Always get memory from vm_stat + sysctl (accurate total from sysctl)
+    const memoryMetrics = await this.getMemoryMetrics();
 
     // Determine source and add warnings
     let source: 'macmon' | 'vm_stat' | 'none';