@appkit/llamacpp-cli 1.5.0 → 1.6.0

package/src/commands/ps.ts

@@ -1,11 +1,14 @@
 import chalk from 'chalk';
 import Table from 'cli-table3';
+import blessed from 'blessed';
 import { stateManager } from '../lib/state-manager';
 import { statusChecker } from '../lib/status-checker';
 import { formatUptime, formatBytes } from '../utils/format-utils';
 import { getProcessMemory } from '../utils/process-utils';
+import { createMultiServerMonitorUI } from '../tui/MultiServerMonitorApp.js';
+import { ServerConfig } from '../types/server-config.js';
 
-export async function psCommand(): Promise<void> {
+async function showStaticTable(): Promise<void> {
   const servers = await stateManager.getAllServers();
 
   if (servers.length === 0) {
@@ -52,12 +55,14 @@ export async function psCommand(): Promise<void> {
         ? formatUptime(server.lastStarted)
         : '-';
 
-    // Get memory usage for running servers
+    // Get memory usage for running servers (CPU + Metal GPU memory)
     let memoryText = '-';
     if (server.status === 'running' && server.pid) {
-      const memoryBytes = await getProcessMemory(server.pid);
-      if (memoryBytes !== null) {
-        memoryText = formatBytes(memoryBytes);
+      const cpuMemoryBytes = await getProcessMemory(server.pid);
+      if (cpuMemoryBytes !== null) {
+        const metalMemoryBytes = server.metalMemoryMB ? server.metalMemoryMB * 1024 * 1024 : 0;
+        const totalMemoryBytes = cpuMemoryBytes + metalMemoryBytes;
+        memoryText = formatBytes(totalMemoryBytes);
       }
     }
 
@@ -88,3 +93,81 @@ export async function psCommand(): Promise<void> {
     console.log(chalk.red('\n⚠️  Some servers have crashed. Check logs with: llamacpp server logs <id> --errors'));
   }
 }
+
+export async function psCommand(identifier?: string, options?: { table?: boolean }): Promise<void> {
+  // If --table flag is set, show static table (backward compatibility)
+  if (options?.table) {
+    await showStaticTable();
+    return;
+  }
+
+  // Get all servers and update their statuses
+  const servers = await stateManager.getAllServers();
+
+  if (servers.length === 0) {
+    console.log(chalk.yellow('No servers configured.'));
+    console.log(chalk.dim('\nCreate a server: llamacpp server create <model-filename>'));
+    return;
+  }
+
+  // Update all server statuses
+  const updated = await statusChecker.updateAllServerStatuses();
+
+  // If identifier is provided, find the server and jump to detail view
+  if (identifier) {
+    const server = await findServer(identifier, updated);
+    if (!server) {
+      console.log(chalk.red(`❌ Server not found: ${identifier}`));
+      console.log(chalk.dim('\nAvailable servers:'));
+      updated.forEach((s: ServerConfig) => {
+        console.log(chalk.dim(`  - ${s.id} (port ${s.port})`));
+      });
+      process.exit(1);
+    }
+
+    // Find the server index for direct jump
+    const serverIndex = updated.findIndex(s => s.id === server.id);
+
+    // Launch multi-server TUI with direct jump to detail view
+    const screen = blessed.screen({
+      smartCSR: true,
+      title: 'llama.cpp Multi-Server Monitor',
+      fullUnicode: true,
+    });
+
+    await createMultiServerMonitorUI(screen, updated, true, serverIndex); // fromPs = true, directJumpIndex
+    return;
+  }
+
+  // No identifier - launch multi-server TUI
+  const runningServers = updated.filter((s: ServerConfig) => s.status === 'running');
+
+  // Launch multi-server TUI (shows all servers, not just running ones)
+  const screen = blessed.screen({
+    smartCSR: true,
+    title: 'llama.cpp Multi-Server Monitor',
+    fullUnicode: true,
+  });
+
+  await createMultiServerMonitorUI(screen, updated, true); // fromPs = true
+}
+
+// Helper function to find server by identifier
+async function findServer(identifier: string, servers: ServerConfig[]): Promise<ServerConfig | null> {
+  // Try by port
+  const port = parseInt(identifier);
+  if (!isNaN(port)) {
+    const server = servers.find(s => s.port === port);
+    if (server) return server;
+  }
+
+  // Try by exact ID
+  const byId = servers.find(s => s.id === identifier);
+  if (byId) return byId;
+
+  // Try by partial model name
+  const byModel = servers.find(s => s.modelName.toLowerCase().includes(identifier.toLowerCase()));
+  if (byModel) return byModel;
+
+  return null;
+}

package/src/commands/server-show.ts

@@ -68,9 +68,16 @@ export async function serverShowCommand(identifier: string): Promise<void> {
     }
 
     if (updatedServer.pid) {
-      const memoryBytes = await getProcessMemory(updatedServer.pid);
-      if (memoryBytes !== null) {
-        console.log(`${chalk.bold('Memory:')}         ${formatBytes(memoryBytes)}`);
+      const cpuMemoryBytes = await getProcessMemory(updatedServer.pid);
+      if (cpuMemoryBytes !== null) {
+        const metalMemoryBytes = updatedServer.metalMemoryMB ? updatedServer.metalMemoryMB * 1024 * 1024 : 0;
+        const totalMemoryBytes = cpuMemoryBytes + metalMemoryBytes;
+
+        if (metalMemoryBytes > 0) {
+          console.log(`${chalk.bold('Memory:')}         ${formatBytes(totalMemoryBytes)} (CPU: ${formatBytes(cpuMemoryBytes)}, GPU: ${formatBytes(metalMemoryBytes)})`);
+        } else {
+          console.log(`${chalk.bold('Memory:')}         ${formatBytes(cpuMemoryBytes)} (CPU only)`);
+        }
       }
     }
   }

package/src/commands/start.ts

@@ -2,6 +2,7 @@ import chalk from 'chalk';
 import { stateManager } from '../lib/state-manager';
 import { launchctlManager } from '../lib/launchctl-manager';
 import { statusChecker } from '../lib/status-checker';
+import { parseMetalMemoryFromLog } from '../utils/file-utils';
 
 export async function startCommand(identifier: string): Promise<void> {
   // Initialize state manager
@@ -61,9 +62,21 @@ export async function startCommand(identifier: string): Promise<void> {
   }
 
   // 7. Update server status
-  await statusChecker.updateServerStatus(server);
+  let updatedServer = await statusChecker.updateServerStatus(server);
 
-  // 8. Display success
+  // 8. Parse Metal (GPU) memory allocation if not already captured
+  if (!updatedServer.metalMemoryMB) {
+    console.log(chalk.dim('Detecting Metal (GPU) memory allocation...'));
+    await new Promise(resolve => setTimeout(resolve, 8000)); // 8 second delay
+    const metalMemoryMB = await parseMetalMemoryFromLog(updatedServer.stderrPath);
+    if (metalMemoryMB) {
+      updatedServer = { ...updatedServer, metalMemoryMB };
+      await stateManager.saveServerConfig(updatedServer);
+      console.log(chalk.dim(`Metal memory: ${metalMemoryMB.toFixed(0)} MB`));
+    }
+  }
+
+  // 9. Display success
   console.log();
   console.log(chalk.green('✅ Server started successfully!'));
   console.log();

package/src/lib/history-manager.ts

@@ -0,0 +1,172 @@
+import { mkdir, readFile, writeFile, access, rename } from 'fs/promises';
+import { join } from 'path';
+import { homedir } from 'os';
+import { ServerMetrics, SystemMetrics } from '../types/monitor-types.js';
+import { HistoryData, HistorySnapshot, TIME_WINDOW_HOURS, TimeWindow } from '../types/history-types.js';
+
+export class HistoryManager {
+  private serverId: string;
+  private historyDir: string;
+  private historyPath: string;
+  private readonly MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
+
+  constructor(serverId: string) {
+    this.serverId = serverId;
+    this.historyDir = join(homedir(), '.llamacpp', 'history');
+    this.historyPath = join(this.historyDir, `${serverId}.json`);
+  }
+
+  /**
+   * Append a new snapshot to history (with auto-pruning)
+   */
+  async appendSnapshot(serverMetrics: ServerMetrics, systemMetrics?: SystemMetrics): Promise<void> {
+    try {
+      // Ensure history directory exists
+      await mkdir(this.historyDir, { recursive: true });
+
+      // Load existing history
+      const historyData = await this.loadHistoryData();
+
+      // Create new snapshot
+      const snapshot: HistorySnapshot = {
+        timestamp: Date.now(),
+        server: {
+          healthy: serverMetrics.healthy,
+          uptime: serverMetrics.uptime,
+          activeSlots: serverMetrics.activeSlots,
+          idleSlots: serverMetrics.idleSlots,
+          totalSlots: serverMetrics.totalSlots,
+          avgPromptSpeed: serverMetrics.avgPromptSpeed,
+          avgGenerateSpeed: serverMetrics.avgGenerateSpeed,
+          processMemory: serverMetrics.processMemory,
+          processCpuUsage: serverMetrics.processCpuUsage,
+        },
+        system: systemMetrics ? {
+          gpuUsage: systemMetrics.gpuUsage,
+          cpuUsage: systemMetrics.cpuUsage,
+          aneUsage: systemMetrics.aneUsage,
+          temperature: systemMetrics.temperature,
+          memoryUsed: systemMetrics.memoryUsed,
+          memoryTotal: systemMetrics.memoryTotal,
+        } : undefined,
+      };
+
+      // Append new snapshot
+      historyData.snapshots.push(snapshot);
+
+      // Prune old snapshots (keep only last 24h)
+      historyData.snapshots = this.pruneOldSnapshots(historyData.snapshots, this.MAX_AGE_MS);
+
+      // Atomic write: write to temp file in same directory, then rename
+      // This prevents read collisions during concurrent access
+      // IMPORTANT: temp file MUST be in same directory as destination for rename to work across filesystems
+      const tempPath = join(this.historyDir, `.${this.serverId}-${Date.now()}.tmp`);
+      await writeFile(tempPath, JSON.stringify(historyData, null, 2), 'utf-8');
+      await rename(tempPath, this.historyPath);
+    } catch (error) {
+      // Silent failure - don't interrupt monitoring
+      // Don't throw - just return silently to avoid polluting console
+      return;
+    }
+  }
+
+  /**
+   * Load all snapshots within specified time window
+   */
+  async loadHistory(windowHours: number): Promise<HistorySnapshot[]> {
+    // Retry logic for file I/O collisions during concurrent read/write
+    const maxRetries = 3;
+    let lastError: Error | null = null;
+
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      try {
+        const historyData = await this.loadHistoryData();
+        return this.filterByTimeWindow(historyData.snapshots, windowHours);
+      } catch (error) {
+        lastError = error as Error;
+        // Wait briefly before retry (exponential backoff)
+        if (attempt < maxRetries - 1) {
+          await new Promise(resolve => setTimeout(resolve, 50 * Math.pow(2, attempt)));
+        }
+      }
+    }
+
+    // All retries failed - throw error so it can be handled upstream
+    throw new Error(`Failed to load history after ${maxRetries} attempts: ${lastError?.message || 'Unknown error'}`);
+  }
+
+  /**
+   * Load history for specific time window type
+   */
+  async loadHistoryByWindow(window: TimeWindow): Promise<HistorySnapshot[]> {
+    return this.loadHistory(TIME_WINDOW_HOURS[window]);
+  }
+
+  /**
+   * Get file path for server history
+   */
+  getHistoryPath(): string {
+    return this.historyPath;
+  }
+
+  /**
+   * Check if history file exists
+   */
+  async hasHistory(): Promise<boolean> {
+    try {
+      await access(this.historyPath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Clear all history for server
+   */
+  async clearHistory(): Promise<void> {
+    const emptyHistory: HistoryData = {
+      serverId: this.serverId,
+      snapshots: [],
+    };
+
+    await mkdir(this.historyDir, { recursive: true });
+
+    // Atomic write - temp file in same directory as destination
+    const tempPath = join(this.historyDir, `.${this.serverId}-${Date.now()}.tmp`);
+    await writeFile(tempPath, JSON.stringify(emptyHistory, null, 2), 'utf-8');
+    await rename(tempPath, this.historyPath);
+  }
+
+  /**
+   * Load full history data from file
+   */
+  private async loadHistoryData(): Promise<HistoryData> {
+    try {
+      const content = await readFile(this.historyPath, 'utf-8');
+      return JSON.parse(content) as HistoryData;
+    } catch (error) {
+      // File doesn't exist or is corrupted, return empty history
+      return {
+        serverId: this.serverId,
+        snapshots: [],
+      };
+    }
+  }
+
+  /**
+   * Prune snapshots older than maxAge
+   */
+  private pruneOldSnapshots(snapshots: HistorySnapshot[], maxAgeMs: number): HistorySnapshot[] {
+    const cutoff = Date.now() - maxAgeMs;
+    return snapshots.filter(s => s.timestamp >= cutoff);
+  }
+
+  /**
+   * Filter snapshots by time window
+   */
+  private filterByTimeWindow(snapshots: HistorySnapshot[], windowHours: number): HistorySnapshot[] {
+    const cutoff = Date.now() - (windowHours * 60 * 60 * 1000);
+    return snapshots.filter(s => s.timestamp >= cutoff);
+  }
+}

package/src/lib/metrics-aggregator.ts

@@ -2,7 +2,7 @@ import { ServerConfig } from '../types/server-config.js';
 import { ServerMetrics, SlotInfo, MonitorData } from '../types/monitor-types.js';
 import { statusChecker } from './status-checker.js';
 import { systemCollector } from './system-collector.js';
-import { getProcessMemory } from '../utils/process-utils.js';
+import { getProcessMemory, getProcessCpu } from '../utils/process-utils.js';
 
 /**
  * Aggregates metrics from llama.cpp server API endpoints
@@ -128,10 +128,12 @@ export class MetricsAggregator {
    * Aggregate all server metrics
    * @param server - Server configuration
    * @param processMemory - Optional pre-fetched process memory (for batch collection)
+   * @param processCpuUsage - Optional pre-fetched process CPU usage (for batch collection)
    */
   async collectServerMetrics(
     server: ServerConfig,
-    processMemory?: number | null
+    processMemory?: number | null,
+    processCpuUsage?: number | null
   ): Promise<ServerMetrics> {
     const now = Date.now();
 
@@ -167,14 +169,17 @@ export class MetricsAggregator {
     }
 
     // Fetch detailed metrics in parallel
-    // If processMemory was pre-fetched (batch mode), use it; otherwise fetch individually
-    const [healthy, props, slots, fetchedMemory] = await Promise.all([
+    // If processMemory/CPU were pre-fetched (batch mode), use them; otherwise fetch individually
+    const [healthy, props, slots, fetchedMemory, fetchedCpu] = await Promise.all([
       this.getHealth(),
       this.getProps(),
       this.getSlots(),
       processMemory !== undefined
         ? Promise.resolve(processMemory)
         : (server.pid ? getProcessMemory(server.pid) : Promise.resolve(null)),
+      processCpuUsage !== undefined
+        ? Promise.resolve(processCpuUsage)
+        : (server.pid ? getProcessCpu(server.pid) : Promise.resolve(null)),
     ]);
 
     // Calculate slot statistics
@@ -201,6 +206,13 @@ export class MetricsAggregator {
           ) / processingSlots.length
         : undefined;
 
+    // Calculate total memory (CPU + Metal GPU memory if available)
+    let totalMemory = fetchedMemory ?? undefined;
+    if (totalMemory !== undefined && server.metalMemoryMB) {
+      // Add Metal memory (convert MB to bytes)
+      totalMemory += server.metalMemoryMB * 1024 * 1024;
+    }
+
     return {
       server,
       healthy,
@@ -214,7 +226,8 @@ export class MetricsAggregator {
       slots,
       avgPromptSpeed,
       avgGenerateSpeed,
-      processMemory: fetchedMemory ?? undefined,
+      processMemory: totalMemory,
+      processCpuUsage: fetchedCpu ?? undefined,
       timestamp: now,
       stale: false,
     };

package/src/lib/system-collector.ts

@@ -178,16 +178,12 @@ export class SystemCollector {
    */
   private parseVmStatOutput(output: string): {
     memoryUsed: number;
-    memoryTotal: number;
   } {
     const lines = output.split('\n');
     const pageSize = 16384; // 16KB on Apple Silicon
     let pagesActive = 0;
     let pagesWired = 0;
     let pagesCompressed = 0;
-    let pagesFree = 0;
-    let pagesInactive = 0;
-    let pagesSpeculative = 0;
 
     for (const line of lines) {
       const match = line.match(/Pages (.*?):\s+(\d+)\./);
@@ -198,49 +194,56 @@ export class SystemCollector {
         if (name === 'active') pagesActive = value;
         else if (name === 'wired down') pagesWired = value;
         else if (name === 'compressed') pagesCompressed = value;
-        else if (name === 'free') pagesFree = value;
-        else if (name === 'inactive') pagesInactive = value;
-        else if (name === 'speculative') pagesSpeculative = value;
       }
     }
 
     // Calculate used memory (active + wired + compressed)
+    // This matches what Activity Monitor and macmon report as "used"
     const usedPages = pagesActive + pagesWired + pagesCompressed;
     const memoryUsed = usedPages * pageSize;
 
-    // Calculate total memory (used + free + inactive + speculative)
-    const totalPages =
-      pagesActive +
-      pagesWired +
-      pagesCompressed +
-      pagesFree +
-      pagesInactive +
-      pagesSpeculative;
-    const memoryTotal = totalPages * pageSize;
-
-    return { memoryUsed, memoryTotal };
+    return { memoryUsed };
+  }
+
+  /**
+   * Get total system memory from sysctl
+   * Returns installed RAM size in bytes
+   */
+  private async getTotalMemory(): Promise<number> {
+    try {
+      const output = await execCommand('sysctl -n hw.memsize 2>/dev/null');
+      return parseInt(output.trim(), 10) || 0;
+    } catch {
+      return 0;
+    }
   }
 
   /**
-   * Collect vm_stat memory metrics
+   * Collect vm_stat memory metrics + total system memory from sysctl
    */
-  private async getVmStatMetrics(): Promise<{
+  private async getMemoryMetrics(): Promise<{
     memoryUsed: number;
     memoryTotal: number;
   }> {
     try {
-      const output = await execCommand('vm_stat 2>/dev/null');
-      return this.parseVmStatOutput(output);
+      // Get used memory from vm_stat
+      const vmStatOutput = await execCommand('vm_stat 2>/dev/null');
+      const { memoryUsed } = this.parseVmStatOutput(vmStatOutput);
+
+      // Get total installed RAM from sysctl (this is accurate)
+      const memoryTotal = await this.getTotalMemory();
+
+      return { memoryUsed, memoryTotal };
     } catch {
-      // Fallback to zeros if vm_stat fails
+      // Fallback to zeros if commands fail
       return { memoryUsed: 0, memoryTotal: 0 };
     }
   }
 
   /**
    * Collect all system metrics
-   * Attempts macmon first (GPU/CPU/ANE + memory), falls back to vm_stat (memory only)
-   * Caches results for 1.5s to prevent spawning multiple macmon processes
+   * Attempts macmon first (GPU/CPU/ANE), always gets memory from vm_stat + sysctl
+   * Caches results for 4s to prevent spawning multiple macmon processes
    */
   async collectSystemMetrics(): Promise<SystemMetrics> {
     const now = Date.now();
@@ -276,11 +279,11 @@ export class SystemCollector {
     const warnings: string[] = [];
     const now = Date.now();
 
-    // Try macmon first
+    // Try macmon first for GPU/CPU/ANE
     const macmonMetrics = await this.getMacmonMetrics();
 
-    // Always get memory from vm_stat (more reliable than macmon)
-    const memoryMetrics = await this.getVmStatMetrics();
+    // Always get memory from vm_stat + sysctl (accurate total from sysctl)
+    const memoryMetrics = await this.getMemoryMetrics();
 
     // Determine source and add warnings
     let source: 'macmon' | 'vm_stat' | 'none';