@appkit/llamacpp-cli 1.5.0 → 1.7.0

package/src/tui/MultiServerMonitorApp.ts

@@ -3,6 +3,8 @@ import { ServerConfig } from '../types/server-config.js';
 import { MetricsAggregator } from '../lib/metrics-aggregator.js';
 import { SystemCollector } from '../lib/system-collector.js';
 import { MonitorData, SystemMetrics } from '../types/monitor-types.js';
+import { HistoryManager } from '../lib/history-manager.js';
+import { createHistoricalUI, createMultiServerHistoricalUI } from './HistoricalMonitorApp.js';
 
 type ViewMode = 'list' | 'detail';
 
@@ -14,14 +16,19 @@ interface ServerMonitorData {
 
 export async function createMultiServerMonitorUI(
   screen: blessed.Widgets.Screen,
-  servers: ServerConfig[]
+  servers: ServerConfig[],
+  _fromPs: boolean = false,
+  directJumpIndex?: number
 ): Promise<void> {
   let updateInterval = 2000;
   let intervalId: NodeJS.Timeout | null = null;
-  let viewMode: ViewMode = 'list';
-  let selectedServerIndex = 0;
+  let viewMode: ViewMode = directJumpIndex !== undefined ? 'detail' : 'list';
+  let selectedServerIndex = directJumpIndex ?? 0;
+  let selectedRowIndex = directJumpIndex ?? 0; // Track which row is highlighted in list view
   let isLoading = false;
   let lastSystemMetrics: SystemMetrics | null = null;
+  let cameFromDirectJump = directJumpIndex !== undefined; // Track if we entered via ps <id>
+  let inHistoricalView = false; // Track whether we're in historical view to prevent key conflicts
 
   // Spinner animation
   const spinnerFrames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
@@ -30,11 +37,13 @@ export async function createMultiServerMonitorUI(
 
   const systemCollector = new SystemCollector();
   const aggregators = new Map<string, MetricsAggregator>();
+  const historyManagers = new Map<string, HistoryManager>();
   const serverDataMap = new Map<string, ServerMonitorData>();
 
-  // Initialize aggregators for each server
+  // Initialize aggregators and history managers for each server
   for (const server of servers) {
     aggregators.set(server.id, new MetricsAggregator(server));
+    historyManagers.set(server.id, new HistoryManager(server.id));
     serverDataMap.set(server.id, {
       server,
       data: null,
@@ -70,7 +79,7 @@ export async function createMultiServerMonitorUI(
     return '[' + '█'.repeat(Math.max(0, filled)) + '░'.repeat(Math.max(0, empty)) + ']';
   }
 
-  // Render system resources section
+  // Render system resources section (system-wide for list view)
   function renderSystemResources(systemMetrics: SystemMetrics | null): string {
     let content = '';
 
@@ -120,6 +129,97 @@ export async function createMultiServerMonitorUI(
     return content;
   }
 
+  // Render aggregate model resources (all running servers in list view)
+  function renderAggregateModelResources(): string {
+    let content = '';
+
+    content += '{bold}Model Resources{/bold}\n';
+    const termWidth = (screen.width as number) || 80;
+    const divider = '─'.repeat(termWidth - 2);
+    content += divider + '\n';
+
+    // Aggregate CPU and memory across all running servers (skip stopped servers)
+    let totalCpu = 0;
+    let totalMemoryBytes = 0;
+    let serverCount = 0;
+
+    for (const serverData of serverDataMap.values()) {
+      // Only count running servers with valid data
+      if (serverData.server.status === 'running' && serverData.data?.server && !serverData.data.server.stale) {
+        if (serverData.data.server.processCpuUsage !== undefined) {
+          totalCpu += serverData.data.server.processCpuUsage;
+          serverCount++;
+        }
+        if (serverData.data.server.processMemory !== undefined) {
+          totalMemoryBytes += serverData.data.server.processMemory;
+        }
+      }
+    }
+
+    if (serverCount === 0) {
+      content += '{gray-fg}No running servers{/gray-fg}\n';
+      return content;
+    }
+
+    // CPU: Sum of all process CPU percentages
+    const cpuBar = createProgressBar(Math.min(totalCpu, 100));
+    content += `CPU:    {cyan-fg}${cpuBar}{/cyan-fg} ${Math.round(totalCpu)}%`;
+    content += ` {gray-fg}(${serverCount} ${serverCount === 1 ? 'server' : 'servers'}){/gray-fg}\n`;
+
+    // Memory: Sum of all process memory
+    const totalMemoryGB = totalMemoryBytes / (1024 ** 3);
+    const estimatedMaxGB = serverCount * 8; // Assume ~8GB per server max
+    const memoryPercentage = Math.min((totalMemoryGB / estimatedMaxGB) * 100, 100);
+    const memoryBar = createProgressBar(memoryPercentage);
+    content += `Memory: {cyan-fg}${memoryBar}{/cyan-fg} ${totalMemoryGB.toFixed(2)} GB`;
+    content += ` {gray-fg}(${serverCount} ${serverCount === 1 ? 'server' : 'servers'}){/gray-fg}\n`;
+
+    return content;
+  }
+
+  // Render model resources section (per-process for detail view)
+  function renderModelResources(data: MonitorData): string {
+    let content = '';
+
+    content += '{bold}Model Resources{/bold}\n';
+    const termWidth = (screen.width as number) || 80;
+    const divider = '─'.repeat(termWidth - 2);
+    content += divider + '\n';
+
+    // GPU: System-wide (can't get per-process on macOS)
+    if (data.system && data.system.gpuUsage !== undefined) {
+      const bar = createProgressBar(data.system.gpuUsage);
+      content += `GPU:    {cyan-fg}${bar}{/cyan-fg} ${Math.round(data.system.gpuUsage)}% {gray-fg}(system){/gray-fg}`;
+
+      if (data.system.temperature !== undefined) {
+        content += ` - ${Math.round(data.system.temperature)}°C`;
+      }
+
+      content += '\n';
+    }
+
+    // CPU: Per-process
+    if (data.server.processCpuUsage !== undefined) {
+      const bar = createProgressBar(data.server.processCpuUsage);
+      content += `CPU:    {cyan-fg}${bar}{/cyan-fg} ${Math.round(data.server.processCpuUsage)}%\n`;
+    }
+
+    // Memory: Per-process
+    if (data.server.processMemory !== undefined) {
+      const memoryGB = data.server.processMemory / (1024 ** 3);
+      const estimatedMax = 8;
+      const memoryPercentage = Math.min((memoryGB / estimatedMax) * 100, 100);
+      const bar = createProgressBar(memoryPercentage);
+      content += `Memory: {cyan-fg}${bar}{/cyan-fg} ${memoryGB.toFixed(2)} GB\n`;
+    }
+
+    if (data.system && data.system.warnings && data.system.warnings.length > 0) {
+      content += `\n{yellow-fg}⚠ ${data.system.warnings.join(', ')}{/yellow-fg}\n`;
+    }
+
+    return content;
+  }
+
   // Show loading spinner
   function showLoading(): void {
     if (isLoading) return; // Already loading
@@ -169,53 +269,76 @@ export async function createMultiServerMonitorUI(
     let content = '';
 
     // Header
-    content += '{bold}{blue-fg}═══ llama.cpp Multi-Server Monitor ═══{/blue-fg}{/bold}\n';
-
-    // Status line with optional spinner
-    const statusPlainText = 'Press 1-9 for details | [F] Filter | [Q] Quit';
-    const spinnerChar = isLoading ? spinnerFrames[spinnerFrameIndex] : '';
-    const spinnerText = spinnerChar ? `  {cyan-fg}${spinnerChar}{/cyan-fg}` : '';
-
-    content += `{gray-fg}${statusPlainText}${spinnerText}{/gray-fg}\n\n`;
+    content += '{bold}{blue-fg}═══ llama.cpp{/blue-fg}{/bold}\n\n';
 
     // System resources
     content += renderSystemResources(systemMetrics);
     content += '\n';
 
+    // Aggregate model resources (CPU + memory for all running servers)
+    content += renderAggregateModelResources();
+    content += '\n';
+
     // Server list header
     const runningCount = servers.filter(s => s.status === 'running').length;
     const stoppedCount = servers.filter(s => s.status !== 'running').length;
     content += `{bold}Servers (${runningCount} running, ${stoppedCount} stopped){/bold}\n`;
-    content += '{gray-fg}Press number for details{/gray-fg}\n';
+    content += '{gray-fg}Use arrow keys to navigate, Enter to view details{/gray-fg}\n';
     content += divider + '\n';
 
-    // Table header
-    content += '{bold}# │ Server ID        │ Port │ Status │ Slots │ tok/s  │ Memory{/bold}\n';
+    // Calculate Server ID column width (variable based on screen width)
+    // Fixed columns breakdown:
+    // indicator(1) + " │ "(3) + " │ "(3) + port(4) + " │ "(3) + status(6) + "│ "(2) +
+    // slots(5) + " │ "(3) + tok/s(6) + " │ "(3) + memory(7) = 46
+    const fixedColumnsWidth = 48; // Add 2 extra for safety margin
+    const minServerIdWidth = 20;
+    const maxServerIdWidth = 60;
+    const serverIdWidth = Math.max(
+      minServerIdWidth,
+      Math.min(maxServerIdWidth, termWidth - fixedColumnsWidth)
+    );
+
+    // Table header with variable Server ID width
+    const serverIdHeader = 'Server ID'.padEnd(serverIdWidth);
+    content += `{bold}  │ ${serverIdHeader}│ Port │ Status │ Slots │ tok/s  │ Memory{/bold}\n`;
     content += divider + '\n';
 
     // Server rows
     servers.forEach((server, index) => {
       const serverData = serverDataMap.get(server.id);
-      const num = index + 1;
+      const isSelected = index === selectedRowIndex;
+
+      // Selection indicator (arrow for selected row)
+      // Use plain arrow for selected (will be white), colored for unselected indicator
+      const indicator = isSelected ? '►' : ' ';
 
-      // Server ID (truncate if needed)
-      const serverId = server.id.padEnd(16).substring(0, 16);
+      // Server ID (variable width, truncate if longer than available space)
+      const serverId = server.id.padEnd(serverIdWidth).substring(0, serverIdWidth);
 
       // Port
       const port = server.port.toString().padStart(4);
 
-      // Status
+      // Status - Check actual server status first, then health
+      // Build two versions: colored for normal, plain for selected
       let status = '';
-      if (serverData?.data) {
+      let statusPlain = '';
+      if (server.status !== 'running') {
+        // Server is stopped according to config
+        status = '{gray-fg}○ OFF{/gray-fg} ';
+        statusPlain = '○ OFF ';
+      } else if (serverData?.data) {
+        // Server is running and we have data
         if (serverData.data.server.healthy) {
           status = '{green-fg}● RUN{/green-fg} ';
+          statusPlain = '● RUN ';
         } else {
           status = '{red-fg}● ERR{/red-fg} ';
+          statusPlain = '● ERR ';
         }
-      } else if (server.status === 'running') {
-        status = '{yellow-fg}● ...{/yellow-fg} ';
       } else {
-        status = '{gray-fg}○ STOP{/gray-fg}';
+        // Server is running but no data yet (still loading)
+        status = '{yellow-fg}● ...{/yellow-fg} ';
+        statusPlain = '● ... ';
       }
 
       // Slots
@@ -247,13 +370,23 @@ export async function createMultiServerMonitorUI(
         }
       }
 
-      content += `${num} │ ${serverId} │ ${port} │ ${status} │ ${slots} │ ${tokensPerSec} │ ${memory}\n`;
+      // Build row content - use plain status for selected rows
+      let rowContent = '';
+      if (isSelected) {
+        // Use color code 15 (bright white) with cyan background
+        // When white-bg worked, it was probably auto-selecting bright white fg
+        rowContent = `{cyan-bg}{15-fg}${indicator} │ ${serverId} │ ${port} │ ${statusPlain}│ ${slots} │ ${tokensPerSec} │ ${memory}{/15-fg}{/cyan-bg}`;
+      } else {
+        // Use colored status for normal rows
+        rowContent = `${indicator} │ ${serverId} │ ${port} │ ${status}│ ${slots} │ ${tokensPerSec} │ ${memory}`;
+      }
+
+      content += rowContent + '\n';
     });
 
     // Footer
     content += '\n' + divider + '\n';
-    content += `{gray-fg}Updated: ${new Date().toLocaleTimeString()} | `;
-    content += `Interval: ${updateInterval}ms | [R]efresh [+/-]Speed{/gray-fg}`;
+    content += `{gray-fg}Updated: ${new Date().toLocaleTimeString()} | [H]istory [Q]uit{/gray-fg}`;
 
     return content;
   }
@@ -267,18 +400,50 @@ export async function createMultiServerMonitorUI(
     let content = '';
 
     // Header
-    content += `{bold}{blue-fg}═══ Server #${selectedServerIndex + 1}: ${server.id} (${server.port}) ═══{/blue-fg}{/bold}\n`;
+    content += `{bold}{blue-fg}═══ ${server.id} (${server.port}){/blue-fg}{/bold}\n\n`;
 
-    // Status line with optional spinner
-    const statusPlainText = '[ESC] Back to list | [Q] Quit';
-    const spinnerChar = isLoading ? spinnerFrames[spinnerFrameIndex] : '';
-    const spinnerText = spinnerChar ? `  {cyan-fg}${spinnerChar}{/cyan-fg}` : '';
+    // Check if server is stopped
+    if (server.status !== 'running') {
+      // Show stopped server configuration (no metrics)
+      content += '{bold}Server Information{/bold}\n';
+      content += divider + '\n';
+      content += `Status:   {gray-fg}○ STOPPED{/gray-fg}\n`;
+      content += `Model:    ${server.modelName}\n`;
+      const displayHost = server.host || '127.0.0.1';
+      content += `Endpoint: http://${displayHost}:${server.port}\n`;
+      content += '\n';
 
-    content += `{gray-fg}${statusPlainText}${spinnerText}{/gray-fg}\n\n`;
+      content += '{bold}Configuration{/bold}\n';
+      content += divider + '\n';
+      content += `Threads:    ${server.threads}\n`;
+      content += `Context:    ${server.ctxSize} tokens\n`;
+      content += `GPU Layers: ${server.gpuLayers}\n`;
+      if (server.verbose) {
+        content += `Verbose:    Enabled\n`;
+      }
+      if (server.customFlags && server.customFlags.length > 0) {
+        content += `Flags:      ${server.customFlags.join(', ')}\n`;
+      }
+      content += '\n';
 
-    // System resources
-    content += renderSystemResources(systemMetrics);
-    content += '\n';
+      if (server.lastStarted) {
+        content += '{bold}Last Activity{/bold}\n';
+        content += divider + '\n';
+        content += `Started:  ${new Date(server.lastStarted).toLocaleString()}\n`;
+        if (server.lastStopped) {
+          content += `Stopped:  ${new Date(server.lastStopped).toLocaleString()}\n`;
+        }
+        content += '\n';
+      }
+
+      content += '{bold}Quick Actions{/bold}\n';
+      content += divider + '\n';
+      content += `{dim}Start server:  llamacpp server start ${server.port}{/dim}\n`;
+      content += `{dim}Update config: llamacpp server config ${server.port} [options]{/dim}\n`;
+      content += `{dim}View logs:     llamacpp server logs ${server.port}{/dim}\n`;
+
+      return content;
+    }
 
     if (!serverData?.data) {
       content += '{yellow-fg}Loading server data...{/yellow-fg}\n';
@@ -287,6 +452,10 @@ export async function createMultiServerMonitorUI(
 
     const data = serverData.data;
 
+    // Model resources (per-process)
+    content += renderModelResources(data);
+    content += '\n';
+
     // Server Information
     content += '{bold}Server Information{/bold}\n';
     content += divider + '\n';
@@ -308,21 +477,7 @@ export async function createMultiServerMonitorUI(
 
     // Handle null host (legacy configs) by defaulting to 127.0.0.1
     const displayHost = server.host || '127.0.0.1';
-    content += `Endpoint: http://${displayHost}:${server.port}`;
-
-    // Add actual process memory (if available)
-    if (data.server.processMemory) {
-      const bytes = data.server.processMemory;
-      let memStr;
-      if (bytes >= 1024 * 1024 * 1024) {
-        memStr = `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
-      } else {
-        memStr = `${Math.round(bytes / (1024 * 1024))} MB`;
-      }
-      content += `       Memory:  ${memStr}\n`;
-    } else {
-      content += '\n';
-    }
+    content += `Endpoint: http://${displayHost}:${server.port}\n`;
 
     content += `Slots:    ${data.server.activeSlots} active / ${data.server.totalSlots} total\n`;
     content += '\n';
@@ -377,8 +532,7 @@ export async function createMultiServerMonitorUI(
 
     // Footer
     content += divider + '\n';
-    content += `{gray-fg}Updated: ${data.lastUpdated.toLocaleTimeString()} | `;
-    content += `Interval: ${updateInterval}ms | [R]efresh [+/-]Speed{/gray-fg}`;
+    content += `{gray-fg}Updated: ${data.lastUpdated.toLocaleTimeString()} | [H]istory [ESC] Back [Q]uit{/gray-fg}`;
 
     return content;
   }
@@ -390,51 +544,68 @@ export async function createMultiServerMonitorUI(
       // This prevents spawning multiple macmon processes
       const systemMetricsPromise = systemCollector.collectSystemMetrics();
 
-      // Batch collect process memory for ALL servers in one top call
+      // Batch collect process memory and CPU for ALL servers in parallel
       // This prevents spawning multiple top processes (5x speedup)
-      const { getBatchProcessMemory } = await import('../utils/process-utils.js');
+      const { getBatchProcessMemory, getBatchProcessCpu } = await import('../utils/process-utils.js');
       const pids = servers.filter(s => s.pid).map(s => s.pid!);
       const memoryMapPromise = pids.length > 0
         ? getBatchProcessMemory(pids)
         : Promise.resolve(new Map<number, number | null>());
+      const cpuMapPromise = pids.length > 0
+        ? getBatchProcessCpu(pids)
+        : Promise.resolve(new Map<number, number | null>());
 
-      // Wait for memory batch to complete
-      const memoryMap = await memoryMapPromise;
-
-      // Collect server metrics only (NOT system metrics) for each server
-      const promises = servers.map(async (server) => {
-        const aggregator = aggregators.get(server.id)!;
-        try {
-          // Use collectServerMetrics instead of collectMonitorData
-          // to avoid spawning macmon per server
-          // Pass pre-fetched memory to avoid spawning top per server
-          const serverMetrics = await aggregator.collectServerMetrics(
-            server,
-            server.pid ? memoryMap.get(server.pid) ?? null : null
-          );
-
-          // Build MonitorData manually with shared system metrics
-          const data: MonitorData = {
-            server: serverMetrics,
-            system: undefined, // Will be set after system metrics resolve
-            lastUpdated: new Date(),
-            updateInterval,
-            consecutiveFailures: 0,
-          };
+      // Wait for both batches to complete
+      const [memoryMap, cpuMap] = await Promise.all([memoryMapPromise, cpuMapPromise]);
+
+      // Collect server metrics only for RUNNING servers (skip stopped servers)
+      const promises = servers
+        .filter(server => server.status === 'running')
+        .map(async (server) => {
+          const aggregator = aggregators.get(server.id)!;
+          try {
+            // Use collectServerMetrics instead of collectMonitorData
+            // to avoid spawning macmon per server
+            // Pass pre-fetched memory and CPU to avoid spawning top per server
+            const serverMetrics = await aggregator.collectServerMetrics(
+              server,
+              server.pid ? memoryMap.get(server.pid) ?? null : null,
+              server.pid ? cpuMap.get(server.pid) ?? null : null
+            );
+
+            // Build MonitorData manually with shared system metrics
+            const data: MonitorData = {
+              server: serverMetrics,
+              system: undefined, // Will be set after system metrics resolve
+              lastUpdated: new Date(),
+              updateInterval,
+              consecutiveFailures: 0,
+            };
+
+            serverDataMap.set(server.id, {
+              server,
+              data,
+              error: null,
+            });
+          } catch (err) {
+            serverDataMap.set(server.id, {
+              server,
+              data: null,
+              error: err instanceof Error ? err.message : 'Unknown error',
+            });
+          }
+        });
 
-          serverDataMap.set(server.id, {
-            server,
-            data,
-            error: null,
-          });
-        } catch (err) {
+      // Set null data for stopped servers (no metrics collection)
+      servers
+        .filter(server => server.status !== 'running')
+        .forEach(server => {
           serverDataMap.set(server.id, {
             server,
             data: null,
-            error: err instanceof Error ? err.message : 'Unknown error',
+            error: null,
           });
-        }
-      });
+        });
 
       // Wait for both system metrics and server metrics to complete
       const systemMetrics = await systemMetricsPromise;
@@ -450,6 +621,19 @@ export async function createMultiServerMonitorUI(
         }
       }
 
+      // Append to history for each server (silent failure)
+      // Only save history for servers that are healthy and not stale
+      for (const [serverId, serverData] of serverDataMap) {
+        if (serverData.data && !serverData.data.server.stale && serverData.data.server.healthy) {
+          const manager = historyManagers.get(serverId);
+          manager?.appendSnapshot(serverData.data.server, serverData.data.system)
+            .catch(err => {
+              // Don't interrupt monitoring on history write failure
+              console.error(`Failed to save history for ${serverId}:`, err);
+            });
+        }
+      }
+
       // Render once with complete data
       let content = '';
       if (viewMode === 'list') {
@@ -485,12 +669,32 @@ export async function createMultiServerMonitorUI(
     intervalId = setInterval(fetchData, updateInterval);
   }
 
-  // Keyboard shortcuts - List view
-  screen.key(['1', '2', '3', '4', '5', '6', '7', '8', '9'], (ch) => {
-    const index = parseInt(ch, 10) - 1;
-    if (index >= 0 && index < servers.length) {
+  // Keyboard shortcuts - List view navigation with arrow keys
+  screen.key(['up', 'k'], () => {
+    if (viewMode === 'list') {
+      selectedRowIndex = Math.max(0, selectedRowIndex - 1);
+      // Re-render immediately for responsive feel
+      const content = renderListView(lastSystemMetrics);
+      contentBox.setContent(content);
+      screen.render();
+    }
+  });
+
+  screen.key(['down', 'j'], () => {
+    if (viewMode === 'list') {
+      selectedRowIndex = Math.min(servers.length - 1, selectedRowIndex + 1);
+      // Re-render immediately for responsive feel
+      const content = renderListView(lastSystemMetrics);
+      contentBox.setContent(content);
+      screen.render();
+    }
+  });
+
+  // Enter key to view details for selected server
+  screen.key(['enter'], () => {
+    if (viewMode === 'list') {
       showLoading();
-      selectedServerIndex = index;
+      selectedServerIndex = selectedRowIndex;
       viewMode = 'detail';
       fetchData();
     }
@@ -498,27 +702,68 @@ export async function createMultiServerMonitorUI(
 
   // Keyboard shortcuts - Detail view
   screen.key(['escape'], () => {
+    // Don't handle ESC if we're in historical view - let historical view handle it
+    if (inHistoricalView) return;
+
     if (viewMode === 'detail') {
       showLoading();
       viewMode = 'list';
+      cameFromDirectJump = false; // Clear direct jump flag when returning to list
       fetchData();
+    } else if (viewMode === 'list') {
+      // ESC in list view - exit
+      showLoading();
+      if (intervalId) clearInterval(intervalId);
+      if (spinnerIntervalId) clearInterval(spinnerIntervalId);
+      setTimeout(() => {
+        screen.destroy();
+        process.exit(0);
+      }, 100);
     }
   });
 
   // Keyboard shortcuts - Common
-  screen.key(['r', 'R'], () => {
-    showLoading();
-    fetchData();
-  });
 
-  screen.key(['+', '='], () => {
-    updateInterval = Math.max(500, updateInterval - 500);
-    startPolling();
-  });
+  screen.key(['h', 'H'], async () => {
+    // Prevent entering historical view if already there
+    if (inHistoricalView) return;
+
+    // Keep polling in background for live historical updates
+    // Stop spinner if running
+    if (spinnerIntervalId) clearInterval(spinnerIntervalId);
 
-  screen.key(['-', '_'], () => {
-    updateInterval = Math.min(10000, updateInterval + 500);
-    startPolling();
+    // Remove current content box
+    screen.remove(contentBox);
+
+    // Mark that we're in historical view
+    inHistoricalView = true;
+
+    if (viewMode === 'list') {
+      // Show multi-server historical view
+      await createMultiServerHistoricalUI(screen, servers, selectedServerIndex, () => {
+        // Mark that we've left historical view
+        inHistoricalView = false;
+        // Re-attach content box when returning from history
+        screen.append(contentBox);
+        // Re-render the list view
+        const content = renderListView(lastSystemMetrics);
+        contentBox.setContent(content);
+        screen.render();
+      });
+    } else {
+      // Show single-server historical view for selected server
+      const selectedServer = servers[selectedServerIndex];
+      await createHistoricalUI(screen, selectedServer, () => {
+        // Mark that we've left historical view
+        inHistoricalView = false;
+        // Re-attach content box when returning from history
+        screen.append(contentBox);
+        // Re-render the detail view
+        const content = renderDetailView(lastSystemMetrics);
+        contentBox.setContent(content);
+        screen.render();
+      });
+    }
   });
 
   screen.key(['q', 'Q', 'C-c'], () => {

package/src/types/history-types.ts

@@ -0,0 +1,39 @@
+// Historical monitoring data types
+
+export interface HistorySnapshot {
+  timestamp: number; // Unix timestamp in milliseconds
+  server: {
+    healthy: boolean;
+    uptime?: string;
+    activeSlots: number;
+    idleSlots: number;
+    totalSlots: number;
+    avgPromptSpeed?: number;       // Tokens per second
+    avgGenerateSpeed?: number;     // Tokens per second
+    processMemory?: number;        // Bytes (RSS)
+    processCpuUsage?: number;      // Percentage (0-100+) from ps
+  };
+  system?: {
+    gpuUsage?: number;             // Percentage (0-100)
+    cpuUsage?: number;             // Percentage (0-100)
+    aneUsage?: number;             // Percentage (0-100)
+    temperature?: number;          // Celsius
+    memoryUsed: number;            // Bytes
+    memoryTotal: number;           // Bytes
+  };
+}
+
+export interface HistoryData {
+  serverId: string;
+  snapshots: HistorySnapshot[];
+}
+
+export type TimeWindow = '1h' | '6h' | '24h';
+
+export const TIME_WINDOW_HOURS: Record<TimeWindow, number> = {
+  '1h': 1,
+  '6h': 6,
+  '24h': 24,
+};
+
+export const TIME_WINDOWS: TimeWindow[] = ['1h', '6h', '24h'];

package/src/types/monitor-types.ts

@@ -123,6 +123,7 @@ export interface ServerMetrics {
 
   // Process metrics
   processMemory?: number;     // Bytes (actual RSS from top command)
+  processCpuUsage?: number;   // Percentage (0-100+) from ps command
 
   // Timestamp
   timestamp: number;

package/src/types/server-config.ts

@@ -22,6 +22,7 @@ export interface ServerConfig {
   createdAt: string;       // ISO timestamp
   lastStarted?: string;    // ISO timestamp
   lastStopped?: string;    // ISO timestamp
+  metalMemoryMB?: number;  // Metal (GPU) memory allocated in MB (parsed from logs)
 
   // launchctl metadata
   plistPath: string;       // Full path to plist file