@flame0510/project-aether 1.7.0 → 1.8.0

package/README.md

@@ -18,7 +18,8 @@
 - **Live event stream** — Real-time session lineage, agent spawning, cost reporting
 - **Workspace browser** — Browse and edit agent files live from the dashboard
 - **Provider proxy** — OpenAI-compatible chat completions proxy that routes through a shared key vault
-- **System health** — runtime sessions, event ingestion and errors, cron jobs and usage cost, with recommendations
+- **System health** — runtime sessions, daemon heartbeat and errors, cron jobs and usage cost, with recommendations
+- **System** — the host machine's CPU, memory, swap and storage, now and over 1h / 24h / 7d / 30d (sampled every 30 s, kept 30 days)
 - **Cost tracking** — Per-session, per-model, per-time-range usage and cost reporting
 - **PWA-ready** — Install on phone for mobile monitoring
 

package/app/api/assistant/route.ts

@@ -18,7 +18,7 @@ interface PageContext {
 const PAGE_CONTEXT: Record<string, PageContext> = {
   '/': {
     title: 'Dashboard',
-    description: 'Main overview with live session list, cost summary by model, health cards for Rev4a\'s own runtime, cron and lineage, and a real-time event feed. It shows no CPU, RAM, disk or load average: those are collected by the daemon but no page displays them.',
+    description: 'Main overview with live session list, cost summary by model, health cards for Rev4a\'s own runtime, cron and lineage, and a real-time event feed. The Machine card shows the server\'s CPU, RAM and fullest disk right now, coloured past a threshold, with a link to the System page.',
     actions: ['Click any session row to open the Session Drawer for full details and tool call history', 'View cost breakdown by model and time period', 'See live events as they happen', 'Take a screenshot of the dashboard (mobile)'],
     hints: 'The Session Drawer shows you the complete tool call history for any session — just click a row.',
   },
@@ -46,6 +46,12 @@ const PAGE_CONTEXT: Record<string, PageContext> = {
     actions: ['See all scheduled cron jobs at a glance', 'Check when each cron last ran and when it will run next', 'View job status (scheduled, running, completed, error)'],
     hints: 'Cron jobs use standard cron syntax. If a job shows "error" status, it means the last execution failed.',
   },
+  '/system': {
+    title: 'System',
+    description: 'The machine Rev4a runs on: CPU (percent, cores, load average 1/5/15 min), memory (used, available, swap) and storage (each disk with used and free space, and whether it is the system disk, Docker\'s data or Rev4a\'s data), now and over 1h, 24h, 7d or 30d. Sampled by the Rev4a daemon every 30 seconds, kept 30 days.',
+    actions: ['Read the current CPU, memory, swap and disk usage', 'Switch the charts between 1h, 24h, 7d and 30d', 'Hover a chart (or focus it and use the arrow keys) to read the average and peak at a point', 'Open a chart\'s Table view for the numbers'],
+    hints: 'Bars turn yellow then red past their thresholds: CPU and memory at 85% and 95%, disks at 80% and 90%. If the page says "Not collecting", the Rev4a daemon is not sampling: check that rev4a serve is running.',
+  },
   '/containers': {
     title: 'Containers',
     description: 'List of all Docker containers on the server, running or stopped (not just agents). Shows container ID, name, image, status, state, ports, and IP address.',
@@ -117,6 +123,7 @@ Navigation links inside Rev4a:
 - Gateway → /gateway
 - Cron Jobs → /crons
 - Containers → /containers
+- System (CPU, RAM, disk of the server) → /system
 - Config → /config
 - Lineage → /lineage?period=7d
 - Memory / Context → /memory

package/app/api/metrics/route.ts

@@ -1,34 +1,137 @@
+import os from 'os';
 import { NextResponse, type NextRequest } from 'next/server';
-import { openDb } from '@/lib/db';
+import { openMetricsDb } from '@/lib/metrics-db';
 import { requireAuthJWT } from '@/lib/rev4a-auth';
 
 export const dynamic = 'force-dynamic';
 
+/** How often daemon.js samples the machine; the page calls a sample older than 3× this stale. */
+const SAMPLE_INTERVAL_S = 30;
+
+/**
+ * History ranges and their span in seconds; each is cut into about POINTS buckets, never
+ * shorter than two samples — a 30 s bucket on a 30 s timer that drifts a second would
+ * leave empty buckets, drawn as gaps, where no sample is missing.
+ */
+const RANGES = { '1h': 3600, '24h': 86_400, '7d': 7 * 86_400, '30d': 30 * 86_400 } as const;
+type Range = keyof typeof RANGES;
+const POINTS = 120;
+
+interface LatestRow {
+  ts: number;
+  cpu: number | null;
+  ram_used_mb: number | null;
+  ram_total_mb: number | null;
+  swap_used_mb: number | null;
+  swap_total_mb: number | null;
+  load_avg_1m: number | null;
+  load_avg_5m: number | null;
+  load_avg_15m: number | null;
+}
+
+interface DiskRow {
+  mount: string;
+  roles: string | null;
+  used_mb: number;
+  avail_mb: number;
+  total_mb: number;
+}
+
+/** Share of a filesystem in use, as `df` computes it: used / (used + avail). */
+function diskPercent(used: number, avail: number): number {
+  return used + avail > 0 ? Math.round((used / (used + avail)) * 100) : 0;
+}
+
+/**
+ * Machine metrics of the host Rev4a runs on: the latest sample, live host facts, and a
+ * bucketed history for one range (average and peak per bucket).
+ */
 export async function GET(request: NextRequest): Promise<NextResponse> {
   const denied = await requireAuthJWT(request); if (denied) return denied as any;
-  const { searchParams } = new URL(request.url);
-  const limit = Math.min(Number.parseInt(searchParams.get('limit') ?? '60', 10), 1000);
+  const param = new URL(request.url).searchParams.get('range') ?? '1h';
+  if (!Object.hasOwn(RANGES, param)) {
+    return NextResponse.json({ error: `range must be one of ${Object.keys(RANGES).join(', ')}` }, { status: 400 });
+  }
+  const range = param as Range;
+  const span = RANGES[range];
+  const bucket = Math.max(2 * SAMPLE_INTERVAL_S, Math.round(span / POINTS));
+  const now = Math.floor(Date.now() / 1000);
+  const since = now - span;
+
+  const host = {
+    hostname: os.hostname(),
+    platform: os.platform(),
+    cores: os.cpus().length,
+    uptime_s: Math.round(os.uptime()),
+  };
+  const base = { host, interval_s: SAMPLE_INTERVAL_S, range, bucket_s: bucket };
+
   try {
-    const db = openDb();
-    const latest = db
-      .prepare(
-        `SELECT cpu_percent as cpu, ram_used_mb, ram_total_mb, disk_used_gb, disk_total_gb,
-         load_avg_1m as load_avg, ts FROM system_metrics ORDER BY ts DESC LIMIT 1`,
-      )
-      .get();
-    const history = db
-      .prepare(
-        'SELECT cpu_percent as cpu, ram_used_mb, ram_total_mb, ts FROM system_metrics ORDER BY ts DESC LIMIT ?',
-      )
-      .all(limit)
-      .reverse();
-    const stats_24h = db
-      .prepare(
-        'SELECT ROUND(AVG(cpu_percent)) as cpu_avg, MAX(cpu_percent) as cpu_max FROM system_metrics WHERE ts > ?',
-      )
-      .get(Math.floor(Date.now() / 1000) - 86400);
-    db.close();
-    return NextResponse.json({ latest, history, stats_24h });
+    const db = openMetricsDb();
+    // No metrics database yet: the daemon has not sampled once.
+    if (!db) return NextResponse.json({ ...base, latest: null, history: [], disk_history: [] });
+    try {
+      const latest = db
+        .prepare(
+          `SELECT ts, cpu_percent AS cpu, ram_used_mb, ram_total_mb, swap_used_mb, swap_total_mb,
+             load_avg_1m, load_avg_5m, load_avg_15m
+           FROM system_metrics ORDER BY ts DESC LIMIT 1`,
+        )
+        .get() as LatestRow | undefined;
+
+      const disks = latest
+        ? (db
+            // One row per mount even if two samples share this ts (a second daemon, a clock step).
+            .prepare('SELECT mount, roles, used_mb, avail_mb, total_mb FROM system_disks WHERE ts = ? GROUP BY mount ORDER BY mount')
+            .all(latest.ts) as DiskRow[])
+        : [];
+
+      const history = db
+        .prepare(
+          `SELECT (ts / CAST(@bucket AS INTEGER)) * CAST(@bucket AS INTEGER) AS ts,
+             ROUND(AVG(cpu_percent), 1) AS cpu_avg,
+             ROUND(MAX(cpu_percent), 1) AS cpu_max,
+             ROUND(AVG(ram_used_mb * 100.0 / NULLIF(ram_total_mb, 0)), 1) AS ram_avg,
+             ROUND(MAX(ram_used_mb * 100.0 / NULLIF(ram_total_mb, 0)), 1) AS ram_max
+           FROM system_metrics WHERE ts > @since
+           GROUP BY ts / CAST(@bucket AS INTEGER) ORDER BY ts`,
+        )
+        .all({ bucket, since });
+
+      const diskRows = db
+        .prepare(
+          `SELECT mount, (ts / CAST(@bucket AS INTEGER)) * CAST(@bucket AS INTEGER) AS ts,
+             ROUND(AVG(used_mb * 100.0 / NULLIF(used_mb + avail_mb, 0)), 1) AS percent
+           FROM system_disks WHERE ts > @since
+           GROUP BY mount, ts / CAST(@bucket AS INTEGER) ORDER BY mount, ts`,
+        )
+        .all({ bucket, since }) as { mount: string; ts: number; percent: number | null }[];
+      const diskHistory = new Map<string, { ts: number; percent: number | null }[]>();
+      for (const r of diskRows) {
+        const points = diskHistory.get(r.mount) ?? [];
+        points.push({ ts: r.ts, percent: r.percent });
+        diskHistory.set(r.mount, points);
+      }
+
+      return NextResponse.json({
+        latest: latest
+          ? {
+              ...latest,
+              age_s: Math.max(0, now - latest.ts),
+              disks: disks.map((d) => ({
+                ...d,
+                roles: d.roles ? d.roles.split(',') : [],
+                percent: diskPercent(d.used_mb, d.avail_mb),
+              })),
+            }
+          : null,
+        ...base,
+        history,
+        disk_history: [...diskHistory].map(([mount, points]) => ({ mount, points })),
+      });
+    } finally {
+      db.close();
+    }
   } catch (e: unknown) {
     return NextResponse.json({ error: (e as Error).message }, { status: 500 });
   }

package/app/api/system-health/route.ts

@@ -1,5 +1,6 @@
 import { NextResponse, type NextRequest } from 'next/server';
 import { openDb } from '@/lib/db';
+import { latestMetricSample } from '@/lib/metrics-db';
 import { maybeSendAlert } from '@/lib/alerts';
 import { summarizeBilling } from '@/lib/billing';
 import { listOpenClawCronJobs } from '@/lib/openclaw-cron';
@@ -83,11 +84,12 @@ export async function GET(request: NextRequest): Promise<NextResponse> {
       source: 'runtime',
     });
 
-    const hasSystemMetrics = tableExists(db, 'system_metrics');
-    const latestMetricRow = hasSystemMetrics
-      ? (db.prepare('SELECT MAX(ts) AS latest FROM system_metrics').get() as { latest: number | null })
-      : null;
-    const latestMetricTs = getSeconds(latestMetricRow?.latest);
+    // The daemon's heartbeat: the age of its newest machine metrics sample (metrics.db, every
+    // 30 s on a timer of its own). It proves the daemon is alive — not that the OpenClaw session
+    // poll is ingesting, which runs separately and may have nothing to poll on this host.
+    const metricSample = latestMetricSample();
+    const hasSystemMetrics = metricSample.available;
+    const latestMetricTs = getSeconds(metricSample.ts);
     const latestMetricAgeSeconds = latestMetricTs ? Math.floor(Date.now() / 1000) - latestMetricTs : null;
     const runtimeFreshnessHealth: Health = !hasSystemMetrics
       ? 'ok'
@@ -98,15 +100,17 @@ export async function GET(request: NextRequest): Promise<NextResponse> {
           : 'ok';
 
     checks.push({
-      id: 'runtime.ingestion',
-      label: 'Runtime ingestion',
+      id: 'runtime.heartbeat',
+      label: 'Daemon heartbeat',
       health: runtimeFreshnessHealth,
       value: hasSystemMetrics ? (latestMetricTs ? 'fresh' : 'missing') : 'n/d',
-      details: !hasSystemMetrics
-        ? 'Heartbeat metrics unavailable in this DB'
-        : latestMetricAgeSeconds === null
-          ? 'No heartbeat metrics detected'
-          : `last heartbeat metrics ${latestMetricAgeSeconds}s ago`,
+      details: metricSample.error
+        ? `metrics.db unreadable: ${metricSample.error}`
+        : !hasSystemMetrics
+          ? 'No machine metrics database yet'
+          : latestMetricAgeSeconds === null
+            ? 'No machine metrics sample yet'
+            : `last machine metrics sample ${latestMetricAgeSeconds}s ago`,
       source: 'runtime',
     });
 
@@ -115,11 +119,11 @@ export async function GET(request: NextRequest): Promise<NextResponse> {
         id: 'runtime.check-freshness',
         severity: 'warning',
         source: 'runtime',
-        title: 'Check Rev4a runtime heartbeat',
+        title: 'Check that the Rev4a daemon is running',
         details: latestMetricAgeSeconds === null
-          ? 'No heartbeat metrics available from runtime.'
-          : `Heartbeat metrics stuck for ${latestMetricAgeSeconds}s (threshold ${staleThresholdSeconds}s).`,
-        actionHref: '/lineage',
+          ? 'No machine metrics sample yet.'
+          : `No machine metrics sample for ${latestMetricAgeSeconds}s (threshold ${staleThresholdSeconds}s).`,
+        actionHref: '/system',
         dismissible: false,
         createdAt,
       });
@@ -130,11 +134,11 @@ export async function GET(request: NextRequest): Promise<NextResponse> {
       kind: 'db-freshness',
       title: 'Rev4a DB freshness',
       message: !hasSystemMetrics
-        ? 'system_metrics table unavailable: freshness runtime not verifiable.'
+        ? 'No machine metrics database: the daemon heartbeat cannot be verified.'
         : latestMetricAgeSeconds === null
-          ? 'No heartbeat metrics detected.'
-          : `Last heartbeat metrics ${latestMetricAgeSeconds}s ago (threshold ${staleThresholdSeconds}s).`,
-      resolvedMessage: 'DB Rev4a fresh again.',
+          ? 'No machine metrics sample yet.'
+          : `Last machine metrics sample ${latestMetricAgeSeconds}s ago (threshold ${staleThresholdSeconds}s).`,
+      resolvedMessage: 'Rev4a daemon sampling again.',
       stale: hasSystemMetrics ? (latestMetricAgeSeconds === null ? true : latestMetricAgeSeconds > staleThresholdSeconds) : false,
     });
 

package/app/components/Sidebar.tsx

@@ -59,6 +59,16 @@ const NAV: NavItem[] = [
       </svg>
     ),
   },
+  {
+    href: '/system',
+    label: 'System',
+    icon: (
+      <svg width="16" height="16" viewBox="0 0 16 16" fill="none">
+        <rect x="1.5" y="2.5" width="13" height="11" stroke="currentColor" strokeWidth="1.3" />
+        <path d="M4 10.5l2.5-3 2 2 3.5-4.5" stroke="currentColor" strokeWidth="1.3" strokeLinecap="round" strokeLinejoin="round" />
+      </svg>
+    ),
+  },
   {
     href: '/workspace',
     label: 'Workspace',

package/app/components/Skeleton.tsx

@@ -11,6 +11,8 @@ type SkeletonProps = {
   inline?: boolean;
   className?: string;
   style?: React.CSSProperties;
+  /** `span` for a placeholder inside text (a `<p>`, a `<span>`), where a `<div>` is invalid HTML and the parser would split the paragraph. */
+  as?: 'div' | 'span';
 };
 
 export function Skeleton({
@@ -21,6 +23,7 @@ export function Skeleton({
   inline = false,
   className,
   style,
+  as: Tag = 'div',
 }: SkeletonProps) {
   const base: React.CSSProperties = {
     display: inline ? 'inline-block' : 'block',
@@ -35,7 +38,7 @@ export function Skeleton({
     verticalAlign: inline ? 'middle' : undefined,
     ...style,
   };
-  return <div className={className} style={base} aria-hidden="true" />;
+  return <Tag className={className} style={base} aria-hidden="true" />;
 }
 
 /** Tree skeleton — simulates file tree structure like workspace sidebar */

package/app/components/SystemCockpit.tsx

@@ -1,11 +1,11 @@
 'use client';
 
-import { useEffect, useMemo, useState } from 'react';
+import { useEffect, useMemo, useRef, useState } from 'react';
 import Link from 'next/link';
 import type { Costs, FilterConfig, Session, SessionEvent } from '@/lib/types';
 import { isSessionActive } from '@/lib/patterns/sessionPresentation';
 import { SkeletonLines, SkeletonMetric } from './Skeleton';
-import { Metric, StatusCard, Surface, toneFromHealth } from './ui';
+import { Metric, StatusCard, Surface, meterLevel, toneFromHealth, type MeterLevel, type Tone } from './ui';
 import { apiFetch } from '@/lib/apiFetch';
 import LiveFeed from './LiveFeed';
 import CostBreakdown from './CostBreakdown';
@@ -38,6 +38,50 @@ interface SystemHealthPayload {
   generatedAt: string;
 }
 
+interface MachineNow {
+  cpu: number | null;
+  ram: number | null;
+  disk: number | null;
+  /** `empty`: no sample yet; `stale`: the newest is older than 3 intervals; `error`: the last refresh failed (figures are the previous ones). */
+  state: 'ok' | 'empty' | 'stale' | 'error';
+}
+
+const LEVEL_TONE: Record<MeterLevel, Tone> = { ok: 'neutral', warning: 'warning', critical: 'danger' };
+
+/** The host machine right now (CPU, RAM, fullest disk) from /api/metrics; details on /system. */
+function useMachineNow(): MachineNow | null {
+  const [now, setNow] = useState<MachineNow | null>(null);
+  const abortRef = useRef<AbortController | null>(null);
+  useEffect(() => {
+    const load = () => {
+      abortRef.current?.abort();
+      const controller = new AbortController();
+      abortRef.current = controller;
+      apiFetch('/api/metrics?range=1h', { signal: controller.signal })
+        .then((r) => (r.ok ? r.json() : Promise.reject(new Error(`HTTP ${r.status}`))))
+        .then((d) => {
+          const l = d.latest;
+          if (!l) { setNow({ cpu: null, ram: null, disk: null, state: 'empty' }); return; }
+          const disks: { percent: number }[] = l.disks ?? [];
+          setNow({
+            cpu: l.cpu === null ? null : Math.round(l.cpu),
+            ram: l.ram_total_mb ? Math.round((l.ram_used_mb / l.ram_total_mb) * 100) : null,
+            disk: disks.length ? Math.max(...disks.map((x) => x.percent)) : null,
+            state: l.age_s > d.interval_s * 3 ? 'stale' : 'ok',
+          });
+        })
+        .catch((e) => {
+          if (e?.name === 'AbortError') return;
+          setNow((prev) => ({ cpu: prev?.cpu ?? null, ram: prev?.ram ?? null, disk: prev?.disk ?? null, state: 'error' }));
+        });
+    };
+    load();
+    const t = setInterval(load, 30_000);
+    return () => { clearInterval(t); abortRef.current?.abort(); };
+  }, []);
+  return now;
+}
+
 interface SystemCockpitProps {
   sessions: Session[];
   events: SessionEvent[];
@@ -75,6 +119,7 @@ export default function SystemCockpit({
   loading = false,
 }: SystemCockpitProps) {
   const [systemHealth, setSystemHealth] = useState<SystemHealthPayload | null>(null);
+  const machine = useMachineNow();
 
   useEffect(() => {
     const load = () => {
@@ -108,6 +153,19 @@ export default function SystemCockpit({
   const health = systemHealth?.health ?? 'warning';
   const recommendations = systemHealth?.recommendations ?? [];
   const runtimeFreshness = runtimeCheck?.details ?? fmtAge(metrics.latest);
+  const pct = (v: number | null | undefined) => (v === null || v === undefined ? '—' : `${v}%`);
+  // Each figure with its level, on the thresholds the System page uses (CPU/RAM 85/95, disk 80/90).
+  const machineItems: { name: string; value: number | null; level: MeterLevel }[] = machine
+    ? [
+        { name: 'CPU', value: machine.cpu, level: machine.cpu === null ? 'ok' : meterLevel(machine.cpu, 85, 95) },
+        { name: 'RAM', value: machine.ram, level: machine.ram === null ? 'ok' : meterLevel(machine.ram, 85, 95) },
+        { name: 'Disk', value: machine.disk, level: machine.disk === null ? 'ok' : meterLevel(machine.disk, 80, 90) },
+      ]
+    : [];
+  const machineIssues = machineItems.filter((i) => i.level !== 'ok');
+  const machineTone: Tone = machine && machine.state !== 'ok'
+    ? 'warning'
+    : LEVEL_TONE[machineIssues.some((i) => i.level === 'critical') ? 'critical' : machineIssues.length ? 'warning' : 'ok'];
 
   return (
     <section className="cockpit">
@@ -121,6 +179,28 @@ export default function SystemCockpit({
           <div className="ui-muted" style={{ marginTop: 10, fontSize: 11 }}>Server-side monitor from `/api/system-health`.</div>
         </Surface>
         <Metric title="Active sessions" value={metrics.active} subtitle={`${sessions.length} total`} loading={loading} />
+        {/* A Surface, not a Metric: a danger Metric paints its whole value red, and only the
+            figure past its threshold should be. */}
+        <Surface as="article" tone={machineTone}>
+          <div className="ui-kicker">Machine</div>
+          <div style={{ display: 'flex', flexWrap: 'wrap', columnGap: 12, marginTop: 14, fontSize: 15, fontWeight: 700 }}>
+            {machineItems.length ? machineItems.map((i) => (
+              <span key={i.name} className={`cockpit__machine-item cockpit__machine-item--${i.level}`}>{i.name} {pct(i.value)}</span>
+            )) : '—'}
+          </div>
+          <div className="ui-muted ui-metric-subtitle">
+            {machine?.state === 'error'
+              ? 'Metrics unavailable. '
+              : machine?.state === 'empty'
+                ? 'No samples yet. '
+                : machine?.state === 'stale'
+                  ? 'Not collecting. '
+                  : machineIssues.length
+                    ? `${machineIssues.map((i) => `${i.name} ${i.level === 'critical' ? 'critical' : 'high'}`).join(', ')}. `
+                    : ''}
+            <Link href="/system" className="ui-link">Open System →</Link>
+          </div>
+        </Surface>
         <Metric title="Usage cost" value={String(costCheck?.value ?? fmtMoney(0))} subtitle={costCheck?.details ?? `DB estimate ${fmtMoney(costs.today)}`} tone="accent" loading={systemLoading} />
       </div>
 

package/app/components/ui/Meter.tsx

@@ -0,0 +1,34 @@
+export type MeterLevel = 'ok' | 'warning' | 'critical';
+
+/** The level a percentage falls in, given its warning and critical thresholds. */
+export function meterLevel(percent: number, warning: number, critical: number): MeterLevel {
+  if (percent >= critical) return 'critical';
+  if (percent >= warning) return 'warning';
+  return 'ok';
+}
+
+const LEVEL_LABEL: Record<MeterLevel, string> = { ok: 'ok', warning: 'high', critical: 'critical' };
+
+/**
+ * A horizontal fill for a 0-100 % share. The fill carries the level (accent → warning →
+ * danger), the track is a dim step behind it, and the level is also written out, so it
+ * never rests on colour alone.
+ */
+export function Meter({ percent, level, label }: { percent: number; level: MeterLevel; label: string }) {
+  const clamped = Math.max(0, Math.min(100, percent));
+  return (
+    <div className="ui-meter">
+      <div
+        className="ui-meter__track"
+        role="meter"
+        aria-label={label}
+        aria-valuemin={0}
+        aria-valuemax={100}
+        aria-valuenow={Math.round(clamped)}
+      >
+        <div className={`ui-meter__fill ui-meter__fill--${level}`} style={{ width: `${clamped}%` }} />
+      </div>
+      {level !== 'ok' && <span className={`ui-meter__level ui-meter__level--${level}`}>{LEVEL_LABEL[level]}</span>}
+    </div>
+  );
+}