@flame0510/project-aether 1.10.0 → 1.11.0

package/lib/docker-socket.ts

@@ -1,90 +1,21 @@
 /**
  * Docker Engine API over its local unix socket.
  *
- * The socket path is NOT the same everywhere. `/var/run/docker.sock` is the
- * Linux default (and what the VPS uses), but Docker Desktop on macOS puts it
- * at `~/.docker/run/docker.sock` and does not create the /var/run symlink
- * unless the user opts in. Hardcoding the Linux path made every read-only
- * Docker route return an empty list on macOS — agents existed and ran, but
- * the dashboard showed nothing, because creation shells out to the `docker`
- * CLI (which reads the context) while listing went through this socket.
+ * Talking to the socket directly is preferred over shelling out for reads: no fork, no
+ * shell quoting, no `--format` template to parse — just JSON.
  *
- * Talking to the socket directly is preferred over shelling out for reads:
- * no fork, no shell quoting, no `--format` template to parse — just JSON.
+ * Where the socket lives (it is NOT the same on Linux and on Docker Desktop) and the
+ * request itself are in lib/docker-socket-path.js, plain CommonJS shared with daemon.js so
+ * there is one copy of those rules; this module keeps the names the routes import.
  */
-import * as http from 'http';
-import * as fs from 'fs';
-import * as os from 'os';
-import * as path from 'path';
-import { execFileSync } from 'child_process';
-
-let cached: string | null = null;
-let lastMissAt = 0;
+import { resolveDockerSocket as resolve, dockerRequestJson } from './docker-socket-path';
 
 /**
- * How long a failed resolution is remembered. Long enough that a Docker outage
- * doesn't spawn a `docker context inspect` per request, short enough that a
- * server which started before the Docker daemon picks it up on its own.
- */
-const MISS_TTL_MS = 5000;
-
-function fromDockerHostEnv(): string | null {
-  const raw = process.env.DOCKER_HOST;
-  if (!raw) return null;
-  // Only unix sockets are usable here; tcp:// would need a different client.
-  if (!raw.startsWith('unix://')) return null;
-  return raw.slice('unix://'.length);
-}
-
-function fromDockerContext(): string | null {
-  try {
-    const out = execFileSync(
-      'docker',
-      ['context', 'inspect', '--format', '{{.Endpoints.docker.Host}}'],
-      { encoding: 'utf-8', timeout: 3000, stdio: ['ignore', 'pipe', 'ignore'] },
-    ).trim();
-    return out.startsWith('unix://') ? out.slice('unix://'.length) : null;
-  } catch {
-    return null;
-  }
-}
-
-function usable(candidate: string | null): string | null {
-  if (!candidate) return null;
-  try {
-    fs.accessSync(candidate, fs.constants.R_OK | fs.constants.W_OK);
-    return candidate;
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Resolve the Docker socket, most authoritative source first. Returns null when
- * Docker isn't reachable at all.
- *
- * A success is cached for the life of the process — the path doesn't move. A
- * failure is only cached for MISS_TTL_MS: Rev4a can legitimately start before
- * the Docker daemon is up (systemd ordering, Docker Desktop still booting), and
- * caching that failure permanently would leave every Docker route returning an
- * empty list until someone restarted the server.
+ * Resolve the Docker socket, most authoritative source first. Returns null when Docker
+ * isn't reachable at all (a failure is remembered for a few seconds only).
  */
 export function resolveDockerSocket(): string | null {
-  if (cached) return cached;
-  if (Date.now() - lastMissAt < MISS_TTL_MS) return null;
-
-  const found =
-    usable(fromDockerHostEnv()) ??
-    usable(fromDockerContext()) ??
-    // Docker Desktop (macOS, and Windows with WSL integration)
-    usable(path.join(os.homedir(), '.docker', 'run', 'docker.sock')) ??
-    // Linux default — the VPS lands here
-    usable('/var/run/docker.sock');
-
-  if (found) cached = found;
-  else lastMissAt = Date.now();
-
-  return found;
+  return resolve();
 }
 
 /** True when a usable Docker socket exists. */
@@ -97,25 +28,5 @@ export function dockerAvailable(): boolean {
  * Rejects when Docker is unreachable or the payload isn't JSON.
  */
 export function dockerFetch<T = any>(method: string, apiPath: string): Promise<T> {
-  return new Promise((resolve, reject) => {
-    const socketPath = resolveDockerSocket();
-    if (!socketPath) {
-      reject(new Error('Docker socket not found'));
-      return;
-    }
-
-    const req = http.request(
-      { socketPath, path: apiPath, method, headers: { Host: 'localhost' } },
-      (res) => {
-        let data = '';
-        res.on('data', (chunk) => { data += chunk; });
-        res.on('end', () => {
-          try { resolve(JSON.parse(data) as T); }
-          catch { reject(new Error('Invalid JSON from Docker')); }
-        });
-      },
-    );
-    req.on('error', reject);
-    req.end();
-  });
+  return dockerRequestJson<T>(method, apiPath);
 }

package/lib/docker-stats.js

@@ -0,0 +1,284 @@
+'use strict';
+
+/**
+ * What each container consumes, from the Docker Engine API — the parsing and the rate
+ * arithmetic, free of any database or timer so it can be tested on its own. daemon.js
+ * owns the timers and the tables (metrics.db); this module turns Docker's answers into
+ * rows.
+ *
+ * Plain CommonJS for the same reason as lib/docker-socket-path.js: the daemon requires it.
+ *
+ * Why the raw API and not `docker stats`: `GET /containers/{id}/stats?one-shot=true`
+ * answers in 10–20 ms with cumulative counters (CPU time in ns, network and block bytes),
+ * while `docker stats --no-stream` waits a second per container for a second sample and
+ * hands back formatted strings. Two samples of the daemon's own timer give the CPU
+ * share over a real window.
+ */
+
+/** A Docker container name, as `docker ps` prints it; anything else is never used. */
+const CONTAINER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
+
+const REQUEST_TIMEOUT_MS = 10_000;
+const DF_TIMEOUT_MS = 60_000;
+/** `docker info` (cores, memory) changes only when Docker Desktop is reconfigured. */
+const HOST_TTL_MS = 10 * 60_000;
+/** Two samples closer than this give a rate dominated by timing noise. */
+const MIN_WINDOW_S = 1;
+const MB = 1024 * 1024;
+
+/**
+ * The container's own name from Docker's `Names`: the one with a single leading slash.
+ * A container that is the target of a legacy link also carries `/other/alias` entries, and
+ * nothing promises which comes first — taking the first blindly would drop it, silently.
+ */
+function primaryName(names) {
+  const own = (Array.isArray(names) ? names : []).find((n) => typeof n === 'string' && /^\/[^/]+$/.test(n));
+  return own ? own.slice(1) : '';
+}
+
+/** A display name without control characters (a tab or a newline in a name), bounded. */
+function cleanName(value, fallback = null) {
+  const s = String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').trim().slice(0, 80);
+  return s || fallback;
+}
+
+const finite = (n) => typeof n === 'number' && Number.isFinite(n);
+
+/**
+ * Working-set memory: usage minus the inactive page cache, what `docker stats` shows (and
+ * what the kernel will not give back under pressure). cgroup v2 names it `inactive_file`,
+ * v1 `total_inactive_file`.
+ */
+function workingSetBytes(memory) {
+  const usage = memory?.usage;
+  if (!finite(usage)) return null;
+  const inactive = memory.stats?.inactive_file ?? memory.stats?.total_inactive_file ?? 0;
+  return Math.max(0, usage - (finite(inactive) ? inactive : 0));
+}
+
+/** Bytes received and sent, summed over the container's networks. */
+function sumNetwork(networks) {
+  let rx = 0;
+  let tx = 0;
+  let seen = false;
+  for (const n of Object.values(networks || {})) {
+    if (finite(n?.rx_bytes)) { rx += n.rx_bytes; seen = true; }
+    if (finite(n?.tx_bytes)) { tx += n.tx_bytes; seen = true; }
+  }
+  return seen ? { rx, tx } : { rx: null, tx: null };
+}
+
+/** Bytes read and written on block devices (`op` is `read`/`write` on v2, `Read`/`Write` on v1). */
+function sumBlockIo(blkio) {
+  let rd = 0;
+  let wr = 0;
+  let seen = false;
+  for (const e of blkio?.io_service_bytes_recursive || []) {
+    if (!finite(e?.value)) continue;
+    const op = String(e.op || '').toLowerCase();
+    if (op === 'read') { rd += e.value; seen = true; }
+    else if (op === 'write') { wr += e.value; seen = true; }
+  }
+  return seen ? { rd, wr } : { rd: null, wr: null };
+}
+
+/**
+ * One stats answer as counters, or null when it holds no CPU figure (a stopped container, or
+ * one that has only just started, answers an empty object).
+ */
+function readCounters(stats, atMs) {
+  const cpuNs = stats?.cpu_stats?.cpu_usage?.total_usage;
+  if (!finite(cpuNs)) return null;
+  const net = sumNetwork(stats.networks);
+  const blk = sumBlockIo(stats.blkio_stats);
+  return {
+    t: atMs,
+    cpuNs,
+    memBytes: workingSetBytes(stats.memory_stats),
+    pids: finite(stats.pids_stats?.current) ? stats.pids_stats.current : null,
+    rx: net.rx, tx: net.tx, rd: blk.rd, wr: blk.wr,
+  };
+}
+
+/**
+ * Per-second rates between two readings of one container. Each counter stands alone: one
+ * that went backwards (the container restarted, the network was re-attached) gives null and
+ * nothing else — never a negative rate, never a spike. No earlier reading, a window under a
+ * second, or one longer than `maxWindowS`: all null. The long window matters: after a laptop
+ * slept or a pass was skipped, the counters moved little over many hours, and dividing by the
+ * whole gap would draw a confident near-zero instead of no figure.
+ */
+function ratesBetween(prev, cur, maxWindowS = Infinity) {
+  const none = { cpuCores: null, rxBps: null, txBps: null, rdBps: null, wrBps: null };
+  if (!prev || !cur) return none;
+  const dt = (cur.t - prev.t) / 1000;
+  if (!(dt >= MIN_WINDOW_S) || dt > maxWindowS) return none;
+  const rate = (a, b) => (finite(a) && finite(b) && b >= a ? (b - a) / dt : null);
+  const cpu = rate(prev.cpuNs, cur.cpuNs);
+  return {
+    cpuCores: cpu === null ? null : cpu / 1e9,
+    rxBps: rate(prev.rx, cur.rx),
+    txBps: rate(prev.tx, cur.tx),
+    rdBps: rate(prev.rd, cur.rd),
+    wrBps: rate(prev.wr, cur.wr),
+  };
+}
+
+const round = (n, digits) => (n === null ? null : Number(n.toFixed(digits)));
+
+/**
+ * The container sampler: holds the previous counters of every container between calls (in
+ * memory only — a daemon restart primes again) and the names it had to inspect.
+ *
+ * `request(method, path, timeoutMs)` is the Docker call (lib/docker-socket-path.js in the
+ * daemon, a fake in a test); `now()` the clock; `maxWindowS` the longest gap between two
+ * readings that still gives a rate (the daemon passes three sampling intervals).
+ */
+function createContainerSampler({ request, now = Date.now, maxWindowS = 180 }) {
+  const previous = new Map(); // container id → counters
+  const described = new Map(); // container id → display name
+  let host = null;
+  let hostAt = 0;
+
+  async function readHost() {
+    if (host && now() - hostAt < HOST_TTL_MS) return host;
+    try {
+      const info = await request('GET', '/info', REQUEST_TIMEOUT_MS);
+      if (finite(info?.NCPU) && finite(info?.MemTotal) && info.NCPU > 0) {
+        host = { ncpu: info.NCPU, memTotalMb: Math.round(info.MemTotal / MB) };
+        hostAt = now();
+      }
+    } catch { /* keep the last answer; none yet → no host row */ }
+    return host;
+  }
+
+  /** The name to show: an agent's AGENT_NAME (read once per container id), else the container's. */
+  async function displayName(c, container) {
+    if (!c.Labels?.AGENT_ID) return container;
+    if (described.has(c.Id)) return described.get(c.Id);
+    let name = container;
+    try {
+      const inspect = await request('GET', `/containers/${encodeURIComponent(c.Id)}/json`, REQUEST_TIMEOUT_MS);
+      const entry = (inspect?.Config?.Env || []).find((e) => typeof e === 'string' && e.startsWith('AGENT_NAME='));
+      name = cleanName(entry?.slice('AGENT_NAME='.length), container);
+    } catch { /* fall back to the container name; asked again next time */ return name; }
+    described.set(c.Id, name);
+    return name;
+  }
+
+  /**
+   * One pass over the running containers. Throws when the list itself cannot be read
+   * (Docker down); a container whose stats fail is skipped, the others are kept.
+   */
+  async function sample() {
+    const list = await request('GET', '/containers/json', REQUEST_TIMEOUT_MS);
+    if (!Array.isArray(list)) throw new Error('unexpected answer from Docker');
+    const atMs = now();
+    const containers = [];
+    for (const c of list) {
+      const container = primaryName(c?.Names);
+      if (!c?.Id || !CONTAINER_NAME_RE.test(container)) continue;
+      containers.push({ c, container });
+    }
+
+    // The name (an inspect, agents only) runs beside the stats call, not after it: a slow
+    // inspect must not stretch the pass by its timeout once per agent.
+    const answers = await Promise.allSettled(containers.map(async ({ c, container }) => {
+      const [stats, name] = await Promise.all([
+        request('GET', `/containers/${encodeURIComponent(c.Id)}/stats?stream=false&one-shot=true`, REQUEST_TIMEOUT_MS),
+        displayName(c, container),
+      ]);
+      return { c, container, name, counters: readCounters(stats, now()) };
+    }));
+
+    const rows = [];
+    const sources = [];
+    // Identity comes from the list itself — a container whose stats call failed was still
+    // seen, so it keeps its place on the page (as stopped) instead of vanishing. Rates come
+    // only from counters. Alive is what Docker lists, not what answered its stats call: one
+    // timeout must not make a container forget its counters (the next pass would have no
+    // rate) and its name.
+    const live = new Set(containers.map(({ c }) => c.Id));
+    for (let i = 0; i < containers.length; i++) {
+      const { c, container } = containers[i];
+      const a = answers[i];
+      const name = a.status === 'fulfilled' ? a.value.name : container;
+      const volumes = (c.Mounts || []).filter((m) => m?.Type === 'volume' && m.Name).map((m) => m.Name);
+      sources.push({
+        container,
+        agent_id: c.Labels?.AGENT_ID ? String(c.Labels.AGENT_ID) : null,
+        name,
+        is_agent: c.Labels?.AGENT_ID ? 1 : 0,
+        volumes: volumes.join(','),
+      });
+      if (a.status !== 'fulfilled') continue;
+      const { counters } = a.value;
+      if (!counters) continue; // just started: no CPU figure yet
+      const rates = ratesBetween(previous.get(c.Id), counters, maxWindowS);
+      previous.set(c.Id, counters);
+      rows.push({
+        container,
+        cpu_cores: round(rates.cpuCores, 4),
+        mem_mb: counters.memBytes === null ? null : Math.round(counters.memBytes / MB),
+        pids: counters.pids,
+        net_rx_bps: round(rates.rxBps, 0),
+        net_tx_bps: round(rates.txBps, 0),
+        blk_read_bps: round(rates.rdBps, 0),
+        blk_write_bps: round(rates.wrBps, 0),
+      });
+    }
+    // A container that is gone takes its counters with it (a new id starts from none).
+    for (const id of previous.keys()) if (!live.has(id)) previous.delete(id);
+    for (const id of described.keys()) if (!live.has(id)) described.delete(id);
+
+    return { ts: Math.floor(atMs / 1000), host: await readHost(), rows, sources };
+  }
+
+  return { sample };
+}
+
+/**
+ * `/system/df` as the readings worth keeping: each named volume (and what no container uses),
+ * each container's writable layer, the images and the build cache in total with what
+ * could be freed. Sizes in bytes; -1 (Docker could not size it) becomes null.
+ * Returns null for anything that is not a df answer (an error body parses as JSON too) —
+ * the caller keeps its last reading instead of writing zeros.
+ */
+function parseStorage(df) {
+  if (!df || typeof df !== 'object' || !('Volumes' in df || 'Containers' in df || 'Images' in df || 'BuildCache' in df)) return null;
+  const size = (n) => (finite(n) && n >= 0 ? n : null);
+  const volumes = (df?.Volumes || [])
+    .filter((v) => v?.Name)
+    .map((v) => ({
+      name: String(v.Name),
+      sizeBytes: size(v.UsageData?.Size),
+      unused: (v.UsageData?.RefCount ?? 0) === 0,
+    }));
+  const layers = (df?.Containers || [])
+    .map((c) => ({ container: primaryName(c?.Names), sizeBytes: size(c?.SizeRw) }))
+    .filter((l) => CONTAINER_NAME_RE.test(l.container));
+  const images = df?.Images || [];
+  const cache = df?.BuildCache || [];
+  const sum = (items, pick) => items.reduce((acc, x) => acc + (size(pick(x)) ?? 0), 0);
+  return {
+    volumes,
+    layers,
+    images: {
+      // LayersSize counts shared layers once, unlike summing each image.
+      sizeBytes: size(df?.LayersSize) ?? sum(images, (i) => i.Size),
+      reclaimableBytes: sum(images.filter((i) => (i.Containers ?? 0) === 0), (i) => i.Size),
+    },
+    buildCache: {
+      sizeBytes: sum(cache, (e) => e.Size),
+      // Docker's own "reclaimable" figure for the cache reads a few KB next to a cache of
+      // 1.7 GB (observed): what `docker builder prune` can free is what no build uses.
+      reclaimableBytes: sum(cache.filter((e) => e.InUse === false), (e) => e.Size),
+    },
+  };
+}
+
+module.exports = {
+  CONTAINER_NAME_RE, DF_TIMEOUT_MS, MB,
+  cleanName, primaryName, workingSetBytes, sumNetwork, sumBlockIo, readCounters, ratesBetween,
+  createContainerSampler, parseStorage,
+};

package/lib/metrics-db.ts

@@ -4,25 +4,34 @@ import path from 'path';
 import Database from 'better-sqlite3';
 import { DB_PATH } from './db';
 
+/** History ranges of the System page and their span in seconds (GET /api/metrics and /api/metrics/containers). */
+export const METRIC_RANGES = { '1h': 3600, '24h': 86_400, '7d': 7 * 86_400, '30d': 30 * 86_400 } as const;
+export type MetricRange = keyof typeof METRIC_RANGES;
+
 /** Next to events.db, as daemon.js places it; its own file so the event log stays clean. */
 export const METRICS_DB_PATH = path.join(path.dirname(DB_PATH), 'metrics.db');
 
 /**
  * Open the metrics database read-only, or null when there is nothing to read yet: no file
- * (first start, or a daemon that never ran) or a file without the daemon's tables (it
- * failed right after creating it). The daemon owns the schema.
+ * (first start, or a daemon that never ran), a file without the daemon's tables (it failed
+ * right after creating it), or a file that cannot be opened at all (corrupt, locked — the
+ * caller answers "no data", never a 500). The daemon owns the schema. Never throws.
  */
 export function openMetricsDb(): Database.Database | null {
   if (!fs.existsSync(METRICS_DB_PATH)) return null;
-  const db = new Database(METRICS_DB_PATH, { readonly: true, fileMustExist: true });
+  let db: Database.Database;
+  try {
+    db = new Database(METRICS_DB_PATH, { readonly: true, fileMustExist: true });
+  } catch {
+    return null;
+  }
   try {
     const tables = db
       .prepare("SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table' AND name IN ('system_metrics', 'system_disks')")
       .get() as { n: number };
     if (tables.n === 2) return db;
-  } catch (e) {
-    db.close();
-    throw e;
+  } catch {
+    /* fall through: unreadable schema counts as nothing to read */
   }
   db.close();
   return null;

package/lib/utils/format.ts

@@ -4,6 +4,32 @@ export function formatUsd(value: number | null | undefined): string {
   return `$${Number(value ?? 0).toFixed(2)}`;
 }
 
+/** Megabytes as MB, GB or TB with the precision a reader wants: '692 MB', '1.6 GB'. */
+export function formatMb(mb: number | null | undefined): string {
+  if (mb === null || mb === undefined || !Number.isFinite(mb)) return '—';
+  if (mb >= 1024 * 1024) return `${(mb / 1024 / 1024).toFixed(2)} TB`;
+  if (mb >= 1024) return `${(mb / 1024).toFixed(1)} GB`;
+  return `${Math.round(mb)} MB`;
+}
+
+/** A CPU share in percent, with the decimals a small figure needs (an idle agent is 0.02 % of Docker's cores). */
+export function formatSharePercent(percent: number | null | undefined): string {
+  if (percent === null || percent === undefined || !Number.isFinite(percent)) return '—';
+  if (percent === 0) return '0%';
+  if (percent < 0.01) return '<0.01%';
+  if (percent < 1) return `${percent.toFixed(2)}%`;
+  if (percent < 10) return `${percent.toFixed(1)}%`;
+  return `${Math.round(percent)}%`;
+}
+
+/** Bytes per second as B/s, KB/s or MB/s. */
+export function formatRate(bytesPerSecond: number | null | undefined): string {
+  if (bytesPerSecond === null || bytesPerSecond === undefined || !Number.isFinite(bytesPerSecond)) return '—';
+  if (bytesPerSecond >= 1024 * 1024) return `${(bytesPerSecond / 1024 / 1024).toFixed(1)} MB/s`;
+  if (bytesPerSecond >= 1024) return `${(bytesPerSecond / 1024).toFixed(1)} KB/s`;
+  return `${Math.round(bytesPerSecond)} B/s`;
+}
+
 /**
  * Dollars with the precision a small figure needs: an agent's day can be a fraction of a
  * cent, which formatUsd would show as $0.00. Used wherever agent costs are shown.
@@ -11,9 +37,9 @@ export function formatUsd(value: number | null | undefined): string {
 export function formatCost(value: number | null | undefined): string {
   const v = Number(value ?? 0);
   if (!v) return '$0.00';
-  if (v < 0.01) return `${v.toFixed(4)}`;
-  if (v < 1) return `${v.toFixed(3)}`;
-  return `${v.toFixed(2)}`;
+  if (v < 0.01) return `$${v.toFixed(4)}`;
+  if (v < 1) return `$${v.toFixed(3)}`;
+  return `$${v.toFixed(2)}`;
 }
 
 export function formatUsdOrDash(value: number | null | undefined): string {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flame0510/project-aether",
-  "version": "1.10.0",
+  "version": "1.11.0",
   "description": "Rev4a — Revolution for Agents. OpenClaw agent fleet orchestrator.",
   "keywords": [
     "openclaw",