@yuuki824/kanshi 0.1.0

package/app/storage.py

@@ -0,0 +1,220 @@
+"""Filelight-style directory size tree.
+
+The walk is a single os.scandir pass per root, run in a worker thread on a slow
+timer and cached — never recomputed per refresh. Sizes come from st_blocks so
+they match `du` (actual blocks on disk) rather than apparent size, and hardlinked
+files are counted once.
+"""
+from __future__ import annotations
+
+import asyncio
+import heapq
+import os
+import time
+
+from .config import config
+
+# Never ship a tile smaller than this, regardless of the fraction threshold.
+MIN_ABSOLUTE = 4 * 1024 * 1024
+# Largest individual files kept per directory; the rest are aggregated.
+TOP_FILES = 12
+
+_state: dict = {"roots": [], "scanned_at": None, "duration": None, "scanning": False, "error": None}
+_lock = asyncio.Lock()
+_last_scan = 0.0
+
+
+def _walk(root_path: str, max_depth: int) -> tuple[dict, str, int]:
+    """Iterative DFS. Returns (nodes-by-path, root-path, unreadable-dir-count).
+
+    Only directories within `max_depth` of the root get a node of their own.
+    Anything deeper is still fully traversed and counted, but its bytes roll up
+    into the nearest retained ancestor — the tree is pruned to this depth before
+    it is served anyway, so materialising a node per directory just burns memory.
+    (On this host that is ~2.3k retained nodes instead of ~96k.)
+    """
+    root_dev = os.stat(root_path).st_dev
+    excluded = set(config.storage_exclude)
+    unreadable = 0
+    # Packed into one int rather than a (dev, ino) tuple: overlay2 hardlinks
+    # everything, so this set reaches six figures and tuples cost ~3x the ints.
+    seen_inodes: set[int] = set()
+    nodes: dict[str, dict] = {}
+    order: list[str] = []
+    # (path, depth, nearest retained ancestor path)
+    stack: list[tuple[str, int, str | None]] = [(root_path, 0, None)]
+
+    while stack:
+        cur, depth, anchor = stack.pop()
+        if cur in excluded:
+            continue
+
+        if depth <= max_depth:
+            node = {
+                "name": os.path.basename(cur.rstrip("/")) or cur,
+                "self": 0,        # bytes of files directly inside this dir
+                "deep": 0,        # bytes below the retained depth
+                "children": [],   # child directory paths that have their own node
+                "files": [],      # min-heap of (size, name), capped at TOP_FILES
+            }
+            nodes[cur] = node
+            order.append(cur)
+            anchor = cur
+        else:
+            node = nodes[anchor]
+
+        try:
+            it = os.scandir(cur)
+        except OSError:
+            # A directory we cannot read would otherwise silently vanish from
+            # the totals — count it so the UI can say the tree is incomplete
+            # rather than quietly under-reporting.
+            unreadable += 1
+            continue
+        with it:
+            while True:
+                try:
+                    entry = next(it)
+                except StopIteration:
+                    break
+                except OSError:
+                    break
+                try:
+                    if entry.is_symlink():
+                        continue  # never follow; the link itself is negligible
+                    st = entry.stat(follow_symlinks=False)
+                    if entry.is_dir(follow_symlinks=False):
+                        # Don't cross into other filesystems — each root is
+                        # walked separately, so we'd otherwise double-count.
+                        if st.st_dev != root_dev:
+                            continue
+                        stack.append((entry.path, depth + 1, anchor))
+                        if depth + 1 <= max_depth:
+                            node["children"].append(entry.path)
+                    elif entry.is_file(follow_symlinks=False):
+                        if st.st_nlink > 1:
+                            key = (st.st_dev << 48) | st.st_ino
+                            if key in seen_inodes:
+                                continue
+                            seen_inodes.add(key)
+                        size = st.st_blocks * 512
+                        if depth <= max_depth:
+                            node["self"] += size
+                            if len(node["files"]) < TOP_FILES:
+                                heapq.heappush(node["files"], (size, entry.name))
+                            elif size > node["files"][0][0]:
+                                heapq.heapreplace(node["files"], (size, entry.name))
+                        else:
+                            node["deep"] += size
+                except OSError:
+                    continue
+
+    # Children always follow their parent in a DFS pre-order, so walking the
+    # order backwards guarantees every child is totalled before its parent.
+    for path in reversed(order):
+        node = nodes[path]
+        node["size"] = node["self"] + node["deep"] + sum(nodes[c]["size"] for c in node["children"])
+    return nodes, root_path, unreadable
+
+
+def _to_tree(nodes: dict, path: str, depth: int) -> dict:
+    node = nodes[path]
+    total = node["size"]
+    out = {"name": node["name"], "path": path, "size": total, "kind": "dir"}
+
+    if depth <= 0 or total <= 0:
+        return out
+
+    entries: list[dict] = []
+    for child in node["children"]:
+        entries.append({"_path": child, "size": nodes[child]["size"], "kind": "dir"})
+    kept_files = sorted(node["files"], reverse=True)
+    file_bytes_kept = 0
+    for size, name in kept_files:
+        entries.append({"name": name, "size": size, "kind": "file"})
+        file_bytes_kept += size
+    remainder = node["self"] + node["deep"] - file_bytes_kept
+    if remainder > 0:
+        entries.append({"name": "other files", "size": remainder, "kind": "rest"})
+
+    entries.sort(key=lambda e: -e["size"])
+    threshold = max(total * config.tree_min_fraction, MIN_ABSOLUTE)
+
+    children, folded, folded_count = [], 0, 0
+    for entry in entries:
+        if entry["size"] < threshold or len(children) >= config.tree_max_children:
+            folded += entry["size"]
+            folded_count += 1
+            continue
+        if entry["kind"] == "dir":
+            children.append(_to_tree(nodes, entry["_path"], depth - 1))
+        else:
+            children.append({"name": entry["name"], "size": entry["size"], "kind": entry["kind"]})
+    if folded > 0:
+        # Keep the aggregate so child sizes still sum to the parent — otherwise
+        # the treemap silently under-reports and the areas lie.
+        children.append({"name": f"{folded_count} smaller items", "size": folded, "kind": "rest"})
+
+    if children:
+        out["children"] = children
+    return out
+
+
+def _scan_sync() -> list[dict]:
+    # The walk is a long burst of syscalls competing with the live poller for
+    # this container's CPU quota. Deprioritise it so a rescan never makes the
+    # at-a-glance numbers stutter — the walk finishing a few seconds later is
+    # invisible, a frozen dashboard is not.
+    try:
+        os.nice(10)
+    except OSError:
+        pass
+
+    trees = []
+    for label, path in config.roots():
+        if not os.path.isdir(path):
+            continue
+        started = time.monotonic()
+        nodes, root, unreadable = _walk(path, config.tree_depth)
+        tree = _to_tree(nodes, root, config.tree_depth)
+        tree["name"] = label
+        tree["root"] = path
+        tree["unreadable"] = unreadable
+        tree["walk_seconds"] = round(time.monotonic() - started, 2)
+        trees.append(tree)
+    return trees
+
+
+async def scan(force: bool = False) -> dict:
+    """Rescan the roots. Rate-limited unless `force` is set by the timer."""
+    global _last_scan
+    if _lock.locked():
+        return _state
+    since = time.monotonic() - _last_scan
+    if not force and _last_scan and since < config.storage_min_rescan:
+        return _state
+
+    async with _lock:
+        _state["scanning"] = True
+        started = time.time()
+        try:
+            trees = await asyncio.to_thread(_scan_sync)
+            _state.update(roots=trees, scanned_at=started, error=None,
+                          duration=round(time.time() - started, 2))
+        except Exception as exc:
+            _state["error"] = f"{type(exc).__name__}: {exc}"
+        finally:
+            _state["scanning"] = False
+            _last_scan = time.monotonic()
+    return _state
+
+
+def snapshot() -> dict:
+    return _state
+
+
+async def loop() -> None:
+    """Background refresher — slow by default (15 min)."""
+    while True:
+        await scan(force=True)
+        await asyncio.sleep(max(60.0, config.storage_interval))

package/app/vitals.py

@@ -0,0 +1,170 @@
+"""Host-wide CPU / memory / disk / network vitals via psutil.
+
+Kanshi runs with `network_mode: host` and without lxcfs, so /proc/stat,
+/proc/meminfo, /proc/diskstats and /proc/net/dev all report real host values
+and psutil needs no special configuration.
+"""
+from __future__ import annotations
+
+import os
+import time
+
+import psutil
+
+from .config import config
+
+_last_net: tuple[float, int, int] | None = None
+_last_disk: tuple[float, int, int] | None = None
+_last_rates: dict[str, tuple[float, float]] = {"net": (0.0, 0.0), "disk": (0.0, 0.0)}
+_last_cpu_call = 0.0
+
+# Below this gap the counter deltas are too small to divide by: a 20ms window
+# turns a routine 2MB read into a fake 100MB/s spike. Hold the previous rate.
+MIN_DT = 0.5
+
+
+def prime() -> None:
+    """Seed psutil's internal counters so the first sample isn't all zeros."""
+    global _last_cpu_call
+    psutil.cpu_percent(percpu=True, interval=None)
+    _last_cpu_call = time.monotonic()
+    _rate_net()
+    _rate_disk()
+
+
+def _cpu_cores() -> list[float]:
+    """Per-core utilisation since the previous call.
+
+    psutil's interval=None form is a delta against the last call, so if that
+    call was moments ago every core reads 0%. When the gap is too short to be
+    meaningful, measure a real (short, blocking) window instead — sample() runs
+    in a worker thread, so this never stalls the event loop.
+    """
+    global _last_cpu_call
+    gap = time.monotonic() - _last_cpu_call
+    cores = psutil.cpu_percent(percpu=True, interval=None if gap >= MIN_DT else 0.25)
+    _last_cpu_call = time.monotonic()
+    return cores
+
+
+def _rate_net() -> tuple[float, float]:
+    global _last_net
+    now = time.monotonic()
+    try:
+        c = psutil.net_io_counters()
+    except Exception:
+        return (0.0, 0.0)
+    prev = _last_net
+    if prev is not None and now - prev[0] < MIN_DT:
+        return _last_rates["net"]
+    _last_net = (now, c.bytes_recv, c.bytes_sent)
+    if prev is None:
+        return (0.0, 0.0)
+    dt = now - prev[0]
+    # Counters are 64-bit but can reset if an interface disappears; clamp.
+    rates = (max(0, c.bytes_recv - prev[1]) / dt, max(0, c.bytes_sent - prev[2]) / dt)
+    _last_rates["net"] = rates
+    return rates
+
+
+def _rate_disk() -> tuple[float, float]:
+    global _last_disk
+    now = time.monotonic()
+    try:
+        c = psutil.disk_io_counters()
+    except Exception:
+        return (0.0, 0.0)
+    if c is None:
+        return (0.0, 0.0)
+    prev = _last_disk
+    if prev is not None and now - prev[0] < MIN_DT:
+        return _last_rates["disk"]
+    _last_disk = (now, c.read_bytes, c.write_bytes)
+    if prev is None:
+        return (0.0, 0.0)
+    dt = now - prev[0]
+    rates = (max(0, c.read_bytes - prev[1]) / dt, max(0, c.write_bytes - prev[2]) / dt)
+    _last_rates["disk"] = rates
+    return rates
+
+
+def _temperature() -> float | None:
+    """Best-effort package temperature; needs /sys mounted (Docker does by default)."""
+    try:
+        temps = psutil.sensors_temperatures()
+    except Exception:
+        return None
+    for key in ("coretemp", "k10temp", "cpu_thermal", "acpitz", "zenpower"):
+        for entry in temps.get(key, []):
+            if entry.current:
+                return round(entry.current, 1)
+    for entries in temps.values():
+        for entry in entries:
+            if entry.current:
+                return round(entry.current, 1)
+    return None
+
+
+def filesystems() -> list[dict]:
+    """Usage for each configured storage root, read straight off statvfs."""
+    out = []
+    for label, path in config.roots():
+        try:
+            st = os.statvfs(path)
+        except OSError:
+            continue
+        total = st.f_blocks * st.f_frsize
+        # f_bavail excludes root-reserved blocks, so used+free won't equal total.
+        # Report "used" the way df does, against the non-reserved capacity.
+        free = st.f_bavail * st.f_frsize
+        used = total - st.f_bfree * st.f_frsize
+        if total <= 0:
+            continue
+        out.append({
+            "label": label,
+            "path": path,
+            "total": total,
+            "used": used,
+            "free": free,
+            "percent": round(used / (used + free) * 100, 1) if used + free else 0.0,
+        })
+    return out
+
+
+def sample() -> dict:
+    per_core = _cpu_cores()
+    mem = psutil.virtual_memory()
+    swap = psutil.swap_memory()
+    rx, tx = _rate_net()
+    read, write = _rate_disk()
+    try:
+        load1, load5, load15 = os.getloadavg()
+    except OSError:
+        load1 = load5 = load15 = 0.0
+
+    return {
+        "ts": time.time(),
+        "cpu": {
+            "percent": round(sum(per_core) / len(per_core), 1) if per_core else 0.0,
+            "cores": [round(c, 1) for c in per_core],
+            "load": [round(load1, 2), round(load5, 2), round(load15, 2)],
+            "count": len(per_core),
+            "temp": _temperature(),
+        },
+        "memory": {
+            "total": mem.total,
+            "used": mem.total - mem.available,
+            "available": mem.available,
+            "cached": getattr(mem, "cached", 0) + getattr(mem, "buffers", 0),
+            "percent": round((mem.total - mem.available) / mem.total * 100, 1) if mem.total else 0.0,
+        },
+        "swap": {
+            "total": swap.total,
+            "used": swap.used,
+            "percent": round(swap.percent, 1),
+        },
+        "network": {"rx": rx, "tx": tx},
+        "diskio": {"read": read, "write": write},
+        "filesystems": filesystems(),
+        "uptime": time.time() - psutil.boot_time(),
+    }

package/bin/kanshi.js

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+
+const { existsSync } = require('node:fs');
+const { spawnSync } = require('node:child_process');
+const { join } = require('node:path');
+
+const packageRoot = join(__dirname, '..');
+const composeFile = join(packageRoot, 'docker-compose.yml');
+const callerDirectory = process.cwd();
+const args = process.argv.slice(2);
+
+if (args.includes('--help') || args.includes('-h')) {
+  console.log(`Usage: npx @yuuki824/kanshi [docker-compose arguments]
+
+Starts Kanshi with Docker Compose. A .env file in the current directory is used
+when present. Common commands:
+  npx @yuuki824/kanshi
+  npx @yuuki824/kanshi logs -f
+  npx @yuuki824/kanshi down
+
+Docker and the Docker Compose v2 plugin are required.`);
+  process.exit(0);
+}
+
+const dockerCheck = spawnSync('docker', ['compose', 'version'], { stdio: 'ignore' });
+if (dockerCheck.error || dockerCheck.status !== 0) {
+  console.error('Kanshi requires Docker with the Docker Compose v2 plugin.');
+  console.error('Install Docker first, then run this command again.');
+  process.exit(1);
+}
+
+const composeArgs = [
+  'compose',
+  '--project-directory', packageRoot,
+  '--project-name', 'kanshi',
+  '--file', composeFile,
+];
+
+const envFile = join(callerDirectory, '.env');
+if (callerDirectory !== packageRoot && existsSync(envFile)) {
+  composeArgs.push('--env-file', envFile);
+}
+
+if (args.length === 0) {
+  composeArgs.push('up', '--detach', '--build');
+} else {
+  composeArgs.push(...args);
+}
+
+const result = spawnSync('docker', composeArgs, { stdio: 'inherit' });
+if (result.error) {
+  console.error(`Could not start Docker: ${result.error.message}`);
+  process.exit(1);
+}
+process.exit(result.status ?? 1);

package/docker-compose.yml

@@ -0,0 +1,59 @@
+services:
+  kanshi:
+    build: .
+    container_name: kanshi
+    restart: unless-stopped
+
+    # Host networking for two reasons: /proc/net/dev is namespaced, so it is the
+    # only way to report real host network throughput, and it lets us bind
+    # straight to the Tailscale address below.
+    network_mode: host
+
+    environment:
+      # Bind to the Tailscale IP only — not the LAN, not the public interface.
+      # Reachable at http://homeserver.tail7ec1d9.ts.net:8100 from any device on
+      # the tailnet. Set to 0.0.0.0 to also expose it on the LAN.
+      KANSHI_HOST: "${KANSHI_HOST:-100.73.243.14}"
+      KANSHI_PORT: "${KANSHI_PORT:-8100}"
+
+      # Live poll cadence. Each tick is one /containers/json plus one /stats per
+      # running container, so keep this conservative.
+      KANSHI_POLL_INTERVAL: "${KANSHI_POLL_INTERVAL:-5}"
+      KANSHI_DOCKER_CONCURRENCY: "${KANSHI_DOCKER_CONCURRENCY:-8}"
+      # Polling stops entirely once no browser has been connected this long.
+      KANSHI_IDLE_TIMEOUT: "${KANSHI_IDLE_TIMEOUT:-30}"
+
+      # Storage walk: label=path pairs. Cached, never recomputed per refresh.
+      KANSHI_STORAGE_ROOTS: "${KANSHI_STORAGE_ROOTS:-/mnt/data=/mnt/data,/=/hostfs}"
+      KANSHI_STORAGE_INTERVAL: "${KANSHI_STORAGE_INTERVAL:-1800}"
+      # Container-side paths to skip. /hostfs/var/lib/docker is ~48 GB across
+      # ~100k overlay2 files and dominates the walk time — uncomment to trim it.
+      KANSHI_STORAGE_EXCLUDE: "${KANSHI_STORAGE_EXCLUDE:-}"
+      KANSHI_TREE_DEPTH: "${KANSHI_TREE_DEPTH:-4}"
+
+    volumes:
+      # Read-only on the socket is a mount flag, not an API restriction — it is
+      # the Tailscale-only bind above that keeps this out of reach.
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /mnt/data:/mnt/data:ro
+      - /:/hostfs:ro
+
+    read_only: true
+    tmpfs:
+      - /tmp:size=16m
+    cap_drop: [ALL]
+    # Read/traverse bypass only — it does NOT grant write bypass (that would be
+    # DAC_OVERRIDE). Without it the walk cannot enter /home/yuuki (0750) and
+    # silently under-reports the root filesystem by ~330 GB.
+    cap_add: [DAC_READ_SEARCH]
+    security_opt:
+      - no-new-privileges:true
+
+    # Stays out of the way of the other ~28 containers on this 4-core box.
+    # Steady state is ~1% of one core; the ceiling is headroom for the walk.
+    cpus: "1.0"
+    mem_limit: 256m
+
+    logging:
+      driver: json-file
+      options: { max-size: "10m", max-file: "3" }

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@yuuki824/kanshi",
+  "version": "0.1.0",
+  "description": "Launch the Kanshi Docker host dashboard with npx.",
+  "license": "UNLICENSED",
+  "bin": {
+    "kanshi": "bin/kanshi.js"
+  },
+  "files": [
+    "app",
+    "web",
+    "bin",
+    "Dockerfile",
+    "docker-compose.yml",
+    "requirements.txt",
+    "README.md",
+    ".env.example"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "pack:check": "npm pack --dry-run",
+    "test": "node --check bin/kanshi.js"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "docker",
+    "dashboard",
+    "monitoring",
+    "tailscale"
+  ]
+}

package/requirements.txt

@@ -0,0 +1,4 @@
+fastapi==0.115.6
+uvicorn[standard]==0.34.0
+psutil==6.1.1
+httpx==0.28.1