modelstat 0.0.44 → 0.0.46

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelstat",
-  "version": "0.0.44",
+  "version": "0.0.46",
   "description": "modelstat companion — reads local AI-tool usage and ships tokenised events to modelstat.",
   "license": "Apache-2.0",
   "type": "module",

package/vendor/tray-mac/Sources/ModelstatTray/main.swift

@@ -13,6 +13,7 @@
 // ~300 LOC, no dependencies beyond AppKit/Foundation.
 
 import AppKit
+import Darwin
 import Foundation
 
 // ── Resolve the `modelstat` CLI on $PATH, then at the install-path
@@ -107,9 +108,20 @@ struct LocalStatsCounters: Decodable {
 final class TrayController: NSObject {
   private let statusItem: NSStatusItem
   private let menu = NSMenu()
-  private let cli: URL?
+  /// Re-resolved by ensureDaemon() when nil — a transient resolution
+  /// failure at tray boot (e.g. the installer is mid-rewrite of
+  /// ~/.modelstat/bin) must not permanently strand the daemon.
+  private var cli: URL?
   private var daemon: Process?
+  /// When the current `daemon` child was spawned — used to detect
+  /// "exited cleanly almost immediately" (another daemon owns the
+  /// lock) so we back off to the watchdog instead of respawning hot.
+  private var daemonSpawnedAt: Date?
   private var paused = false
+  /// Serialises _daemon-health probes; collapses overlapping
+  /// ensureDaemon() calls into one in-flight probe.
+  private let superviseQueue = DispatchQueue(label: "ai.modelstat.tray.supervise", qos: .utility)
+  private var ensureInFlight = false
   private var latest: AgentStats?
   /// Live local heartbeat, read straight from ~/.modelstat/last-status.json
   /// on the fast timer. Decoupled from `latest` (the slower, network-backed
@@ -120,6 +132,7 @@ final class TrayController: NSObject {
   private var spinnerTick = 0
   private var fastTimer: Timer?
   private var slowTimer: Timer?
+  private var watchdogTimer: Timer?
 
   // Menu items we update on every poll
   private let statusMI = NSMenuItem(title: "Loading…", action: nil, keyEquivalent: "")
@@ -140,7 +153,7 @@ final class TrayController: NSObject {
     super.init()
     configureStatusItem()
     buildMenu()
-    startDaemon()
+    ensureDaemon()
     refreshStats()
     tickLocal()
 
@@ -164,6 +177,15 @@ final class TrayController: NSObject {
     }
     RunLoop.main.add(slow, forMode: .common)
     slowTimer = slow
+    // Watchdog: re-converge the daemon every 30s no matter how the
+    // last attempt ended — heals the give-up paths (CLI unresolvable
+    // at boot, spawn throw, adopted daemon died) that used to strand
+    // the pipeline until the user noticed the tray frozen.
+    let watchdog = Timer(timeInterval: 30.0, repeats: true) { [weak self] _ in
+      MainActor.assumeIsolated { self?.ensureDaemon() }
+    }
+    RunLoop.main.add(watchdog, forMode: .common)
+    watchdogTimer = watchdog
   }
 
   private func configureStatusItem() {
@@ -221,24 +243,97 @@ final class TrayController: NSObject {
   }
 
   // ── Daemon lifecycle ─────────────────────────────────────────────
-
-  private func startDaemon() {
+  //
+  // The tray no longer spawns `start --force` blindly. Blind --force
+  // SIGTERMs whatever live daemon owns the singleton lock (see
+  // apps/agent-dev/src/lock.ts), so two briefly-coexisting trays
+  // (kickstart -k racing a reinstall, KeepAlive respawn overlap) had
+  // their daemons kill each other in a loop — observed 2026-06-12
+  // ending with zero daemons and nothing restarting them. Instead,
+  // every (re)start funnels through ensureDaemon(), which asks the CLI
+  // `_daemon-health` (decision logic + tests live in
+  // apps/agent-dev/src/supervise.ts):
+  //   adopt   → a live, heartbeating daemon owns the lock — leave it.
+  //   spawn   → no live owner — plain `start` (a dead owner's stale
+  //             lock is reclaimed without --force by lock.ts).
+  //   replace → live owner that stopped heartbeating — `start --force`.
+  // A 30s watchdog re-runs ensureDaemon() so one-shot failure modes
+  // (CLI unresolvable at boot, Process.run() throw, adopted daemon
+  // dying with no terminationHandler) heal on the next tick instead of
+  // stranding the pipeline forever.
+
+  /// Converge toward "exactly one live daemon". Safe to call from any
+  /// trigger (boot, watchdog, child exit, resume) — overlapping calls
+  /// collapse into one in-flight health probe.
+  private func ensureDaemon() {
+    guard !paused else { return }
+    if let d = daemon, d.isRunning { return }
+    if cli == nil { cli = locateCli() }
     guard let cli else {
-      statusMI.title = "modelstat CLI not found"
+      statusMI.title = "modelstat CLI not found — retrying…"
       return
     }
+    guard !ensureInFlight else { return }
+    ensureInFlight = true
+    superviseQueue.async { [weak self] in
+      // Probe off-main: the health command boots node (~100-300ms).
+      let decision = Self.queryDaemonHealth(cli: cli) ?? "spawn"
+      DispatchQueue.main.async {
+        MainActor.assumeIsolated {
+          guard let self else { return }
+          self.ensureInFlight = false
+          guard !self.paused else { return }
+          if let d = self.daemon, d.isRunning { return }
+          switch decision {
+          case "adopt":
+            // A healthy daemon someone else spawned. Adopt: render its
+            // heartbeat (tickLocal already does) and spawn nothing.
+            break
+          case "replace":
+            self.spawnDaemon(cli: cli, force: true)
+          default:
+            self.spawnDaemon(cli: cli, force: false)
+          }
+        }
+      }
+    }
+  }
+
+  /// Run `modelstat _daemon-health` and return its decision, or nil if
+  /// the command failed (older CLI, node missing) — caller treats nil
+  /// as "spawn", which is safe: an unforced spawn against a healthy
+  /// owner exits 0 in <1s without killing anything.
+  private nonisolated static func queryDaemonHealth(cli: URL) -> String? {
     let p = Process()
-    // --force: the tray owns the daemon. If a stale lock from a prior
-    // run (crash, kill -9, OS reboot mid-write) is left behind, the
-    // unforced `start` exits in <1s with "already running" and the
-    // tray's terminationHandler retries it forever. With --force we
-    // claim the lock unconditionally and become the live daemon.
     if cli.pathExtension == "mjs" {
       p.launchPath = "/usr/bin/env"
-      p.arguments = ["node", cli.path, "start", "--force"]
+      p.arguments = ["node", cli.path, "_daemon-health"]
     } else {
       p.launchPath = cli.path
-      p.arguments = ["start", "--force"]
+      p.arguments = ["_daemon-health"]
+    }
+    let pipe = Pipe()
+    p.standardOutput = pipe
+    p.standardError = Pipe()
+    do { try p.run() } catch { return nil }
+    p.waitUntilExit()
+    guard p.terminationStatus == 0 else { return nil }
+    let data = pipe.fileHandleForReading.readDataToEndOfFile()
+    guard let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
+          let decision = obj["decision"] as? String else { return nil }
+    return decision
+  }
+
+  private func spawnDaemon(cli: URL, force: Bool) {
+    let p = Process()
+    var args = ["start"]
+    if force { args.append("--force") }
+    if cli.pathExtension == "mjs" {
+      p.launchPath = "/usr/bin/env"
+      p.arguments = ["node", cli.path] + args
+    } else {
+      p.launchPath = cli.path
+      p.arguments = args
     }
     // Bolt stdout/stderr onto the same log the launchd plist uses so
     // `modelstat status` still sees the same tail.
@@ -249,29 +344,57 @@ final class TrayController: NSObject {
     p.standardOutput = out
     p.standardError = err
     p.terminationHandler = { [weak self] proc in
-      // Daemon exited — if we didn't pause it intentionally, restart
-      // after 2s so a crash is self-healing without user clicks.
+      // Daemon exited — re-converge via the health check, which adopts
+      // a replacement daemon instead of counter-killing it. A clean
+      // sub-5s exit means "another daemon owns the lock" (or an equally
+      // immediate no-op); skip the hot retry and let the 30s watchdog
+      // re-check, so a stale CLI can't put us in a 2s spawn loop.
+      let status = proc.terminationStatus
       Task { @MainActor in
         guard let self else { return }
-        guard !self.paused else { return }
+        let uptime = self.daemonSpawnedAt.map { Date().timeIntervalSince($0) } ?? .infinity
         self.daemon = nil
+        self.daemonSpawnedAt = nil
+        guard !self.paused else { return }
+        if status == 0 && uptime < 5 { return } // watchdog will re-ensure
         DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { [weak self] in
-          self?.startDaemon()
+          MainActor.assumeIsolated { self?.ensureDaemon() }
         }
-        _ = proc
       }
     }
     do {
       try p.run()
       daemon = p
+      daemonSpawnedAt = Date()
     } catch {
-      statusMI.title = "modelstat start failed: \(error.localizedDescription)"
+      // Watchdog retries in ≤30s — do NOT give up permanently here.
+      statusMI.title = "modelstat start failed (retrying): \(error.localizedDescription)"
     }
   }
 
   private func stopDaemon() {
-    daemon?.terminate()
-    daemon = nil
+    if let d = daemon {
+      d.terminate()
+      daemon = nil
+      daemonSpawnedAt = nil
+      return
+    }
+    // No child of our own — but Pause/Quit must also stop an ADOPTED
+    // daemon (one we found healthy and left alone). SIGTERM the lock
+    // owner; harmless no-op if it's already gone.
+    if let pid = Self.readLockOwnerPid() {
+      kill(pid, SIGTERM)
+    }
+  }
+
+  /// pid from ~/.modelstat/daemon.lock, for stopping an adopted daemon.
+  private nonisolated static func readLockOwnerPid() -> pid_t? {
+    let path = ("~/.modelstat/daemon.lock" as NSString).expandingTildeInPath
+    guard let data = FileManager.default.contents(atPath: path),
+          let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
+          let pid = obj["pid"] as? Int, pid > 0
+    else { return nil }
+    return pid_t(pid)
   }
 
   // ── Live local heartbeat (fast path) ────────────────────────────
@@ -507,7 +630,7 @@ final class TrayController: NSObject {
       statusMI.title = "Paused"
     } else {
       pauseMI.title = "Pause"
-      startDaemon()
+      ensureDaemon()
     }
   }
 
@@ -520,6 +643,7 @@ final class TrayController: NSObject {
     stopDaemon()
     fastTimer?.invalidate()
     slowTimer?.invalidate()
+    watchdogTimer?.invalidate()
     NSApp.terminate(nil)
   }
 }