mishkan-harness 0.2.5 → 0.2.6

package/payload/mishkan/observability/watchd/src/mishkan_watchd/server.py

@@ -36,9 +36,32 @@ class WatchdServer:
         self.lock = asyncio.Lock()
         self._heartbeat_task: asyncio.Task | None = None
 
+    @staticmethod
+    def _socket_is_live(socket_path: Path) -> bool:
+        """Return True if a daemon is already listening on socket_path.
+
+        Attempts a non-blocking connect and an immediate close. If it
+        succeeds the socket has a live owner; if it raises (connection
+        refused, file not found, timeout) the socket is stale or absent.
+        """
+        try:
+            with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
+                s.settimeout(0.5)
+                s.connect(str(socket_path))
+            return True
+        except (OSError, socket.timeout):
+            return False
+
     async def start(self) -> asyncio.AbstractServer:
-        # Ensure parent dir, remove stale socket.
+        # Ensure parent dir exists.
         self.socket_path.parent.mkdir(parents=True, exist_ok=True)
+        # Check liveness BEFORE unlinking: if a daemon is already serving
+        # this socket, do not steal it — return a no-op guard server instead.
+        # Only unlink a socket whose owner is gone (stale file).
+        if self.socket_path.exists() and self._socket_is_live(self.socket_path):
+            raise RuntimeError(
+                f"mishkan-watchd: daemon already running on {self.socket_path}"
+            )
         try:
             self.socket_path.unlink()
         except FileNotFoundError:

package/payload/mishkan/observability/watchd/src/mishkan_watchd/sources/graphify_tail.py

@@ -1,18 +1,22 @@
-"""Graphify daemon source — watch graphify-out/ for scans + queries.
+"""Graphify daemon source — stats-only watcher for graphify-out/.
 
 Graphify produces deterministic AST graphs under `<project>/graphify-out/`.
-This source watches every active session's project for changes in that
-directory and synthesises bus events that the Knowledge tab consumes.
-
-Two event types emitted:
-
-- graphify_scan  : `graph.json` mtime changed → a `graphify update <path>`
-                   completed. Payload: project path, node count, edge
-                   count, community count (parsed from manifest.json
-                   when present).
-- graphify_query : a new file under `graphify-out/memory/` → a
-                   `graphify query` was saved via `graphify save-result`.
-                   Payload: question, answer excerpt (200 chars).
+This source watches every active session's project for graph.json changes
+and surfaces current node/edge/community counts as state refreshes.
+
+STATS-ONLY (as of Fix ③): this source no longer owns scan or query COUNTS.
+Real graphify CLI invocations are detected by post-tool-observe.sh (Bash
+hook, PostToolUse) which emits the authoritative graphify_scan and
+graphify_query events that carry session_id and increment the counters in
+state.py. This source only emits graphify_scan events with stats_only=True
+so state.py updates the size display (nodes/edges/communities) WITHOUT
+incrementing the scan counter. This eliminates:
+  - double-counting (hook + tail both firing for one real scan), and
+  - the phantom scan on every daemon restart (old last_graph_mtime=0.0 init).
+
+The memory/ watcher is also removed: graphify query --save-result is not
+the documented workflow; the hook detects the real `graphify query` CLI
+invocation instead.
 
 Fail-open per the daemon contract. Missing graphify-out/ is silent —
 many projects don't use Graphify and that's fine.
@@ -34,36 +38,34 @@ def _iso(ts: float | None = None) -> str:
 
 
 class _ProjectWatcher:
-    """One watcher per project root that has a graphify-out/ dir."""
+    """One watcher per project root that has a graphify-out/ dir.
+
+    Stats-only: emits graphify_scan with stats_only=True when graph.json
+    mtime advances. This refreshes the node/edge/community display in the
+    Knowledge tab WITHOUT incrementing the scan counter in state.py.
+
+    The mtime baseline is initialised to the CURRENT mtime of graph.json
+    at construction time so an already-built graph does NOT fire a phantom
+    stats event on daemon restart. It fires only when the graph is actually
+    rebuilt after the daemon started.
+    """
 
     def __init__(self, project_path: Path, queue: asyncio.Queue[dict[str, Any]]) -> None:
         self.project_path = project_path
         self.queue = queue
         self.graph_dir = project_path / "graphify-out"
         self.graph_json = self.graph_dir / "graph.json"
-        self.manifest = self.graph_dir / "manifest.json"
-        self.memory_dir = self.graph_dir / "memory"
-        # Start at 0 so the first poll detects the current graph state and
-        # emits a graphify_scan event — otherwise the TUI shows "no scan"
-        # forever for projects whose graph predated the daemon. This costs
-        # one event per project per daemon restart; trivially cheap.
-        self.last_graph_mtime = 0.0
-        # Memory queries DO use seen-state to avoid replaying old queries
-        # on every restart (queries are append-only, replays would spam).
-        self.seen_memory: set[str] = set()
-        if self.memory_dir.is_dir():
-            try:
-                for f in self.memory_dir.iterdir():
-                    if f.is_file():
-                        self.seen_memory.add(f.name)
-            except OSError:
-                pass
+        # Initialise to the current mtime so pre-existing graphs do NOT
+        # produce a phantom stats event on the first poll.
+        try:
+            self.last_graph_mtime: float = self.graph_json.stat().st_mtime
+        except OSError:
+            self.last_graph_mtime = 0.0
 
     async def step(self) -> None:
         if not self.graph_dir.is_dir():
             return
         await self._check_graph()
-        await self._check_memory()
 
     async def _check_graph(self) -> None:
         try:
@@ -84,12 +86,15 @@ class _ProjectWatcher:
             "tool": None,
             "outcome": "completed",
             "payload": {
-                "op": "scan",
+                "op": "stats",
                 "project": str(self.project_path),
                 "nodes": stats.get("nodes"),
                 "edges": stats.get("edges"),
                 "communities": stats.get("communities"),
                 "scanned_at": _iso(mtime),
+                # stats_only=True signals state.py to update size fields
+                # but NOT increment the scan counter.
+                "stats_only": True,
             },
         })
 
@@ -120,37 +125,6 @@ class _ProjectWatcher:
             "communities": len(communities) if isinstance(communities, list) else None,
         }
 
-    async def _check_memory(self) -> None:
-        if not self.memory_dir.is_dir():
-            return
-        try:
-            entries = [f for f in self.memory_dir.iterdir() if f.is_file()]
-        except OSError:
-            return
-        for f in entries:
-            if f.name in self.seen_memory:
-                continue
-            self.seen_memory.add(f.name)
-            payload = {"op": "query", "project": str(self.project_path), "file": f.name}
-            try:
-                data = json.loads(f.read_text())
-                if isinstance(data, dict):
-                    payload["question"] = (data.get("question") or "")[:200]
-                    answer = data.get("answer") or ""
-                    payload["answer_excerpt"] = answer[:200] if isinstance(answer, str) else ""
-                    payload["query_type"] = data.get("type") or data.get("query_type")
-            except Exception:
-                pass
-            await self.queue.put({
-                "ts": _iso(),
-                "session": None,
-                "project": str(self.project_path),
-                "type": "graphify_query",
-                "tool": None,
-                "outcome": "completed",
-                "payload": payload,
-            })
-
 
 def _decode_project(p: str) -> str:
     """Decode Claude Code's encoded project dir name into an absolute path.

package/payload/mishkan/observability/watchd/src/mishkan_watchd/state.py

@@ -32,6 +32,20 @@ def _now() -> str:
 # a hook fires for a brand-new session before its first JSONL flush.
 _PENDING_TTL_S = 15.0
 
+# A workflow run that has received no workflow_* event for this many seconds
+# is transitioned to "stale" in the snapshot. The Workflow tool returns
+# immediately and reports completion via task-notification (not a tool call),
+# so workflow_complete is never emitted by any hook — a TTL is the only
+# durable way to clear lingering "running" entries.
+WORKFLOW_STALE_TTL_S = 900
+
+# A session_stop from session_discover is ignored when the session is "busy":
+# either it still has active agents, or a real bus event was applied within
+# this window. Chosen comfortably above session_discover's 60 s active_window
+# so a session whose transcript goes quiet while a subagent is writing to its
+# own nested JSONL is not torn down mid-run.
+SESSION_KEEPALIVE_S = 90.0
+
 
 @dataclass
 class AgentState:
@@ -62,6 +76,10 @@ class WorkflowState:
     spend_usd: float = 0.0
     tokens_total: int = 0
     started: str = ""
+    # Monotonic timestamp of the last workflow_* event for this run. Used by
+    # the stale-sweep in to_snapshot() to transition runs that have received
+    # no activity for WORKFLOW_STALE_TTL_S to status "stale".
+    last_activity_mono: float = field(default_factory=monotonic)
 
 
 @dataclass
@@ -77,6 +95,18 @@ class SessionState:
     cache_read: int = 0
     cache_write: int = 0
     cost_estimate_usd: float = 0.0
+    # Most-recent turn's input footprint (SET, not accumulated). Equals
+    # cache_read + cache_write + tokens_in from the last token_usage event.
+    # This is what the TUI Usage gauge should divide by the context window to
+    # get a meaningful fill fraction; the cumulative tokens_in alone reads ~0%
+    # under prompt caching because cache_read tokens are not billed as input.
+    last_context_tokens: int = 0
+    # Monotonic timestamp of the last bus event applied to this session.
+    # Used by the session_stop guard to keep a busy session (active agents or
+    # recent activity) alive when session_discover declares it stale because
+    # its parent transcript has not been written to (subagent writes go to
+    # nested subagents/agent-*.jsonl, leaving the parent file quiet).
+    last_event_mono: float = 0.0
 
 
 @dataclass
@@ -163,15 +193,42 @@ class HarnessState:
     def apply(self, event: dict[str, Any]) -> None:
         """Apply a bus-format event to the state.
 
-        Authority gate: session_discover is the source of truth for alive
-        sessions. Only sessions confirmed by session_discover (via
-        session_start) are eligible to have events applied. Events for
-        unknown session_ids are buffered briefly (_PENDING_TTL_S) and
-        flushed if confirmation arrives; otherwise dropped.
-
-        This kills the phantom-session bug at its root: bus_tail can no
-        longer resurrect stopped or never-alive sessions, no matter what
-        old hook events surface from disk.
+        Authority gate — two tiers:
+
+        Tier 1 (session_discover path): session_discover emits session_start
+        when it observes an active JSONL file (transcript mtime < 60 s). That
+        adds the session to _confirmed_alive and flushes any buffered events.
+        session_stop is the stop authority; it removes from _confirmed_alive and
+        tombstones the session in _stopped_recently.
+
+        Tier 2 (live-event path): bus_tail seeks to EOF on startup — there is no
+        historical replay. Any event that arrives for a session_id that is not
+        tombstoned (_stopped_recently) is therefore a genuine live signal. On the
+        first such event for an unconfirmed session we confirm it immediately,
+        create the SessionState, and flush any buffered events, then fall through
+        to apply the triggering event. This fixes the agent-only session blindspot:
+        during an agent run the parent transcript is quiet (subagent writes go to
+        nested subagents/agent-*.jsonl), so session_discover never confirms the
+        session via mtime; but bus events (agent_spawn, tool_call, token_usage)
+        stream in live, and tier 2 picks them up on the first one.
+
+        Tombstone resurrection: because bus_tail seeks to EOF there is no
+        historical replay, so a fresh event for a tombstoned session is genuine
+        new activity — the session was spuriously stopped by session_discover
+        when its parent transcript went quiet during an agent run, not actually
+        dead. The tombstone branch un-tombstones, re-confirms, and falls through
+        so the triggering event is applied. A truly-ended session produces no new
+        events, so it stays tombstoned naturally. Lagging hook events (the
+        original phantom-resurrection concern) are distinguished by the busy-guard
+        on session_stop: a session that was genuinely busy was never tombstoned in
+        the first place. This branch therefore only fires for sessions that were
+        spuriously stopped while quiet-but-active (the quiet-then-active pattern).
+
+        Liveness is refreshed by bus activity (last_event_mono). A session_stop
+        from session_discover is ignored when the session is busy (agents_active
+        non-empty OR last_event_mono within SESSION_KEEPALIVE_S); session_discover
+        re-polls and re-emits session_stop, so idle sessions are still cleaned up
+        normally.
         """
         try:
             self._sweep_pending()
@@ -185,19 +242,39 @@ class HarnessState:
                     self._ensure_session(session_id, event.get("project") or "unknown")
                     self._flush_pending(session_id)
                 elif etype == "session_stop":
-                    # Honoured below; remove from confirmed + tombstone.
+                    # Busy-guard handled below; fall through.
                     pass
                 elif session_id in self._stopped_recently:
-                    # Stopped session — drop silently. Tombstone window handles
-                    # lagging hook events for sessions whose stop just propagated.
-                    return
+                    # A live event for a tombstoned session. bus_tail seeks to
+                    # EOF (no historical replay), so this is genuine new activity
+                    # — the session was spuriously stopped by session_discover
+                    # when its parent transcript went quiet during an agent run,
+                    # not actually dead. Un-tombstone and re-confirm so its
+                    # agents/tokens reappear. (A truly-ended session produces no
+                    # new events, so it stays tombstoned.)
+                    try:
+                        self._stopped_recently.remove(session_id)
+                    except ValueError:
+                        pass
+                    self._confirmed_alive.add(session_id)
+                    self._ensure_session(session_id, event.get("project") or "unknown")
+                    self._flush_pending(session_id)
+                    # fall through (NO return) so this event is applied below
                 elif session_id not in self._confirmed_alive:
-                    # Unknown session. Buffer with TTL — if session_discover
-                    # confirms within _PENDING_TTL_S we replay in order;
-                    # otherwise the sweep drops it.
-                    buf = self._pending.setdefault(session_id, deque(maxlen=200))
-                    buf.append((monotonic(), event))
-                    return
+                    # bus_tail seeks to EOF (no historical replay), so a live
+                    # event for a non-tombstoned, unconfirmed session is a
+                    # genuine alive signal. Confirm on this first event rather
+                    # than waiting for session_discover's transcript-mtime poll
+                    # — which never fires for an agent-only session whose parent
+                    # transcript stays quiet while subagents write to their own
+                    # nested JSONL files. The _stopped_recently tombstone check
+                    # above still blocks resurrection of recently-stopped sessions.
+                    self._confirmed_alive.add(session_id)
+                    self._ensure_session(session_id, event.get("project") or "unknown")
+                    self._flush_pending(session_id)
+                    # Fall through (NO return) so this event reaches the
+                    # dispatch below and agents_active / tokens / etc. populate
+                    # immediately.
 
             if etype == "agent_spawn":
                 self._on_agent_spawn(session_id, event)
@@ -217,10 +294,29 @@ class HarnessState:
                 self._on_cognee_op(event)
             elif etype in ("graphify_scan", "graphify_query"):
                 self._on_graphify(event)
+            elif etype in ("workflow_start", "workflow_update"):
+                self._on_workflow_event(session_id, event)
             elif etype == "session_start":
                 pass  # ensured above
             elif etype == "session_stop":
                 if session_id:
+                    sess = self.sessions.get(session_id)
+                    busy = (
+                        sess is not None
+                        and (
+                            bool(sess.agents_active)
+                            or (monotonic() - sess.last_event_mono) < SESSION_KEEPALIVE_S
+                        )
+                    )
+                    if busy:
+                        # Session still has running agents or received a real
+                        # bus event recently — transcript staleness from
+                        # session_discover does not reflect true liveness here
+                        # (subagent writes go to nested JSONL files). Ignore
+                        # this stop; session_discover will re-emit on the next
+                        # poll cycle and the guard re-evaluates then.
+                        return
+                    # Genuinely idle: no active agents and no recent bus events.
                     self.sessions.pop(session_id, None)
                     self._confirmed_alive.discard(session_id)
                     self._pending.pop(session_id, None)
@@ -230,6 +326,10 @@ class HarnessState:
 
             if session_id and session_id in self.sessions:
                 self.sessions[session_id].recent_events.append(event)
+                # Stamp liveness so the session_stop busy-guard has a fresh
+                # monotonic reference. Any real bus event refreshes the window,
+                # keeping a session alive through a transcript-quiet subagent run.
+                self.sessions[session_id].last_event_mono = monotonic()
         except Exception:
             return  # fail-open on any state error
 
@@ -299,11 +399,15 @@ class HarnessState:
         if not sid:
             return
         sess = self.sessions[sid]
-        agent_name = event.get("agent") or (event.get("payload") or {}).get("subagent_type")
-        if not agent_name:
-            return
         payload = event.get("payload") or {}
-        sess.agents_active[agent_name] = AgentState(
+        # Key by tool_use_id so two concurrent agents of the same subagent_type
+        # do not collide. Fall back to agent_name when tool_use_id is absent
+        # (e.g. replayed legacy events that predate the new schema).
+        key = payload.get("tool_use_id") or event.get("agent") or payload.get("subagent_type")
+        if not key:
+            return
+        agent_name = (event.get("agent") or payload.get("subagent_type") or key)
+        sess.agents_active[key] = AgentState(
             name=agent_name,
             started=event.get("ts") or _now(),
             last_tool=event.get("tool"),
@@ -317,10 +421,13 @@ class HarnessState:
         sess = self.sessions.get(sid)
         if not sess:
             return
-        agent_name = event.get("agent")
-        if not agent_name:
+        payload = event.get("payload") or {}
+        # Mirror _on_agent_spawn: pop by tool_use_id first, fall back to
+        # agent field for legacy events.
+        key = payload.get("tool_use_id") or event.get("agent")
+        if not key:
             return
-        sess.agents_active.pop(agent_name, None)
+        sess.agents_active.pop(key, None)
 
     def _on_tool_call(self, sid: Optional[str], event: dict[str, Any]) -> None:
         if not sid:
@@ -341,6 +448,52 @@ class HarnessState:
         sess.cache_read += int(p.get("cache_read") or 0)
         sess.cache_write += int(p.get("cache_write") or 0)
         sess.cost_estimate_usd += float(p.get("cost_estimate_usd") or 0.0)
+        # SET (do not accumulate): the per-turn context footprint that the
+        # Usage gauge should display. Under prompt caching the vast majority
+        # of the context lives in cache_read, not tokens_in, so cumulative
+        # tokens_in alone would read ~0% of the context window.
+        sess.last_context_tokens = (
+            int(p.get("cache_read") or 0)
+            + int(p.get("cache_write") or 0)
+            + int(p.get("tokens_in") or 0)
+        )
+
+    def _on_workflow_event(self, sid: Optional[str], event: dict[str, Any]) -> None:
+        """Record a workflow_start or workflow_update into the owning session.
+
+        Stamps last_activity_mono on every call so the stale-sweep in
+        to_snapshot() has a fresh monotonic reference to work from.
+        workflow_complete is never emitted (the Workflow tool is fire-and-forget
+        from the hook's perspective); TTL-based staling in to_snapshot() is the
+        durable path to clearing "running" entries.
+        """
+        if not sid:
+            return
+        sess = self.sessions.get(sid)
+        if not sess:
+            return
+        p = event.get("payload") or {}
+        run_id = p.get("run_id") or p.get("workflow_id")
+        if not run_id:
+            return
+        etype = event.get("type")
+        if etype == "workflow_start":
+            sess.workflows_active[run_id] = WorkflowState(
+                name=p.get("name") or p.get("scriptPath") or run_id,
+                run_id=run_id,
+                phase="running",
+                started=event.get("ts") or _now(),
+                last_activity_mono=monotonic(),
+            )
+        elif etype == "workflow_update":
+            wf = sess.workflows_active.get(run_id)
+            if wf:
+                wf.phase = p.get("phase") or wf.phase
+                wf.phases_total = int(p.get("phases_total") or wf.phases_total)
+                wf.phases_done = int(p.get("phases_done") or wf.phases_done)
+                wf.spend_usd = float(p.get("spend_usd") or wf.spend_usd)
+                wf.tokens_total = int(p.get("tokens_total") or wf.tokens_total)
+                wf.last_activity_mono = monotonic()
 
     def _on_worktree(self, event: dict[str, Any]) -> None:
         p = event.get("payload") or {}
@@ -390,24 +543,39 @@ class HarnessState:
     def _on_graphify(self, event: dict[str, Any]) -> None:
         """Aggregate Graphify scan/query events into a single rollup.
 
-        We don't break this down per-project here — the TUI cards show the
-        most recent scan's totals + a query counter, which is what a single
-        rollup needs. The snapshot serializer flattens this back to a dict
-        so the existing TUI Knowledge tab can read it directly.
+        Event source split (as of Fix ③):
+        - graphify_scan / graphify_query COUNTS come from the Bash hook
+          (post-tool-observe.sh), which detects real CLI invocations.
+          These events carry session_id so they also appear in session
+          recent_events correctly.
+        - graphify_tail is STATS-ONLY: it reads graph.json node/edge counts
+          and emits graphify_scan events that carry nodes/edges/communities
+          but whose 'stats_only' flag is set. Those update the size stats
+          WITHOUT incrementing the scan counter. This prevents double-counting
+          and eliminates the phantom-scan-on-restart from the old mtime=0 init.
+
+        The snapshot serializer flattens this to a dict for the TUI
+        Knowledge tab to read directly.
         """
         p = event.get("payload") or {}
         etype = event.get("type")
         g = self.graphify
         if etype == "graphify_scan":
+            # Always update node/edge/community stats when present —
+            # both hook events and graphify_tail stats events carry these.
             if p.get("nodes") is not None:
                 g.nodes = int(p["nodes"])
             if p.get("edges") is not None:
                 g.edges = int(p["edges"])
             if p.get("communities") is not None:
                 g.communities = int(p["communities"])
-            g.scans += 1
-            g.last_scan_project = p.get("project")
-            g.last_scan_at = p.get("scanned_at") or event.get("ts")
+            # Only increment the scan counter for real invocations (hook events).
+            # graphify_tail stats events set stats_only=True to signal they are
+            # size refreshes, not new scan invocations.
+            if not p.get("stats_only"):
+                g.scans += 1
+                g.last_scan_project = p.get("project")
+                g.last_scan_at = p.get("scanned_at") or event.get("ts")
         elif etype == "graphify_query":
             g.queries += 1
             g.last_query_project = p.get("project")
@@ -419,6 +587,25 @@ def _dc_to_dict(dc) -> dict[str, Any]:
     return {k: v for k, v in dc.__dict__.items()}
 
 
+def _workflow_dict(wf: WorkflowState) -> dict[str, Any]:
+    """Serialise a WorkflowState, substituting "stale" for any non-terminal
+    status whose last_activity_mono is older than WORKFLOW_STALE_TTL_S.
+
+    The live WorkflowState object is never mutated here — the substitution is
+    snapshot-only so the in-memory record stays authoritative if a late event
+    arrives for the same run_id.
+    """
+    d = _dc_to_dict(wf)
+    # last_activity_mono is an internal float; strip it from the wire shape.
+    d.pop("last_activity_mono", None)
+    terminal = {"stale", "completed", "failed", "cancelled"}
+    if d.get("phase") not in terminal:
+        age = monotonic() - wf.last_activity_mono
+        if age > WORKFLOW_STALE_TTL_S:
+            d["phase"] = "stale"
+    return d
+
+
 def _session_dict(s: SessionState) -> dict[str, Any]:
     # Truncate recent_events for the snapshot frame so the daemon doesn't
     # send 200 events × N sessions on connect — that easily exceeds the
@@ -432,11 +619,12 @@ def _session_dict(s: SessionState) -> dict[str, Any]:
         "project": s.project,
         "started": s.started,
         "agents_active": {k: _dc_to_dict(v) for k, v in s.agents_active.items()},
-        "workflows_active": {k: _dc_to_dict(v) for k, v in s.workflows_active.items()},
+        "workflows_active": {k: _workflow_dict(v) for k, v in s.workflows_active.items()},
         "recent_events": recent,
         "tokens_in": s.tokens_in,
         "tokens_out": s.tokens_out,
         "cache_read": s.cache_read,
         "cache_write": s.cache_write,
         "cost_estimate_usd": s.cost_estimate_usd,
+        "last_context_tokens": s.last_context_tokens,
     }