loki-mode 7.78.0 → 7.80.0

package/dashboard/registry.py

@@ -525,6 +525,218 @@ def get_cross_project_tasks(project_ids: Optional[list[str]] = None) -> list[dic
     return all_tasks
 
 
+def _pid_alive(pid) -> bool:
+    """Return True if pid is a positive int naming a live process.
+
+    Mirrors the liveness probe used by the dashboard's running-projects view:
+    signal 0 delivered -> alive; EPERM (owned by another user) -> alive; ESRCH
+    -> dead. Never raises.
+    """
+    if not isinstance(pid, int) or pid <= 0:
+        return False
+    try:
+        os.kill(pid, 0)
+        return True
+    except PermissionError:
+        return True
+    except (ProcessLookupError, OSError):
+        return False
+
+
+def _read_project_run_snapshot(path: str) -> dict:
+    """Read a single project's live run snapshot from its .loki/ state files.
+
+    HONEST SCOPE: this polls the shared on-disk metadata that `loki start`
+    already writes per project (no new store, no controller). It reads:
+      - .loki/dashboard-state.json (phase, iteration; written ~every 2s)
+      - .loki/session.json         (status, startedAt fallback)
+      - .loki/metrics/efficiency/*.json (summed cost_usd) with a
+        .loki/context/tracking.json fallback (totals.total_cost_usd)
+
+    Returns a dict with phase, iteration, cost_usd, started_at, and ended_at
+    (best-effort; missing values default to None/0). Never raises: any file
+    problem degrades the affected field to its default.
+    """
+    snap = {
+        "phase": "",
+        "iteration": 0,
+        "cost_usd": 0.0,
+        "started_at": None,
+        "ended_at": None,
+    }
+    if not path:
+        return snap
+    loki_dir = Path(path) / ".loki"
+
+    # Phase + iteration from dashboard-state.json (the live writer).
+    state_file = loki_dir / "dashboard-state.json"
+    if state_file.exists():
+        try:
+            state = json.loads(state_file.read_text())
+            if isinstance(state, dict):
+                _p = state.get("phase", "")
+                snap["phase"] = _p if isinstance(_p, str) else ""
+                _i = state.get("iteration", 0)
+                snap["iteration"] = _i if isinstance(_i, int) else 0
+        except (json.JSONDecodeError, OSError, ValueError):
+            pass
+
+    # Timestamps from session.json (startedAt; endedAt when present).
+    session_file = loki_dir / "session.json"
+    if session_file.exists():
+        try:
+            sd = json.loads(session_file.read_text())
+            if isinstance(sd, dict):
+                _sa = sd.get("startedAt") or sd.get("started_at")
+                snap["started_at"] = _sa if isinstance(_sa, str) else None
+                _ea = sd.get("endedAt") or sd.get("ended_at")
+                snap["ended_at"] = _ea if isinstance(_ea, str) else None
+        except (json.JSONDecodeError, OSError, ValueError):
+            pass
+
+    # Cost: sum per-iteration efficiency files; fall back to context tracking.
+    cost = 0.0
+    found_cost = False
+    eff_dir = loki_dir / "metrics" / "efficiency"
+    if eff_dir.is_dir():
+        try:
+            for eff_file in eff_dir.glob("*.json"):
+                try:
+                    data = json.loads(eff_file.read_text())
+                    if not isinstance(data, dict):
+                        continue
+                    c = data.get("cost_usd")
+                    if isinstance(c, (int, float)):
+                        cost += float(c)
+                        found_cost = True
+                except (json.JSONDecodeError, OSError, ValueError):
+                    continue
+        except OSError:
+            pass
+    if not found_cost:
+        ctx_file = loki_dir / "context" / "tracking.json"
+        if ctx_file.exists():
+            try:
+                ctx = json.loads(ctx_file.read_text())
+                if isinstance(ctx, dict):
+                    totals = ctx.get("totals", {})
+                    if isinstance(totals, dict):
+                        tc = totals.get("total_cost_usd")
+                        if isinstance(tc, (int, float)):
+                            cost = float(tc)
+            except (json.JSONDecodeError, OSError, ValueError):
+                pass
+    snap["cost_usd"] = round(cost, 6)
+    return snap
+
+
+def get_fleet_runs(include_inactive: bool = True) -> list[dict]:
+    """Build a fleet-wide view of builds across ALL registered projects.
+
+    v1 SCOPE (honest): this polls the shared metadata store that `loki start`
+    already maintains -- the machine-global registry (~/.loki/dashboard/
+    projects.json) plus each project's own .loki/ state files. There is NO
+    controller, CRD, or Job-watcher here; a real k8s operator watching Jobs is
+    future work. One registered project maps to one "run" entry (its current /
+    most-recent build), which is the granularity the registry tracks.
+
+    Each entry carries: id, name, path, status (running|stopped|<registry
+    status>), running (live pid probe), phase, iteration, cost_usd, started_at,
+    duration_seconds, port. Never raises: registry problems degrade to an empty
+    list and per-project read problems degrade that entry's fields.
+    """
+    try:
+        projects = list_projects(include_inactive=include_inactive)
+    except Exception:
+        return []
+
+    out = []
+    for p in projects:
+        path = p.get("path", "")
+        pid = p.get("pid")
+        running = _pid_alive(pid)
+        snap = _read_project_run_snapshot(path)
+
+        # A live pid is authoritative for "running"; otherwise reflect the
+        # registry status (stopped/active/missing). This mirrors the
+        # running-projects endpoint's pid-first precedence.
+        if running:
+            status = "running"
+        else:
+            status = p.get("status") or "unknown"
+
+        # Duration: wall time from started_at to ended_at (or now if running).
+        duration_seconds = None
+        started_at = snap.get("started_at")
+        if started_at:
+            try:
+                st = datetime.fromisoformat(str(started_at).replace("Z", "+00:00"))
+                if st.tzinfo is None:
+                    st = st.replace(tzinfo=timezone.utc)
+                end_ref = None
+                ended_at = snap.get("ended_at")
+                if ended_at and not running:
+                    try:
+                        end_ref = datetime.fromisoformat(
+                            str(ended_at).replace("Z", "+00:00")
+                        )
+                        if end_ref.tzinfo is None:
+                            end_ref = end_ref.replace(tzinfo=timezone.utc)
+                    except (ValueError, TypeError):
+                        end_ref = None
+                if end_ref is None:
+                    end_ref = datetime.now(timezone.utc)
+                duration_seconds = max(0, int((end_ref - st).total_seconds()))
+            except (ValueError, TypeError):
+                duration_seconds = None
+
+        out.append({
+            "id": p.get("id"),
+            "name": p.get("name") or (os.path.basename(path) if path else "project"),
+            "path": path,
+            "status": status,
+            "running": running,
+            "phase": snap.get("phase", ""),
+            "iteration": snap.get("iteration", 0),
+            "cost_usd": snap.get("cost_usd", 0.0),
+            "started_at": started_at,
+            "duration_seconds": duration_seconds,
+            "port": p.get("port"),
+        })
+
+    # Running builds first, then by most-recent start time.
+    out.sort(
+        key=lambda r: (
+            0 if r.get("running") else 1,
+            r.get("started_at") or "",
+        ),
+        reverse=False,
+    )
+    # Within the same running-group, most recent start first.
+    out.sort(key=lambda r: r.get("started_at") or "", reverse=True)
+    out.sort(key=lambda r: 0 if r.get("running") else 1)
+    return out
+
+
+def get_fleet_summary(include_inactive: bool = True) -> dict:
+    """Aggregate fleet-wide totals from get_fleet_runs().
+
+    Returns counts (total / running / stopped) and a summed cost across all
+    registered projects. v1 polls the shared metadata store (see
+    get_fleet_runs); not a controller. Never raises.
+    """
+    runs = get_fleet_runs(include_inactive=include_inactive)
+    total = len(runs)
+    running = sum(1 for r in runs if r.get("running"))
+    total_cost = round(sum(float(r.get("cost_usd") or 0.0) for r in runs), 6)
+    return {
+        "total_runs": total,
+        "running_runs": running,
+        "stopped_runs": total - running,
+        "total_cost_usd": total_cost,
+    }
+
+
 def get_cross_project_learnings() -> dict:
     """
     Get learnings from the global learnings database.

package/dashboard/server.py

@@ -2389,6 +2389,118 @@ async def get_cross_project_learnings():
     return learnings
 
 
+# =============================================================================
+# Fleet Observability (v1: poll the shared metadata store)
+# =============================================================================
+#
+# HONEST SCOPE: these endpoints aggregate the data `loki start` already writes
+# -- the machine-global registry (~/.loki/dashboard/projects.json) plus each
+# project's own .loki/ state files. There is NO controller, CRD, or Kubernetes
+# Job-watcher; a real operator watching Jobs is future work. One registered
+# project maps to one fleet "run" (its current / most-recent build), which is
+# the granularity the registry tracks. Cancel reuses the same STOP-file + pid
+# teardown as the per-project switcher Stop. Retry is intentionally NOT exposed
+# here: there is no clean cross-project re-launch primitive in the registry
+# path (the original spec source lives only in each project's CWD), so retry is
+# a follow-up.
+
+
+class FleetRunResponse(BaseModel):
+    """One fleet run = one registered project's current/most-recent build."""
+    id: Optional[str] = None
+    name: str
+    path: str
+    status: str
+    running: bool
+    phase: str = ""
+    iteration: int = 0
+    cost_usd: float = 0.0
+    started_at: Optional[str] = None
+    duration_seconds: Optional[int] = None
+    port: Optional[int] = None
+
+
+class FleetSummaryResponse(BaseModel):
+    """Fleet-wide totals across all registered projects."""
+    total_runs: int
+    running_runs: int
+    stopped_runs: int
+    total_cost_usd: float
+
+
+@app.get(
+    "/api/fleet/runs",
+    response_model=list[FleetRunResponse],
+    dependencies=[Depends(auth.require_scope("read"))],
+)
+async def list_fleet_runs(include_inactive: bool = True):
+    """List all builds across every registered project (fleet view).
+
+    v1 polls the shared metadata store (registry + per-project .loki/ state);
+    it is not a controller. Never raises: registry problems degrade to [].
+    """
+    if not _read_limiter.check("fleet_runs"):
+        raise HTTPException(status_code=429, detail="Rate limit exceeded")
+    return await asyncio.to_thread(registry.get_fleet_runs, include_inactive)
+
+
+@app.get(
+    "/api/fleet/summary",
+    response_model=FleetSummaryResponse,
+    dependencies=[Depends(auth.require_scope("read"))],
+)
+async def get_fleet_summary(include_inactive: bool = True):
+    """Fleet-wide totals (counts + summed cost) across registered projects."""
+    if not _read_limiter.check("fleet_summary"):
+        raise HTTPException(status_code=429, detail="Rate limit exceeded")
+    return await asyncio.to_thread(registry.get_fleet_summary, include_inactive)
+
+
+@app.get(
+    "/api/fleet/runs/{identifier}",
+    response_model=FleetRunResponse,
+    dependencies=[Depends(auth.require_scope("read"))],
+)
+async def get_fleet_run(identifier: str):
+    """Get a single fleet run by registry id / path / alias.
+
+    Resolves the identifier through the registry (never a caller-supplied
+    arbitrary path), then returns that project's current build snapshot.
+    """
+    project = await asyncio.to_thread(registry.get_project, identifier)
+    if not project:
+        raise HTTPException(status_code=404, detail="Run not found in fleet")
+    pid = project.get("pid")
+    running = registry._pid_alive(pid)
+    snap = registry._read_project_run_snapshot(project.get("path", ""))
+    duration_seconds = None
+    started_at = snap.get("started_at")
+    if started_at:
+        try:
+            st = datetime.fromisoformat(str(started_at).replace("Z", "+00:00"))
+            if st.tzinfo is None:
+                st = st.replace(tzinfo=timezone.utc)
+            duration_seconds = max(
+                0, int((datetime.now(timezone.utc) - st).total_seconds())
+            )
+        except (ValueError, TypeError):
+            duration_seconds = None
+    path = project.get("path", "")
+    return FleetRunResponse(
+        id=project.get("id"),
+        name=project.get("name") or (os.path.basename(path) if path else "project"),
+        path=path,
+        status="running" if running else (project.get("status") or "unknown"),
+        running=running,
+        phase=snap.get("phase", ""),
+        iteration=snap.get("iteration", 0),
+        cost_usd=snap.get("cost_usd", 0.0),
+        started_at=started_at,
+        duration_seconds=duration_seconds,
+        port=project.get("port"),
+    )
+
+
 # =============================================================================
 # Active Project Focus (for AI Chat / cross-directory usage)
 # =============================================================================
@@ -3204,6 +3316,130 @@ async def stop_running_project(request: Request, body: RunningProjectStopRequest
     }
 
 
+@app.post(
+    "/api/fleet/runs/{identifier}/cancel",
+    dependencies=[Depends(auth.require_scope("control"))],
+)
+async def cancel_fleet_run(request: Request, identifier: str):
+    """Cancel ONE build in the fleet view.
+
+    Resolves the run via the registry (by id / path / alias), then runs the
+    same teardown the per-project switcher Stop uses: write a STOP file into the
+    registry-resolved .loki dir (the only path ever written -- never a
+    caller-supplied one) for a clean runner exit, SIGTERM->poll->SIGKILL the
+    recorded orchestrator pid, group-kill + cwd-scoped reap as backstop, then
+    mark the registry/session stopped.
+
+    Retry is intentionally NOT exposed: there is no clean cross-project
+    re-launch primitive in the registry path (the original spec source lives
+    only in each project's CWD). Retry is a documented follow-up.
+    """
+    if not _control_limiter.check("control"):
+        raise HTTPException(status_code=429, detail="Rate limit exceeded")
+
+    project = registry.get_project(identifier)
+    if not project:
+        raise HTTPException(status_code=404, detail="Run not found in fleet")
+
+    project_id = project.get("id")
+    audit.log_event(
+        action="cancel",
+        resource_type="fleet_run",
+        details={"source": "api", "project_id": project_id},
+        ip_address=request.client.host if request.client else None,
+    )
+
+    # Only ever operate on the registry-stored path.
+    path = project.get("path", "")
+    loki_dir = None
+    if path:
+        p = _Path(path)
+        if p.is_dir() and (p / ".loki").is_dir():
+            loki_dir = p / ".loki"
+
+    # STOP file: clean runner teardown (also stops a containerized `loki docker`
+    # build polling the bind-mounted .loki/STOP).
+    stop_signaled = False
+    if loki_dir is not None:
+        try:
+            (loki_dir / "STOP").write_text(datetime.now(timezone.utc).isoformat())
+            stop_signaled = True
+        except OSError:
+            pass
+
+    pid = project.get("pid")
+    stopped = False
+    # Ownership guard (hardening): only direct-kill the registry pid if it is STILL
+    # a process whose cwd is this project's path. A stale pid reused by an unrelated
+    # host process after a crash must NOT be SIGKILLed. If ownership cannot be
+    # confirmed, skip the direct kill and let the cwd-scoped reaper below handle it.
+    if isinstance(pid, int) and pid > 0:
+        _owned = True
+        try:
+            _cwd = _pid_cwd(pid)
+            if _cwd is not None and path:
+                _owned = os.path.realpath(_cwd) == os.path.realpath(path)
+        except Exception:
+            _owned = True  # best-effort: if we cannot tell, preserve prior behavior
+        if not _owned:
+            pid = None  # do not direct-kill a pid that is not this project's
+    if isinstance(pid, int) and pid > 0:
+        try:
+            os.kill(pid, 15)  # SIGTERM
+            for _ in range(10):
+                await asyncio.sleep(0.5)
+                try:
+                    os.kill(pid, 0)
+                except OSError:
+                    stopped = True
+                    break
+            if not stopped:
+                try:
+                    os.kill(pid, 9)  # SIGKILL
+                    stopped = True
+                except (OSError, ProcessLookupError):
+                    stopped = True
+        except (ValueError, OSError, ProcessLookupError):
+            stopped = True
+    else:
+        # No host pid (genuinely stopped, or a docker build): the STOP write is
+        # the cancel signal. Treat a successful STOP write as cancelled.
+        stopped = stop_signaled
+
+    # Group-kill + cwd-scoped reaper backstop against a stale pid.
+    if loki_dir is not None:
+        proj_dir = loki_dir.parent
+        _pgid = _read_pgid(loki_dir)
+        if _pgid is not None:
+            await asyncio.to_thread(
+                _killpg_project, _pgid, _collect_protected_pids(loki_dir)
+            )
+        found_any, all_gone = await asyncio.to_thread(
+            _reap_orchestrators_until_clear, proj_dir, str(proj_dir)
+        )
+        if found_any:
+            stopped = all_gone
+
+        # Mark session.json stopped.
+        session_file = loki_dir / "session.json"
+        if session_file.exists():
+            try:
+                sd = json.loads(session_file.read_text())
+                sd["status"] = "stopped"
+                atomic_write_json(session_file, sd, use_lock=True)
+            except Exception:
+                pass
+
+    registry.mark_project_stopped(project_id)
+
+    return {
+        "success": True,
+        "project_id": project_id,
+        "cancelled": stopped,
+        "stop_signaled": stop_signaled,
+    }
+
+
 # =============================================================================
 # Enterprise Features (Optional - enabled via environment variables)
 # =============================================================================
@@ -4008,7 +4244,7 @@ async def get_memory_summary():
     return summary
 
 
-@app.get("/api/memory/episodes")
+@app.get("/api/memory/episodes", dependencies=[Depends(auth.require_scope("read"))])
 async def list_episodes(limit: int = Query(default=50, ge=1, le=1000)):
     """List episodic memory entries."""
     # Both backends below are blocking (SQLite queries / a glob+read loop over
@@ -4079,7 +4315,7 @@ async def get_episode(episode_id: str):
     raise HTTPException(status_code=404, detail="Episode not found")
 
 
-@app.get("/api/memory/patterns")
+@app.get("/api/memory/patterns", dependencies=[Depends(auth.require_scope("read"))])
 async def list_patterns():
     """List semantic patterns."""
     # Try SQLite first
@@ -4118,7 +4354,7 @@ async def get_pattern(pattern_id: str):
     raise HTTPException(status_code=404, detail="Pattern not found")
 
 
-@app.get("/api/memory/skills")
+@app.get("/api/memory/skills", dependencies=[Depends(auth.require_scope("read"))])
 async def list_skills():
     """List procedural skills."""
     # Blocking SQLite query / glob+read loop; offload the whole read so the
@@ -4343,7 +4579,7 @@ async def retrieve_memory(query: dict = None):
         raise HTTPException(status_code=503, detail=f"Retrieval unavailable: {e}")
 
 
-@app.get("/api/memory/index")
+@app.get("/api/memory/index", dependencies=[Depends(auth.require_scope("read"))])
 async def get_memory_index():
     """Get memory index (Layer 1 - lightweight discovery)."""
     index_file = _get_loki_dir() / "memory" / "index.json"
@@ -4355,7 +4591,7 @@ async def get_memory_index():
     return {"topics": [], "lastUpdated": None}
 
 
-@app.get("/api/memory/timeline")
+@app.get("/api/memory/timeline", dependencies=[Depends(auth.require_scope("read"))])
 async def get_memory_timeline():
     """Get memory timeline (Layer 2 - progressive disclosure)."""
     timeline_file = _get_loki_dir() / "memory" / "timeline.json"
@@ -4550,7 +4786,7 @@ def _get_memory_storage():
         return None
 
 
-@app.get("/api/memory/search")
+@app.get("/api/memory/search", dependencies=[Depends(auth.require_scope("read"))])
 async def search_memory(
     q: str = Query(..., min_length=1, max_length=500, description="Search query"),
     collection: str = Query(default="all", pattern="^(episodes|patterns|skills|all)$"),
@@ -4588,7 +4824,7 @@ async def search_memory(
         raise HTTPException(status_code=500, detail=f"Search failed: {e}")
 
 
-@app.get("/api/memory/stats")
+@app.get("/api/memory/stats", dependencies=[Depends(auth.require_scope("read"))])
 async def get_memory_stats():
     """Get memory system statistics (counts, size, backend info)."""
     # SQLite stats query or a directory-walk over many JSON files; both block,
@@ -6860,6 +7096,62 @@ except ImportError as e:
     logger.debug(f"Collaboration module not available: {e}")
 
 
+class _CollabWsAuthMiddleware:
+    """ASGI middleware that auth-gates the native /ws/collab WebSocket.
+
+    The collaboration module registers @app.websocket("/ws/collab") inside
+    create_collab_routes() and performs NO token validation: it accepts any
+    connection and trusts a client-supplied ?user_id=. With enterprise auth or
+    OIDC enabled this native WS is therefore reachable UNAUTHENTICATED, exposing
+    user presence, shared state, and operation sync to any client. The dashboard
+    cannot rely on route dependencies for WebSockets (FastAPI Depends() is not
+    supported on @app.websocket routes), so this middleware validates the token
+    on the /ws/collab handshake before the route runs, mirroring the native /ws
+    gate and the _MountAuthGuard WS logic.
+
+    Scope: the collab WS handle_message path applies state operations (writes)
+    via ws_manager.handle_message -> sync.apply_operation, so a valid but
+    read-only token must not be admitted. This requires the "control" scope to
+    match the _MountAuthGuard WS scope-check pattern (a valid token alone is not
+    enough), so a read-only token is closed 1008 even though it authenticates.
+
+    When enterprise auth and OIDC are both OFF this is a pass-through, so local
+    default-mode behavior is unchanged. Non-websocket scopes and other websocket
+    paths (the native /ws self-guards in-route) are passed through untouched.
+    Added via app.add_middleware(), so app stays a FastAPI instance and all
+    later route registrations are unaffected.
+    """
+
+    def __init__(self, app) -> None:
+        self._app = app
+
+    async def __call__(self, scope, receive, send) -> None:
+        if scope.get("type") != "websocket" or scope.get("path") != "/ws/collab":
+            await self._app(scope, receive, send)
+            return
+        if not auth.is_enterprise_mode() and not auth.is_oidc_mode():
+            await self._app(scope, receive, send)
+            return
+        token_str = _MountAuthGuard._ws_token_from_scope(scope)
+        token_info = _MountAuthGuard._validate_ws_token(token_str)
+        if token_info is None or not auth.has_scope(token_info, "control"):
+            # Accept-then-close is the portable ASGI way to surface a policy
+            # violation to the client before any route code runs. 1008 = policy
+            # violation, matching the native /ws and _MountAuthGuard behavior.
+            # "control" (not just a valid token) is required because the collab
+            # WS path performs state writes; a read-only token is rejected here.
+            await send({"type": "websocket.accept"})
+            await send({"type": "websocket.close", "code": 1008})
+            return
+        await self._app(scope, receive, send)
+
+
+# Gate the /ws/collab handshake before the unauthenticated collab route runs.
+# Registered as middleware so app remains a FastAPI instance (route decorators
+# below keep working).
+app.add_middleware(_CollabWsAuthMiddleware)
+
+
 # =============================================================================
 # Secrets / Credential Status
 # =============================================================================