claude-finops 0.7.2 → 0.8.0

package/finops/api.py

@@ -4,10 +4,11 @@ the warehouse contains your full prompt text.
 import json
 import io
 import csv
+import logging
 import os
-import sqlite3
+import sys
 import threading
-import traceback
+from datetime import datetime
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from urllib.parse import urlparse, parse_qs
 
@@ -17,6 +18,49 @@ from .paths import DB_PATH, LOCAL_SETTINGS_PATH, LOGFILE, PIDFILE, WEB_DIR, ensu
 WEB = WEB_DIR
 _lock = threading.Lock()
 A = None
+log = logging.getLogger("finops")
+
+
+class BadRequest(Exception):
+    pass
+
+
+def _int(qs, key, default, lo=0, hi=1_000_000):
+    raw = qs.get(key, [None])[0]
+    if raw in (None, ""):
+        return default
+    try:
+        v = int(raw)
+    except ValueError:
+        raise BadRequest(f"{key} must be an integer")
+    return max(lo, min(hi, v))
+
+
+SETTINGS_SHAPE = {
+    "budgets": dict, "limits": dict, "alert_thresholds_pct": list, "waste_rules": dict,
+    "anomaly": dict, "scorecard": dict, "account": dict, "billing_period": dict,
+}
+
+
+def _is_num(v):
+    return isinstance(v, (int, float)) and not isinstance(v, bool)
+
+
+def _validate_numeric_section(name, v):
+    """budgets/limits: every leaf a number or null, per_*_usd a dict of numbers."""
+    for k, leaf in v.items():
+        if k.startswith("_"):
+            continue
+        if k in ("per_project_usd", "per_model_usd"):
+            if not isinstance(leaf, dict) or not all(_is_num(x) for x in leaf.values()):
+                raise BadRequest(f"{name}.{k} must be a dict of numbers")
+        elif not (leaf is None or _is_num(leaf)):
+            raise BadRequest(f"{name}.{k} must be a number or null")
+
+
+def _validate_thresholds(v):
+    if not all(_is_num(x) and 0 <= x <= 1000 for x in v):
+        raise BadRequest("alert_thresholds_pct must be a list of numbers 0..1000")
 
 MIME = {".html": "text/html; charset=utf-8", ".js": "text/javascript; charset=utf-8",
         ".css": "text/css; charset=utf-8", ".json": "application/json",
@@ -30,15 +74,39 @@ def filters_from(qs):
         for item in v:
             out += [x for x in item.split(",") if x]
         return out
+    try:
+        projects = [int(x) for x in lst("projects")]
+    except ValueError:
+        raise BadRequest("projects must be integers")
+
+    def _date(k):
+        v = qs.get(k, [None])[0] or None
+        if v is None:
+            return None
+        try:
+            datetime.strptime(v, "%Y-%m-%d")
+        except ValueError:
+            raise BadRequest(f"{k} must be YYYY-MM-DD")
+        return v
+
+    def _num(k, cast):
+        v = qs.get(k, [None])[0] or None
+        if v is None:
+            return None
+        try:
+            return cast(v)
+        except ValueError:
+            raise BadRequest(f"{k} must be a number")
+
     f = {
-        "start": qs.get("start", [None])[0] or None,
-        "end": qs.get("end", [None])[0] or None,
-        "agents": lst("agents"), "models": lst("models"), "projects": lst("projects"),
+        "start": _date("start"),
+        "end": _date("end"),
+        "agents": lst("agents"), "models": lst("models"), "projects": projects,
         "sessions": lst("sessions"), "categories": lst("categories"),
         "include_sandbox": qs.get("include_sandbox", ["1"])[0] != "0",
-        "min_cost": qs.get("min_cost", [None])[0] or None,
-        "max_cost": qs.get("max_cost", [None])[0] or None,
-        "min_tokens": qs.get("min_tokens", [None])[0] or None,
+        "min_cost": _num("min_cost", float),
+        "max_cost": _num("max_cost", float),
+        "min_tokens": _num("min_tokens", int),
     }
     return f
 
@@ -91,42 +159,65 @@ class Handler(BaseHTTPRequestHandler):
         self.send_response(code)
         self.send_header("Content-Type", "application/json")
         self.send_header("Content-Length", str(len(body)))
+        self.send_header("X-Content-Type-Options", "nosniff")
+        self.send_header("Referrer-Policy", "no-referrer")
         self.end_headers()
         self.wfile.write(body)
 
-    def send_text(self, body, ctype, filename=None, code=200):
+    def send_text(self, body, ctype, filename=None, code=200, no_store=False):
         if isinstance(body, str):
             body = body.encode()
         self.send_response(code)
         self.send_header("Content-Type", ctype)
+        if no_store:
+            self.send_header("Cache-Control", "no-store, must-revalidate")
         if filename:
             self.send_header("Content-Disposition", f'attachment; filename="{filename}"')
         self.send_header("Content-Length", str(len(body)))
+        self.send_header("X-Content-Type-Options", "nosniff")
+        self.send_header("Referrer-Policy", "no-referrer")
         self.end_headers()
         self.wfile.write(body)
 
+    def _host_ok(self):
+        host = (self.headers.get("Host") or "").split(":")[0]
+        return host in ("127.0.0.1", "localhost", "[::1]", "::1")
+
     def do_POST(self):
-        n = int(self.headers.get("Content-Length") or 0)
-        payload = json.loads(self.rfile.read(n) or b"{}")
+        if not self._host_ok():
+            return self.send_json({"error": "forbidden host"}, 403)
         path = urlparse(self.path).path
         try:
+            try:
+                n = max(0, min(int(self.headers.get("Content-Length") or 0), 5_000_000))
+                payload = json.loads(self.rfile.read(n) or b"{}")
+                if not isinstance(payload, dict):
+                    raise ValueError
+            except ValueError:
+                return self.send_json({"error": "body must be a JSON object"}, 400)
             if path.startswith("/api/live/"):
                 self._payload = payload
                 return self.session_action(path)
-            if path == "/api/trial/run":
-                # Spends real money and drives a real agent, so it is guarded like
-                # the other side-effecting routes and only ever reached by a click.
-                if not self._same_origin():
-                    return self.send_json({"ok": False, "error": "forbidden"}, 403)
-                from .trial import run
-                return self.send_json(run(payload.get("category"), payload.get("model"),
-                                          payload.get("prompts") or [],
-                                          cwd=payload.get("cwd") or None))
             if path.startswith("/api/do/"):
                 if not self._same_origin():
                     return self.send_json({"ok": False, "error": "forbidden"}, 403)
                 return self.do_action(path[len("/api/do/"):].strip("/").split("/"), payload)
             if path == "/api/settings":
+                if not self._same_origin():
+                    return self.send_json({"ok": False, "error": "forbidden"}, 403)
+                for k, v in payload.items():
+                    if k not in SETTINGS_SHAPE:
+                        return self.send_json({"error": f"unknown key {k}"}, 400)
+                    if not isinstance(v, SETTINGS_SHAPE[k]):
+                        return self.send_json({"error": f"{k} must be a {SETTINGS_SHAPE[k].__name__}"}, 400)
+                bp = payload.get("billing_period") or {}
+                if "anchor_day" in bp and not (isinstance(bp["anchor_day"], int) and 1 <= bp["anchor_day"] <= 28):
+                    return self.send_json({"error": "anchor_day must be 1..28"}, 400)
+                for section in ("budgets", "limits"):
+                    if section in payload:
+                        _validate_numeric_section(section, payload[section])
+                if "alert_thresholds_pct" in payload:
+                    _validate_thresholds(payload["alert_thresholds_pct"])
                 # UI edits go to the gitignored per-machine file, never the shared defaults
                 local = {}
                 if os.path.exists(LOCAL_SETTINGS_PATH):
@@ -146,8 +237,11 @@ class Handler(BaseHTTPRequestHandler):
                     A.settings = cur
                 return self.send_json({"ok": True, "settings": cur})
             self.send_json({"error": "unknown endpoint"}, 404)
+        except BadRequest as e:
+            self.send_json({"error": str(e)}, 400)
         except Exception:
-            self.send_json({"error": traceback.format_exc()}, 500)
+            log.exception("POST %s failed", path)
+            self.send_json({"error": "internal error; see data/server.log"}, 500)
 
     def _same_origin(self):
         origin = self.headers.get("Origin")
@@ -239,6 +333,8 @@ class Handler(BaseHTTPRequestHandler):
         return self.send_json(act(int(parts[2]), parts[3]))
 
     def do_GET(self):
+        if not self._host_ok():
+            return self.send_json({"error": "forbidden host"}, 403)
         u = urlparse(self.path)
         path, qs = u.path, parse_qs(u.query)
         try:
@@ -246,15 +342,22 @@ class Handler(BaseHTTPRequestHandler):
                 return self.api(path[5:], qs)
             rel = "index.html" if path in ("/", "") else path.lstrip("/")
             fp = os.path.normpath(os.path.join(WEB, rel))
-            if not fp.startswith(WEB) or not os.path.isfile(fp):
+            if os.path.commonpath([fp, WEB]) != WEB or not os.path.isfile(fp):
                 return self.send_text("not found", "text/plain", code=404)
             ext = os.path.splitext(fp)[1]
             with open(fp, "rb") as fh:
-                self.send_text(fh.read(), MIME.get(ext, "application/octet-stream"))
+                # Served from disk on every request and revalidated every time: this is
+                # localhost, so the fetch is free, and a stale cached app.js after an
+                # upgrade looks exactly like "the new feature is missing".
+                self.send_text(fh.read(), MIME.get(ext, "application/octet-stream"),
+                               no_store=True)
         except BrokenPipeError:
             pass
+        except BadRequest as e:
+            self.send_json({"error": str(e) or "bad request"}, 400)
         except Exception:
-            self.send_json({"error": traceback.format_exc()}, 500)
+            log.exception("GET %s failed", path)
+            self.send_json({"error": "internal error; see data/server.log"}, 500)
 
     def api(self, route, qs):
         if route == "update":
@@ -287,29 +390,34 @@ class Handler(BaseHTTPRequestHandler):
         if route == "projects":
             return self.send_json(a.projects(f))
         if route == "sessions":
-            return self.send_json(a.sessions(f, int(g("limit", 200)), g("order", "cost")))
+            limit = _int(qs, "limit", 200, 1, 5000)
+            offset = _int(qs, "offset", 0, 0)
+            return self.send_json({"rows": a.sessions(f, limit, g("order", "cost"), offset),
+                                    "total": a.sessions_total(f)})
         if route == "prompts":
-            return self.send_json(a.prompts(f, int(g("limit", 200)), int(g("offset", 0)),
-                                            g("order", "cost"), g("q")))
+            limit = _int(qs, "limit", 200, 1, 5000)
+            offset = _int(qs, "offset", 0, 0)
+            search = g("q")
+            return self.send_json({"rows": a.prompts(f, limit, offset, g("order", "cost"), search),
+                                    "total": a.prompts_total(f, search)})
         if route == "leaderboards":
-            return self.send_json(a.leaderboards(f, int(g("n", 20))))
+            return self.send_json(a.leaderboards(f, _int(qs, "n", 20)))
         if route == "categories":
             return self.send_json(a.categories(f))
         if route == "efficiency":
             return self.send_json(a.efficiency(f))
+        if route == "long_context_pricing":
+            return self.send_json(a.long_context_pricing(f))
+        if route == "ttl_replay":
+            return self.send_json(a.ttl_replay(f))
+        if route == "hygiene":
+            return self.send_json(a.hygiene(f, top=_int(qs, "top", 12)))
         if route == "context":
             return self.send_json(a.context_analysis(f))
         if route == "waste":
             return self.send_json(a.waste(f))
-        if route == "trial":
-            from .trial import samples, available
-            return self.send_json({"available": available(),
-                                   "samples": samples(qs.get("category", [""])[0],
-                                                      int(qs.get("limit", ["3"])[0]))})
-        if route == "model_evidence":
-            return self.send_json(a.model_evidence(f))
-        if route == "model_switch":
-            return self.send_json(a.model_switch(f))
+        if route == "context_window_fit":
+            return self.send_json(a.context_window_fit(f))
         if route == "recommendations":
             return self.send_json(a.recommendations(f))
         if route == "forecast":
@@ -329,7 +437,7 @@ class Handler(BaseHTTPRequestHandler):
             from .diagnose import Diagnoser
             return self.send_json(Diagnoser(a).breakdown(f))
         if route == "live":
-            from .procs import list_sessions, list_agent_sessions
+            from .procs import list_sessions, list_agent_sessions, ACTIONS
             want = [x for x in (g("agents", "") or "").split(",") if x]
             claude = [dict(s, agent="claude", signalable=True,
                            resume=f"claude --resume {s.get('session_id')}")
@@ -337,27 +445,20 @@ class Handler(BaseHTTPRequestHandler):
             others = list_agent_sessions(a.pricing, [x for x in want if x != "claude"] or None) \
                 if not want or any(x != "claude" for x in want) else []
             rows = sorted(claude + others, key=lambda x: -(x.get("context") or 0))
-            # Live model advice, while the session can still act on it. One cached
-            # evidence read for the whole list, and never fatal to the view.
-            try:
-                from .advisor import advise, evidence
-                ev = evidence()
-                for r in rows:
-                    r["advice"] = advise(model=r.get("model"), transcript=r.get("transcript"),
-                                         ev=ev) if ev else None
-            except Exception:
-                for r in rows:
-                    r.setdefault("advice", None)
-            return self.send_json({"sessions": rows})
+            return self.send_json({"sessions": rows, "actions": list(ACTIONS)})
         if route == "cloud":
             from .cloud import report
-            return self.send_json(report(a, int(g("days", 30))))
+            return self.send_json(report(a, _int(qs, "days", 30)))
         if route == "developer":
             return self.send_json(a.developer(f))
         if route == "search":
-            return self.send_json(a.search(g("q", ""), int(g("limit", 40))))
+            return self.send_json(a.search(g("q", ""), _int(qs, "limit", 40)))
         if route.startswith("prompt/"):
-            return self.send_json(a.prompt_detail(int(route.split("/")[1])))
+            try:
+                pid = int(route.split("/")[1])
+            except ValueError:
+                raise BadRequest("id must be an integer")
+            return self.send_json(a.prompt_detail(pid))
         if route.startswith("session/"):
             return self.send_json(a.session_detail(route.split("/", 1)[1]))
         if route == "bundle":
@@ -449,7 +550,7 @@ def _sync_job(log):
     res = X.sync(log)
     with _lock:
         try:
-            A.db.close()          # Windows can't replace a file that's still open
+            A.close()             # Windows can't replace a file that's still open
         except Exception:
             pass
         X.finish_sync(res["tmp"])
@@ -465,12 +566,21 @@ def _notify_update():
 
 def serve(port=8787, db=DB_PATH, background=None):
     global A
+    ensure_dirs()
+    if not log.handlers:
+        fh = logging.FileHandler(LOGFILE)
+        fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
+        log.addHandler(fh)
+        log.setLevel(logging.ERROR)
     if not os.path.exists(db):
         raise SystemExit(f"No warehouse at {db}. Run:  python3 -m finops.etl")
     if background is None:
         background = os.environ.get("FINOPS_DETACH") == "1" or _under_claude()
     if background and hasattr(os, "fork"):
         detach(port)
+    from .etl import needs_rebuild
+    if needs_rebuild(db):
+        print("Warehouse schema is out of date. Run:  claude-finops --rebuild", file=sys.stderr)
     A = Analytics(db)
     srv = Server(("127.0.0.1", port), Handler)
     print(f"Claude FinOps Command Center -> http://127.0.0.1:{port}")
@@ -478,11 +588,20 @@ def serve(port=8787, db=DB_PATH, background=None):
     print(f"  data      : {A.first_day} .. {A.last_day}")
     print("  Ctrl-C to stop.")
     # Off the main thread: a slow registry must never delay the dashboard.
-    threading.Thread(target=_notify_update, daemon=True).start()
+    from .update import disabled
+    if not disabled():
+        threading.Thread(target=_notify_update, daemon=True).start()
+    with open(PIDFILE, "w") as fh:
+        fh.write(str(os.getpid()))
     try:
         srv.serve_forever()
     except KeyboardInterrupt:
         print("\nbye")
+    finally:
+        try:
+            os.remove(PIDFILE)
+        except OSError:
+            pass
 
 
 if __name__ == "__main__":

package/finops/diagnose.py

@@ -154,7 +154,7 @@ class Diagnoser:
         a = self.a
         if not a.last_day:
             return None
-        end = _d(f.get("end") or a.last_day)
+        end = _d(f.get("end")) if f.get("end") else (a.today() - timedelta(days=1))
         ranges = {"current": (end - timedelta(days=6), end),
                   "previous": (end - timedelta(days=13), end - timedelta(days=7))}
         res = {}
@@ -189,7 +189,6 @@ class Diagnoser:
         t = drv["total"]
         if not t.get("n"):
             return recs
-        cr_rate_cost = a.pricing.estimate(self._main_model(f), cache_read=t["cr"] or 0)
 
         if d.get("context_reread", {}).get("share_pct", 0) > 60:
             recs.append({
@@ -198,27 +197,27 @@ class Diagnoser:
                 "how": f"Run {v['clear']} when you switch to an unrelated task, and {v['compact']} once a "
                        "session passes ~100K context. Start one session per ticket rather than "
                        "one per day.",
-                "est_savings_usd": round(cr_rate_cost * 0.3, 2),
-                "savings_basis": "30% fewer cache-read tokens, priced at your main model's rate"})
+                # No dollar figure: the old one assumed 30% fewer cache-read tokens, a
+                # number that came from nowhere in the data.
+            })
         if d.get("long_sessions", {}).get("share_pct", 0) > 30:
             recs.append({
                 "priority": 1, "title": "Break up marathon sessions",
                 "why": d["long_sessions"]["detail"],
                 "how": f"Finish a unit of work, write the state to a file or memory, then {v['clear']}. "
                        "Use the Sessions view sorted by cost to find the worst ones.",
-                "est_savings_usd": None, "savings_basis": None})
+            })
         fm = d.get("frontier_model")
         if fm and fm["share_pct"] > 70:
-            mr = a.recommendations(f)
-            save = sum(r["estimated_savings_usd"] for r in mr["recommendations"]
-                       if r["type"] == "model_downgrade")
+            # No dollar figure: the repriced model-switch estimate was removed because it
+            # held turn counts fixed and ignored the per-model cache. The observation
+            # (most spend is on the frontier model) stands; the saving is not knowable
+            # without running the work on both models.
             recs.append({
                 "priority": 2, "title": v["cheaper"],
                 "why": fm["detail"],
                 "how": v["model_how"],
-                "est_savings_usd": round(save, 2) if save else None,
-                "savings_basis": "Model-downgrade estimate from the Recommendations view; "
-                                 "quality not modelled"})
+            })
         th = d.get("tool_heavy")
         if th and th.get("evidence"):
             bash = next((x for x in th["evidence"] if x["name"] in v["shell"]), None)
@@ -230,7 +229,7 @@ class Diagnoser:
                     "how": "Ask for `| head`/`| tail`, quiet flags (`-q`, `--silent`), and run "
                            "test suites with a reporter that prints failures only. Add these "
                            "habits to " + v["md"] + " so the agent does it unprompted.",
-                    "est_savings_usd": None, "savings_basis": None})
+                })
             mcp = [x for x in th["evidence"] if x["name"].startswith("mcp__")]
             if mcp:
                 recs.append({
@@ -240,7 +239,7 @@ class Diagnoser:
                     "how": "Prefer targeted reads (find / get_page_text) over screenshots, and "
                            "do browser checks in a subagent so the payload doesn't stay in the "
                            "main session.",
-                    "est_savings_usd": None, "savings_basis": None})
+                })
         sub = d.get("subagents")
         if sub and sub["share_pct"] > 25:
             recs.append({
@@ -248,7 +247,7 @@ class Diagnoser:
                 "why": sub["detail"],
                 "how": "Use subagents for wide searches only; for known files, read them "
                        "directly. Set `model: sonnet` or `haiku` in custom agent frontmatter.",
-                "est_savings_usd": None, "savings_basis": None})
+            })
         n_fix = sum(1 for pj in projects if pj["issues"])
         if n_fix:
             recs.append({
@@ -256,8 +255,8 @@ class Diagnoser:
                 "why": "Missing or oversized CLAUDE.md, oversized memory, repeated file "
                        "re-reads or reads of generated files — see the project table below.",
                 "how": "Apply the per-project fixes listed below.",
-                "est_savings_usd": None, "savings_basis": None})
-        recs.sort(key=lambda r: (r["priority"], -(r["est_savings_usd"] or 0)))
+            })
+        recs.sort(key=lambda r: r["priority"])
         return recs
 
     def _main_model(self, f):
@@ -538,7 +537,7 @@ class Diagnoser:
         a = self.a
         f = f or {}
         w, p = a.where(f)
-        rate = a.pricing.rates(self._main_model(f)).get("cache_read", 0.3) / 1e6
+        rate = (a.pricing.rates(self._main_model(f)).get("cache_read") or 0) / 1e6
         tcw = f"tc.request_pk IN (SELECT r.id FROM requests r WHERE {w})"
 
         sessions = a.q(f"""SELECT r.session_id, s.title, pj.name project,
@@ -692,7 +691,6 @@ class Diagnoser:
             FROM requests r JOIN sessions s ON s.id=r.session_id JOIN projects pj ON pj.id=r.project_id
             WHERE {w} GROUP BY r.session_id HAVING peak >= ? ORDER BY over_base DESC LIMIT ?""",
                    p + [self.CTX_WARN, limit])
-        rate = a.pricing.rates(self._main_model(f)).get("cache_read", 0.3) / 1e6
         compacts = {r["session_id"]: r["n"] for r in a.q(
             "SELECT session_id, COUNT(*) n FROM prompts WHERE source='slash:/compact' GROUP BY 1")}
         for r in rows:
@@ -702,8 +700,7 @@ class Diagnoser:
             # NULL used to take the whole diagnose page down with a 500.
             over = r["over_base"] or 0
             r["over_base"] = over
-            r["avoidable_tokens"] = over
-            r["avoidable_cost"] = over * rate
+            r["tokens_above_100k"] = over     # re-read above a 100K baseline; not a saving
             if not compacts.get(sid):
                 fixes.append(f"Never compacted. {r['heavy_steps']:,} steps ran above "
                              f"{self.CTX_WARN // 1000}K context; /compact (or /clear between the "