claude-finops 0.7.2 → 0.8.0

package/web/charts.js

@@ -4,6 +4,9 @@ const NS = 'http://www.w3.org/2000/svg';
 export const SERIES = ['--s1','--s2','--s3','--s4','--s5','--s6','--s7','--s8'];
 export const seriesVar = i => `var(${SERIES[i % SERIES.length]})`;
 
+export const esc = s => String(s ?? '').replace(/[&<>"']/g, c =>
+  ({'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;'}[c]));
+
 let tipEl = null;
 function tip() {
   if (!tipEl) { tipEl = document.createElement('div'); tipEl.className = 'tip';
@@ -140,8 +143,8 @@ export function timeSeries(host, opts) {
     cross.setAttribute('x2', type === 'bar' ? cx(i) : px(i));
     const body = active.map(s => `<div class="row"><span class="k">
       <span class="swatch" style="background:${s.color || seriesVar(series.indexOf(s))}"></span>
-      ${s.label}</span><span class="v">${(s.fmt || fmt)(+r[s.key] || 0)}</span></div>`).join('');
-    showTip(`<div class="t">${xLabel(r[xk])}</div>${body}`, ev.clientX, ev.clientY);
+      ${esc(s.label)}</span><span class="v">${(s.fmt || fmt)(+r[s.key] || 0)}</span></div>`).join('');
+    showTip(`<div class="t">${esc(xLabel(r[xk]))}</div>${body}`, ev.clientX, ev.clientY);
   });
   hit.addEventListener('mouseleave', () => { hideTip(); cross.setAttribute('opacity', 0); });
   if (onClick) hit.addEventListener('click', ev => onClick(rows[idxAt(ev)]));
@@ -170,7 +173,7 @@ export function barsH(host, opts) {
     grp.appendChild(el('rect', {x: lw, y: y0 + 4, width: w, height: 11, rx: 4, fill: col}));
     grp.appendChild(el('text', {x: W - 2, y: y0 + 13, 'text-anchor': 'end', class: 'val'}, fmt(v)));
     grp.addEventListener('mousemove', ev => showTip(
-      `<div class="t">${label(r)}</div><div class="row"><span class="k">
+      `<div class="t">${esc(label(r))}</div><div class="row"><span class="k">
        <span class="swatch" style="background:${col}"></span>Value</span>
        <span class="v">${fmt(v)}</span></div>${sub ? sub(r) : ''}`, ev.clientX, ev.clientY));
     grp.addEventListener('mouseleave', hideTip);
@@ -205,7 +208,7 @@ export function donut(host, opts) {
       L${x3},${y3} A${r0},${r0} 0 ${large} 0 ${x4},${y4} Z`, fill: col,
       style: onClick ? 'cursor:pointer' : ''});
     p.addEventListener('mousemove', ev => showTip(
-      `<div class="t">${label(row)}</div><div class="row"><span class="k">
+      `<div class="t">${esc(label(row))}</div><div class="row"><span class="k">
        <span class="swatch" style="background:${col}"></span>Estimated</span>
        <span class="v">${fmt(v)}</span></div><div class="row"><span class="k">Share</span>
        <span class="v">${(100 * v / total).toFixed(1)}%</span></div>`, ev.clientX, ev.clientY));
@@ -255,23 +258,29 @@ export function forecastFan(host, {history, scenarios, remainingDays, height = 2
   let cum = 0;
   const hist = history.map(d => ({day: d.day, v: (cum += d.cost)}));
   const base = cum, n = hist.length, total = n + remainingDays;
+  // scenarios may only carry 'expected' (insufficient_history): iterate only the
+  // keys actually present so a caller that forgets to check that flag cannot crash.
+  const keys = ['conservative', 'expected', 'high'].filter(k => scenarios[k]);
   const paths = {};
-  for (const k of ['conservative', 'expected', 'high']) {
+  for (const k of keys) {
     const rate = scenarios[k].daily_rate;
     paths[k] = Array.from({length: remainingDays + 1}, (_, i) => base + rate * i);
   }
-  const max = nice(Math.max(base, ...paths.high) || 1);
+  const hasBand = paths.conservative && paths.high;
+  const max = nice(Math.max(base, ...(paths.high || paths.expected)) || 1);
   const X = i => m.l + (total <= 1 ? 0 : i * (iw / (total - 1)));
   const Y = v => m.t + ih - (v / max) * ih;
   const svg = el('svg', {viewBox: `0 0 ${W} ${H}`, height: H});
   const gi = el('g', {transform: `translate(${m.l},0)`});
   axisLeft(gi, Y, max, iw, 4, fmtUSD); svg.appendChild(gi);
 
-  const band = [];
-  paths.high.forEach((v, i) => band.push(`${i ? 'L' : 'M'}${X(n - 1 + i)},${Y(v)}`));
-  for (let i = paths.conservative.length - 1; i >= 0; i--)
-    band.push(`L${X(n - 1 + i)},${Y(paths.conservative[i])}`);
-  svg.appendChild(el('path', {d: band.join(' ') + ' Z', fill: 'var(--s1)', 'fill-opacity': .13}));
+  if (hasBand) {
+    const band = [];
+    paths.high.forEach((v, i) => band.push(`${i ? 'L' : 'M'}${X(n - 1 + i)},${Y(v)}`));
+    for (let i = paths.conservative.length - 1; i >= 0; i--)
+      band.push(`L${X(n - 1 + i)},${Y(paths.conservative[i])}`);
+    svg.appendChild(el('path', {d: band.join(' ') + ' Z', fill: 'var(--s1)', 'fill-opacity': .13}));
+  }
 
   const line = (pts, color, dash) => el('path', {
     d: pts.map((p, i) => (i ? 'L' : 'M') + p[0] + ',' + p[1]).join(' '), fill: 'none',
@@ -286,8 +295,10 @@ export function forecastFan(host, {history, scenarios, remainingDays, height = 2
     svg.appendChild(el('circle', {cx: x, cy: y, r: 3.5, fill: c, stroke: 'var(--surface)',
       'stroke-width': 2}));
   });
-  svg.appendChild(el('text', {x: X(total - 1), y: Y(paths.high.at(-1)) - 6, 'text-anchor': 'end',
-    class: 'val'}, 'High ' + fmtUSD(paths.high.at(-1))));
+  if (paths.high) {
+    svg.appendChild(el('text', {x: X(total - 1), y: Y(paths.high.at(-1)) - 6, 'text-anchor': 'end',
+      class: 'val'}, 'High ' + fmtUSD(paths.high.at(-1))));
+  }
   svg.appendChild(el('text', {x: X(total - 1), y: Y(paths.expected.at(-1)) - 6, 'text-anchor': 'end',
     class: 'val'}, 'Expected ' + fmtUSD(paths.expected.at(-1))));
   svg.appendChild(el('text', {x: X(0), y: H - 8}, history[0].day));
@@ -304,7 +315,7 @@ export function forecastFan(host, {history, scenarios, remainingDays, height = 2
       ev.clientX, ev.clientY);
     else {
       const j = i - n + 1;
-      showTip(`<div class="t">Day +${j} · forecast</div>` + ['conservative','expected','high']
+      showTip(`<div class="t">Day +${j} · forecast</div>` + keys
         .map(k => `<div class="row"><span class="k">${k}</span>
           <span class="v">${fmtUSD(paths[k][j])}</span></div>`).join(''), ev.clientX, ev.clientY);
     }
@@ -333,7 +344,7 @@ export function legend(host, items, onToggle) {
   items.forEach(it => {
     const s = document.createElement('span');
     s.className = 'it' + (it.off ? ' off' : '');
-    s.innerHTML = `<span class="swatch" style="background:${it.color}"></span>${it.label}`;
+    s.innerHTML = `<span class="swatch" style="background:${it.color}"></span>${esc(it.label)}`;
     if (onToggle) { s.style.cursor = 'pointer'; s.onclick = () => onToggle(it.key); }
     host.appendChild(s);
   });

package/finops/advisor.py

@@ -1,188 +0,0 @@
-"""Live model advice: what to switch to, while the session is still running.
-
-The back-test in analytics.model_evidence() answers "what should I have used?",
-which is the wrong tense once a session is under way. This module answers it in
-the present: given the prompts a session has actually sent and the model it is
-on, say whether your own history already shows a cheaper model doing this kind
-of work without taking more turns.
-
-It is built to run three ways, so the advice is the same wherever you meet it:
-
-  * the dashboard's Running sessions view (per live session)
-  * a UserPromptSubmit hook (per prompt, before the turn runs)
-  * a statusline command (continuously, in a few characters)
-
-The hook and statusline run on every prompt, so the evidence is computed once
-and cached; a warehouse query per keystroke would be felt. Everything here fails
-soft and silent — advice that breaks your terminal is worse than no advice.
-"""
-import json
-import os
-import time
-
-from .classify import classify
-from .paths import DATA_DIR, DB_PATH, ensure_dirs
-
-CACHE = os.path.join(DATA_DIR, "advice_cache.json")
-TTL_S = 3600
-RECENT_PROMPTS = 12      # how much of the session counts as "what it is doing now"
-MIN_SAVING_PCT = 25      # below this, interrupting someone is not worth it
-
-# `/model <name>` takes a family name, not the pricing table's id.
-ALIASES = ("opus", "sonnet", "haiku", "fable")
-
-
-def model_alias(model_id):
-    """'claude-fable-5-1' -> 'fable'. Falls back to the full id we were given."""
-    low = str(model_id or "").lower()
-    for a in ALIASES:
-        if a in low:
-            return a
-    return model_id
-
-
-# ---------------------------------------------------------------- evidence ----
-
-def _build_evidence():
-    """Pull the back-test into the small shape the live surfaces need."""
-    from .analytics import Analytics
-    a = Analytics(DB_PATH)
-    ev = a.model_evidence({})
-    out = {}
-    for c in ev["categories"]:
-        cands = [{"model": x["model"], "name": x["name"], "verdict": x["verdict"],
-                  "cost_per_prompt": round(x["cost_per_prompt"], 3),
-                  "savings_pct": x["savings_pct"], "turn_ratio": x["turn_ratio"],
-                  "prompts": x["prompts"], "why": x["why"]}
-                 for x in c["candidates"]]
-        out[c["category"]] = {"current": c["current"]["model"],
-                              "current_name": c["current"]["name"],
-                              "current_cost_per_prompt": round(c["current"]["cost_per_prompt"], 3),
-                              "agent": c.get("agent", "claude"),
-                              "candidates": cands}
-    return out
-
-
-def evidence(force=False):
-    """Cached evidence table, keyed by category. Never raises."""
-    try:
-        with open(CACHE) as fh:
-            c = json.load(fh)
-        if not force and time.time() - c.get("built_at", 0) < TTL_S:
-            return c.get("categories") or {}
-    except (OSError, ValueError):
-        pass
-    try:
-        cats = _build_evidence()
-    except Exception:
-        return {}
-    try:
-        ensure_dirs()
-        tmp = CACHE + ".tmp"
-        with open(tmp, "w") as fh:
-            json.dump({"built_at": int(time.time()), "categories": cats}, fh)
-        os.replace(tmp, CACHE)
-    except OSError:
-        pass
-    return cats
-
-
-# ---------------------------------------------------------------- session ----
-
-def recent_prompts(transcript, n=RECENT_PROMPTS):
-    """The last n human prompts in a live transcript, newest last.
-
-    Reads the tail only: an active session's JSONL runs to tens of megabytes and
-    this is on the path of every prompt you type.
-    """
-    try:
-        size = os.path.getsize(transcript)
-        with open(transcript, "rb") as fh:
-            fh.seek(max(0, size - 400_000))
-            lines = fh.read().decode("utf-8", "replace").splitlines()[1:]
-    except OSError:
-        return []
-    out = []
-    for line in reversed(lines):
-        if '"type":"user"' not in line and '"type": "user"' not in line:
-            continue
-        try:
-            d = json.loads(line)
-        except ValueError:
-            continue
-        if d.get("isMeta") or d.get("isSidechain"):
-            continue
-        msg = d.get("message") or {}
-        content = msg.get("content")
-        if isinstance(content, list):
-            content = " ".join(b.get("text", "") for b in content
-                               if isinstance(b, dict) and b.get("type") == "text")
-        if not isinstance(content, str) or not content.strip():
-            continue
-        if content.lstrip().startswith(("<", "[Request interrupted")):
-            continue          # tool results and interrupt markers are not prompts
-        out.append(content)
-        if len(out) >= n:
-            break
-    return list(reversed(out))
-
-
-def category_of(texts):
-    """The category this stretch of work is in, by weight of confidence."""
-    scores = {}
-    for t in texts:
-        cat, conf, _ = classify(t)
-        scores[cat] = scores.get(cat, 0) + max(conf, 0.1)
-    if not scores:
-        return "other", 0.0
-    cat = max(scores, key=scores.get)
-    return cat, round(scores[cat] / sum(scores.values()), 2)
-
-
-def advise(model=None, transcript=None, prompt=None, ev=None):
-    """What to say about a session that is running right now.
-
-    model      the model the session is on (pricing-table id)
-    transcript path to the live JSONL, for reading what it has been doing
-    prompt     the prompt about to be sent, when we are called from a hook
-    Returns None when there is nothing worth saying.
-    """
-    ev = evidence() if ev is None else ev
-    if not ev:
-        return None
-    texts = list(recent_prompts(transcript)) if transcript else []
-    if prompt:
-        # The prompt in hand is what the next turn will cost, so it leads.
-        texts = texts[-4:] + [prompt] * 3
-    if not texts:
-        return None
-    cat, conf = category_of(texts)
-    row = ev.get(cat)
-    if not row:
-        return None
-    # Only speak when they are on the model the evidence is about. Advising a
-    # switch away from a model you already left would be noise.
-    if model and row["current"] and model_alias(model) != model_alias(row["current"]):
-        return None
-    usable = [c for c in row["candidates"]
-              if c["verdict"] in ("supported", "caution") and c["savings_pct"] >= MIN_SAVING_PCT]
-    if not usable:
-        return None
-    best = usable[0]
-    trial = best["verdict"] == "caution"
-    return {
-        "category": cat, "confidence": conf,
-        "current_model": row["current"], "current_name": row["current_name"],
-        "current_cost_per_prompt": row["current_cost_per_prompt"],
-        "model": best["model"], "name": best["name"], "alias": model_alias(best["model"]),
-        "cost_per_prompt": best["cost_per_prompt"], "savings_pct": best["savings_pct"],
-        "turn_ratio": best["turn_ratio"], "prompts": best["prompts"],
-        "verdict": best["verdict"], "trial": trial, "why": best["why"],
-        "command": f"/model {model_alias(best['model'])}",
-        "line": (f"{cat.replace('_', ' ')} on {row['current_name']} "
-                 f"(${row['current_cost_per_prompt']:.2f}/prompt). Your last {best['prompts']} "
-                 f"ran on {best['name']} at ${best['cost_per_prompt']:.2f} in "
-                 f"{best['turn_ratio']}x the turns"
-                 + (" — worth a trial here." if trial else ".")),
-        "short": f"try {model_alias(best['model'])} (-{best['savings_pct']:.0f}%)",
-    }

package/finops/trial.py

@@ -1,169 +0,0 @@
-"""Run the trial the evidence asks for, instead of only recommending one.
-
-model_evidence() ends with an honest caveat: your prompts were never randomly
-assigned to models, so a category can simply have been easier on one of them.
-That caveat can only be closed by actually running the same prompt on both
-models and looking at what came back.
-
-So: take real prompts out of your own history, re-run them headlessly on the
-candidate model, and compare what they cost against what they cost the first
-time. `claude -p ... --output-format json` reports cost, turns and errors for
-exactly this purpose.
-
-Two things this deliberately does not do:
-
-  * It never runs anything on its own. A trial spends real money and drives a
-    real agent, so it happens on an explicit click, with the bill shown first.
-  * It runs in a scratch directory by default, not your repo. Headless Claude
-    cannot ask for permission, so tools that need it are denied and recorded —
-    but a trial that edits your working tree to prove a point is not a trial
-    anyone wants.
-"""
-import json
-import os
-import shutil
-import subprocess
-import tempfile
-import time
-
-from .advisor import model_alias
-from .paths import DB_PATH
-
-TIMEOUT_S = 600
-MAX_PROMPTS = 5
-MAX_CHARS = 6000          # a prompt longer than this is usually a paste, not a task
-
-
-def samples(category, limit=3):
-    """Real prompts from this category, with what they actually cost first time.
-
-    Picks around the middle of the cost distribution: the cheapest prompts are
-    usually "yes"/"continue" and the most expensive are outliers, and neither
-    tells you much about a model.
-    """
-    from .analytics import Analytics
-    a = Analytics(DB_PATH)
-    rows = a.q("""SELECT p.id, p.text, p.models model, p.est_cost_usd cost,
-        p.request_count turns, p.tool_calls tools, p.session_id, p.day
-        FROM prompts p
-        WHERE COALESCE(p.category,'other') = ? AND p.agent = 'claude'
-          AND p.models IS NOT NULL AND p.models NOT LIKE '%,%'
-          AND p.models != '<synthetic>' AND p.est_cost_usd > 0
-          AND LENGTH(p.text) BETWEEN 40 AND ?
-          -- Things you actually typed. System turns, tool results, queued
-          -- follow-ups and SDK traffic are not tasks anyone would re-run, and
-          -- a trial built from task-notification XML measures nothing.
-          AND p.source = 'typed'
-          AND p.text NOT LIKE '<%' AND p.text NOT LIKE '[Request interrupted%'
-          AND p.text NOT LIKE 'Caveat:%' AND p.text NOT LIKE '%<local-command%' 
-        ORDER BY p.est_cost_usd""", (category, MAX_CHARS))
-    if not rows:
-        return []
-    lo = int(len(rows) * 0.35)
-    hi = max(lo + 1, int(len(rows) * 0.85))
-    mid = rows[lo:hi] or rows
-    step = max(1, len(mid) // limit)
-    picked = mid[::step][:limit]
-    return [{"prompt_id": r["id"], "text": r["text"], "baseline_model": r["model"],
-             "baseline_name": a.pricing.display_name(r["model"]),
-             "baseline_cost_usd": round(r["cost"], 4),
-             "baseline_turns": r["turns"], "baseline_tools": r["tools"],
-             "day": r["day"]} for r in picked]
-
-
-def available():
-    return bool(shutil.which("claude"))
-
-
-def run_one(text, model, cwd=None, timeout=TIMEOUT_S):
-    """One headless run. Returns what it cost and whether it got there."""
-    exe = shutil.which("claude")
-    if not exe:
-        return {"ok": False, "error": "The `claude` CLI is not on PATH."}
-    sandbox = cwd or tempfile.mkdtemp(prefix="finops-trial-")
-    started = time.time()
-    try:
-        p = subprocess.run([exe, "-p", text, "--model", model_alias(model),
-                            "--output-format", "json"],
-                           capture_output=True, text=True, timeout=timeout, cwd=sandbox)
-    except subprocess.TimeoutExpired:
-        return {"ok": False, "error": f"Gave up after {timeout}s.",
-                "elapsed_s": round(time.time() - started, 1)}
-    except OSError as e:
-        return {"ok": False, "error": str(e)}
-    try:
-        d = json.loads(p.stdout)
-    except ValueError:
-        return {"ok": False, "error": (p.stderr or p.stdout or "no output")[:400],
-                "elapsed_s": round(time.time() - started, 1)}
-    usage = d.get("usage") or {}
-    return {
-        "ok": not d.get("is_error"),
-        "cost_usd": d.get("total_cost_usd"),
-        "turns": d.get("num_turns"),
-        "elapsed_s": round(time.time() - started, 1),
-        "duration_api_ms": d.get("duration_api_ms"),
-        "output_tokens": usage.get("output_tokens"),
-        "input_tokens": usage.get("input_tokens"),
-        "denials": len(d.get("permission_denials") or []),
-        "stop_reason": d.get("stop_reason") or d.get("terminal_reason"),
-        "result": (d.get("result") or "")[:4000],
-        "session_id": d.get("session_id"),
-        "sandbox": sandbox,
-    }
-
-
-def _verdict(runs, baseline_cost):
-    """What the runs actually showed, in the same language as the back-test."""
-    done = [r for r in runs if r.get("ok") and r.get("cost_usd") is not None]
-    if not done:
-        return "failed", ("Every run errored or produced nothing, so this says nothing about "
-                          "cost. Check the errors below before reading anything into it.")
-    got = sum(r["cost_usd"] for r in done)
-    base = sum(baseline_cost)
-    denials = sum(r.get("denials") or 0 for r in done)
-    if base <= 0:
-        return "unclear", "No usable baseline cost for these prompts."
-    pct = 100.0 * (1 - got / base)
-    tail = (f" {denials} tool call(s) were denied because a headless run cannot ask for "
-            f"permission, so the real task may be larger than this." if denials else "")
-    if len(done) < len(runs):
-        tail += f" {len(runs) - len(done)} of {len(runs)} runs failed and are excluded."
-    if pct >= 25:
-        return "confirmed", (f"Re-running your own prompts cost {pct:.0f}% less on the cheaper "
-                             f"model (${got:.2f} against ${base:.2f} the first time).{tail}")
-    if pct >= 0:
-        return "marginal", (f"Only {pct:.0f}% cheaper on a re-run (${got:.2f} against "
-                            f"${base:.2f}). Not the saving the history suggested.{tail}")
-    return "contradicted", (f"The re-run cost {abs(pct):.0f}% *more* (${got:.2f} against "
-                            f"${base:.2f}). The history overstated this switch.{tail}")
-
-
-def run(category, model, prompts, cwd=None):
-    """Run a set of sample prompts on a candidate model and judge the result."""
-    if not prompts:
-        return {"ok": False, "error": "Nothing to run."}
-    prompts = prompts[:MAX_PROMPTS]
-    runs, baseline = [], []
-    for item in prompts:
-        text = (item.get("text") or "").strip()
-        if not text:
-            continue
-        r = run_one(text, model, cwd=cwd)
-        r["prompt"] = text[:300]
-        r["baseline_cost_usd"] = item.get("baseline_cost_usd") or 0
-        r["baseline_turns"] = item.get("baseline_turns")
-        runs.append(r)
-        baseline.append(r["baseline_cost_usd"])
-    verdict, why = _verdict(runs, baseline)
-    done = [r for r in runs if r.get("ok") and r.get("cost_usd") is not None]
-    return {
-        "ok": True, "category": category, "model": model, "alias": model_alias(model),
-        "runs": runs, "verdict": verdict, "why": why,
-        "measured_cost_usd": round(sum(r["cost_usd"] for r in done), 4),
-        "baseline_cost_usd": round(sum(baseline), 4),
-        "ran_at": int(time.time()),
-        "note": ("Measured by re-running these exact prompts headlessly. The agent had no "
-                 "permission to use tools that ask, and worked in a scratch directory, so a "
-                 "task that needs your repo will look smaller here than it is."),
-    }