claude-finops 0.6.0 → 0.7.0

package/README.md

@@ -287,6 +287,23 @@ place for you to delete once you are happy.
 
 ---
 
+## Running the trial, not just recommending one
+
+The evidence table ends by admitting its own limit: your prompts were never
+randomly assigned to models, so a category can simply have been easier on one of
+them. **Try it →** on any row closes that gap. It pulls prompts you actually
+typed in that category, re-runs them headlessly on the candidate model, and
+prices the result against what they cost the first time.
+
+Nothing runs on its own: a trial spends real money and drives a real agent, so
+it takes two clicks and shows the first-time bill before you commit. Runs happen
+in a scratch directory, and headless Claude cannot ask for permission — so tools
+that need it are denied and counted, and a task that needs your repo will look
+smaller than it is. The verdict says which way it went: *confirmed*, *smaller
+than advertised*, or *history overstated it*.
+
+---
+
 ## Live model advice
 
 The back-test tells you what to use next time. These tell you mid-session, while

package/finops/api.py

@@ -113,6 +113,15 @@ class Handler(BaseHTTPRequestHandler):
             if path.startswith("/api/live/"):
                 self._payload = payload
                 return self.session_action(path)
+            if path == "/api/trial/run":
+                # Spends real money and drives a real agent, so it is guarded like
+                # the other side-effecting routes and only ever reached by a click.
+                if not self._same_origin():
+                    return self.send_json({"ok": False, "error": "forbidden"}, 403)
+                from .trial import run
+                return self.send_json(run(payload.get("category"), payload.get("model"),
+                                          payload.get("prompts") or [],
+                                          cwd=payload.get("cwd") or None))
             if path.startswith("/api/do/"):
                 if not self._same_origin():
                     return self.send_json({"ok": False, "error": "forbidden"}, 403)
@@ -292,6 +301,11 @@ class Handler(BaseHTTPRequestHandler):
             return self.send_json(a.context_analysis(f))
         if route == "waste":
             return self.send_json(a.waste(f))
+        if route == "trial":
+            from .trial import samples, available
+            return self.send_json({"available": available(),
+                                   "samples": samples(qs.get("category", [""])[0],
+                                                      int(qs.get("limit", ["3"])[0]))})
         if route == "model_evidence":
             return self.send_json(a.model_evidence(f))
         if route == "model_switch":

package/finops/trial.py

@@ -0,0 +1,169 @@
+"""Run the trial the evidence asks for, instead of only recommending one.
+
+model_evidence() ends with an honest caveat: your prompts were never randomly
+assigned to models, so a category can simply have been easier on one of them.
+That caveat can only be closed by actually running the same prompt on both
+models and looking at what came back.
+
+So: take real prompts out of your own history, re-run them headlessly on the
+candidate model, and compare what they cost against what they cost the first
+time. `claude -p ... --output-format json` reports cost, turns and errors for
+exactly this purpose.
+
+Two things this deliberately does not do:
+
+  * It never runs anything on its own. A trial spends real money and drives a
+    real agent, so it happens on an explicit click, with the bill shown first.
+  * It runs in a scratch directory by default, not your repo. Headless Claude
+    cannot ask for permission, so tools that need it are denied and recorded —
+    but a trial that edits your working tree to prove a point is not a trial
+    anyone wants.
+"""
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+
+from .advisor import model_alias
+from .paths import DB_PATH
+
+TIMEOUT_S = 600
+MAX_PROMPTS = 5
+MAX_CHARS = 6000          # a prompt longer than this is usually a paste, not a task
+
+
+def samples(category, limit=3):
+    """Real prompts from this category, with what they actually cost first time.
+
+    Picks around the middle of the cost distribution: the cheapest prompts are
+    usually "yes"/"continue" and the most expensive are outliers, and neither
+    tells you much about a model.
+    """
+    from .analytics import Analytics
+    a = Analytics(DB_PATH)
+    rows = a.q("""SELECT p.id, p.text, p.models model, p.est_cost_usd cost,
+        p.request_count turns, p.tool_calls tools, p.session_id, p.day
+        FROM prompts p
+        WHERE COALESCE(p.category,'other') = ? AND p.agent = 'claude'
+          AND p.models IS NOT NULL AND p.models NOT LIKE '%,%'
+          AND p.models != '<synthetic>' AND p.est_cost_usd > 0
+          AND LENGTH(p.text) BETWEEN 40 AND ?
+          -- Things you actually typed. System turns, tool results, queued
+          -- follow-ups and SDK traffic are not tasks anyone would re-run, and
+          -- a trial built from task-notification XML measures nothing.
+          AND p.source = 'typed'
+          AND p.text NOT LIKE '<%' AND p.text NOT LIKE '[Request interrupted%'
+          AND p.text NOT LIKE 'Caveat:%' AND p.text NOT LIKE '%<local-command%' 
+        ORDER BY p.est_cost_usd""", (category, MAX_CHARS))
+    if not rows:
+        return []
+    lo = int(len(rows) * 0.35)
+    hi = max(lo + 1, int(len(rows) * 0.85))
+    mid = rows[lo:hi] or rows
+    step = max(1, len(mid) // limit)
+    picked = mid[::step][:limit]
+    return [{"prompt_id": r["id"], "text": r["text"], "baseline_model": r["model"],
+             "baseline_name": a.pricing.display_name(r["model"]),
+             "baseline_cost_usd": round(r["cost"], 4),
+             "baseline_turns": r["turns"], "baseline_tools": r["tools"],
+             "day": r["day"]} for r in picked]
+
+
+def available():
+    return bool(shutil.which("claude"))
+
+
+def run_one(text, model, cwd=None, timeout=TIMEOUT_S):
+    """One headless run. Returns what it cost and whether it got there."""
+    exe = shutil.which("claude")
+    if not exe:
+        return {"ok": False, "error": "The `claude` CLI is not on PATH."}
+    sandbox = cwd or tempfile.mkdtemp(prefix="finops-trial-")
+    started = time.time()
+    try:
+        p = subprocess.run([exe, "-p", text, "--model", model_alias(model),
+                            "--output-format", "json"],
+                           capture_output=True, text=True, timeout=timeout, cwd=sandbox)
+    except subprocess.TimeoutExpired:
+        return {"ok": False, "error": f"Gave up after {timeout}s.",
+                "elapsed_s": round(time.time() - started, 1)}
+    except OSError as e:
+        return {"ok": False, "error": str(e)}
+    try:
+        d = json.loads(p.stdout)
+    except ValueError:
+        return {"ok": False, "error": (p.stderr or p.stdout or "no output")[:400],
+                "elapsed_s": round(time.time() - started, 1)}
+    usage = d.get("usage") or {}
+    return {
+        "ok": not d.get("is_error"),
+        "cost_usd": d.get("total_cost_usd"),
+        "turns": d.get("num_turns"),
+        "elapsed_s": round(time.time() - started, 1),
+        "duration_api_ms": d.get("duration_api_ms"),
+        "output_tokens": usage.get("output_tokens"),
+        "input_tokens": usage.get("input_tokens"),
+        "denials": len(d.get("permission_denials") or []),
+        "stop_reason": d.get("stop_reason") or d.get("terminal_reason"),
+        "result": (d.get("result") or "")[:4000],
+        "session_id": d.get("session_id"),
+        "sandbox": sandbox,
+    }
+
+
+def _verdict(runs, baseline_cost):
+    """What the runs actually showed, in the same language as the back-test."""
+    done = [r for r in runs if r.get("ok") and r.get("cost_usd") is not None]
+    if not done:
+        return "failed", ("Every run errored or produced nothing, so this says nothing about "
+                          "cost. Check the errors below before reading anything into it.")
+    got = sum(r["cost_usd"] for r in done)
+    base = sum(baseline_cost)
+    denials = sum(r.get("denials") or 0 for r in done)
+    if base <= 0:
+        return "unclear", "No usable baseline cost for these prompts."
+    pct = 100.0 * (1 - got / base)
+    tail = (f" {denials} tool call(s) were denied because a headless run cannot ask for "
+            f"permission, so the real task may be larger than this." if denials else "")
+    if len(done) < len(runs):
+        tail += f" {len(runs) - len(done)} of {len(runs)} runs failed and are excluded."
+    if pct >= 25:
+        return "confirmed", (f"Re-running your own prompts cost {pct:.0f}% less on the cheaper "
+                             f"model (${got:.2f} against ${base:.2f} the first time).{tail}")
+    if pct >= 0:
+        return "marginal", (f"Only {pct:.0f}% cheaper on a re-run (${got:.2f} against "
+                            f"${base:.2f}). Not the saving the history suggested.{tail}")
+    return "contradicted", (f"The re-run cost {abs(pct):.0f}% *more* (${got:.2f} against "
+                            f"${base:.2f}). The history overstated this switch.{tail}")
+
+
+def run(category, model, prompts, cwd=None):
+    """Run a set of sample prompts on a candidate model and judge the result."""
+    if not prompts:
+        return {"ok": False, "error": "Nothing to run."}
+    prompts = prompts[:MAX_PROMPTS]
+    runs, baseline = [], []
+    for item in prompts:
+        text = (item.get("text") or "").strip()
+        if not text:
+            continue
+        r = run_one(text, model, cwd=cwd)
+        r["prompt"] = text[:300]
+        r["baseline_cost_usd"] = item.get("baseline_cost_usd") or 0
+        r["baseline_turns"] = item.get("baseline_turns")
+        runs.append(r)
+        baseline.append(r["baseline_cost_usd"])
+    verdict, why = _verdict(runs, baseline)
+    done = [r for r in runs if r.get("ok") and r.get("cost_usd") is not None]
+    return {
+        "ok": True, "category": category, "model": model, "alias": model_alias(model),
+        "runs": runs, "verdict": verdict, "why": why,
+        "measured_cost_usd": round(sum(r["cost_usd"] for r in done), 4),
+        "baseline_cost_usd": round(sum(baseline), 4),
+        "ran_at": int(time.time()),
+        "note": ("Measured by re-running these exact prompts headlessly. The agent had no "
+                 "permission to use tools that ask, and worked in a scratch directory, so a "
+                 "task that needs your repo will look smaller here than it is."),
+    }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-finops",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Local FinOps dashboard for Claude Code: what you used, what it cost, why it cost that much, and what to change. Reads your own transcripts, no API key, no data leaves the machine.",
   "bin": {
     "claude-finops": "bin/claude-finops.js"

package/web/app.js

@@ -1354,6 +1354,8 @@ function evidenceBody(ev) {
       statusChip(VERDICT[r.x.verdict][2], VERDICT[r.x.verdict][1])}</span>`},
     {h: 'Would save', num: 1, f: r => r.x.verdict === 'supported'
       ? `<b>${fmtUSD(r.x.estimated_savings_usd)}</b>` : `<span class="note">${fmtUSD(r.x.estimated_savings_usd)}</span>`},
+    {h: '', f: r => `<button class="act ghost trial-btn" data-cat="${esc(r.c.category)}"
+       data-model="${esc(r.x.model)}" data-name="${esc(r.x.name)}">Try it →</button>`},
   ], rows) + `<div class="note" style="padding:10px 14px">${esc(ev.method)}</div>`;
 }
 
@@ -1416,6 +1418,7 @@ VIEWS.modelswitch = async (page) => {
     {h: 'Could save', num: 1, f: r => fmtUSD(r.estimated_savings_usd)},
     {h: 'Why', f: r => esc(r.why)},
   ], m.projects);
+  wireTrials(page);
   addChart(page, 'Savings by switch', el => C.barsH(el, {
     rows: m.switches.slice(0, 10), label: r => clip(`${r.scope} → ${r.recommended_name}`, 48),
     value: r => r.estimated_savings_usd,
@@ -1426,6 +1429,116 @@ VIEWS.modelswitch = async (page) => {
     {badge: BADGE.recommendation, hint: 'Colour = confidence (green high, blue medium, amber low)'});
 };
 
+/* ---------- trial: stop recommending, start measuring ----------
+   The evidence ends at "strong evidence for a trial, not proof". This runs the
+   trial: real prompts out of your own history, re-run headlessly on the
+   candidate model, priced against what they cost the first time. It spends real
+   money, so nothing happens without two clicks. */
+function trialPanelHTML(cat, model, name, s) {
+  if (!s.available) {
+    return `<div class="empty">The <code>claude</code> CLI is not on PATH, so a trial cannot be
+      run from here.</div>`;
+  }
+  if (!s.samples.length) {
+    return `<div class="empty">No prompt you actually typed in this category is short enough to
+      re-run safely.</div>`;
+  }
+  const base = s.samples.reduce((a, x) => a + (x.baseline_cost_usd || 0), 0);
+  return `
+    <div class="dt">These are ${s.samples.length} prompts you really sent in
+      <b>${esc(cat.replace('_', ' '))}</b>. Running them again on <b>${esc(name)}</b> costs money —
+      they cost ${fmtUSD(base)} the first time, and the cheaper model should come in under that.</div>
+    <div class="stack trial-samples">${s.samples.map((x, i) => `
+      <div class="dt trial-s" data-i="${i}">
+        <span class="note">${esc(x.day)} · ${esc(x.baseline_name)} · ${fmtUSD(x.baseline_cost_usd)} ·
+          ${fmtInt(x.baseline_turns)} turns</span>
+        <div class="trial-text">${esc(x.text.slice(0, 400))}${x.text.length > 400 ? '…' : ''}</div>
+      </div>`).join('')}</div>
+    <div class="dt note">Runs headlessly in a scratch directory. Tools that need permission are
+      denied, because a headless agent cannot ask — so a task that needs your repo will look
+      smaller here than it really is.</div>
+    <div class="live-actions">
+      <button class="act trial-run">▶ Run ${s.samples.length} prompts on ${esc(name)}</button>
+      <button class="act ghost trial-copy">Copy the first prompt instead</button>
+      <span class="trial-msg note"></span>
+    </div>
+    <div class="trial-out"></div>`;
+}
+
+const TRIAL_VERDICT = {
+  confirmed:    ['healthy', 'Confirmed by running it'],
+  marginal:     ['high', 'Smaller than advertised'],
+  contradicted: ['critical', 'History overstated it'],
+  failed:       ['critical', 'Runs failed'],
+  unclear:      ['high', 'Inconclusive'],
+};
+
+function trialResultHTML(r) {
+  const v = TRIAL_VERDICT[r.verdict] || TRIAL_VERDICT.unclear;
+  return `<div class="dt"><b>${statusChip(v[0], v[1])}</b> ${esc(r.why)}</div>` + table([
+    {h: 'Prompt', trunc: 1, f: x => esc(x.prompt)},
+    {h: 'First time', num: 1, f: x => fmtUSD(x.baseline_cost_usd)},
+    {h: 'On ' + esc(r.alias), num: 1, f: x => x.ok ? fmtUSD(x.cost_usd) : '—'},
+    {h: 'Turns', num: 1, f: x => x.ok ? fmtInt(x.turns) : '—'},
+    {h: 'Took', num: 1, f: x => x.ok ? x.elapsed_s + 's' : '—'},
+    {h: 'Denied', num: 1, f: x => x.denials ? fmtInt(x.denials) : ''},
+    {h: 'Result', f: x => x.ok ? `<span class="note">${esc((x.result || '').slice(0, 120))}</span>`
+      : `<span class="status critical">${esc((x.error || 'failed').slice(0, 120))}</span>`},
+  ], r.runs) + `<div class="note" style="padding:8px 14px">${esc(r.note)}</div>`;
+}
+
+function wireTrials(page) {
+  page.querySelectorAll('.trial-btn').forEach(b => b.onclick = async () => {
+    const row = b.closest('tr');
+    if (row.nextElementSibling?.classList.contains('trial-row')) {
+      row.nextElementSibling.remove(); return;
+    }
+    const {cat, model, name} = b.dataset;
+    const tr = h(`<tr class="trial-row"><td colspan="10"><div class="trial-panel">
+      <div class="empty">Finding prompts you sent…</div></div></td></tr>`);
+    row.after(tr);
+    const host = tr.querySelector('.trial-panel');
+    let s;
+    try {
+      s = await fetch(`/api/trial?category=${encodeURIComponent(cat)}&limit=3`).then(r => r.json());
+    } catch (e) { host.innerHTML = `<div class="empty">${esc(e.message)}</div>`; return; }
+    host.innerHTML = trialPanelHTML(cat, model, name, s);
+    const msg = host.querySelector('.trial-msg');
+    host.querySelector('.trial-copy')?.addEventListener('click', async () => {
+      try { await navigator.clipboard.writeText(s.samples[0].text); msg.textContent = 'Copied.'; }
+      catch { msg.textContent = 'Could not copy.'; }
+    });
+    host.querySelector('.trial-run')?.addEventListener('click', async ev => {
+      const btn = ev.currentTarget;
+      if (btn.dataset.armed !== '1') {
+        btn.dataset.armed = '1';
+        btn.textContent = 'Click again to spend real money';
+        btn.classList.add('warn');
+        return;
+      }
+      btn.disabled = true;
+      btn.textContent = 'Running…';
+      msg.textContent = 'Each prompt runs to completion; this can take a few minutes.';
+      try {
+        const r = await fetch('/api/trial/run', {
+          method: 'POST',
+          headers: {'X-FinOps-Action': '1', 'Content-Type': 'application/json'},
+          body: JSON.stringify({category: cat, model, prompts: s.samples}),
+        }).then(x => x.json());
+        host.querySelector('.trial-out').innerHTML = r.ok
+          ? trialResultHTML(r) : `<div class="empty">${esc(r.error || 'failed')}</div>`;
+        msg.textContent = '';
+      } catch (e) {
+        msg.textContent = e.message;
+      }
+      btn.disabled = false;
+      btn.classList.remove('warn');
+      btn.textContent = '▶ Run again';
+      btn.dataset.armed = '';
+    });
+  });
+}
+
 /* ---------- waste ---------- */
 VIEWS.waste = async (page) => {
   const w = await api('waste');

package/web/styles.css

@@ -425,3 +425,13 @@ table.tbl .sub { font-size:10.5px; color:var(--muted); }
    so it reads as a note with an accent edge rather than a warning. */
 .switch-tip { border-left: 2px solid var(--accent, #eb6834); padding-left: 9px; }
 .switch-tip .act { margin-left: 8px; }
+
+/* Trial panel: an experiment you are about to pay for, so it reads as a quiet
+   workbench rather than another recommendation. */
+.trial-row > td { padding: 0 !important; background: var(--surface-2); }
+.trial-panel { padding: 12px 14px; display: grid; gap: 8px;
+               border-left: 2px solid var(--accent, #eb6834); }
+.trial-text { font-size: 12px; margin-top: 3px; white-space: pre-wrap;
+              max-height: 76px; overflow: auto; color: var(--text-2); }
+.trial-s { border-top: 1px solid var(--border); padding-top: 7px; }
+.trial-out:empty { display: none; }