memindex 1.4.4 → 1.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/.qoder-plugin/plugin.json +1 -1
- package/README.md +9 -1
- package/dist/bug-report/index.js +94 -83
- package/dist/npx-cli/index.js +250 -249
- package/dist/pi-extension/TYPEBOX-PROVENANCE.json +4 -4
- package/dist/pi-extension/index.js +9 -9
- package/integrations/dsh/package.json +1 -1
- package/integrations/openclaw/openclaw.plugin.json +1 -5
- package/integrations/pi/TYPEBOX-PROVENANCE.json +4 -4
- package/package.json +5 -5
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/.qoder-plugin/plugin.json +1 -1
- package/plugin/package.json +1 -1
- package/plugin/scripts/context-generator.cjs +62 -61
- package/plugin/scripts/context-generator.cjs.map +3 -3
- package/plugin/scripts/mcp-server.cjs +2 -2
- package/plugin/scripts/transcript-watcher.cjs +2 -2
- package/plugin/scripts/transcript-watcher.cjs.map +1 -1
- package/plugin/scripts/worker-service.cjs +677 -669
- package/plugin/scripts/worker-service.cjs.map +3 -3
- package/plugin/skills/distill/SKILL.md +22 -0
- package/plugin/skills/how-it-works/SKILL.md +1 -1
- package/plugin/skills/how-it-works/onboarding-explainer.md +1 -1
- package/plugin/skills/import-chats/SKILL.md +22 -0
- package/plugin/skills/import-jira/SKILL.md +22 -0
- package/plugin/skills/import-reviews/SKILL.md +22 -0
- package/plugin/skills/pin/SKILL.md +26 -0
- package/plugin/skills/savings/SKILL.md +22 -0
- package/integrations/openclaw/skills/agent-cost-report/CHECKSUMS.txt +0 -50
- package/integrations/openclaw/skills/agent-cost-report/SKILL.md +0 -241
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/__init__.py +0 -1
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/behavior.py +0 -303
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/classify.py +0 -83
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/costs.py +0 -119
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/devices.py +0 -88
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/evidence.py +0 -160
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/labels.py +0 -115
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/measure.py +0 -89
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/mistakes.py +0 -310
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/patterns.py +0 -391
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/period.py +0 -98
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/prices.py +0 -95
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/render.py +0 -586
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/rollup.py +0 -355
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/rules.json +0 -4
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/rules.py +0 -76
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/sample.py +0 -45
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/snapshot.py +0 -53
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/sync.py +0 -79
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/transcripts.py +0 -126
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/wins.py +0 -341
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr.py +0 -219
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/_db.py +0 -49
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/_paths.py +0 -9
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/codex/sessions/2026/09/18/rollout-codex-1.jsonl +0 -4
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/make_report_fixtures.py +0 -46
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/prices_sample.json +0 -6
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/projects/-workspace-proj-a/sess-aaaa.jsonl +0 -8
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/projects/-workspace-proj-b/sess-bbbb.jsonl +0 -1
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/report-1day.json +0 -823
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/report-30day.json +0 -2213
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/report-7day.json +0 -1252
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/report-empty.json +0 -536
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/report-session.json +0 -662
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_behavior.py +0 -299
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_devices.py +0 -99
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_labels.py +0 -96
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_measure.py +0 -120
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_pdf_windows.py +0 -54
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_period.py +0 -137
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_prices.py +0 -126
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_render.py +0 -238
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_render_encoding.py +0 -70
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_rollup.py +0 -263
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_snapshot.py +0 -63
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_sync.py +0 -52
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_transcript_encoding.py +0 -109
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_transcripts.py +0 -151
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_wins.py +0 -222
- package/plugin/skills/agent-cost-report/CHECKSUMS.txt +0 -50
- package/plugin/skills/agent-cost-report/SKILL.md +0 -241
- package/plugin/skills/agent-cost-report/scripts/acr/__init__.py +0 -1
- package/plugin/skills/agent-cost-report/scripts/acr/behavior.py +0 -303
- package/plugin/skills/agent-cost-report/scripts/acr/classify.py +0 -83
- package/plugin/skills/agent-cost-report/scripts/acr/costs.py +0 -119
- package/plugin/skills/agent-cost-report/scripts/acr/devices.py +0 -88
- package/plugin/skills/agent-cost-report/scripts/acr/evidence.py +0 -160
- package/plugin/skills/agent-cost-report/scripts/acr/labels.py +0 -115
- package/plugin/skills/agent-cost-report/scripts/acr/measure.py +0 -89
- package/plugin/skills/agent-cost-report/scripts/acr/mistakes.py +0 -310
- package/plugin/skills/agent-cost-report/scripts/acr/patterns.py +0 -391
- package/plugin/skills/agent-cost-report/scripts/acr/period.py +0 -98
- package/plugin/skills/agent-cost-report/scripts/acr/prices.py +0 -95
- package/plugin/skills/agent-cost-report/scripts/acr/render.py +0 -586
- package/plugin/skills/agent-cost-report/scripts/acr/rollup.py +0 -355
- package/plugin/skills/agent-cost-report/scripts/acr/rules.json +0 -4
- package/plugin/skills/agent-cost-report/scripts/acr/rules.py +0 -76
- package/plugin/skills/agent-cost-report/scripts/acr/sample.py +0 -45
- package/plugin/skills/agent-cost-report/scripts/acr/snapshot.py +0 -53
- package/plugin/skills/agent-cost-report/scripts/acr/sync.py +0 -79
- package/plugin/skills/agent-cost-report/scripts/acr/transcripts.py +0 -126
- package/plugin/skills/agent-cost-report/scripts/acr/wins.py +0 -341
- package/plugin/skills/agent-cost-report/scripts/acr.py +0 -219
- package/plugin/skills/agent-cost-report/scripts/tests/_db.py +0 -49
- package/plugin/skills/agent-cost-report/scripts/tests/_paths.py +0 -9
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/codex/sessions/2026/09/18/rollout-codex-1.jsonl +0 -4
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/make_report_fixtures.py +0 -46
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/prices_sample.json +0 -6
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/projects/-workspace-proj-a/sess-aaaa.jsonl +0 -8
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/projects/-workspace-proj-b/sess-bbbb.jsonl +0 -1
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/report-1day.json +0 -823
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/report-30day.json +0 -2213
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/report-7day.json +0 -1252
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/report-empty.json +0 -536
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/report-session.json +0 -662
- package/plugin/skills/agent-cost-report/scripts/tests/test_behavior.py +0 -299
- package/plugin/skills/agent-cost-report/scripts/tests/test_devices.py +0 -99
- package/plugin/skills/agent-cost-report/scripts/tests/test_labels.py +0 -96
- package/plugin/skills/agent-cost-report/scripts/tests/test_measure.py +0 -120
- package/plugin/skills/agent-cost-report/scripts/tests/test_pdf_windows.py +0 -54
- package/plugin/skills/agent-cost-report/scripts/tests/test_period.py +0 -137
- package/plugin/skills/agent-cost-report/scripts/tests/test_prices.py +0 -126
- package/plugin/skills/agent-cost-report/scripts/tests/test_render.py +0 -238
- package/plugin/skills/agent-cost-report/scripts/tests/test_render_encoding.py +0 -70
- package/plugin/skills/agent-cost-report/scripts/tests/test_rollup.py +0 -263
- package/plugin/skills/agent-cost-report/scripts/tests/test_snapshot.py +0 -63
- package/plugin/skills/agent-cost-report/scripts/tests/test_sync.py +0 -52
- package/plugin/skills/agent-cost-report/scripts/tests/test_transcript_encoding.py +0 -109
- package/plugin/skills/agent-cost-report/scripts/tests/test_transcripts.py +0 -151
- package/plugin/skills/agent-cost-report/scripts/tests/test_wins.py +0 -222
- package/plugin/skills/babysit/SKILL.md +0 -87
- package/plugin/skills/design-is/SKILL.md +0 -312
- package/plugin/skills/do/SKILL.md +0 -45
- package/plugin/skills/learn-codebase/SKILL.md +0 -21
- package/plugin/skills/make-plan/SKILL.md +0 -67
- package/plugin/skills/oh-my-issues/SKILL.md +0 -226
- package/plugin/skills/pathfinder/SKILL.md +0 -111
- package/plugin/skills/screenpipe/SKILL.md +0 -78
- package/plugin/skills/screenpipe/screenpipe.mjs +0 -196
- package/plugin/skills/smart-explore/SKILL.md +0 -193
- package/plugin/skills/standup/SKILL.md +0 -142
- package/plugin/skills/standup/agent-brief.md +0 -47
- package/plugin/skills/standup/standup.mjs +0 -586
- package/plugin/skills/what-the/SKILL.md +0 -6
- package/plugin/skills/wowerpoint/SKILL.md +0 -205
|
@@ -1,355 +0,0 @@
|
|
|
1
|
-
"""Session rollup -> report.json, line-items.csv, evidence.json, labels.review.json (plan Phase 2).
|
|
2
|
-
|
|
3
|
-
Inputs: <out>/usage.json (collect), <out>/prices.json (prices), a fresh read-only snapshot at
|
|
4
|
-
<out>/snapshot.db. Everything derived from the observer's discovery_tokens stays in the separate
|
|
5
|
-
observer block / spend.observer_note_taker_est_usd; it never feeds agent_estimated_usd, headline_usd,
|
|
6
|
-
by_category or by_day. agent_measured_usd is null with measured_status "unavailable" (never $0).
|
|
7
|
-
"""
|
|
8
|
-
import collections
|
|
9
|
-
import csv
|
|
10
|
-
import datetime as dt
|
|
11
|
-
import json
|
|
12
|
-
import os
|
|
13
|
-
|
|
14
|
-
from . import costs, devices, evidence, labels, measure, mistakes, period, snapshot, wins
|
|
15
|
-
from . import prices as prices_mod
|
|
16
|
-
from .evidence import LOCAL
|
|
17
|
-
|
|
18
|
-
FINISHED = ("shipped", "completed")
|
|
19
|
-
HEADLINE_LABEL = "estimated at OpenRouter list prices from measured tokens"
|
|
20
|
-
GROK_BOT_UNAVAILABLE = dict(status="unavailable", reason="no documented API or export for Cursor Grok Bot seat usage",
|
|
21
|
-
checked_sources=["https://cursor.com/docs/grok-bot", "https://cursor.com/pricing", "https://docs.x.ai/developers/rest-api-reference/management/billing"],
|
|
22
|
-
note="Cursor exposes seat usage only on the plan screen; the xAI Management API covers xAI API keys only and Grok Bot seats carry no xAI key")
|
|
23
|
-
RULE_1H = "explicit input_cache_write_1h when listed, else 2 x input (weekly_report.py:95)"
|
|
24
|
-
CSV_COLUMNS = ("work_item_id", "title", "scope", "project", "worktree", "session_ids", "content_session_id", "status", "category",
|
|
25
|
-
"failure_type", "failure_signals", "cost_measured", "cost_estimated", "cost_extrapolated", "attributed_usd", "wasted_cost",
|
|
26
|
-
"recovery_cost", "productive_cost", "risk_exposure", "evidence_ids", "summary_ids", "agent_tokens", "model",
|
|
27
|
-
"model_prices_usd_per_mtok", "cost_basis", "label_source", "device", "recommended_action", "confidence", "date_pt",
|
|
28
|
-
"active_minutes", "has_transcript", "trivial", "notes")
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
def _days(s_ms, e_ms):
|
|
32
|
-
d, end = evidence.pt(s_ms).date(), evidence.pt(e_ms).date(); out = []
|
|
33
|
-
while d < end: out.append(d.isoformat()); d += dt.timedelta(days=1)
|
|
34
|
-
return out
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def _ts_ms(iso):
|
|
38
|
-
return int(dt.datetime.fromisoformat(iso).timestamp() * 1000)
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
def status_of(e, s):
|
|
42
|
-
if e["ship_titles"]: return "shipped"
|
|
43
|
-
if e["completed_sums"]: return "completed"
|
|
44
|
-
return "abandoned" if s["status"] in ("completed", "failed") else "in_progress"
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
def waste_split(li):
|
|
48
|
-
"""Draft money split (SKILL.md:88-109): abandoned work is wasted; 'Recovery after miss' spend is recovery.
|
|
49
|
-
Once the Phase 2B behavior pass has run, its same-session union is the only waste figure (2B.9)."""
|
|
50
|
-
if "behavior_counts" in li: return
|
|
51
|
-
c = li["attributed_usd"]
|
|
52
|
-
li["wasted_cost"] = round(c, 2) if li["status"] == "abandoned" else 0.0
|
|
53
|
-
li["recovery_cost"] = round(c, 2) if ("Recovery after miss" in li["failure_signals"] and not li["wasted_cost"]) else 0.0
|
|
54
|
-
li["productive_cost"] = round(c - li["wasted_cost"] - li["recovery_cost"], 2)
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def build(usage, prices, db, scope, window_block, now=None, use_gh=True, gh=wins.gh_pr_view, remote=wins.git_remote, behavior=True, classify=None, rules_dir=None, glob_pattern=None, device_exports=(), precision=None, wins_repos=wins.WINS_REPOS, gh_list=wins.gh_merged_prs):
|
|
58
|
-
now = now or period.now_pt(); pricer = costs.Pricer(prices)
|
|
59
|
-
sess = evidence.load_sessions(db, scope)
|
|
60
|
-
ev, dev_labels, counts, obs_unpriced = evidence.load_evidence(db, scope, sess, pricer.observer_input_rate)
|
|
61
|
-
for m, tok in obs_unpriced.items(): pricer.unpriced[m]["observer"]["tokens"] += tok
|
|
62
|
-
rows = usage["rows"]
|
|
63
|
-
device_merge = devices.merge(rows, list(device_exports), sess, window_block) if device_exports else []
|
|
64
|
-
if scope.kind == "project":
|
|
65
|
-
keep = {v["content_session_id"] for v in sess.values()}; rows = [r for r in rows if r["session"] in keep]
|
|
66
|
-
elif scope.kind == "session":
|
|
67
|
-
rows = [r for r in rows if r["session"] == scope.session]
|
|
68
|
-
for r in rows: r["ts_ms"] = _ts_ms(r["ts"])
|
|
69
|
-
rows_outside = 0
|
|
70
|
-
if scope.S is not None:
|
|
71
|
-
kept = [r for r in rows if scope.S <= r["ts_ms"] < scope.E]; rows_outside = len(rows) - len(kept); rows = kept
|
|
72
|
-
by_session, by_model, _ = costs.price_rows(rows, pricer)
|
|
73
|
-
matched, unmatched = evidence.join_transcripts(rows, sess)
|
|
74
|
-
for r in rows: r["x1e6"] = costs.api_equiv_x1e6(r, pricer.rate(r["model"]))
|
|
75
|
-
items = []; evid = []; review = []; all_stamps = []; n = 0
|
|
76
|
-
for n, (mid, s) in enumerate(sorted(sess.items(), key=lambda kv: kv[1]["started_at_epoch"]), 1):
|
|
77
|
-
e = ev[mid]; cs = s["content_session_id"]; t = by_session.get(cs) or costs._zero(); trs = matched.get(mid, [])
|
|
78
|
-
stamps = e["stamps"] + [r["ts_ms"] for r in trs] + [s["started_at_epoch"]] + ([s["completed_at_epoch"]] if s["completed_at_epoch"] else [])
|
|
79
|
-
all_stamps += stamps; active = labels.active_minutes(stamps); has_tx = bool(t["calls"])
|
|
80
|
-
draft = labels.classify(e, s); status = status_of(e, s); model = costs.dominant_model(t)
|
|
81
|
-
devs = sorted({dev_labels.get(d, d) for d in e["devices"]} | {r.get("device") or LOCAL for r in trs})
|
|
82
|
-
title = (s["custom_title"] or (s["user_prompt"] or (e["prompts"][0] if e["prompts"] else "") or (e["text"][0] if e["text"] else "")))[:120].replace("\n", " ")
|
|
83
|
-
proj = s["project"] or ""
|
|
84
|
-
li = dict(work_item_id=f"WI-{n}", title=title, scope=scope.kind, project=proj.split("/")[0], worktree=proj.split("/", 1)[1] if "/" in proj else "",
|
|
85
|
-
session_ids=[mid], content_session_id=cs, status=status, category=draft["category"], work_category=draft["category"],
|
|
86
|
-
failure_type=draft["failure_signals"][0] if draft["failure_signals"] else "", failure_signals=draft["failure_signals"],
|
|
87
|
-
cost_measured="unavailable", cost_estimated=costs.usd(t["usd_x1e6"]) if has_tx else None, cost_extrapolated=None,
|
|
88
|
-
risk_exposure="none", evidence_ids=e["obs_ids"], summary_ids=e["sum_ids"],
|
|
89
|
-
agent_tokens={f: t[f] for f in costs.TOKEN_FIELDS}, agent_calls=t["calls"], unpriced_calls=t["unpriced_calls"],
|
|
90
|
-
model=model, model_prices_usd_per_mtok=costs.price_block(pricer.rate(model)) if model else None,
|
|
91
|
-
cost_basis="estimated_usage" if has_tx else "extrapolated", label_source="keyword", device=devs,
|
|
92
|
-
recommended_action="", confidence=("high" if has_tx and status in FINISHED and not t["unpriced_calls"] else "medium" if has_tx else "low"),
|
|
93
|
-
date_pt=evidence.day_pt(s["started_at_epoch"]), active_minutes=round(active, 1), has_transcript=has_tx,
|
|
94
|
-
trivial=(e["obs"] == 0 and not has_tx and active < 1), platform=s["platform_source"], session_status=s["status"],
|
|
95
|
-
observations=e["obs"], summaries=e["sums"], ship_events=len(set(e["ship_titles"])), ship_titles=sorted(set(e["ship_titles"])),
|
|
96
|
-
notes=(f"{status.replace('_', ' ')}; {e['obs']} observations, {e['sums']} summaries, {len(e['prompts'])} prompts; "
|
|
97
|
-
+ (("tokens measured from the transcript on this box" if all((r.get("device") or LOCAL) == LOCAL for r in trs) else "tokens measured from a device export: " + ", ".join(sorted({r["device"] for r in trs if r.get("device")}))) if has_tx else "no transcript on this box (remote device)")),
|
|
98
|
-
observer_tokens=e["obs_tok_dedup"], cost_x1e6=t["usd_x1e6"])
|
|
99
|
-
items.append(li); review.append(labels.review_entry(li, draft))
|
|
100
|
-
evid.append(dict(work_item_id=li["work_item_id"], memory_session_id=mid, content_session_id=cs, observation_ids=e["obs_ids"],
|
|
101
|
-
summary_ids=e["sum_ids"], titles=[x[:100] for x in e["text"][:20]], ship_titles=li["ship_titles"], ship_ids=e["ship_ids"],
|
|
102
|
-
observer_tokens=e["obs_tok_dedup"], observer_tokens_raw=e["obs_tok"], observer_est_usd=costs.usd(e["obs_cost_x1e6"]),
|
|
103
|
-
generated_by_model=sorted(m for m in e["models"] if m), devices=[evidence.device_hash(d) for d in sorted(e["devices"])]))
|
|
104
|
-
for cs, rs in sorted(unmatched.items(), key=lambda kv: min(r["ts"] for r in kv[1])):
|
|
105
|
-
n += 1; t = by_session.get(cs) or costs._zero(); model = costs.dominant_model(t); rs = sorted(rs, key=lambda r: r["ts"])
|
|
106
|
-
d0 = rs[0].get("dir") or ""; proj = (os.path.basename(rs[0]["cwd"]) if rs[0].get("cwd") else d0.strip("-").split("-")[-1] or "unknown")
|
|
107
|
-
active = labels.active_minutes([r["ts_ms"] for r in rs]); all_stamps += [r["ts_ms"] for r in rs]
|
|
108
|
-
draft = dict(category="Investigation", failure_signals=[], matched=dict(category_where="default"), head="", tail="")
|
|
109
|
-
li = dict(work_item_id=f"WI-{n}", title=f"Transcript with no memindex session ({rs[0]['src']}, {d0 or proj})", scope=scope.kind, project=proj, worktree="",
|
|
110
|
-
session_ids=[f"nomem-{cs}"], content_session_id=cs, status="in_progress", category="Investigation", work_category="Investigation", failure_type="", failure_signals=[],
|
|
111
|
-
cost_measured="unavailable", cost_estimated=costs.usd(t["usd_x1e6"]), cost_extrapolated=None, risk_exposure="none", evidence_ids=[], summary_ids=[],
|
|
112
|
-
agent_tokens={f: t[f] for f in costs.TOKEN_FIELDS}, agent_calls=t["calls"], unpriced_calls=t["unpriced_calls"], model=model,
|
|
113
|
-
model_prices_usd_per_mtok=costs.price_block(pricer.rate(model)) if model else None, cost_basis="estimated_usage", label_source="keyword",
|
|
114
|
-
device=sorted({r.get("device") or LOCAL for r in rs}), recommended_action="", confidence="low", date_pt=evidence.day_pt(rs[0]["ts_ms"]), active_minutes=round(active, 1),
|
|
115
|
-
has_transcript=True, trivial=False, orphan=True, platform=rs[0]["src"], session_status="unknown", observations=0, summaries=0, ship_events=0, ship_titles=[],
|
|
116
|
-
notes=f"no memindex session for this transcript ({len(rs)} API calls); outcome unknown, counted in the totals", observer_tokens=0, cost_x1e6=t["usd_x1e6"])
|
|
117
|
-
items.append(li); review.append(labels.review_entry(li, draft))
|
|
118
|
-
evid.append(dict(work_item_id=li["work_item_id"], memory_session_id=None, content_session_id=cs, observation_ids=[], summary_ids=[], titles=[], ship_titles=[], ship_ids=[],
|
|
119
|
-
observer_tokens=0, observer_tokens_raw=0, observer_est_usd=0.0, generated_by_model=[], devices=li["device"]))
|
|
120
|
-
extrapolated_usd, basis, ratio = costs.extrapolation(items)
|
|
121
|
-
for li in items:
|
|
122
|
-
if not li["has_transcript"]: li["cost_extrapolated"] = costs.usd(li["extrapolated_x1e6"]) if ratio is not None else None
|
|
123
|
-
li["attributed_usd"] = li["cost_estimated"] if li["has_transcript"] else (li["cost_extrapolated"] or 0.0)
|
|
124
|
-
li["cost_status"] = "estimated" if li["has_transcript"] else "extrapolated" if ratio is not None else "unmeasured"
|
|
125
|
-
waste_split(li)
|
|
126
|
-
agent_x1e6 = sum(m["usd_x1e6"] for m in by_model.values())
|
|
127
|
-
observer_x1e6 = sum(ev[mid]["obs_cost_x1e6"] for mid in sess)
|
|
128
|
-
agent_usd = costs.usd(agent_x1e6)
|
|
129
|
-
spend = dict(agent_estimated_usd=agent_usd, agent_estimated_label="ESTIMATED", agent_measured_usd=None, measured_status="unavailable",
|
|
130
|
-
extrapolated_unmeasured_usd=extrapolated_usd, extrapolated_label="EXTRAPOLATED (low confidence)", extrapolation_basis=basis,
|
|
131
|
-
observer_note_taker_est_usd=costs.usd(observer_x1e6), grok_bot_usage=GROK_BOT_UNAVAILABLE,
|
|
132
|
-
headline_usd=agent_usd, headline_label=HEADLINE_LABEL, total_estimate_usd=round(agent_usd + (extrapolated_usd or 0), 2))
|
|
133
|
-
if scope.kind == "session":
|
|
134
|
-
ds = [evidence.day_pt(x) for x in all_stamps] or [li["date_pt"] for li in items]
|
|
135
|
-
days = _days(period.epoch_ms(period.parse_day(min(ds))), period.epoch_ms(period.parse_day(max(ds)) + dt.timedelta(days=1))) if ds else []
|
|
136
|
-
else:
|
|
137
|
-
days = _days(scope.S, scope.E)
|
|
138
|
-
by_day = {d: dict(day_pt=d, sessions_started=0, agent_calls=0, tokens=0, agent_estimated_usd=0, extrapolated_usd=0, ship_events=0,
|
|
139
|
-
finished_outcomes=0, events=0) for d in days}
|
|
140
|
-
for r in rows:
|
|
141
|
-
d = by_day.get(r["ts"][:10])
|
|
142
|
-
if d is None: continue
|
|
143
|
-
d["agent_calls"] += 1; d["tokens"] += sum(r[f] for f in costs.TOKEN_FIELDS); d["agent_estimated_usd"] += r["x1e6"] or 0
|
|
144
|
-
for li in items:
|
|
145
|
-
d = by_day.get(li["date_pt"])
|
|
146
|
-
if d is None: continue
|
|
147
|
-
d["sessions_started"] += 1; d["ship_events"] += li["ship_events"]; d["extrapolated_usd"] += li.get("extrapolated_x1e6", 0)
|
|
148
|
-
for x in all_stamps:
|
|
149
|
-
d = by_day.get(evidence.day_pt(x))
|
|
150
|
-
if d is not None: d["events"] += 1
|
|
151
|
-
for d in by_day.values():
|
|
152
|
-
d["agent_estimated_usd"] = costs.usd(d["agent_estimated_usd"]); d["extrapolated_usd"] = costs.usd(d["extrapolated_usd"])
|
|
153
|
-
d["no_agent_work"] = not (d["sessions_started"] or d["agent_calls"] or d["events"])
|
|
154
|
-
ship_obs = []
|
|
155
|
-
for mid, s in sess.items():
|
|
156
|
-
e = ev[mid]
|
|
157
|
-
for oid, t_ in zip(e["ship_ids"], e["ship_titles"]):
|
|
158
|
-
ship_obs.append(dict(id=oid, title=t_, ts_ms=None, project=s["project"], memory_session_id=mid, content_session_id=s["content_session_id"]))
|
|
159
|
-
if ship_obs:
|
|
160
|
-
ids = [o["id"] for o in ship_obs]; stamp = {}
|
|
161
|
-
for i in range(0, len(ids), 200):
|
|
162
|
-
chunk = ids[i:i + 200]
|
|
163
|
-
for r in db.execute(f"select id, created_at_epoch from observations where id in ({','.join('?' * len(chunk))})", chunk): stamp[r[0]] = r[1]
|
|
164
|
-
for o in ship_obs: o["ts_ms"] = stamp.get(o["id"])
|
|
165
|
-
sessions_by_cs = {li["content_session_id"]: dict(project=li["project"], memory_session_id=li["session_ids"][0],
|
|
166
|
-
has_transcript=li["has_transcript"], observer_tokens=li["observer_tokens"],
|
|
167
|
-
cwd=next((r["cwd"] for r in matched.get(li["session_ids"][0], []) if r.get("cwd")), None)) for li in items}
|
|
168
|
-
rows_by_cs = collections.defaultdict(list)
|
|
169
|
-
for r in rows: rows_by_cs[r["session"]].append(r)
|
|
170
|
-
files = sorted({r["file"] for r in rows if r.get("file")})
|
|
171
|
-
wins_block, wins_by_day = wins.build(files, scope.S, scope.E, ship_obs, sessions_by_cs, rows_by_cs, ratio, days, gh=gh, remote=remote, use_gh=use_gh, repos=tuple(wins_repos), gh_list=gh_list)
|
|
172
|
-
if wins_block["cost_status"] == "session_linked":
|
|
173
|
-
wins_block["unattributed_usd"] = round(max(spend["total_estimate_usd"] - wins_block["total_attributed_usd"], 0.0), 2)
|
|
174
|
-
behavior_block = None; mistakes_by_day = []
|
|
175
|
-
if behavior:
|
|
176
|
-
session_projects = {li["content_session_id"]: li["project"] for li in items}
|
|
177
|
-
behavior_block, mspend, mistakes_by_day = mistakes.run(db, scope, window_block, pricer, session_projects, items, classify=classify, rules_dir=rules_dir, usage_rows=rows, glob_pattern=glob_pattern, precision=precision)
|
|
178
|
-
spend.update(mspend)
|
|
179
|
-
um = [dict(session=cs, dir=rs[0]["dir"], src=rs[0]["src"], calls=len(rs), tokens=sum(sum(r[f] for f in costs.TOKEN_FIELDS) for r in rs),
|
|
180
|
-
usd=costs.usd(sum(r["x1e6"] or 0 for r in rs)), first_ts=min(r["ts"] for r in rs)) for cs, rs in unmatched.items()]
|
|
181
|
-
um.sort(key=lambda x: -x["usd"])
|
|
182
|
-
unmatched_block = dict(count=len(um), usd=round(sum(x["usd"] for x in um), 2), tokens=sum(x["tokens"] for x in um),
|
|
183
|
-
by_dir=dict(collections.Counter(x["dir"] for x in um)), sessions=um,
|
|
184
|
-
note="transcripts on this box that matched no sdk_sessions row (work memindex never saw); counted in the totals above")
|
|
185
|
-
tok = {f: sum(r[f] for r in rows) for f in costs.TOKEN_FIELDS}; tok["total"] = sum(tok.values()); tok["api_calls"] = len(rows)
|
|
186
|
-
by_model_list = []
|
|
187
|
-
for (src, m), c in sorted(by_model.items(), key=lambda kv: -kv[1]["usd_x1e6"]):
|
|
188
|
-
rt = pricer.rate(m)
|
|
189
|
-
by_model_list.append(dict(source=src, model=m, calls=c["calls"], **{f: c[f] for f in costs.TOKEN_FIELDS}, agent_estimated_usd=costs.usd(c["usd_x1e6"]),
|
|
190
|
-
unpriced_calls=c["unpriced_calls"], priced=pricer.usable(m), prices_usd_per_mtok=costs.price_block(rt)))
|
|
191
|
-
by_device = collections.defaultdict(lambda: dict(sessions=0, agent_estimated_usd=0, extrapolated_usd=0))
|
|
192
|
-
for li in items:
|
|
193
|
-
for d in li["device"]:
|
|
194
|
-
b = by_device[d]; b["sessions"] += 1; b["agent_estimated_usd"] += li["cost_x1e6"] if len(li["device"]) == 1 or d == LOCAL else 0
|
|
195
|
-
b["extrapolated_usd"] += li.get("extrapolated_x1e6", 0) if d != LOCAL else 0
|
|
196
|
-
hashes = {lbl: evidence.device_hash(d) for d, lbl in dev_labels.items()}
|
|
197
|
-
by_device_list = [dict(device=d, id_hash=hashes.get(d, d), sessions=b["sessions"], agent_estimated_usd=costs.usd(b["agent_estimated_usd"]),
|
|
198
|
-
extrapolated_usd=costs.usd(b["extrapolated_usd"])) for d, b in sorted(by_device.items())]
|
|
199
|
-
observer = dict(tokens_dedup=sum(li["observer_tokens"] for li in items), est_usd=spend["observer_note_taker_est_usd"],
|
|
200
|
-
models=sorted({m for x in evid for m in x["generated_by_model"]}),
|
|
201
|
-
unpriced_tokens=dict(obs_unpriced), basis="ESTIMATED: observer model's own tokens (deduped per reply) x its input list price; "
|
|
202
|
-
"kept separate from the agent figures above", rows_in_scope=dict(counts))
|
|
203
|
-
report = dict(window=dict(window_block, usage_rows_outside_window=rows_outside) if scope.S is not None else window_block, scope=scope.block(), generated_at_pt=now.strftime("%Y-%m-%d %H:%M PT"), spend=spend,
|
|
204
|
-
totals=dict(sessions=sum(1 for li in items if not li.get("orphan")), transcript_only_sessions=sum(1 for li in items if li.get("orphan")),
|
|
205
|
-
agent_hours=round(sum(li["active_minutes"] for li in items if not li.get("orphan")) / 60, 1), transcript_only_hours=round(sum(li["active_minutes"] for li in items if li.get("orphan")) / 60, 1),
|
|
206
|
-
wall_clock_hours=round(labels.active_minutes(all_stamps) / 60, 1), tokens=tok,
|
|
207
|
-
hours_method="sum of gaps <=15 min between consecutive events (prompts, observations, summaries, tool calls, transcript API calls); "
|
|
208
|
-
"parallel sessions add up in agent_hours but not in wall_clock_hours"),
|
|
209
|
-
by_day=list(by_day.values()), by_model=by_model_list, by_device=by_device_list, line_items=items,
|
|
210
|
-
labels=dict(reviewed=0, total=len(items)), pricing=dict(source=prices.get("source"), fetched=prices.get("fetched"),
|
|
211
|
-
loaded_from=prices.get("loaded_from"), rule_1h=RULE_1H, models_listed=len(prices["models"])),
|
|
212
|
-
unmatched_transcripts=unmatched_block, unpriced_models=pricer.unpriced_list(), wins=wins_block, timeline=dict(wins_by_day=wins_by_day, mistakes_by_day=mistakes_by_day),
|
|
213
|
-
behavior=behavior_block,
|
|
214
|
-
observer=observer, devices=[dict(device=lbl, id_hash=evidence.device_hash(d)) for d, lbl in dev_labels.items()], device_exports=device_merge,
|
|
215
|
-
trivial_definition="no observations, no transcript on this box, and <1 active minute")
|
|
216
|
-
aggregate(report)
|
|
217
|
-
raw = (behavior_block or {}).pop("_raw", None)
|
|
218
|
-
if raw is not None: report["_behavior_raw"] = raw
|
|
219
|
-
return report, evid, dict(generated_at_pt=report["generated_at_pt"], scope=scope.block(), items=review)
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
def aggregate(report):
|
|
223
|
-
"""Everything that follows from the line items' labels; re-run after review --apply."""
|
|
224
|
-
items = report["line_items"]; real = [li for li in items if not li["trivial"] and not li.get("orphan")]
|
|
225
|
-
for li in items: waste_split(li)
|
|
226
|
-
finished = [li for li in real if li["status"] in FINISHED]
|
|
227
|
-
byd = {d["day_pt"]: d for d in report["by_day"]}
|
|
228
|
-
for d in byd.values(): d["finished_outcomes"] = 0
|
|
229
|
-
for li in finished:
|
|
230
|
-
if li["date_pt"] in byd: byd[li["date_pt"]]["finished_outcomes"] += 1
|
|
231
|
-
total = sum(li["attributed_usd"] for li in items); wasted = sum(li["wasted_cost"] for li in items); rec = sum(li["recovery_cost"] for li in items)
|
|
232
|
-
t = report["totals"]
|
|
233
|
-
t.update(real_work_sessions=len(real), trivial_sessions=sum(1 for li in items if li["trivial"]),
|
|
234
|
-
projects=len({(li["project"], li["worktree"]) for li in items if not li.get("orphan")}), repos=len({li["project"] for li in items if not li.get("orphan")}),
|
|
235
|
-
transcript_only_projects=len({li["project"] for li in items if li.get("orphan")}),
|
|
236
|
-
devices=len({d for li in items for d in li["device"]}), finished_outcomes=len(finished),
|
|
237
|
-
ship_events=len({x for li in items for x in li["ship_titles"]}),
|
|
238
|
-
cost_per_completed_outcome=round(sum(li["attributed_usd"] for li in finished) / len(finished), 2) if finished else None,
|
|
239
|
-
waste_rate=round(wasted / total, 4) if total else None, recovery_share=round(rec / total, 4) if total else None,
|
|
240
|
-
wasted_usd=round(wasted, 2), recovery_usd=round(rec, 2))
|
|
241
|
-
cats = collections.defaultdict(lambda: dict(sessions=0, finished=0, agent_x1e6=0, extrapolated_x1e6=0, wasted_usd=0.0, recovery_usd=0.0, active_minutes=0.0))
|
|
242
|
-
for li in real:
|
|
243
|
-
c = cats[li["category"]]; c["sessions"] += 1; c["finished"] += li["status"] in FINISHED
|
|
244
|
-
c["agent_x1e6"] += li["cost_x1e6"]; c["extrapolated_x1e6"] += li.get("extrapolated_x1e6", 0)
|
|
245
|
-
c["wasted_usd"] += li["wasted_cost"]; c["recovery_usd"] += li["recovery_cost"]; c["active_minutes"] += li["active_minutes"]
|
|
246
|
-
report["by_category"] = [dict(category=k, sessions=c["sessions"], finished=c["finished"], agent_estimated_usd=costs.usd(c["agent_x1e6"]),
|
|
247
|
-
extrapolated_usd=costs.usd(c["extrapolated_x1e6"]), wasted_usd=round(c["wasted_usd"], 2),
|
|
248
|
-
recovery_usd=round(c["recovery_usd"], 2), active_minutes=round(c["active_minutes"], 1))
|
|
249
|
-
for k, c in sorted(cats.items(), key=lambda kv: -(kv[1]["agent_x1e6"] + kv[1]["extrapolated_x1e6"]))]
|
|
250
|
-
fe = collections.defaultdict(lambda: dict(sessions=0, attributed_usd=0.0, wasted_usd=0.0, recovery_usd=0.0, work_item_ids=[]))
|
|
251
|
-
for li in real:
|
|
252
|
-
for f in li["failure_signals"]:
|
|
253
|
-
x = fe[f]; x["sessions"] += 1; x["attributed_usd"] += li["attributed_usd"]; x["wasted_usd"] += li["wasted_cost"]
|
|
254
|
-
x["recovery_usd"] += li["recovery_cost"]; x["work_item_ids"].append(li["work_item_id"])
|
|
255
|
-
report["failure_economics"] = [dict(failure_type=k, sessions=v["sessions"], attributed_usd=round(v["attributed_usd"], 2), wasted_usd=round(v["wasted_usd"], 2),
|
|
256
|
-
recovery_usd=round(v["recovery_usd"], 2), work_item_ids=v["work_item_ids"]) for k, v in sorted(fe.items(), key=lambda kv: -kv[1]["attributed_usd"])]
|
|
257
|
-
att = []
|
|
258
|
-
b = report.get("behavior")
|
|
259
|
-
if b:
|
|
260
|
-
ob = b["outbound"]
|
|
261
|
-
if ob["incidents"]: att.append(dict(kind="bad_outbound", text=f"{ob['incidents']} send(s) reached {ob['recipients']} recipient(s) without your approval"))
|
|
262
|
-
bad_rule = next((r for r in b["rule_effectiveness"] if r.get("card")), None)
|
|
263
|
-
if bad_rule and len(att) < 2: att.append(dict(kind="rule_not_stopped", rule_key=bad_rule["rule_key"], text=f"Rule '{bad_rule['name']}' ({bad_rule['landed_pt']}) did not stop repeats: {bad_rule['before']['per_100']} -> {bad_rule['after']['per_100']} per 100 human prompts"))
|
|
264
|
-
top = max((p for p in b["patterns"] if p["low_usd"]), key=lambda p: p["low_usd"], default=None)
|
|
265
|
-
if top and len(att) < 2: att.append(dict(kind="pattern", pattern=top["key"], usd=top["low_usd"], text=f"Largest behavior cost: {top['name']} at ${top['low_usd']:.2f} ESTIMATED (heuristic, {top['count']} found)"))
|
|
266
|
-
w = max((li for li in real if li["wasted_cost"]), key=lambda li: li["wasted_cost"], default=None)
|
|
267
|
-
if w: att.append(dict(kind="waste", work_item_id=w["work_item_id"], usd=w["wasted_cost"], text=f"Biggest waste: {w['work_item_id']} ({w['status']}, {w['category']}) at ${w['wasted_cost']:.2f} ESTIMATED"))
|
|
268
|
-
u = max((li for li in real if li["status"] == "in_progress"), key=lambda li: li["attributed_usd"], default=None)
|
|
269
|
-
if u: att.append(dict(kind="unfinished", work_item_id=u["work_item_id"], usd=u["attributed_usd"], text=f"Biggest unfinished item: {u['work_item_id']} ({u['category']}) at ${u['attributed_usd']:.2f} ESTIMATED, still in progress"))
|
|
270
|
-
unp = sorted({x["model"] for x in report["unpriced_models"] if x["tokens"]})
|
|
271
|
-
if unp: att.append(dict(kind="unpriced", text="Unpriced model(s): " + ", ".join(unp) + " (tokens counted, no dollars)"))
|
|
272
|
-
report["attention"] = att[:3]
|
|
273
|
-
report["labels"] = dict(reviewed=sum(1 for li in items if li["label_source"] != "keyword"), total=len(items))
|
|
274
|
-
report["trivial_sessions"] = [dict(work_item_id=li["work_item_id"], session_ids=li["session_ids"], title=li["title"]) for li in items if li["trivial"]]
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
def write_outputs(outdir, report, evid=None, review=None):
|
|
278
|
-
os.makedirs(outdir, exist_ok=True)
|
|
279
|
-
raw = report.pop("_behavior_raw", None)
|
|
280
|
-
if raw is not None:
|
|
281
|
-
with open(os.path.join(outdir, "behavior.json"), "w", encoding="utf-8") as fh: json.dump(dict(window=report["window"], scope=report["scope"], **raw), fh, indent=1, default=str)
|
|
282
|
-
with open(os.path.join(outdir, "report.json"), "w", encoding="utf-8") as fh: json.dump(report, fh, indent=1, default=str)
|
|
283
|
-
with open(os.path.join(outdir, "line-items.csv"), "w", encoding="utf-8", newline="") as fh:
|
|
284
|
-
w = csv.DictWriter(fh, fieldnames=CSV_COLUMNS, extrasaction="ignore"); w.writeheader()
|
|
285
|
-
for li in report["line_items"]:
|
|
286
|
-
w.writerow({k: (";".join(map(str, v)) if isinstance(v, list) else json.dumps(v) if isinstance(v, dict) else v) for k, v in li.items()})
|
|
287
|
-
if evid is not None:
|
|
288
|
-
with open(os.path.join(outdir, "evidence.json"), "w", encoding="utf-8") as fh: json.dump(evid, fh, indent=1)
|
|
289
|
-
if review is not None:
|
|
290
|
-
with open(os.path.join(outdir, "labels.review.json"), "w", encoding="utf-8") as fh: json.dump(review, fh, indent=1)
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
def resolve_scope(args, usage):
|
|
294
|
-
"""--session | --project [--start/--end] | --start/--end. Returns (scope, window_block)."""
|
|
295
|
-
if args.session:
|
|
296
|
-
return evidence.Scope("session", session=args.session), (usage["window"] if usage["window"].get("session") else period.session_block(args.session))
|
|
297
|
-
w = period.resolve_window(args.start, args.end)
|
|
298
|
-
kind = "project" if args.project else "period"
|
|
299
|
-
return evidence.Scope(kind, S=w.start_epoch_ms, E=w.end_epoch_ms, project=args.project), w.block()
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
def input_path(outdir, name):
|
|
303
|
-
"""<out>/<name>, else the parent directory's copy, so `--session` / `--project` runs under a period's --out
|
|
304
|
-
can share its collect + prices run (plan 2.6). Returns the <out> path when neither exists (for the error)."""
|
|
305
|
-
p = os.path.join(outdir, name)
|
|
306
|
-
if os.path.exists(p): return p
|
|
307
|
-
q = os.path.join(os.path.dirname(os.path.abspath(outdir)), name)
|
|
308
|
-
return q if os.path.exists(q) else p
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
def run_rollup(args):
|
|
312
|
-
usage_path = input_path(args.out, "usage.json")
|
|
313
|
-
if not os.path.exists(usage_path): raise SystemExit(f"acr.py rollup: {usage_path} not found; run `acr.py collect` first")
|
|
314
|
-
with open(usage_path, encoding="utf-8") as fh: usage = json.load(fh)
|
|
315
|
-
prices = prices_mod.load(args.prices or input_path(args.out, "prices.json"))
|
|
316
|
-
scope, window_block = resolve_scope(args, usage)
|
|
317
|
-
if scope.kind == "session" and usage["window"].get("session") not in (None, args.session):
|
|
318
|
-
raise SystemExit("acr.py rollup: usage.json was collected for a different --session")
|
|
319
|
-
exports = []
|
|
320
|
-
for f in getattr(args, "device_usage", None) or []:
|
|
321
|
-
try: exports.append(devices.load(f))
|
|
322
|
-
except (OSError, ValueError) as ex: raise SystemExit(f"acr.py rollup --device-usage: {ex}")
|
|
323
|
-
snap = snapshot.snapshot(args.out, live=getattr(args, "db", None)); db = snapshot.open_snapshot(snap)
|
|
324
|
-
try:
|
|
325
|
-
precision = None
|
|
326
|
-
if getattr(args, "precision", None):
|
|
327
|
-
with open(args.precision, encoding="utf-8-sig") as fh: precision = json.load(fh)
|
|
328
|
-
cl = dict(enabled=True, cap_usd=getattr(args, "classify_budget", None) or mistakes.classify_mod.CAP_USD, model=getattr(args, "classify_model", None)) if getattr(args, "classify", False) else None
|
|
329
|
-
report, evid, review = build(usage, prices, db, scope, window_block, use_gh=not getattr(args, "no_gh", False),
|
|
330
|
-
behavior=not getattr(args, "no_behavior", False), classify=cl, rules_dir=getattr(args, "rules_dir", None), device_exports=exports, precision=precision, wins_repos=tuple(getattr(args, "wins_repo", None) or wins.WINS_REPOS))
|
|
331
|
-
finally:
|
|
332
|
-
db.close()
|
|
333
|
-
mp = getattr(args, "measured", None) or os.path.join(args.out, "measured.json")
|
|
334
|
-
measured = None
|
|
335
|
-
if os.path.exists(mp):
|
|
336
|
-
with open(mp, encoding="utf-8") as fh: measured = json.load(fh)
|
|
337
|
-
measure.apply(report, measured)
|
|
338
|
-
report["inputs"] = dict(usage=usage_path, prices=prices.get("loaded_from"), snapshot=snap, measured=mp if measured else None)
|
|
339
|
-
write_outputs(args.out, report, evid, review)
|
|
340
|
-
t, s = report["totals"], report["spend"]
|
|
341
|
-
print(f"rollup: {scope.kind} sessions={t['sessions']} real_work={t['real_work_sessions']} projects={t['projects']} devices={t['devices']} "
|
|
342
|
-
f"finished={t['finished_outcomes']} ship_events={t['ship_events']} agent_hours={t['agent_hours']} | agent_est=${s['agent_estimated_usd']} "
|
|
343
|
-
f"extrapolated=${s['extrapolated_unmeasured_usd']} observer=${s['observer_note_taker_est_usd']} measured={s['measured_status']} | "
|
|
344
|
-
f"wins={len(report['wins']['items'])} ({report['wins']['cost_status']}) unpriced={len(report['unpriced_models'])}"
|
|
345
|
-
+ (f" | mistakes=${s['mistakes_estimated_usd']} low ({s['mistakes_turns_n']} turns, {s['mistakes_episodes_n']} episodes, {s['mistakes_unmeasured_n']} unmeasured)" if 'mistakes_estimated_usd' in s else "")
|
|
346
|
-
+ f" -> {args.out}/report.json")
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
def run_review(args):
|
|
350
|
-
rp = os.path.join(args.out, "report.json")
|
|
351
|
-
with open(rp, encoding="utf-8") as fh: report = json.load(fh)
|
|
352
|
-
with open(args.apply, encoding="utf-8-sig") as fh: reviewed = json.load(fh)
|
|
353
|
-
n = labels.apply_review(report["line_items"], reviewed, period.now_pt().strftime("%Y-%m-%d %H:%M PT"))
|
|
354
|
-
aggregate(report); write_outputs(args.out, report)
|
|
355
|
-
print(f"review: applied {n} labels; {report['labels']['reviewed']} of {report['labels']['total']} reviewed -> {rp}")
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
"""Rule effectiveness (plan 2B.10, MI:34-35): for each HARD rule, same-pattern episodes per 100 human
|
|
2
|
-
prompts in the N=7 days before and after `rule_landed_at`, minimum 50 human prompts per side (G15).
|
|
3
|
-
Rules come from `acr/rules.json` (empty as shipped; add your own) plus any rule file that carries a
|
|
4
|
-
"HARD (YYYY-MM-DD" or "HARD — <owner> YYYY-MM-DD" header when a --rules-dir is given. Human prompts are counted only from
|
|
5
|
-
human-tagged turns (R2); bot and unknown turns are never in the denominator.
|
|
6
|
-
"""
|
|
7
|
-
import datetime as dt
|
|
8
|
-
import glob
|
|
9
|
-
import json
|
|
10
|
-
import os
|
|
11
|
-
import re
|
|
12
|
-
|
|
13
|
-
from .period import PT
|
|
14
|
-
|
|
15
|
-
N_DAYS = 7
|
|
16
|
-
MIN_PROMPTS = 50
|
|
17
|
-
HARD_HEADER = re.compile(r"^#+\s*(?P<name>.+?)\s*\(HARD\s*[—–-]*\s*(?:[^\W\d]\w*\s+)?(?P<date>\d{4}-\d{2}-\d{2})", re.M)
|
|
18
|
-
PATTERN_MAP = {"invented_gates_asking": "P1_invented_gates", "broke_things": "P2_broke_things", "wrong_model_or_spend": "P3_wrong_model",
|
|
19
|
-
"over_engineering": "P4_over_engineering", "ignored_instructions": "P5_not_asked", "fake_output": "P6_fake_output",
|
|
20
|
-
"skipped_steps_false_done": "P7_false_done", "wrong_tool_account_contact": "P8_wrong_tool", "bad_outbound": "P9_bad_outbound",
|
|
21
|
-
"memory_loss": "P10_memory_loss", "jargon_verbose_unclear": "P11_jargon", "unattributed": "P12_unclear"}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def seed_rules():
|
|
25
|
-
with open(os.path.join(os.path.dirname(__file__), "rules.json"), encoding="utf-8") as fh: return json.load(fh)["rules"]
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def scan_rule_files(rules_dir):
|
|
29
|
-
"""Dated HARD headers in rule files; pattern is left None unless rules.json names the same rule."""
|
|
30
|
-
out = []
|
|
31
|
-
for f in sorted(glob.glob(os.path.join(rules_dir, "**", "*.md"), recursive=True)):
|
|
32
|
-
try:
|
|
33
|
-
with open(f, encoding="utf-8-sig", errors="ignore") as fh: text = fh.read(200_000)
|
|
34
|
-
except OSError: continue
|
|
35
|
-
for m in HARD_HEADER.finditer(text):
|
|
36
|
-
out.append(dict(rule_key=re.sub(r"\W+", "_", m.group("name").lower()).strip("_")[:60], name=m.group("name").strip(), landed_pt=m.group("date"), pattern=None, source=os.path.basename(f)))
|
|
37
|
-
return out
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def _day_ms(day):
|
|
41
|
-
return int(dt.datetime.combine(dt.date.fromisoformat(day), dt.time(), tzinfo=PT).timestamp() * 1000)
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def _side(episodes, msgs, pattern, a_ms, b_ms):
|
|
45
|
-
eps = [e for e in episodes if e["pattern"] == pattern and a_ms <= _ep_ms(e) < b_ms]
|
|
46
|
-
prompts = sum(1 for ms in msgs.values() for m in ms if m["author"] == "human" and a_ms <= m["ts_ms"] < b_ms)
|
|
47
|
-
return dict(episodes=len(eps), human_prompts=prompts, per_100=round(100 * len(eps) / prompts, 1) if prompts >= MIN_PROMPTS else None), [e["episode_id"] for e in eps]
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
def _ep_ms(e):
|
|
51
|
-
return int(dt.datetime.fromisoformat(e["ts_pt"]).timestamp() * 1000)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
def effectiveness(episodes, msgs, s_ms, e_ms, rules_dir=None, n_days=N_DAYS):
|
|
55
|
-
"""Rows for rules that landed inside the window or within N days before it."""
|
|
56
|
-
rows = []; seen = set()
|
|
57
|
-
if s_ms is None or e_ms is None: return rows
|
|
58
|
-
all_rules = seed_rules() + (scan_rule_files(rules_dir) if rules_dir else [])
|
|
59
|
-
for r in all_rules:
|
|
60
|
-
pat = PATTERN_MAP.get(r.get("pattern"), r.get("pattern"))
|
|
61
|
-
landed = _day_ms(r["landed_pt"])
|
|
62
|
-
if not (s_ms - n_days * 86_400_000 <= landed < e_ms): continue
|
|
63
|
-
key = (r["rule_key"], r["landed_pt"])
|
|
64
|
-
if key in seen: continue
|
|
65
|
-
seen.add(key)
|
|
66
|
-
if not pat:
|
|
67
|
-
rows.append(dict(rule_key=r["rule_key"], name=r.get("name"), landed_pt=r["landed_pt"], pattern=None, n_days=n_days, before=None, after=None, verdict="not_enough_data", note="rule not mapped to a pattern", episode_ids_after=[])); continue
|
|
68
|
-
before, _ = _side(episodes, msgs, pat, landed - n_days * 86_400_000, landed)
|
|
69
|
-
after, ids = _side(episodes, msgs, pat, landed, landed + n_days * 86_400_000)
|
|
70
|
-
if before["per_100"] is None or after["per_100"] is None:
|
|
71
|
-
verdict = "not_enough_data"
|
|
72
|
-
elif after["per_100"] >= before["per_100"] and after["episodes"] >= 2: verdict = "not_stopped"
|
|
73
|
-
else: verdict = "stopped"
|
|
74
|
-
rows.append(dict(rule_key=r["rule_key"], name=r.get("name"), landed_pt=r["landed_pt"], pattern=pat, n_days=n_days, before=before, after=after, verdict=verdict,
|
|
75
|
-
card=(verdict == "not_stopped"), episode_ids_after=ids, source=r.get("source", "rules.json")))
|
|
76
|
-
return rows
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
"""Phase 8.5: `behavior-sample` writes a Markdown spot-check sheet from <run>/behavior.json and report.json:
|
|
2
|
-
N random human-tagged and N bot-tagged user turns (R2 check), N flagged turns or episodes per summary-tile
|
|
3
|
-
pattern, and 10 random unflagged sessions. Each line cites the session and time so a reviewer can mark it
|
|
4
|
-
true / false / unsure. Precision per pattern goes back into `rollup --precision <json>` (tile gate, 70%).
|
|
5
|
-
"""
|
|
6
|
-
import collections
|
|
7
|
-
import json
|
|
8
|
-
import os
|
|
9
|
-
import random
|
|
10
|
-
|
|
11
|
-
from . import patterns
|
|
12
|
-
|
|
13
|
-
TILE_GROUPS = [("P1_invented_gates", ["P1_invented_gates"]), ("P6+P7 made it up / false done", ["P6_fake_output", "P7_false_done"]),
|
|
14
|
-
("P2_broke_things", ["P2_broke_things"]), ("P3_wrong_model", ["P3_wrong_model"])]
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def write(report_path, out_path, per_metric=20, seed=7):
|
|
18
|
-
with open(report_path, encoding="utf-8") as fh: d = json.load(fh)
|
|
19
|
-
bp = os.path.join(os.path.dirname(os.path.abspath(report_path)), "behavior.json")
|
|
20
|
-
with open(bp, encoding="utf-8") as fh: b = json.load(fh)
|
|
21
|
-
rnd = random.Random(seed); L = [f"# Behavior spot-check · {d['window'].get('start_pt')} → {d['window'].get('end_exclusive_pt')} · seed {seed}", "",
|
|
22
|
-
"Mark each line `true`, `false` or `unsure` in the first column. Excerpts are scrubbed and cut to 160 chars.", ""]
|
|
23
|
-
turns = b.get("user_turns", [])
|
|
24
|
-
for author in ("human", "bot"):
|
|
25
|
-
pool = [t for t in turns if t["author"] == author]; pick = rnd.sample(pool, min(per_metric, len(pool)))
|
|
26
|
-
L += [f"## User turns tagged {author} ({len(pick)} of {len(pool)})", "", "| verdict | session | time | source | excerpt |", "|---|---|---|---|---|"]
|
|
27
|
-
L += [f"| | `{t['session'][:12]}` | {t['ts_pt']} | {t['source']} | {t['excerpt'].replace('|', '/')} |" for t in pick]
|
|
28
|
-
L.append("")
|
|
29
|
-
flags = [f for f in b.get("flags", []) if not f.get("unconfirmed")]; eps = b.get("episodes", [])
|
|
30
|
-
for name, keys in TILE_GROUPS:
|
|
31
|
-
pool = [dict(kind="turn", **f) for f in flags if f["pattern"] in keys] + [dict(kind="episode", **e) for e in eps if e["pattern"] in keys]
|
|
32
|
-
pick = rnd.sample(pool, min(per_metric, len(pool)))
|
|
33
|
-
L += [f"## {name} ({len(pick)} of {len(pool)} flagged)", "", "| verdict | kind | session | time | basis | excerpt |", "|---|---|---|---|---|---|"]
|
|
34
|
-
for x in pick:
|
|
35
|
-
sid = x.get("session") or ""; ts = x.get("ts_pt") or ""
|
|
36
|
-
L.append(f"| | {x['kind']} | `{sid[:12]}` | {ts} | {(x.get('basis') or x.get('cost_status') or '').replace('|', '/')} | {(x.get('excerpt') or '').replace('|', '/')} |")
|
|
37
|
-
L.append("")
|
|
38
|
-
flagged_sessions = {f["session"] for f in flags} | {e["session"] for e in eps}
|
|
39
|
-
unflagged = [s for s in b.get("sessions", []) if s["session"] not in flagged_sessions]
|
|
40
|
-
pick = rnd.sample(unflagged, min(10, len(unflagged)))
|
|
41
|
-
L += [f"## Unflagged sessions ({len(pick)} of {len(unflagged)}) — look for obvious misses", "", "| missed? | session | tag | turns | tool errors | last message excerpt |", "|---|---|---|---|---|---|"]
|
|
42
|
-
L += [f"| | `{s['session'][:12]}` | {s['tag']} | {s['turns']} | {s['tool_errors']} | {(s.get('last_text') or '').replace('|', '/')} |" for s in pick]
|
|
43
|
-
L += ["", "## Precision (fill in)", "", "| pattern | true | false | unsure | precision |", "|---|---|---|---|---|"] + [f"| {name} | | | | |" for name, _ in TILE_GROUPS] + [""]
|
|
44
|
-
with open(out_path, "w", encoding="utf-8") as fh: fh.write("\n".join(L))
|
|
45
|
-
return out_path, dict(human=sum(1 for t in turns if t["author"] == "human"), bot=sum(1 for t in turns if t["author"] == "bot"), flags=len(flags), episodes=len(eps), unflagged=len(unflagged))
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
"""Read-only snapshot of the memindex SQLite database (plan Phase 1.3).
|
|
2
|
-
|
|
3
|
-
The live DB runs in WAL mode (src/services/sqlite/connection.ts:53-55), so a plain file copy
|
|
4
|
-
is wrong; the sqlite3 backup API is used instead. The live connection is opened with
|
|
5
|
-
`mode=ro` and closed right after the backup. Nothing here ever writes to the live DB.
|
|
6
|
-
"""
|
|
7
|
-
import os
|
|
8
|
-
import sqlite3
|
|
9
|
-
|
|
10
|
-
DB_FILE = "memindex.db"
|
|
11
|
-
SNAPSHOT_FILE = "snapshot.db"
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def data_dir(env=None):
|
|
15
|
-
"""Mirrors src/shared/paths.ts:20-66: $MEMINDEX_DATA_DIR (then $AGENT_MEM_DATA_DIR) if set, else ~/.memindex.
|
|
16
|
-
settings.json is never consulted."""
|
|
17
|
-
env = os.environ if env is None else env
|
|
18
|
-
d = env.get("MEMINDEX_DATA_DIR") or env.get("AGENT_MEM_DATA_DIR")
|
|
19
|
-
return os.path.expanduser(d) if d else os.path.expanduser(os.path.join("~", ".memindex"))
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def db_path(env=None):
|
|
23
|
-
return os.path.join(data_dir(env), DB_FILE)
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def snapshot(outdir, live=None):
|
|
27
|
-
"""Copy the live DB into <outdir>/snapshot.db via the backup API; return the snapshot path."""
|
|
28
|
-
live = live or db_path()
|
|
29
|
-
if not os.path.exists(live):
|
|
30
|
-
raise FileNotFoundError(f"memindex database not found at {live} "
|
|
31
|
-
f"(set MEMINDEX_DATA_DIR if it lives elsewhere)")
|
|
32
|
-
os.makedirs(outdir, exist_ok=True)
|
|
33
|
-
snap = os.path.join(outdir, SNAPSHOT_FILE)
|
|
34
|
-
for ext in ("", "-wal", "-shm", "-journal"):
|
|
35
|
-
if os.path.exists(snap + ext):
|
|
36
|
-
os.remove(snap + ext)
|
|
37
|
-
src = sqlite3.connect(f"file:{live}?mode=ro", uri=True)
|
|
38
|
-
try:
|
|
39
|
-
dst = sqlite3.connect(snap)
|
|
40
|
-
try:
|
|
41
|
-
src.backup(dst)
|
|
42
|
-
finally:
|
|
43
|
-
dst.close()
|
|
44
|
-
finally:
|
|
45
|
-
src.close()
|
|
46
|
-
return snap
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def open_snapshot(snap):
|
|
50
|
-
"""Open a snapshot for querying (row_factory=Row, as weekly_report.py:18)."""
|
|
51
|
-
db = sqlite3.connect(snap)
|
|
52
|
-
db.row_factory = sqlite3.Row
|
|
53
|
-
return db
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
"""Copy + checksum sync of the skill (SKILL.md, CHECKSUMS.txt and every file under scripts/) from the
|
|
2
|
-
plugin dir (source of truth) to the four mirror plugins (mirrors carry the skill as byte copies, tracked
|
|
3
|
-
in git). `sync-check` reports drift and exits 1; `--write` copies
|
|
4
|
-
plugin -> destinations and re-checks. Destinations live in one place (DESTINATIONS) and can be
|
|
5
|
-
overridden with --dest. Nothing outside that list is ever written.
|
|
6
|
-
"""
|
|
7
|
-
import hashlib
|
|
8
|
-
import os
|
|
9
|
-
import shutil
|
|
10
|
-
|
|
11
|
-
SKILL_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
12
|
-
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(SKILL_DIR)))
|
|
13
|
-
if os.path.basename(os.path.dirname(os.path.dirname(SKILL_DIR))) != "plugin":
|
|
14
|
-
REPO_ROOT = os.path.dirname(REPO_ROOT)
|
|
15
|
-
MIRRORS = ("integrations/cursor", "integrations/grok-bot", "integrations/cowork", "integrations/openclaw")
|
|
16
|
-
CHECKSUMS = "CHECKSUMS.txt"
|
|
17
|
-
SKIP_DIRS = {"__pycache__", ".chrome-profile"}
|
|
18
|
-
SKIP_FILES = {CHECKSUMS}
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def destinations(repo_root=REPO_ROOT):
|
|
22
|
-
return [os.path.join(repo_root, m, "skills", "agent-cost-report") for m in MIRRORS]
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def shipped_files(root=SKILL_DIR):
|
|
26
|
-
"""Relative paths of every shipped file: SKILL.md plus scripts/** (no caches, no checksum file)."""
|
|
27
|
-
out = []
|
|
28
|
-
for dirpath, dirs, files in os.walk(root):
|
|
29
|
-
dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS and (dirpath != root or d == "scripts"))
|
|
30
|
-
for f in sorted(files):
|
|
31
|
-
rel = os.path.relpath(os.path.join(dirpath, f), root)
|
|
32
|
-
if dirpath == root and f not in ("SKILL.md",): continue
|
|
33
|
-
if f in SKIP_FILES or f.endswith(".pyc"): continue
|
|
34
|
-
out.append(rel)
|
|
35
|
-
return out
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
def sha256(path):
|
|
39
|
-
h = hashlib.sha256()
|
|
40
|
-
with open(path, "rb") as fh:
|
|
41
|
-
for chunk in iter(lambda: fh.read(65536), b""): h.update(chunk)
|
|
42
|
-
return h.hexdigest()
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
def write_checksums(root=SKILL_DIR):
|
|
46
|
-
"""sha256sum-compatible CHECKSUMS.txt next to SKILL.md so drift shows in git diffs."""
|
|
47
|
-
lines = [f"{sha256(os.path.join(root, rel))} {rel}" for rel in shipped_files(root)]
|
|
48
|
-
with open(os.path.join(root, CHECKSUMS), "w", encoding="utf-8", newline="\n") as fh: fh.write("\n".join(lines) + "\n")
|
|
49
|
-
return lines
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def compare(src, dst, files):
|
|
53
|
-
"""Per-file status: same | differs | missing."""
|
|
54
|
-
rows = []
|
|
55
|
-
for rel in files + [CHECKSUMS]:
|
|
56
|
-
a, b = os.path.join(src, rel), os.path.join(dst, rel)
|
|
57
|
-
if not os.path.exists(b): rows.append((rel, "missing"))
|
|
58
|
-
elif sha256(a) != sha256(b): rows.append((rel, "differs"))
|
|
59
|
-
else: rows.append((rel, "same"))
|
|
60
|
-
return rows
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
def copy_tree(src, dst, files):
|
|
64
|
-
for rel in files + [CHECKSUMS]:
|
|
65
|
-
d = os.path.join(dst, rel); os.makedirs(os.path.dirname(d), exist_ok=True)
|
|
66
|
-
shutil.copyfile(os.path.join(src, rel), d)
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
def run(write=False, dests=None, src=SKILL_DIR, out=print):
|
|
70
|
-
files = shipped_files(src)
|
|
71
|
-
write_checksums(src)
|
|
72
|
-
dests = list(dests) if dests else destinations()
|
|
73
|
-
drift = 0
|
|
74
|
-
for d in dests:
|
|
75
|
-
if write: copy_tree(src, d, files)
|
|
76
|
-
rows = compare(src, d, files); bad = [r for r in rows if r[1] != "same"]
|
|
77
|
-
drift += len(bad)
|
|
78
|
-
out(f"{'ok ' if not bad else 'DRIFT'} {d}: {len(rows) - len(bad)}/{len(rows)} files identical" + (f"; {len(bad)} {'still ' if write else ''}differ: " + ", ".join(f"{r} ({s})" for r, s in bad[:6]) if bad else ""))
|
|
79
|
-
return drift
|