hstack 0.7.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +277 -0
- package/README.md +39 -13
- package/VERSION +1 -1
- package/dist/commands/doctor.js +51 -1
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/update.js +8 -2
- package/dist/commands/update.js.map +1 -1
- package/dist/lib/descriptions.js +167 -0
- package/dist/lib/descriptions.js.map +1 -0
- package/dist/lib/diff.js +1 -1
- package/dist/lib/git.js +16 -0
- package/dist/lib/git.js.map +1 -1
- package/dist/lib/wire.js +108 -4
- package/dist/lib/wire.js.map +1 -1
- package/dist/manifest.js +17 -2
- package/dist/manifest.js.map +1 -1
- package/package.json +3 -1
- package/template/.claude/agents/adversarial-reviewer.md +16 -64
- package/template/.claude/agents/app-architect.md +12 -49
- package/template/.claude/agents/data-architect.md +13 -51
- package/template/.claude/agents/data-specialist.md +5 -50
- package/template/.claude/agents/implementer.md +8 -65
- package/template/.claude/agents/kernel-fit-analyst.md +7 -68
- package/template/.claude/agents/planner.md +7 -42
- package/template/.claude/agents/product-discovery.md +12 -48
- package/template/.claude/agents/product-manager.md +8 -43
- package/template/.claude/agents/researcher.md +5 -41
- package/template/.claude/agents/security-reviewer.md +19 -54
- package/template/.claude/agents/spec-author.md +18 -52
- package/template/.claude/agents/stack-architect.md +14 -43
- package/template/.claude/agents/test-strategist.md +16 -57
- package/template/.claude/agents/ui-ux-briefer.md +6 -36
- package/template/.claude/agents/verifier.md +13 -45
- package/template/.claude/skills/hstack-adr-new/SKILL.md +6 -33
- package/template/.claude/skills/hstack-adversarial-review/SKILL.md +31 -52
- package/template/.claude/skills/hstack-adversarial-review/references/finding-categories.md +157 -0
- package/template/.claude/skills/hstack-app-architecture/SKILL.md +2 -29
- package/template/.claude/skills/hstack-branch/SKILL.md +4 -31
- package/template/.claude/skills/hstack-brownfield-init/SKILL.md +10 -37
- package/template/.claude/skills/hstack-change-new/SKILL.md +4 -31
- package/template/.claude/skills/hstack-change-plan/SKILL.md +21 -32
- package/template/.claude/skills/hstack-commit/SKILL.md +7 -35
- package/template/.claude/skills/hstack-configure/SKILL.md +7 -34
- package/template/.claude/skills/hstack-coord/SKILL.md +3 -39
- package/template/.claude/skills/hstack-data-architecture/SKILL.md +4 -30
- package/template/.claude/skills/hstack-data-review/SKILL.md +3 -42
- package/template/.claude/skills/hstack-finalize/SKILL.md +30 -49
- package/template/.claude/skills/hstack-flag/SKILL.md +9 -48
- package/template/.claude/skills/hstack-greenfield-init/SKILL.md +9 -36
- package/template/.claude/skills/hstack-help/SKILL.md +11 -37
- package/template/.claude/skills/hstack-implement/SKILL.md +28 -58
- package/template/.claude/skills/hstack-kernel-fit-promote/SKILL.md +7 -46
- package/template/.claude/skills/hstack-kernel-fit-scan/SKILL.md +5 -60
- package/template/.claude/skills/hstack-kernel-fit-scan/references/slack-setup.md +42 -0
- package/template/.claude/skills/hstack-kernel-fit-triage/SKILL.md +12 -50
- package/template/.claude/skills/hstack-module-spec/SKILL.md +5 -32
- package/template/.claude/skills/hstack-product-discovery/SKILL.md +5 -31
- package/template/.claude/skills/hstack-research/SKILL.md +3 -33
- package/template/.claude/skills/hstack-scaffold/SKILL.md +2 -29
- package/template/.claude/skills/hstack-security-review/SKILL.md +5 -43
- package/template/.claude/skills/hstack-ship/SKILL.md +43 -53
- package/template/.claude/skills/hstack-stack-decide/SKILL.md +3 -30
- package/template/.claude/skills/hstack-story-draft/SKILL.md +6 -33
- package/template/.claude/skills/hstack-tech-debt-new/SKILL.md +4 -31
- package/template/.claude/skills/hstack-tech-debt-resolve/SKILL.md +9 -44
- package/template/.claude/skills/hstack-tech-debt-stale/SKILL.md +10 -37
- package/template/.claude/skills/hstack-tech-debt-wontfix/SKILL.md +8 -35
- package/template/.claude/skills/hstack-telemetry/SKILL.md +5 -30
- package/template/.claude/skills/hstack-test-plan/SKILL.md +23 -46
- package/template/.claude/skills/hstack-ui-brief/SKILL.md +3 -30
- package/template/.claude/skills/hstack-verify/SKILL.md +26 -48
- package/template/KERNEL.md +410 -0
- package/template/scripts/compute-merge-readiness.mjs +780 -0
- package/template/scripts/coord/coord_scan.py +121 -22
- package/template/scripts/run-gates.sh +388 -0
- package/template/scripts/telemetry/insights/kernel_fit.py +1 -1
- package/template/scripts/telemetry/insights/token_economics.py +181 -8
- package/template/scripts/telemetry/parsers/sidecars.py +61 -0
- package/template/scripts/telemetry/parsers/transcripts.py +135 -22
- package/template/scripts/telemetry/render.py +68 -3
- package/template/scripts/telemetry/report.py +16 -4
- package/template/scripts/telemetry/run_kernel_fit.py +6 -2
- package/template/scripts/telemetry/session_id.py +139 -0
- package/template/scripts/validate-spec.mjs +3303 -0
- package/template/templates/adr.md +7 -0
- package/template/templates/adversarial-review.md +5 -5
- package/template/templates/ci-cd.md +14 -0
- package/template/templates/coord-message.md +3 -2
- package/template/templates/data-architecture.md +3 -6
- package/template/templates/kernel-fit-finding.md +2 -2
- package/template/templates/kernel-fit-flag.md +2 -2
- package/template/templates/plan.md +4 -0
- package/template/templates/product-brief.md +2 -2
- package/template/templates/roadmap.md +41 -0
- package/template/templates/security-review.md +1 -1
- package/template/templates/telemetry-sidecar.md +56 -13
- package/template/templates/test-plan.md +1 -1
- package/template/CLAUDE.md +0 -443
- package/template/templates/mvp-scope.md +0 -34
|
@@ -1,24 +1,49 @@
|
|
|
1
|
-
"""Token-economics insights: TE-1 cost per
|
|
2
|
-
subagent, TE-3 subagent entry-tax amortization
|
|
1
|
+
"""Token-economics insights: TE-1 cost per Skill, TE-2 cache-hit ratio per
|
|
2
|
+
subagent, TE-3 subagent entry-tax amortization, TE-4 cost per phase, TE-5 cost
|
|
3
|
+
per change (ADR-0009).
|
|
4
|
+
|
|
5
|
+
TE-1/TE-2/TE-3 are *session-scoped*: they attribute a whole session to the first
|
|
6
|
+
Skill it invoked, because a Skill has a start marker and no end marker. TE-4/TE-5
|
|
7
|
+
are *phase-scoped*: they read the sidecar's `[phase_opened_at, phase_closed_at]`
|
|
8
|
+
window and sum only the turns inside it. Where a sidecar exists, TE-4/TE-5
|
|
9
|
+
supersede TE-1.
|
|
10
|
+
"""
|
|
3
11
|
|
|
4
12
|
from __future__ import annotations
|
|
5
13
|
|
|
6
14
|
from collections import defaultdict
|
|
7
15
|
|
|
16
|
+
from telemetry.parsers.transcripts import phase_usage
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
#: The five Skills that emit sidecars (ADR-0001 § v1 emission list). Every other
|
|
20
|
+
#: Skill is invisible to TE-4/TE-5 — which is what the coverage fraction says.
|
|
21
|
+
EMITTING_SKILLS = (
|
|
22
|
+
"hstack-test-plan", "hstack-implement", "hstack-verify",
|
|
23
|
+
"hstack-adversarial-review", "hstack-finalize",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
UNATTRIBUTED = "(unattributed)"
|
|
27
|
+
|
|
8
28
|
|
|
9
|
-
def compute(session_rows: list[dict], changes: dict) -> dict:
|
|
10
|
-
"""Compute the
|
|
29
|
+
def compute(session_rows: list[dict], changes: dict, sidecars: list[dict] | None = None) -> dict:
|
|
30
|
+
"""Compute the five TE metrics.
|
|
11
31
|
|
|
12
32
|
Args:
|
|
13
33
|
session_rows: from transcripts.collect_session_rows
|
|
14
34
|
changes: from frontmatter.load_change_artifacts
|
|
35
|
+
sidecars: from sidecars.load_sidecars (empty/None → TE-4/TE-5 report no
|
|
36
|
+
coverage rather than silently vanishing)
|
|
15
37
|
|
|
16
38
|
Returns a dict ready for rendering.
|
|
17
39
|
"""
|
|
40
|
+
phases = _te_4(sidecars or [])
|
|
18
41
|
return {
|
|
19
42
|
"te_1_cost_per_change": _te_1(session_rows, changes),
|
|
20
43
|
"te_2_cache_hit_per_subagent": _te_2(session_rows),
|
|
21
44
|
"te_3_subagent_entry_tax": _te_3(session_rows),
|
|
45
|
+
"te_4_cost_per_phase": phases,
|
|
46
|
+
"te_5_cost_per_change": _te_5(phases["rows"]),
|
|
22
47
|
}
|
|
23
48
|
|
|
24
49
|
|
|
@@ -44,7 +69,10 @@ def _te_1(session_rows: list[dict], changes: dict) -> dict:
|
|
|
44
69
|
}
|
|
45
70
|
cost_total = defaultdict(int)
|
|
46
71
|
session_counts = defaultdict(int)
|
|
72
|
+
unattributed = 0
|
|
47
73
|
for s in session_rows:
|
|
74
|
+
if s["skill"] is None:
|
|
75
|
+
unattributed += 1
|
|
48
76
|
if s["skill"] in per_change_skills:
|
|
49
77
|
# Heuristic: we don't have a structured change-id-per-session yet,
|
|
50
78
|
# so accumulate by skill until sidecars exist. The (skill, total)
|
|
@@ -62,9 +90,16 @@ def _te_1(session_rows: list[dict], changes: dict) -> dict:
|
|
|
62
90
|
})
|
|
63
91
|
return {
|
|
64
92
|
"rows": rows,
|
|
93
|
+
"unattributed_sessions": unattributed,
|
|
65
94
|
"note": (
|
|
66
|
-
"
|
|
67
|
-
"
|
|
95
|
+
"Session-scoped, not phase-scoped: a Skill has a start marker and no "
|
|
96
|
+
"end marker, so everything a session spends after the invocation "
|
|
97
|
+
"lands in the first bucket — including later phases and unrelated "
|
|
98
|
+
"work. Superseded by TE-4/TE-5 for any change that carries sidecars. "
|
|
99
|
+
"Attribution reads structured invocation markers only (<command-name> "
|
|
100
|
+
"tags, Skill tool_use blocks); sessions with no marker are "
|
|
101
|
+
f"unattributed ({unattributed} of {len(session_rows)} in window) "
|
|
102
|
+
"rather than credited to whichever Skill their prompt mentioned."
|
|
68
103
|
),
|
|
69
104
|
}
|
|
70
105
|
|
|
@@ -76,7 +111,7 @@ def _te_2(session_rows: list[dict]) -> dict:
|
|
|
76
111
|
"""
|
|
77
112
|
per_skill = defaultdict(lambda: {"cache_read": 0, "cache_creation": 0, "turns": 0})
|
|
78
113
|
for s in session_rows:
|
|
79
|
-
key = s["skill"] or
|
|
114
|
+
key = s["skill"] or UNATTRIBUTED
|
|
80
115
|
t = s["totals"]
|
|
81
116
|
per_skill[key]["cache_read"] += t.get("cache_read_input_tokens", 0)
|
|
82
117
|
per_skill[key]["cache_creation"] += t.get("cache_creation_input_tokens", 0)
|
|
@@ -92,7 +127,16 @@ def _te_2(session_rows: list[dict]) -> dict:
|
|
|
92
127
|
"cache_creation": agg["cache_creation"],
|
|
93
128
|
"ratio": ratio,
|
|
94
129
|
})
|
|
95
|
-
return {
|
|
130
|
+
return {
|
|
131
|
+
"rows": rows,
|
|
132
|
+
"note": (
|
|
133
|
+
"Session-scoped, same caveat as TE-1 — the whole session's cache "
|
|
134
|
+
f"behaviour is credited to its first Skill. `{UNATTRIBUTED}` holds "
|
|
135
|
+
"every session with no structured hstack invocation marker, "
|
|
136
|
+
"including plain non-hstack work. Superseded by TE-4/TE-5 wherever "
|
|
137
|
+
"sidecars exist."
|
|
138
|
+
),
|
|
139
|
+
}
|
|
96
140
|
|
|
97
141
|
|
|
98
142
|
def _te_3(session_rows: list[dict]) -> dict:
|
|
@@ -127,3 +171,132 @@ def _te_3(session_rows: list[dict]) -> dict:
|
|
|
127
171
|
"timestamps will sharpen this."
|
|
128
172
|
),
|
|
129
173
|
}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _te_4(sidecars: list[dict]) -> dict:
|
|
177
|
+
"""TE-4: cost per phase — the sidecar's window, summed from the transcript.
|
|
178
|
+
|
|
179
|
+
One row per sidecar. A row is *measured* when the sidecar carries a phase
|
|
180
|
+
window (schema_version ≥ 2) whose session transcript is still on disk;
|
|
181
|
+
otherwise `tokens` is `None` and the row is unmeasured. Never zero: a phase
|
|
182
|
+
whose transcript was swept spent tokens we can no longer count, and printing
|
|
183
|
+
0 would fold it into the average as if it were free.
|
|
184
|
+
"""
|
|
185
|
+
rows = []
|
|
186
|
+
for sc in sidecars:
|
|
187
|
+
usage = phase_usage(sc.get("data") or {})
|
|
188
|
+
data = sc.get("data") or {}
|
|
189
|
+
rows.append({
|
|
190
|
+
"skill": sc.get("skill"),
|
|
191
|
+
"change": sc.get("change_id"),
|
|
192
|
+
"phase_id": sc.get("phase_id"),
|
|
193
|
+
"sidecar": sc.get("file"),
|
|
194
|
+
"schema_version": sc.get("schema_version"),
|
|
195
|
+
"session_id": data.get("session_id"),
|
|
196
|
+
"opened_at": data.get("phase_opened_at"),
|
|
197
|
+
"closed_at": data.get("phase_closed_at"),
|
|
198
|
+
"measured": usage is not None,
|
|
199
|
+
"unmeasured_reason": None if usage is not None else _unmeasured_reason(sc),
|
|
200
|
+
"tokens": usage["total_tokens"] if usage else None,
|
|
201
|
+
"cost_score": usage["cost_score"] if usage else None,
|
|
202
|
+
"turns": usage["turns"] if usage else None,
|
|
203
|
+
"wall_clock_h": round(usage["wall_clock_s"] / 3600, 2) if usage else None,
|
|
204
|
+
})
|
|
205
|
+
rows.sort(key=lambda r: (-(r["tokens"] or 0), r["change"] or "", r["sidecar"] or ""))
|
|
206
|
+
measured = [r for r in rows if r["measured"]]
|
|
207
|
+
return {
|
|
208
|
+
"rows": rows,
|
|
209
|
+
"phases_emitted": len(rows),
|
|
210
|
+
"phases_measured": len(measured),
|
|
211
|
+
"coverage_fraction": (len(measured) / len(rows)) if rows else None,
|
|
212
|
+
"note": (
|
|
213
|
+
"Phase-scoped: tokens are summed over assistant turns whose "
|
|
214
|
+
"timestamp falls inside the sidecar's [phase_opened_at, "
|
|
215
|
+
"phase_closed_at] window. Only the five sidecar-emitting Skills "
|
|
216
|
+
f"({', '.join(EMITTING_SKILLS)}) appear here at all — every other "
|
|
217
|
+
"Skill is invisible, and subagent spend lands in its host's window "
|
|
218
|
+
"(isSidechain=False). Unmeasured rows are phases whose window or "
|
|
219
|
+
"transcript could not be read; they are never counted as zero. "
|
|
220
|
+
"Sidecars are not window-filtered — every change folder on disk is "
|
|
221
|
+
"read, unlike the session and git tables above. Read "
|
|
222
|
+
"this table next to QO-4 (observed vs promised) and WS-2 (gate "
|
|
223
|
+
"findings density): cost without an outcome beside it can only "
|
|
224
|
+
"argue for spending less, never for spending well."
|
|
225
|
+
),
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _unmeasured_reason(sidecar: dict) -> str:
|
|
230
|
+
data = sidecar.get("data") or {}
|
|
231
|
+
if not data.get("phase_opened_at") or not data.get("phase_closed_at"):
|
|
232
|
+
version = sidecar.get("schema_version")
|
|
233
|
+
return ("pre-ADR-0009 sidecar (schema_version "
|
|
234
|
+
f"{version if version is not None else '?'}) — no phase window")
|
|
235
|
+
if not data.get("session_id"):
|
|
236
|
+
return "session id unresolved at write time"
|
|
237
|
+
return "transcript not found (retention sweep, or written on another machine)"
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _te_5(phase_rows: list[dict]) -> dict:
|
|
241
|
+
"""TE-5: cost per change — the sum of that change's measured phases.
|
|
242
|
+
|
|
243
|
+
The coverage fraction is not decoration. Five of the 27 Skills emit
|
|
244
|
+
sidecars, so `tokens` is a sum over a subset by construction: the spec, the
|
|
245
|
+
plan, the security- and data-reviews, the ship gate and the whole configure
|
|
246
|
+
family are absent, and so is any phase whose transcript has aged out. A
|
|
247
|
+
reader who takes this column for a change's total cost will read it low.
|
|
248
|
+
"""
|
|
249
|
+
per_change: dict[str, dict] = {}
|
|
250
|
+
for r in phase_rows:
|
|
251
|
+
change = r.get("change") or "(unknown)"
|
|
252
|
+
agg = per_change.setdefault(change, {
|
|
253
|
+
"change": change,
|
|
254
|
+
"phases_emitted": 0,
|
|
255
|
+
"phases_measured": 0,
|
|
256
|
+
"skills": set(),
|
|
257
|
+
"tokens": 0,
|
|
258
|
+
"cost_score": 0,
|
|
259
|
+
"turns": 0,
|
|
260
|
+
"wall_clock_h": 0.0,
|
|
261
|
+
})
|
|
262
|
+
agg["phases_emitted"] += 1
|
|
263
|
+
if r.get("skill"):
|
|
264
|
+
agg["skills"].add(r["skill"])
|
|
265
|
+
if not r["measured"]:
|
|
266
|
+
continue
|
|
267
|
+
agg["phases_measured"] += 1
|
|
268
|
+
agg["tokens"] += r["tokens"] or 0
|
|
269
|
+
agg["cost_score"] += r["cost_score"] or 0
|
|
270
|
+
agg["turns"] += r["turns"] or 0
|
|
271
|
+
agg["wall_clock_h"] += r["wall_clock_h"] or 0.0
|
|
272
|
+
rows = []
|
|
273
|
+
for agg in per_change.values():
|
|
274
|
+
measured = agg["phases_measured"]
|
|
275
|
+
rows.append({
|
|
276
|
+
"change": agg["change"],
|
|
277
|
+
"phases_measured": measured,
|
|
278
|
+
"phases_emitted": agg["phases_emitted"],
|
|
279
|
+
"coverage_fraction": (measured / agg["phases_emitted"]) if agg["phases_emitted"] else None,
|
|
280
|
+
"skills_measured": sorted(agg["skills"]),
|
|
281
|
+
"tokens": agg["tokens"] if measured else None,
|
|
282
|
+
"cost_score": agg["cost_score"] if measured else None,
|
|
283
|
+
"turns": agg["turns"] if measured else None,
|
|
284
|
+
"wall_clock_h": round(agg["wall_clock_h"], 2) if measured else None,
|
|
285
|
+
})
|
|
286
|
+
rows.sort(key=lambda r: (-(r["tokens"] or 0), r["change"]))
|
|
287
|
+
total_emitted = sum(r["phases_emitted"] for r in rows)
|
|
288
|
+
total_measured = sum(r["phases_measured"] for r in rows)
|
|
289
|
+
return {
|
|
290
|
+
"rows": rows,
|
|
291
|
+
"phases_emitted": total_emitted,
|
|
292
|
+
"phases_measured": total_measured,
|
|
293
|
+
"coverage_fraction": (total_measured / total_emitted) if total_emitted else None,
|
|
294
|
+
"note": (
|
|
295
|
+
"A subset, not a total. Coverage fraction = measured phases / "
|
|
296
|
+
"emitted sidecars, and sidecars are emitted by five Skills only — "
|
|
297
|
+
"change-new, change-plan, security-review, data-review, ship and the "
|
|
298
|
+
"configure family contribute nothing to these sums. Pair with QO-4 "
|
|
299
|
+
"and the adversarial-review findings density before concluding that "
|
|
300
|
+
"an expensive change was a wasteful one."
|
|
301
|
+
),
|
|
302
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Load the per-change telemetry sidecars written by the five emitting Skills.
|
|
2
|
+
|
|
3
|
+
Sidecars live at `hstack/specs/changes/<change-id>/.telemetry/*.json` and are
|
|
4
|
+
derivative, gitignored, and never authoritative (ADR-0001). This parser only
|
|
5
|
+
reads them; a malformed or absent sidecar is skipped, never repaired.
|
|
6
|
+
|
|
7
|
+
Schema versions (see `templates/telemetry-sidecar.md`):
|
|
8
|
+
1 — no phase window. Reads as *unmeasured*: `phase_usage` returns None.
|
|
9
|
+
2 — carries `session_id`, `phase_opened_at`, `phase_closed_at` (ADR-0009).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def load_sidecars(hstack_root: Path) -> list[dict]:
|
|
19
|
+
"""Every sidecar under every change folder, sorted by change then filename.
|
|
20
|
+
|
|
21
|
+
Each entry: {path, change_id, file, skill, phase_id, schema_version, data}.
|
|
22
|
+
`skill` and `phase_id` fall back to the filename when the payload omits them
|
|
23
|
+
(`implement-<phase-id>.json`), so a hand-truncated sidecar still groups.
|
|
24
|
+
"""
|
|
25
|
+
out: list[dict] = []
|
|
26
|
+
changes_dir = hstack_root / "specs" / "changes"
|
|
27
|
+
if not changes_dir.is_dir():
|
|
28
|
+
return out
|
|
29
|
+
for change_dir in sorted(changes_dir.iterdir()):
|
|
30
|
+
if not change_dir.is_dir():
|
|
31
|
+
continue
|
|
32
|
+
telemetry_dir = change_dir / ".telemetry"
|
|
33
|
+
if not telemetry_dir.is_dir():
|
|
34
|
+
continue
|
|
35
|
+
for f in sorted(telemetry_dir.glob("*.json")):
|
|
36
|
+
data = _read_json(f)
|
|
37
|
+
if data is None:
|
|
38
|
+
continue
|
|
39
|
+
stem = f.stem
|
|
40
|
+
skill = data.get("skill") or f"hstack-{stem.split('-')[0]}"
|
|
41
|
+
phase_id = data.get("phase_id")
|
|
42
|
+
if phase_id is None and stem.startswith("implement-"):
|
|
43
|
+
phase_id = stem[len("implement-"):]
|
|
44
|
+
out.append({
|
|
45
|
+
"path": f,
|
|
46
|
+
"file": f.name,
|
|
47
|
+
"change_id": data.get("change_id") or change_dir.name,
|
|
48
|
+
"skill": skill,
|
|
49
|
+
"phase_id": phase_id,
|
|
50
|
+
"schema_version": data.get("schema_version"),
|
|
51
|
+
"data": data,
|
|
52
|
+
})
|
|
53
|
+
return out
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _read_json(path: Path) -> dict | None:
|
|
57
|
+
try:
|
|
58
|
+
loaded = json.loads(path.read_text(encoding="utf-8", errors="replace"))
|
|
59
|
+
except (OSError, json.JSONDecodeError):
|
|
60
|
+
return None
|
|
61
|
+
return loaded if isinstance(loaded, dict) else None
|
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
"""Walk Claude Code transcript JSONL files for token economics and halt signals.
|
|
2
2
|
|
|
3
|
-
Generalized from scripts/audit-token-usage.py. Same cost-score weighting
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
Generalized from scripts/audit-token-usage.py. Same cost-score weighting. Adds:
|
|
4
|
+
halt-sentinel detection, per-session start/end timestamps, cwd grouping for
|
|
5
|
+
Conductor worktree dedup, and (ADR-0009) per-phase usage summation bounded by a
|
|
6
|
+
sidecar's phase window.
|
|
7
|
+
|
|
8
|
+
Session→Skill attribution reads *structured invocation markers only* — the
|
|
9
|
+
`<command-name>` tag the harness writes for a slash command, and `Skill`
|
|
10
|
+
tool_use blocks. Free text is never matched: a prompt that merely mentions
|
|
11
|
+
`/hstack:coord` used to capture the whole session, which credited `coord` with
|
|
12
|
+
roughly half of all measured cache-read tokens (ADR-0009 § Context, defect 2).
|
|
6
13
|
"""
|
|
7
14
|
|
|
8
15
|
from __future__ import annotations
|
|
@@ -10,18 +17,27 @@ from __future__ import annotations
|
|
|
10
17
|
import json
|
|
11
18
|
import re
|
|
12
19
|
from collections import defaultdict
|
|
13
|
-
from datetime import datetime
|
|
20
|
+
from datetime import datetime, timezone
|
|
14
21
|
from pathlib import Path
|
|
15
22
|
from typing import Iterable
|
|
16
23
|
|
|
24
|
+
from telemetry.session_id import transcript_for_session
|
|
25
|
+
|
|
17
26
|
|
|
18
27
|
PROJECTS_ROOT = Path.home() / ".claude" / "projects"
|
|
19
28
|
|
|
20
|
-
|
|
21
|
-
|
|
29
|
+
# Structured markers only. `<command-name>` is emitted by the harness around a
|
|
30
|
+
# slash-command invocation; both the `/hstack:foo` and `/hstack-foo` spellings
|
|
31
|
+
# appear in real transcripts.
|
|
32
|
+
HSTACK_CMD_TAG = re.compile(r"<command-name>/?hstack[:\-]([a-z][a-z0-9\-]*)</command-name>")
|
|
33
|
+
# `Skill` tool_use input, optionally namespaced by a plugin prefix.
|
|
34
|
+
HSTACK_SKILL_NAME = re.compile(r"^(?:[A-Za-z0-9_.\-]+:)?hstack[:\-]([a-z][a-z0-9\-]*)$")
|
|
22
35
|
SUBAGENT_TYPE = re.compile(r'"subagent_type"\s*:\s*"([a-z][a-z0-9\-]*)"')
|
|
23
36
|
HALT_SENTINEL = re.compile(r"HSTACK-HALT:\s*reason=([a-z-]+)", re.IGNORECASE)
|
|
24
37
|
|
|
38
|
+
USAGE_KEYS = ("input_tokens", "cache_creation_input_tokens",
|
|
39
|
+
"cache_read_input_tokens", "output_tokens")
|
|
40
|
+
|
|
25
41
|
|
|
26
42
|
def message_text(msg) -> str:
|
|
27
43
|
if msg is None:
|
|
@@ -91,20 +107,58 @@ def load_session(path: Path) -> list[dict]:
|
|
|
91
107
|
return out
|
|
92
108
|
|
|
93
109
|
|
|
110
|
+
def skill_tool_uses(msg) -> list[str]:
|
|
111
|
+
"""hstack Skill names invoked via a `Skill` tool_use block in this message.
|
|
112
|
+
|
|
113
|
+
Structural walk, not a regex over flattened text: the marker is the block's
|
|
114
|
+
`type`/`name`/`input.skill` triple, which prose cannot forge.
|
|
115
|
+
"""
|
|
116
|
+
if not isinstance(msg, dict):
|
|
117
|
+
return []
|
|
118
|
+
content = msg.get("content")
|
|
119
|
+
if not isinstance(content, list):
|
|
120
|
+
return []
|
|
121
|
+
out = []
|
|
122
|
+
for blk in content:
|
|
123
|
+
if not isinstance(blk, dict):
|
|
124
|
+
continue
|
|
125
|
+
if blk.get("type") != "tool_use" or blk.get("name") != "Skill":
|
|
126
|
+
continue
|
|
127
|
+
inp = blk.get("input")
|
|
128
|
+
name = inp.get("skill") if isinstance(inp, dict) else None
|
|
129
|
+
if not isinstance(name, str):
|
|
130
|
+
continue
|
|
131
|
+
m = HSTACK_SKILL_NAME.match(name.strip())
|
|
132
|
+
if m:
|
|
133
|
+
out.append(m.group(1))
|
|
134
|
+
return out
|
|
135
|
+
|
|
136
|
+
|
|
94
137
|
def classify_session(records: list[dict]) -> tuple[str | None, set[str], list[str]]:
|
|
95
|
-
"""Return (skill_name, set of subagent types invoked, halt reasons).
|
|
138
|
+
"""Return (skill_name, set of subagent types invoked, halt reasons).
|
|
139
|
+
|
|
140
|
+
`skill_name` is the first hstack Skill invoked through a structured marker —
|
|
141
|
+
a `<command-name>` tag or a `Skill` tool_use block. A session carrying
|
|
142
|
+
neither returns `None` and is reported as *unattributed* (ADR-0009 piece 3);
|
|
143
|
+
it is never captured by whatever hstack command its prompt happened to name.
|
|
144
|
+
"""
|
|
96
145
|
skill = None
|
|
97
146
|
subagents: set[str] = set()
|
|
98
147
|
halts: list[str] = []
|
|
99
148
|
for r in records:
|
|
100
149
|
t = r.get("type")
|
|
101
|
-
if t == "user"
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
150
|
+
if t == "user":
|
|
151
|
+
if skill is None:
|
|
152
|
+
m = HSTACK_CMD_TAG.search(message_text(r.get("message")))
|
|
153
|
+
if m:
|
|
154
|
+
skill = m.group(1)
|
|
106
155
|
elif t == "assistant":
|
|
107
|
-
|
|
156
|
+
msg = r.get("message")
|
|
157
|
+
if skill is None:
|
|
158
|
+
invoked = skill_tool_uses(msg)
|
|
159
|
+
if invoked:
|
|
160
|
+
skill = invoked[0]
|
|
161
|
+
text = message_text(msg)
|
|
108
162
|
for sa in SUBAGENT_TYPE.findall(text):
|
|
109
163
|
subagents.add(sa)
|
|
110
164
|
for h in HALT_SENTINEL.findall(text):
|
|
@@ -112,15 +166,24 @@ def classify_session(records: list[dict]) -> tuple[str | None, set[str], list[st
|
|
|
112
166
|
return skill, subagents, halts
|
|
113
167
|
|
|
114
168
|
|
|
169
|
+
def parse_ts(value) -> datetime | None:
|
|
170
|
+
"""Parse an ISO-8601 stamp to an aware UTC datetime. None on anything else."""
|
|
171
|
+
if not isinstance(value, str) or not value:
|
|
172
|
+
return None
|
|
173
|
+
try:
|
|
174
|
+
dt = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
|
175
|
+
except ValueError:
|
|
176
|
+
return None
|
|
177
|
+
if dt.tzinfo is None:
|
|
178
|
+
dt = dt.replace(tzinfo=timezone.utc)
|
|
179
|
+
return dt.astimezone(timezone.utc)
|
|
180
|
+
|
|
181
|
+
|
|
115
182
|
def session_bounds(records: list[dict]) -> tuple[datetime | None, datetime | None]:
|
|
116
183
|
first = last = None
|
|
117
184
|
for r in records:
|
|
118
|
-
|
|
119
|
-
if
|
|
120
|
-
continue
|
|
121
|
-
try:
|
|
122
|
-
dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
|
123
|
-
except (ValueError, AttributeError):
|
|
185
|
+
dt = parse_ts(r.get("timestamp"))
|
|
186
|
+
if dt is None:
|
|
124
187
|
continue
|
|
125
188
|
if first is None or dt < first:
|
|
126
189
|
first = dt
|
|
@@ -129,19 +192,69 @@ def session_bounds(records: list[dict]) -> tuple[datetime | None, datetime | Non
|
|
|
129
192
|
return first, last
|
|
130
193
|
|
|
131
194
|
|
|
132
|
-
def tally_usage(records: list[dict]
|
|
195
|
+
def tally_usage(records: list[dict], start: datetime | None = None,
|
|
196
|
+
end: datetime | None = None) -> dict[str, int]:
|
|
197
|
+
"""Sum assistant-turn usage. With `start`/`end`, only turns whose timestamp
|
|
198
|
+
falls inside the closed interval are counted."""
|
|
133
199
|
totals: dict[str, int] = defaultdict(int)
|
|
200
|
+
for k in USAGE_KEYS:
|
|
201
|
+
totals[k] = 0
|
|
202
|
+
totals["turns"] = 0
|
|
134
203
|
for r in records:
|
|
135
204
|
if r.get("type") != "assistant":
|
|
136
205
|
continue
|
|
206
|
+
if start is not None or end is not None:
|
|
207
|
+
ts = parse_ts(r.get("timestamp"))
|
|
208
|
+
if ts is None:
|
|
209
|
+
continue
|
|
210
|
+
if start is not None and ts < start:
|
|
211
|
+
continue
|
|
212
|
+
if end is not None and ts > end:
|
|
213
|
+
continue
|
|
137
214
|
usage = (r.get("message") or {}).get("usage") or {}
|
|
138
|
-
for k in
|
|
139
|
-
"cache_read_input_tokens", "output_tokens"):
|
|
215
|
+
for k in USAGE_KEYS:
|
|
140
216
|
totals[k] += usage.get(k, 0) or 0
|
|
141
217
|
totals["turns"] += 1
|
|
142
218
|
return dict(totals)
|
|
143
219
|
|
|
144
220
|
|
|
221
|
+
def phase_usage(sidecar: dict, projects_root: Path | None = None) -> dict | None:
|
|
222
|
+
"""Token usage inside one phase window, per ADR-0009 piece 2.
|
|
223
|
+
|
|
224
|
+
Reads the transcript named by `sidecar["session_id"]` and sums assistant-turn
|
|
225
|
+
usage over records whose `timestamp` falls in
|
|
226
|
+
`[phase_opened_at, phase_closed_at]`.
|
|
227
|
+
|
|
228
|
+
Returns `None` — *unmeasured* — whenever the window cannot be honoured: a
|
|
229
|
+
schema_version-1 sidecar with no window, a null session id, a transcript
|
|
230
|
+
swept by `cleanupPeriodDays` or living on another machine, an unparseable or
|
|
231
|
+
inverted window. Never zero: zero is a measurement, and a phase whose
|
|
232
|
+
transcript is gone was not measured. Read-only; nothing is written.
|
|
233
|
+
"""
|
|
234
|
+
if not isinstance(sidecar, dict):
|
|
235
|
+
return None
|
|
236
|
+
opened = parse_ts(sidecar.get("phase_opened_at"))
|
|
237
|
+
closed = parse_ts(sidecar.get("phase_closed_at"))
|
|
238
|
+
if opened is None or closed is None or closed < opened:
|
|
239
|
+
return None
|
|
240
|
+
path = transcript_for_session(sidecar.get("session_id"), projects_root=projects_root)
|
|
241
|
+
if path is None:
|
|
242
|
+
return None
|
|
243
|
+
records = load_session(path)
|
|
244
|
+
if not records:
|
|
245
|
+
return None
|
|
246
|
+
totals = tally_usage(records, start=opened, end=closed)
|
|
247
|
+
total_tokens = sum(totals.get(k, 0) for k in USAGE_KEYS)
|
|
248
|
+
return {
|
|
249
|
+
**totals,
|
|
250
|
+
"total_tokens": total_tokens,
|
|
251
|
+
"cost_score": cost_score(totals),
|
|
252
|
+
"wall_clock_s": (closed - opened).total_seconds(),
|
|
253
|
+
"session_id": sidecar.get("session_id"),
|
|
254
|
+
"transcript": str(path),
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
|
|
145
258
|
def cost_score(totals: dict[str, int]) -> int:
|
|
146
259
|
"""Same weighting as scripts/audit-token-usage.py: input + 1.25*cache_creation
|
|
147
260
|
+ 0.10*cache_read + 5*output. Approximates billable $-weight for ranking."""
|
|
@@ -61,7 +61,7 @@ def _render_token_economics(lines: list[str], te: dict) -> None:
|
|
|
61
61
|
_h(lines, 2, "Token economics")
|
|
62
62
|
|
|
63
63
|
te1 = te.get("te_1_cost_per_change", {})
|
|
64
|
-
_h(lines, 3, "TE-1 — cost-score per Skill (
|
|
64
|
+
_h(lines, 3, "TE-1 — cost-score per Skill (session-scoped)")
|
|
65
65
|
_p(lines, te1.get("note", ""))
|
|
66
66
|
_table(
|
|
67
67
|
lines,
|
|
@@ -71,9 +71,10 @@ def _render_token_economics(lines: list[str], te: dict) -> None:
|
|
|
71
71
|
)
|
|
72
72
|
|
|
73
73
|
te2 = te.get("te_2_cache_hit_per_subagent", {})
|
|
74
|
-
_h(lines, 3, "TE-2 — cache-hit ratio (per Skill,
|
|
74
|
+
_h(lines, 3, "TE-2 — cache-hit ratio (per Skill, session-scoped)")
|
|
75
75
|
_p(lines, "ratio = cache_read / (cache_read + cache_creation). Below 0.5 → context "
|
|
76
76
|
"is being rebuilt rather than reused. Above 0.8 → cross-session reuse is healthy.")
|
|
77
|
+
_p(lines, te2.get("note", ""))
|
|
77
78
|
_table(
|
|
78
79
|
lines,
|
|
79
80
|
["skill", "turns", "cache_read", "cache_creation", "ratio"],
|
|
@@ -93,6 +94,61 @@ def _render_token_economics(lines: list[str], te: dict) -> None:
|
|
|
93
94
|
for r in te3.get("rows", [])],
|
|
94
95
|
)
|
|
95
96
|
|
|
97
|
+
te4 = te.get("te_4_cost_per_phase", {})
|
|
98
|
+
_h(lines, 3, "TE-4 — cost per phase (sidecar phase window)")
|
|
99
|
+
_p(lines, _coverage_line(te4, "phase"))
|
|
100
|
+
_p(lines, te4.get("note", ""))
|
|
101
|
+
rows4 = te4.get("rows", [])
|
|
102
|
+
measured4 = [r for r in rows4 if r["measured"]]
|
|
103
|
+
_table(
|
|
104
|
+
lines,
|
|
105
|
+
["skill", "change", "phase", "tokens", "turns", "wall-clock (h)"],
|
|
106
|
+
[[r["skill"], r["change"], r["phase_id"] or "-",
|
|
107
|
+
f"{r['tokens']:,}", r["turns"], r["wall_clock_h"]]
|
|
108
|
+
for r in measured4[:40]],
|
|
109
|
+
)
|
|
110
|
+
unmeasured4 = [r for r in rows4 if not r["measured"]]
|
|
111
|
+
if unmeasured4:
|
|
112
|
+
by_reason: dict[str, int] = {}
|
|
113
|
+
for r in unmeasured4:
|
|
114
|
+
by_reason[r["unmeasured_reason"] or "unknown"] = by_reason.get(r["unmeasured_reason"] or "unknown", 0) + 1
|
|
115
|
+
_p(lines, f"**{len(unmeasured4)} unmeasured phase(s)** — counted as nothing, never as zero:")
|
|
116
|
+
_table(
|
|
117
|
+
lines,
|
|
118
|
+
["reason", "phases"],
|
|
119
|
+
[[reason, count] for reason, count in sorted(by_reason.items(), key=lambda kv: -kv[1])],
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
te5 = te.get("te_5_cost_per_change", {})
|
|
123
|
+
_h(lines, 3, "TE-5 — cost per change (sum of measured phases)")
|
|
124
|
+
_p(lines, _coverage_line(te5, "phase"))
|
|
125
|
+
_p(lines, te5.get("note", ""))
|
|
126
|
+
_table(
|
|
127
|
+
lines,
|
|
128
|
+
["change", "tokens", "turns", "wall-clock (h)", "phases measured / emitted", "coverage"],
|
|
129
|
+
[[r["change"],
|
|
130
|
+
f"{r['tokens']:,}" if r["tokens"] is not None else "unmeasured",
|
|
131
|
+
r["turns"] if r["turns"] is not None else "-",
|
|
132
|
+
r["wall_clock_h"] if r["wall_clock_h"] is not None else "-",
|
|
133
|
+
f"{r['phases_measured']} / {r['phases_emitted']}",
|
|
134
|
+
f"{r['coverage_fraction']:.0%}" if r["coverage_fraction"] is not None else "-"]
|
|
135
|
+
for r in te5.get("rows", [])[:20]],
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _coverage_line(block: dict, unit: str) -> str:
|
|
140
|
+
emitted = block.get("phases_emitted", 0)
|
|
141
|
+
measured = block.get("phases_measured", 0)
|
|
142
|
+
frac = block.get("coverage_fraction")
|
|
143
|
+
if not emitted:
|
|
144
|
+
return (f"**Coverage: 0 sidecars in this repo.** No {unit} is measurable — "
|
|
145
|
+
"either no change has reached a sidecar-emitting terminal state yet, "
|
|
146
|
+
"or the sidecars pre-date ADR-0009's phase window.")
|
|
147
|
+
return (f"**Coverage: {measured} of {emitted} emitted sidecars measurable"
|
|
148
|
+
+ (f" ({frac:.0%})" if frac is not None else "") + ".** "
|
|
149
|
+
"Emitted sidecars come from five Skills only, so this is a subset of the "
|
|
150
|
+
"change's real cost — read the fraction before reading the totals.")
|
|
151
|
+
|
|
96
152
|
|
|
97
153
|
def _render_workflow_shape(lines: list[str], ws: dict) -> None:
|
|
98
154
|
_h(lines, 2, "Workflow shape")
|
|
@@ -274,7 +330,7 @@ def _render_contract_drift(lines: list[str], cd: dict) -> None:
|
|
|
274
330
|
|
|
275
331
|
def _render_kernel_fit(lines: list[str], kf: dict) -> None:
|
|
276
332
|
_h(lines, 2, "Kernel-fit candidates")
|
|
277
|
-
_p(lines, "Patterns suggesting the kernel itself (
|
|
333
|
+
_p(lines, "Patterns suggesting the kernel itself (KERNEL.md, templates, validators, Skill "
|
|
278
334
|
"flows) may need revision. Each fired pattern is also written as a durable finding "
|
|
279
335
|
"by `/hstack:kernel-fit-scan` at `hstack/kernel-fit/findings/KF-NNNN-*.md`. The "
|
|
280
336
|
"table below is a rollup; the findings are the canonical artifact. See ADR-0004.")
|
|
@@ -345,6 +401,15 @@ def watch_items(metrics: dict) -> list[str]:
|
|
|
345
401
|
if r.get("ratio") is not None and r["ratio"] < 0.5 and r["turns"] > 5:
|
|
346
402
|
items.append(f"Low cache-hit on `{r['skill']}` ({r['ratio']:.0%}) — context is being rebuilt.")
|
|
347
403
|
|
|
404
|
+
# TE-4: emitted sidecars the parser could not measure
|
|
405
|
+
te4 = metrics.get("token_economics", {}).get("te_4_cost_per_phase", {})
|
|
406
|
+
emitted = te4.get("phases_emitted", 0)
|
|
407
|
+
measured = te4.get("phases_measured", 0)
|
|
408
|
+
if emitted and measured < emitted:
|
|
409
|
+
items.append(f"{emitted - measured} of {emitted} phase sidecars are unmeasured "
|
|
410
|
+
"(no phase window, or the session transcript is gone) — TE-4/TE-5 "
|
|
411
|
+
"totals are a subset, not a total.")
|
|
412
|
+
|
|
348
413
|
# QO-2 smells
|
|
349
414
|
smells = metrics.get("quality_outcomes", {}).get("qo_2_severity_resolution_mix", {}).get("high_severity_in_prose_smells", [])
|
|
350
415
|
if smells:
|