agentwrangler 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +191 -0
- package/README.md +116 -0
- package/dist/apply/jobs.js +429 -0
- package/dist/apply/open-terminal-child.mjs +98 -0
- package/dist/apply/open-terminal.js +221 -0
- package/dist/apply/settings-gen.js +35 -0
- package/dist/cli/agentwrangler.js +18 -0
- package/dist/daemon/config.js +51 -0
- package/dist/daemon/http.js +258 -0
- package/dist/daemon/index.js +372 -0
- package/dist/daemon/outcomes-pass.js +82 -0
- package/dist/daemon/readiness.js +15 -0
- package/dist/daemon/router.js +756 -0
- package/dist/daemon/static.js +146 -0
- package/dist/db/migrate.js +72 -0
- package/dist/db/migrations/001_observe.sql +196 -0
- package/dist/db/migrations/002_indexes.sql +6 -0
- package/dist/db/migrations/003_context_inventory_history.sql +20 -0
- package/dist/db/migrations/004_apply_jobs.sql +17 -0
- package/dist/db/migrations/005_tool_event_metadata.sql +17 -0
- package/dist/db/migrations/006_d7_query_indexes.sql +9 -0
- package/dist/db/migrations/007_work_item_branch_keys.sql +11 -0
- package/dist/db/migrations/008_thinking_tokens.sql +1 -0
- package/dist/db/migrations/009_user_turn_count.sql +1 -0
- package/dist/db/migrations/010_workspace_cwd.sql +1 -0
- package/dist/db/migrations/011_reports.sql +1 -0
- package/dist/db/migrations/012_reconcile_indexes.sql +2 -0
- package/dist/db/migrations/013_friction_fields.sql +5 -0
- package/dist/db/migrations/014_session_churn.sql +11 -0
- package/dist/db/migrations/015_gap_aggregates.sql +6 -0
- package/dist/db/open.js +30 -0
- package/dist/detector/benchmark-anchors.js +36 -0
- package/dist/detector/calibration.js +302 -0
- package/dist/detector/context-history-retention.js +312 -0
- package/dist/detector/context-probe.js +574 -0
- package/dist/detector/d1-source-identity.js +25 -0
- package/dist/detector/detectors/d10_catalog_footprint.js +146 -0
- package/dist/detector/detectors/d1_ctx_always_loaded.js +203 -0
- package/dist/detector/detectors/d2_session_long_full_context.js +119 -0
- package/dist/detector/detectors/d4_model_mismatch.js +258 -0
- package/dist/detector/detectors/d5_limit_burn_forecast.js +138 -0
- package/dist/detector/detectors/d6_tool_result_bloat.js +301 -0
- package/dist/detector/detectors/d7_loop_retry_waste.js +345 -0
- package/dist/detector/detectors/d8_cache_write_churn.js +201 -0
- package/dist/detector/detectors/d9_idle_background_session.js +101 -0
- package/dist/detector/engine.js +88 -0
- package/dist/detector/index.js +17 -0
- package/dist/detector/measurement.js +426 -0
- package/dist/detector/practice-registry.js +259 -0
- package/dist/detector/registry.js +32 -0
- package/dist/detector/savings.js +249 -0
- package/dist/detector/types.js +14 -0
- package/dist/evidence/common/approved-input.js +632 -0
- package/dist/evidence/common/boundary.js +84 -0
- package/dist/evidence/common/canonical.js +55 -0
- package/dist/evidence/common/redaction.js +321 -0
- package/dist/evidence/common/sqlite.js +25 -0
- package/dist/evidence/common/state.js +29 -0
- package/dist/evidence/cond1/cli.js +289 -0
- package/dist/evidence/cond1/packet.js +407 -0
- package/dist/evidence/cond1/prepare.js +295 -0
- package/dist/evidence/cond1/score.js +349 -0
- package/dist/evidence/cond1/types.js +1 -0
- package/dist/evidence/create-approval.js +365 -0
- package/dist/evidence/create-scratch.js +542 -0
- package/dist/evidence/d7/cli.js +113 -0
- package/dist/evidence/d7/measure.js +193 -0
- package/dist/evidence/d7/types.js +1 -0
- package/dist/evidence/discover-approval.js +492 -0
- package/dist/evidence/g2/adjudicate.js +20 -0
- package/dist/evidence/g2/cli.js +207 -0
- package/dist/evidence/g2/kappa.js +39 -0
- package/dist/evidence/g2/pipeline.js +92 -0
- package/dist/evidence/g2/store.js +14 -0
- package/dist/evidence/github/client.js +1 -0
- package/dist/evidence/github/gh-cli-client.js +301 -0
- package/dist/evidence/r3/cli.js +209 -0
- package/dist/evidence/r3/evaluate.js +417 -0
- package/dist/evidence/r3/packet.js +162 -0
- package/dist/evidence/r3/prepare.js +405 -0
- package/dist/evidence/r3/score.js +341 -0
- package/dist/evidence/r3/transcript.js +155 -0
- package/dist/evidence/r3/types.js +4 -0
- package/dist/hook/context-budget-hook.mjs +138 -0
- package/dist/hook/danger-guard-denylist.json +27 -0
- package/dist/hook/danger-guard-hook.mjs +167 -0
- package/dist/hook/install.js +0 -0
- package/dist/hook/limit-burn-hook.mjs +127 -0
- package/dist/hook/loop-guard-hook.mjs +104 -0
- package/dist/hook/precompact-checkpoint-hook.mjs +123 -0
- package/dist/ingest/churn-collector.js +122 -0
- package/dist/ingest/detector-hook.js +52 -0
- package/dist/ingest/discovery.js +207 -0
- package/dist/ingest/health.js +43 -0
- package/dist/ingest/index.js +28 -0
- package/dist/ingest/ingestor.js +509 -0
- package/dist/ingest/parser.js +344 -0
- package/dist/ingest/pricing.js +153 -0
- package/dist/ingest/reconcile.js +52 -0
- package/dist/ingest/tail.js +152 -0
- package/dist/ingest/types.js +24 -0
- package/dist/ingest/workspace-mapping.js +114 -0
- package/dist/oauth/anthropic-api-key.js +88 -0
- package/dist/oauth/count-tokens.js +86 -0
- package/dist/oauth/credentials.js +171 -0
- package/dist/oauth/judge-g2-client.js +154 -0
- package/dist/oauth/usage.js +167 -0
- package/dist/outcomes/branch-key.js +49 -0
- package/dist/outcomes/conclusions.js +45 -0
- package/dist/outcomes/derive.js +94 -0
- package/dist/outcomes/finding-extractors.js +131 -0
- package/dist/outcomes/findings.js +237 -0
- package/dist/outcomes/github/client.js +367 -0
- package/dist/outcomes/github/credential.js +195 -0
- package/dist/outcomes/github/gh-cli-client.js +340 -0
- package/dist/outcomes/linker.js +486 -0
- package/dist/outcomes/pool.js +24 -0
- package/dist/outcomes/sync.js +276 -0
- package/dist/query/api/agents-liveness.js +182 -0
- package/dist/query/api/burn-status.js +50 -0
- package/dist/query/api/context-budget.js +114 -0
- package/dist/query/api/context-composition.js +67 -0
- package/dist/query/api/cost-per-success.js +104 -0
- package/dist/query/api/delivery.js +92 -0
- package/dist/query/api/effectiveness.js +254 -0
- package/dist/query/api/efficiency-headroom.js +74 -0
- package/dist/query/api/headroom-trend.js +105 -0
- package/dist/query/api/hook-config.js +75 -0
- package/dist/query/api/hook-install.js +8 -0
- package/dist/query/api/hot-sessions.js +17 -0
- package/dist/query/api/idle-sessions.js +52 -0
- package/dist/query/api/index.js +40 -0
- package/dist/query/api/loop-guard.js +90 -0
- package/dist/query/api/offload-share.js +41 -0
- package/dist/query/api/outcomes.js +218 -0
- package/dist/query/api/overview.js +535 -0
- package/dist/query/api/rec-prompt.js +138 -0
- package/dist/query/api/recommendations-ledger.js +111 -0
- package/dist/query/api/recommendations.js +514 -0
- package/dist/query/api/reports.js +78 -0
- package/dist/query/api/self-churn.js +77 -0
- package/dist/query/api/self-percentiles.js +109 -0
- package/dist/query/api/session-drivers.js +153 -0
- package/dist/query/api/settings.js +85 -0
- package/dist/query/api/spend-flavor.js +234 -0
- package/dist/query/api/trends.js +155 -0
- package/dist/query/cap-weighted.js +119 -0
- package/dist/query/db-context.js +42 -0
- package/dist/query/envelope.js +71 -0
- package/dist/query/forecast.js +191 -0
- package/dist/query/settings-store.js +441 -0
- package/dist/query/spend.js +171 -0
- package/dist/query/trends.js +194 -0
- package/dist/ui/assets/index-DnRKgc21.css +1 -0
- package/dist/ui/assets/index-h1Q1wWq5.js +168 -0
- package/dist/ui/index.html +39 -0
- package/package.json +59 -0
|
@@ -0,0 +1,426 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/detector/measurement.ts — W4 Impact Ledger measurement pass.
|
|
3
|
+
*
|
|
4
|
+
* Owns the ADOPTED → MEASURING → MEASURED_* lifecycle transitions and the
|
|
5
|
+
* pre-adoption baseline snapshot for `recommendation_effects`:
|
|
6
|
+
*
|
|
7
|
+
* - snapshotBeforeValue(db, rec): reads the realized baseline signal
|
|
8
|
+
* (context_inventory_history token delta for D1; session floor context for
|
|
9
|
+
* D2; routing-adherence score for D4; cache read/creation ratio for D8 —
|
|
10
|
+
* RI9). NEVER derives a value from recommendations.modeled_savings_u_per_wk.
|
|
11
|
+
* - runMeasurementPass(db, now, opts?): probe-driven pass called after each
|
|
12
|
+
* context-probe run (wired via the post-probe seam in detector-hook.ts).
|
|
13
|
+
*
|
|
14
|
+
* Honesty rails (W4 design §6):
|
|
15
|
+
* - Realized signal = observed history-table delta, never Claude self-report.
|
|
16
|
+
* - Every transition UPDATE is guarded on the expected predecessor state;
|
|
17
|
+
* zero changes ⇒ log + skip, never throw.
|
|
18
|
+
* - Warning-class recs (D5 / LIMIT) never enter MEASURING and get no effect row.
|
|
19
|
+
* - Unknown target_metric handlers log and leave the rec in MEASURING.
|
|
20
|
+
*
|
|
21
|
+
* Injected `now: Date` everywhere — never call new Date() in this module.
|
|
22
|
+
*/
|
|
23
|
+
import { GLOBAL_WORKSPACE_ID } from "./context-probe.js";
|
|
24
|
+
import { isD1SourceBackedRecommendation, parseD1SourceIdentity } from "./d1-source-identity.js";
|
|
25
|
+
/** Wait ≥1 day after adoption before entering MEASURING. */
|
|
26
|
+
export const MIN_SETTLING_DAYS = 1;
|
|
27
|
+
/** After-window length: once elapsed, the pass closes the measurement and writes a verdict. */
|
|
28
|
+
export const AFTER_WINDOW_DAYS = 14;
|
|
29
|
+
/**
|
|
30
|
+
* Minimum interval between measurement passes (design §10 Q5): the probe can
|
|
31
|
+
* fire every ~30s while tailing; most passes are no-ops. Throttled via
|
|
32
|
+
* user_config 'last_measurement_run_at'. Bypass with opts.force (tests).
|
|
33
|
+
*/
|
|
34
|
+
export const MEASUREMENT_MIN_INTERVAL_MS = 60 * 60 * 1000;
|
|
35
|
+
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
36
|
+
const SIGNAL_WINDOW_DAYS = 14;
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// Schema guards
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
function tableExists(db, name) {
|
|
41
|
+
const row = db
|
|
42
|
+
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = ?")
|
|
43
|
+
.get(name);
|
|
44
|
+
return row !== undefined;
|
|
45
|
+
}
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// Signal queries per detector class (design §2)
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
/** Latest context_inventory_history row for one source (any time ≤ upperBoundIso). */
|
|
50
|
+
function latestHistoryRow(db, workspaceId, component, fileRef, upperBoundIso) {
|
|
51
|
+
const row = db
|
|
52
|
+
.prepare(`SELECT tokens, observed_at FROM context_inventory_history
|
|
53
|
+
WHERE workspace_id = ? AND component = ? AND file_ref = ? AND observed_at <= ?
|
|
54
|
+
ORDER BY observed_at DESC, id DESC LIMIT 1`)
|
|
55
|
+
.get(workspaceId, component, fileRef, upperBoundIso);
|
|
56
|
+
return row ?? null;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* D2 signal: avg of per-session MIN(context_tokens) (cold-cache floor proxy)
|
|
60
|
+
* over RECONCILED sessions whose last_turn_at falls in [fromIso, toIso).
|
|
61
|
+
* Returns null when there are no qualifying sessions (not enough data).
|
|
62
|
+
*/
|
|
63
|
+
function avgFloorContext(db, scopeWorkspaceId, fromIso, toIso) {
|
|
64
|
+
const scoped = scopeWorkspaceId !== null;
|
|
65
|
+
const row = db
|
|
66
|
+
.prepare(`SELECT COUNT(avg_floor_ctx) AS n,
|
|
67
|
+
AVG(avg_floor_ctx) AS avg_floor_ctx
|
|
68
|
+
FROM (
|
|
69
|
+
SELECT (SELECT MIN(t2.context_tokens) FROM turns t2
|
|
70
|
+
WHERE t2.session_id = s.session_id AND t2.provisional = 0) AS avg_floor_ctx
|
|
71
|
+
FROM sessions s
|
|
72
|
+
WHERE s.last_turn_at >= ? AND s.last_turn_at < ?
|
|
73
|
+
AND s.state = 'RECONCILED'
|
|
74
|
+
${scoped ? "AND s.workspace_id = ?" : "AND s.workspace_id != ?"}
|
|
75
|
+
)
|
|
76
|
+
WHERE avg_floor_ctx IS NOT NULL`)
|
|
77
|
+
.get(fromIso, toIso, scopeWorkspaceId ?? GLOBAL_WORKSPACE_ID);
|
|
78
|
+
if (row.n === 0 || row.avg_floor_ctx === null)
|
|
79
|
+
return null;
|
|
80
|
+
return { value: row.avg_floor_ctx, n: row.n };
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* D4 signal: routing-adherence score = ROUND(100 × (1 − premium_share)) over
|
|
84
|
+
* non-sidechain reconciled turns in [fromIso, toIso), where premium = model
|
|
85
|
+
* contains 'opus'. The data-model's "mechanical turn" classifier is not yet
|
|
86
|
+
* shipped, so all reconciled turns are the mechanical set (conservative proxy —
|
|
87
|
+
* documented in the W4 build digest).
|
|
88
|
+
*/
|
|
89
|
+
function routingAdherenceScore(db, scopeWorkspaceId, fromIso, toIso) {
|
|
90
|
+
const scoped = scopeWorkspaceId !== null;
|
|
91
|
+
const sql = `SELECT COUNT(*) AS n,
|
|
92
|
+
AVG(CASE WHEN model LIKE '%opus%' THEN 1.0 ELSE 0.0 END) AS premium_share
|
|
93
|
+
FROM turns
|
|
94
|
+
WHERE is_sidechain = 0 AND provisional = 0 AND ts >= ? AND ts < ? ${scoped ? "AND workspace_id = ?" : ""}`;
|
|
95
|
+
const params = scoped ? [fromIso, toIso, scopeWorkspaceId] : [fromIso, toIso];
|
|
96
|
+
const row = db.prepare(sql).get(...params);
|
|
97
|
+
if (row.n === 0 || row.premium_share === null)
|
|
98
|
+
return null;
|
|
99
|
+
return { value: Math.round(100 * (1 - row.premium_share)), n: row.n };
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* D8 signal (RI9 cache-churn metric): cache read-to-creation ratio =
|
|
103
|
+
* SUM(cache_read_tokens) / SUM(cache_write_5m + cache_write_1h + cache_write_other)
|
|
104
|
+
* over non-sidechain reconciled turns in [fromIso, toIso). A higher ratio means
|
|
105
|
+
* more warm reads per creation (less full-price re-write churn) → improvement.
|
|
106
|
+
* Returns null when no qualifying turns exist or total creation is zero (the
|
|
107
|
+
* ratio is undefined — "not enough data", never a divide-by-zero).
|
|
108
|
+
*/
|
|
109
|
+
function cacheReadToCreationRatio(db, scopeWorkspaceId, fromIso, toIso) {
|
|
110
|
+
const scoped = scopeWorkspaceId !== null;
|
|
111
|
+
const sql = `SELECT COUNT(*) AS n,
|
|
112
|
+
SUM(cache_read_tokens) AS reads,
|
|
113
|
+
SUM(cache_write_5m + cache_write_1h + cache_write_other) AS creation
|
|
114
|
+
FROM turns
|
|
115
|
+
WHERE is_sidechain = 0 AND provisional = 0 AND ts >= ? AND ts < ? ${scoped ? "AND workspace_id = ?" : ""}`;
|
|
116
|
+
const params = scoped ? [fromIso, toIso, scopeWorkspaceId] : [fromIso, toIso];
|
|
117
|
+
const row = db.prepare(sql).get(...params);
|
|
118
|
+
if (row.n === 0 || row.creation === null || row.creation === 0)
|
|
119
|
+
return null;
|
|
120
|
+
return { value: (row.reads ?? 0) / row.creation, n: row.n };
|
|
121
|
+
}
|
|
122
|
+
/** True when a detector class has a measurement handler (design §2a–§2c; RI9 adds D8). */
|
|
123
|
+
function handlerFor(rec) {
|
|
124
|
+
const tm = rec.target_metric ?? "";
|
|
125
|
+
if (isD1SourceBackedRecommendation(rec))
|
|
126
|
+
return "D1";
|
|
127
|
+
if (rec.detector_id === "D2" || tm.startsWith("CACHE_READ_TOKENS_PER_WK"))
|
|
128
|
+
return "D2";
|
|
129
|
+
if (rec.detector_id === "D4" || tm.startsWith("ROUTING_ADHERENCE_SCORE"))
|
|
130
|
+
return "D4";
|
|
131
|
+
if (rec.detector_id === "D8" || tm.startsWith("cache_read_to_creation_ratio"))
|
|
132
|
+
return "D8";
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
/** Warning-class recs (D5 / LIMIT / explicit NONE) skip the lifecycle entirely (design §2d). */
|
|
136
|
+
export function isWarningClass(rec) {
|
|
137
|
+
return (rec.category === "LIMIT" ||
|
|
138
|
+
rec.target_metric === "NONE" ||
|
|
139
|
+
rec.target_metric === "forecast_margin");
|
|
140
|
+
}
|
|
141
|
+
// ---------------------------------------------------------------------------
|
|
142
|
+
// snapshotBeforeValue — design §4c
|
|
143
|
+
// ---------------------------------------------------------------------------
|
|
144
|
+
/**
|
|
145
|
+
* Read the pre-adoption baseline for a rec. Returns null when no signal is
|
|
146
|
+
* available for the class (the adopt path still writes an effect row with
|
|
147
|
+
* before_value = NULL and proceeds — design §4a error handling).
|
|
148
|
+
*/
|
|
149
|
+
export function snapshotBeforeValue(db, rec, now) {
|
|
150
|
+
const nowIso = now.toISOString();
|
|
151
|
+
const handler = handlerFor(rec);
|
|
152
|
+
if (handler === null || isWarningClass(rec))
|
|
153
|
+
return null;
|
|
154
|
+
if (handler === "D1" && !tableExists(db, "context_inventory_history"))
|
|
155
|
+
return null;
|
|
156
|
+
if (handler === "D1") {
|
|
157
|
+
const src = parseD1SourceIdentity(rec.evidence_json);
|
|
158
|
+
if (src === null)
|
|
159
|
+
return null;
|
|
160
|
+
const workspaceId = rec.scope_workspace_id ?? GLOBAL_WORKSPACE_ID;
|
|
161
|
+
const row = latestHistoryRow(db, workspaceId, src.component, src.fileRef, nowIso);
|
|
162
|
+
if (row === null)
|
|
163
|
+
return null;
|
|
164
|
+
// History appends one row per distinct version → the snapshot grain is one
|
|
165
|
+
// observation (the latest version's token count).
|
|
166
|
+
return { value: row.tokens, from_ts: row.observed_at, n: 1 };
|
|
167
|
+
}
|
|
168
|
+
if (handler === "D2") {
|
|
169
|
+
const fromIso = new Date(now.getTime() - SIGNAL_WINDOW_DAYS * MS_PER_DAY).toISOString();
|
|
170
|
+
const r = avgFloorContext(db, rec.scope_workspace_id, fromIso, nowIso);
|
|
171
|
+
return r === null ? null : { value: r.value, from_ts: fromIso, n: r.n };
|
|
172
|
+
}
|
|
173
|
+
if (handler === "D4") {
|
|
174
|
+
const fromIso = new Date(now.getTime() - SIGNAL_WINDOW_DAYS * MS_PER_DAY).toISOString();
|
|
175
|
+
const r = routingAdherenceScore(db, rec.scope_workspace_id, fromIso, nowIso);
|
|
176
|
+
return r === null ? null : { value: r.value, from_ts: fromIso, n: r.n };
|
|
177
|
+
}
|
|
178
|
+
// D8
|
|
179
|
+
const fromIso = new Date(now.getTime() - SIGNAL_WINDOW_DAYS * MS_PER_DAY).toISOString();
|
|
180
|
+
const r = cacheReadToCreationRatio(db, rec.scope_workspace_id, fromIso, nowIso);
|
|
181
|
+
return r === null ? null : { value: r.value, from_ts: fromIso, n: r.n };
|
|
182
|
+
}
|
|
183
|
+
function d1Verdict(beforeValue, afterValue) {
|
|
184
|
+
if (beforeValue === null || afterValue === null || beforeValue === 0) {
|
|
185
|
+
return { delta_pct: null, verdict: "INCONCLUSIVE" };
|
|
186
|
+
}
|
|
187
|
+
const deltaPct = ((afterValue - beforeValue) / beforeValue) * 100;
|
|
188
|
+
// Threshold is exclusive: < −5% ⇒ EFFECTIVE; [−5%, 0%] too small; growth ⇒ NO_EFFECT.
|
|
189
|
+
return { delta_pct: deltaPct, verdict: deltaPct < -5 ? "EFFECTIVE" : "NO_EFFECT" };
|
|
190
|
+
}
|
|
191
|
+
function d2Verdict(beforeValue, afterValue) {
|
|
192
|
+
if (beforeValue === null || afterValue === null || beforeValue === 0) {
|
|
193
|
+
return { delta_pct: null, verdict: "INCONCLUSIVE" };
|
|
194
|
+
}
|
|
195
|
+
const deltaPct = ((afterValue - beforeValue) / beforeValue) * 100;
|
|
196
|
+
// Noisier signal than D1 → higher bar (−15%).
|
|
197
|
+
return { delta_pct: deltaPct, verdict: deltaPct < -15 ? "EFFECTIVE" : "NO_EFFECT" };
|
|
198
|
+
}
|
|
199
|
+
function d4Verdict(beforeValue, afterValue) {
|
|
200
|
+
if (beforeValue === null || afterValue === null) {
|
|
201
|
+
return { delta_pct: null, verdict: "INCONCLUSIVE" };
|
|
202
|
+
}
|
|
203
|
+
// Score units are points (0–100); "delta_pct" carries the point delta here.
|
|
204
|
+
const points = afterValue - beforeValue;
|
|
205
|
+
return { delta_pct: points, verdict: points > 10 ? "EFFECTIVE" : "NO_EFFECT" };
|
|
206
|
+
}
|
|
207
|
+
function d8Verdict(beforeValue, afterValue) {
|
|
208
|
+
if (beforeValue === null || afterValue === null || beforeValue === 0) {
|
|
209
|
+
return { delta_pct: null, verdict: "INCONCLUSIVE" };
|
|
210
|
+
}
|
|
211
|
+
const deltaPct = ((afterValue - beforeValue) / beforeValue) * 100;
|
|
212
|
+
// A rising read/creation ratio = warmer cache = improvement; noisy signal → +15% bar.
|
|
213
|
+
return { delta_pct: deltaPct, verdict: deltaPct > 15 ? "EFFECTIVE" : "NO_EFFECT" };
|
|
214
|
+
}
|
|
215
|
+
// ---------------------------------------------------------------------------
|
|
216
|
+
// After-value readers for the close step
|
|
217
|
+
// ---------------------------------------------------------------------------
|
|
218
|
+
function d1AfterValue(db, rec, adoptedIso, deadlineIso) {
|
|
219
|
+
const src = parseD1SourceIdentity(rec.evidence_json);
|
|
220
|
+
if (src === null)
|
|
221
|
+
return null;
|
|
222
|
+
const workspaceId = rec.scope_workspace_id ?? GLOBAL_WORKSPACE_ID;
|
|
223
|
+
// History rows exist only for DISTINCT versions (append-on-hash-change), so any
|
|
224
|
+
// post-adoption row IS a change. Take the last one in-window (net final state).
|
|
225
|
+
const row = db
|
|
226
|
+
.prepare(`SELECT tokens, observed_at FROM context_inventory_history
|
|
227
|
+
WHERE workspace_id = ? AND component = ? AND file_ref = ?
|
|
228
|
+
AND observed_at > ? AND observed_at <= ?
|
|
229
|
+
ORDER BY observed_at DESC, id DESC LIMIT 1`)
|
|
230
|
+
.get(workspaceId, src.component, src.fileRef, adoptedIso, deadlineIso);
|
|
231
|
+
if (row === undefined)
|
|
232
|
+
return null;
|
|
233
|
+
return { value: row.tokens, to_ts: row.observed_at, n: 1 };
|
|
234
|
+
}
|
|
235
|
+
function d2AfterValue(db, rec, adoptedMs, deadlineMs) {
|
|
236
|
+
const r = avgFloorContext(db, rec.scope_workspace_id, new Date(adoptedMs).toISOString(), new Date(deadlineMs).toISOString());
|
|
237
|
+
if (r === null)
|
|
238
|
+
return null;
|
|
239
|
+
return { value: r.value, to_ts: new Date(deadlineMs).toISOString(), n: r.n };
|
|
240
|
+
}
|
|
241
|
+
function d4AfterValue(db, rec, adoptedMs, deadlineMs) {
|
|
242
|
+
const r = routingAdherenceScore(db, rec.scope_workspace_id, new Date(adoptedMs).toISOString(), new Date(deadlineMs).toISOString());
|
|
243
|
+
if (r === null)
|
|
244
|
+
return null;
|
|
245
|
+
return { value: r.value, to_ts: new Date(deadlineMs).toISOString(), n: r.n };
|
|
246
|
+
}
|
|
247
|
+
function d8AfterValue(db, rec, adoptedMs, deadlineMs) {
|
|
248
|
+
const r = cacheReadToCreationRatio(db, rec.scope_workspace_id, new Date(adoptedMs).toISOString(), new Date(deadlineMs).toISOString());
|
|
249
|
+
if (r === null)
|
|
250
|
+
return null;
|
|
251
|
+
return { value: r.value, to_ts: new Date(deadlineMs).toISOString(), n: r.n };
|
|
252
|
+
}
|
|
253
|
+
// ---------------------------------------------------------------------------
|
|
254
|
+
// Close step — MEASURING → MEASURED_*
|
|
255
|
+
// ---------------------------------------------------------------------------
|
|
256
|
+
function closeMeasurement(db, rec, now, result) {
|
|
257
|
+
const adoptedMs = Date.parse(rec.adopted_at);
|
|
258
|
+
if (Number.isNaN(adoptedMs))
|
|
259
|
+
return;
|
|
260
|
+
const deadlineMs = adoptedMs + AFTER_WINDOW_DAYS * MS_PER_DAY;
|
|
261
|
+
// The after-window closes at its deadline (or now, if the pass runs later).
|
|
262
|
+
const effectiveDeadlineMs = Math.min(deadlineMs, now.getTime());
|
|
263
|
+
const adoptedIso = rec.adopted_at;
|
|
264
|
+
const deadlineIso = new Date(effectiveDeadlineMs).toISOString();
|
|
265
|
+
// Baseline row as written by adoptRecommendation(): measured_at = adopted_at.
|
|
266
|
+
const effect = db
|
|
267
|
+
.prepare("SELECT before_value FROM recommendation_effects WHERE rec_id = ? AND measured_at = ?")
|
|
268
|
+
.get(rec.rec_id, adoptedIso);
|
|
269
|
+
if (effect === undefined) {
|
|
270
|
+
console.warn(`W4: no effect row for ${rec.rec_id}; cannot close measurement — skipped`);
|
|
271
|
+
result.skipped++;
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
const handler = handlerFor(rec);
|
|
275
|
+
let computed;
|
|
276
|
+
let afterValue = null;
|
|
277
|
+
let afterN = null;
|
|
278
|
+
let afterTo = deadlineIso;
|
|
279
|
+
if (handler === "D1") {
|
|
280
|
+
const a = d1AfterValue(db, rec, adoptedIso, deadlineIso);
|
|
281
|
+
if (a === null) {
|
|
282
|
+
// Probe ran but the source never changed within the window.
|
|
283
|
+
computed = { delta_pct: null, verdict: "INCONCLUSIVE" };
|
|
284
|
+
}
|
|
285
|
+
else {
|
|
286
|
+
afterValue = a.value;
|
|
287
|
+
afterN = a.n;
|
|
288
|
+
afterTo = a.to_ts;
|
|
289
|
+
computed = d1Verdict(effect.before_value, a.value);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
else if (handler === "D2") {
|
|
293
|
+
const a = d2AfterValue(db, rec, adoptedMs, effectiveDeadlineMs);
|
|
294
|
+
if (a === null) {
|
|
295
|
+
computed = { delta_pct: null, verdict: "INCONCLUSIVE" };
|
|
296
|
+
}
|
|
297
|
+
else {
|
|
298
|
+
afterValue = a.value;
|
|
299
|
+
afterN = a.n;
|
|
300
|
+
afterTo = a.to_ts;
|
|
301
|
+
computed = d2Verdict(effect.before_value, a.value);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
else if (handler === "D4") {
|
|
305
|
+
const a = d4AfterValue(db, rec, adoptedMs, effectiveDeadlineMs);
|
|
306
|
+
if (a === null) {
|
|
307
|
+
computed = { delta_pct: null, verdict: "INCONCLUSIVE" };
|
|
308
|
+
}
|
|
309
|
+
else {
|
|
310
|
+
afterValue = a.value;
|
|
311
|
+
afterN = a.n;
|
|
312
|
+
afterTo = a.to_ts;
|
|
313
|
+
computed = d4Verdict(effect.before_value, a.value);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
else if (handler === "D8") {
|
|
317
|
+
const a = d8AfterValue(db, rec, adoptedMs, effectiveDeadlineMs);
|
|
318
|
+
if (a === null) {
|
|
319
|
+
computed = { delta_pct: null, verdict: "INCONCLUSIVE" };
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
afterValue = a.value;
|
|
323
|
+
afterN = a.n;
|
|
324
|
+
afterTo = a.to_ts;
|
|
325
|
+
computed = d8Verdict(effect.before_value, a.value);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
else {
|
|
329
|
+
console.info(`W4: no measurement handler for target_metric ${rec.target_metric}; staying MEASURING`);
|
|
330
|
+
result.skipped++;
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
// Write the verdict onto the adoption-cycle effect row (guarded: only when still open).
|
|
334
|
+
const upd = db
|
|
335
|
+
.prepare(`UPDATE recommendation_effects
|
|
336
|
+
SET after_from = ?, after_to = ?, after_value = ?, after_n = ?, delta_pct = ?, verdict = ?
|
|
337
|
+
WHERE rec_id = ? AND measured_at = ? AND verdict IS NULL`)
|
|
338
|
+
.run(adoptedIso, afterTo, afterValue, afterN, computed.delta_pct, computed.verdict, rec.rec_id, adoptedIso);
|
|
339
|
+
if (upd.changes === 0) {
|
|
340
|
+
console.warn(`W4: effect row for ${rec.rec_id} already closed or missing — skipped`);
|
|
341
|
+
result.skipped++;
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
// INCONCLUSIVE maps to the NO_EFFECT state (the state enum has no INCONCLUSIVE);
|
|
345
|
+
// the effect row's verdict column keeps the honest distinction for the ledger.
|
|
346
|
+
const nextState = computed.verdict === "EFFECTIVE" ? "MEASURED_EFFECTIVE" : "MEASURED_NO_EFFECT";
|
|
347
|
+
const trans = db
|
|
348
|
+
.prepare("UPDATE recommendations SET state = ? WHERE rec_id = ? AND state = 'MEASURING'")
|
|
349
|
+
.run(nextState, rec.rec_id);
|
|
350
|
+
if (trans.changes === 0) {
|
|
351
|
+
console.warn(`W4: rec ${rec.rec_id} not in MEASURING at close — skipped`);
|
|
352
|
+
result.skipped++;
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
result.verdicts++;
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* Probe-driven measurement pass. Idempotent, guarded, never throws:
|
|
359
|
+
* ADOPTED → MEASURING when now ≥ adopted_at + MIN_SETTLING_DAYS
|
|
360
|
+
* MEASURING → MEASURED_* when now ≥ adopted_at + AFTER_WINDOW_DAYS
|
|
361
|
+
* Warning-class recs (D5/LIMIT) are skipped entirely; unknown target_metric
|
|
362
|
+
* handlers log and stay in MEASURING.
|
|
363
|
+
*/
|
|
364
|
+
export function runMeasurementPass(db, now, opts) {
|
|
365
|
+
const result = { to_measuring: 0, verdicts: 0, skipped: 0 };
|
|
366
|
+
if (!tableExists(db, "context_inventory_history")) {
|
|
367
|
+
console.warn("W4: context_inventory_history not available; measurement pass skipped");
|
|
368
|
+
return result;
|
|
369
|
+
}
|
|
370
|
+
// Throttle (design §10 Q5): skip when the last run is younger than the interval.
|
|
371
|
+
if (opts?.force !== true && tableExists(db, "user_config")) {
|
|
372
|
+
const last = db
|
|
373
|
+
.prepare("SELECT value FROM user_config WHERE key = 'last_measurement_run_at'")
|
|
374
|
+
.get();
|
|
375
|
+
if (last !== undefined && last.value !== null && last.value !== "") {
|
|
376
|
+
const lastMs = Date.parse(last.value);
|
|
377
|
+
if (!Number.isNaN(lastMs) && now.getTime() - lastMs < MEASUREMENT_MIN_INTERVAL_MS) {
|
|
378
|
+
return result;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
try {
|
|
383
|
+
const rows = db
|
|
384
|
+
.prepare(`SELECT rec_id, detector_id, category, scope_workspace_id, evidence_json,
|
|
385
|
+
target_metric, state, adopted_at
|
|
386
|
+
FROM recommendations
|
|
387
|
+
WHERE state IN ('ADOPTED', 'MEASURING') AND adopted_at IS NOT NULL
|
|
388
|
+
ORDER BY adopted_at ASC, rec_id ASC`)
|
|
389
|
+
.all();
|
|
390
|
+
for (const rec of rows) {
|
|
391
|
+
if (isWarningClass(rec))
|
|
392
|
+
continue; // D5 stays ADOPTED; no MEASURING, no effect rows
|
|
393
|
+
const adoptedMs = Date.parse(rec.adopted_at);
|
|
394
|
+
if (Number.isNaN(adoptedMs))
|
|
395
|
+
continue;
|
|
396
|
+
let state = rec.state;
|
|
397
|
+
if (state === "ADOPTED") {
|
|
398
|
+
if (now.getTime() < adoptedMs + MIN_SETTLING_DAYS * MS_PER_DAY)
|
|
399
|
+
continue;
|
|
400
|
+
const r = db
|
|
401
|
+
.prepare("UPDATE recommendations SET state = 'MEASURING' WHERE rec_id = ? AND state = 'ADOPTED'")
|
|
402
|
+
.run(rec.rec_id);
|
|
403
|
+
if (r.changes === 0) {
|
|
404
|
+
console.warn(`W4: rec ${rec.rec_id} not in ADOPTED for MEASURING transition — skipped`);
|
|
405
|
+
result.skipped++;
|
|
406
|
+
continue;
|
|
407
|
+
}
|
|
408
|
+
state = "MEASURING";
|
|
409
|
+
result.to_measuring++;
|
|
410
|
+
}
|
|
411
|
+
if (state !== "MEASURING")
|
|
412
|
+
continue;
|
|
413
|
+
if (now.getTime() < adoptedMs + AFTER_WINDOW_DAYS * MS_PER_DAY)
|
|
414
|
+
continue; // window still open
|
|
415
|
+
closeMeasurement(db, rec, now, result);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
finally {
|
|
419
|
+
// Record the run regardless of partial failures so the throttle holds.
|
|
420
|
+
if (tableExists(db, "user_config")) {
|
|
421
|
+
db.prepare(`INSERT INTO user_config (key, value, updated_at) VALUES ('last_measurement_run_at', ?, ?)
|
|
422
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`).run(now.toISOString(), now.toISOString());
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
return result;
|
|
426
|
+
}
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/detector/practice-registry.ts — BM1 published-best-practice scorecard.
|
|
3
|
+
*
|
|
4
|
+
* The registry is DATA, not a code path: a typed list of the published Claude
|
|
5
|
+
* Code best practices, each carrying its own citation (source_url + source_date)
|
|
6
|
+
* so a practice that drifts with harness versions is visibly stale and updating
|
|
7
|
+
* one is a data edit. Status is COMPUTED on request from existing aggregates
|
|
8
|
+
* (turns / sessions / context_inventory / recommendations / detector statuses)
|
|
9
|
+
* and is never stored.
|
|
10
|
+
*
|
|
11
|
+
* §0.5 tier compliance: this is a measurement surface, not a rec — it emits zero
|
|
12
|
+
* new advice sentences. ATTENTION rows deep-link to an existing artifact via
|
|
13
|
+
* `artifact_link` (null when none exists); the UI renders status + citation only.
|
|
14
|
+
*
|
|
15
|
+
* SEC-101: every entry field is static public metadata (URLs, dates, numbers)
|
|
16
|
+
* and every computed input is an existing count/ratio — no transcript content.
|
|
17
|
+
*
|
|
18
|
+
* Excluded from v1 (spec §BM1): the effort/thinking practice (spec-prohibited
|
|
19
|
+
* advisory — spec-d11-effort-mismatch.md:8-10); prompt structural-ordering (not
|
|
20
|
+
* observable locally); "agent teams use 7× tokens" (no local lever).
|
|
21
|
+
*/
|
|
22
|
+
import { readFileSync } from "node:fs";
|
|
23
|
+
import { getOffloadShare } from "../query/api/offload-share.js";
|
|
24
|
+
import { getDetectorStatuses } from "./index.js";
|
|
25
|
+
// ── Thresholds (registry-declared; UNVALIDATED defaults, tooltip-surfaced) ────
|
|
26
|
+
/** P1: ATTENTION when this week's cache-read ratio is this many points below the trailing-median. */
|
|
27
|
+
const P1_ATTENTION_DROP_PTS = 10;
|
|
28
|
+
/** P1: minimum priced-input volume (input + cache_read tokens) for the current week to be judged. */
|
|
29
|
+
const P1_MIN_WEEK_VOLUME_TOKENS = 50_000;
|
|
30
|
+
/** P1: minimum number of prior weeks with data before a median comparison is trustworthy. */
|
|
31
|
+
const P1_MIN_PRIOR_WEEKS = 4;
|
|
32
|
+
/** P2: ATTENTION when more than this fraction of non-trivial sessions switched models. */
|
|
33
|
+
const P2_ATTENTION_FRACTION = 0.2;
|
|
34
|
+
/** P2: a session is "non-trivial" at or above this many non-sidechain turns in the window. */
|
|
35
|
+
const P2_MIN_SESSION_TURNS = 5;
|
|
36
|
+
/** P5: official CLAUDE.md line ceiling (Source C, code.claude.com/docs/en/costs). */
|
|
37
|
+
const P5_MAX_CLAUDE_MD_LINES = 200;
|
|
38
|
+
const WEEK_MS = 7 * 24 * 60 * 60 * 1000;
|
|
39
|
+
// ── Editorial registry (authored direct — statements + citations) ─────────────
|
|
40
|
+
/**
|
|
41
|
+
* Static half of each entry. `status` is filled by getPractices; the literal
|
|
42
|
+
* "NO_DATA" here is a placeholder that is always overwritten before return.
|
|
43
|
+
*/
|
|
44
|
+
const REGISTRY = [
|
|
45
|
+
{
|
|
46
|
+
practice_id: "P1",
|
|
47
|
+
statement: "Watch cache-read health — a few points of cache-miss rate dramatically affect cost and latency.",
|
|
48
|
+
source_url: "https://claude.com/blog/lessons-from-building-claude-code-prompt-caching-is-everything",
|
|
49
|
+
source_date: "2026-04-30",
|
|
50
|
+
threshold: {
|
|
51
|
+
value: P1_ATTENTION_DROP_PTS,
|
|
52
|
+
rationale: "Trend-only (no published absolute threshold): ATTENTION when this week's cache-read ratio is >10 points below the trailing-8-week median; NO_DATA under minimum weekly volume.",
|
|
53
|
+
},
|
|
54
|
+
signal: "cache_read_ratio_wk",
|
|
55
|
+
artifact_link: null,
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
practice_id: "P2",
|
|
59
|
+
statement: "Don't switch models mid-session — switching forces a cache rebuild.",
|
|
60
|
+
source_url: "https://claude.com/blog/lessons-from-building-claude-code-prompt-caching-is-everything",
|
|
61
|
+
source_date: "2026-04-30",
|
|
62
|
+
threshold: {
|
|
63
|
+
value: P2_ATTENTION_FRACTION * 100,
|
|
64
|
+
rationale: "ATTENTION when >20% of the week's non-trivial sessions (≥5 non-sidechain turns) used more than one model.",
|
|
65
|
+
},
|
|
66
|
+
signal: "distinct_model_per_session_wk",
|
|
67
|
+
artifact_link: null,
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
practice_id: "P3",
|
|
71
|
+
statement: "Respect the cache-TTL cadence — an idle gap longer than the TTL re-writes the whole context at base price.",
|
|
72
|
+
source_url: "https://claude.com/blog/lessons-from-building-claude-code-prompt-caching-is-everything",
|
|
73
|
+
source_date: "2026-04-30",
|
|
74
|
+
threshold: {
|
|
75
|
+
value: null,
|
|
76
|
+
rationale: "Detector-owned threshold: status follows the D8 CACHE_WRITE_CHURN detector, which flags TTL-crossing cache-write spikes (subscription TTL 1h / API 5m).",
|
|
77
|
+
},
|
|
78
|
+
signal: "D8",
|
|
79
|
+
artifact_link: "/settings",
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
practice_id: "P4",
|
|
83
|
+
statement: "Use /clear between tasks — stale context bills on every subsequent message.",
|
|
84
|
+
source_url: "https://code.claude.com/docs/en/costs",
|
|
85
|
+
source_date: "2026-09-02",
|
|
86
|
+
threshold: {
|
|
87
|
+
value: null,
|
|
88
|
+
rationale: "Binary: ATTENTION whenever the D2 SESSION_LONG_FULL_CONTEXT detector is firing in-window.",
|
|
89
|
+
},
|
|
90
|
+
signal: "D2",
|
|
91
|
+
artifact_link: "/recommendations",
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
practice_id: "P5",
|
|
95
|
+
statement: "Keep CLAUDE.md lean — the docs recommend under 200 lines; it loads into context every session.",
|
|
96
|
+
source_url: "https://code.claude.com/docs/en/costs",
|
|
97
|
+
source_date: "2026-09-02",
|
|
98
|
+
threshold: {
|
|
99
|
+
value: P5_MAX_CLAUDE_MD_LINES,
|
|
100
|
+
rationale: "PASS at or below the official 200-line CLAUDE.md ceiling (Source C), ATTENTION above it.",
|
|
101
|
+
},
|
|
102
|
+
signal: "claude_md_line_count",
|
|
103
|
+
artifact_link: null,
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
practice_id: "P6",
|
|
107
|
+
statement: "Right-size the model — Opus costs ~5× more than Sonnet per token; reserve it for complex tasks.",
|
|
108
|
+
source_url: "https://code.claude.com/docs/en/costs",
|
|
109
|
+
source_date: "2026-09-02",
|
|
110
|
+
threshold: {
|
|
111
|
+
value: null,
|
|
112
|
+
rationale: "Binary: ATTENTION whenever the D4 MODEL_MISMATCH detector is firing in-window.",
|
|
113
|
+
},
|
|
114
|
+
signal: "D4",
|
|
115
|
+
artifact_link: "/recommendations",
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
practice_id: "P7",
|
|
119
|
+
statement: "Offload to subagents — specialized subagents return 1–2k-token summaries despite consuming tens of thousands internally.",
|
|
120
|
+
source_url: "https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents",
|
|
121
|
+
source_date: "2025-09-29",
|
|
122
|
+
threshold: {
|
|
123
|
+
value: null,
|
|
124
|
+
rationale: "Trend-only: the source gives a shape, not a target, so no threshold is claimed — the observed offload share is rendered alongside the citation (PASS when observable, NO_DATA otherwise).",
|
|
125
|
+
},
|
|
126
|
+
signal: "getOffloadShare",
|
|
127
|
+
artifact_link: null,
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
practice_id: "P8",
|
|
131
|
+
statement: "Keep the tool catalog stable and small — any tool-set change invalidates the cache.",
|
|
132
|
+
source_url: "https://claude.com/blog/lessons-from-building-claude-code-prompt-caching-is-everything",
|
|
133
|
+
source_date: "2026-04-30",
|
|
134
|
+
threshold: {
|
|
135
|
+
value: null,
|
|
136
|
+
rationale: "Partial: status is computed from the installed tool-catalog footprint only (D10 CATALOG_FOOTPRINT, config-state); the loaded-vs-installed gap is a UI tooltip disclosure.",
|
|
137
|
+
},
|
|
138
|
+
signal: "D10",
|
|
139
|
+
artifact_link: null,
|
|
140
|
+
},
|
|
141
|
+
];
|
|
142
|
+
// ── Status computation ────────────────────────────────────────────────────────
|
|
143
|
+
/** Map a live detector status to a practice status. */
|
|
144
|
+
export function detectorPracticeStatus(kind) {
|
|
145
|
+
if (kind === "ACTIVE")
|
|
146
|
+
return "ATTENTION";
|
|
147
|
+
if (kind === "INACTIVE")
|
|
148
|
+
return "PASS";
|
|
149
|
+
return "NO_DATA"; // NOT_EVALUATED | BLOCKED | absent
|
|
150
|
+
}
|
|
151
|
+
/** Cache-read ratio and its priced-input volume over [fromIso, toIso). */
|
|
152
|
+
function cacheReadRatio(db, fromIso, toIso) {
|
|
153
|
+
const row = db
|
|
154
|
+
.prepare(`SELECT COALESCE(SUM(cache_read_tokens),0) AS read,
|
|
155
|
+
COALESCE(SUM(input_tokens + cache_read_tokens),0) AS base
|
|
156
|
+
FROM turns
|
|
157
|
+
WHERE ts >= ? AND ts < ? AND provisional = 0`)
|
|
158
|
+
.get(fromIso, toIso);
|
|
159
|
+
return { ratio: row.base === 0 ? null : row.read / row.base, volume: row.base };
|
|
160
|
+
}
|
|
161
|
+
/** P1: current-week cache-read ratio vs trailing-8-week median. */
|
|
162
|
+
function computeP1(db, toIso) {
|
|
163
|
+
const end = new Date(toIso).getTime();
|
|
164
|
+
const current = cacheReadRatio(db, new Date(end - WEEK_MS).toISOString(), toIso);
|
|
165
|
+
if (current.ratio === null || current.volume < P1_MIN_WEEK_VOLUME_TOKENS)
|
|
166
|
+
return "NO_DATA";
|
|
167
|
+
const priors = [];
|
|
168
|
+
for (let i = 1; i <= 8; i++) {
|
|
169
|
+
const wTo = new Date(end - i * WEEK_MS).toISOString();
|
|
170
|
+
const wFrom = new Date(end - (i + 1) * WEEK_MS).toISOString();
|
|
171
|
+
const r = cacheReadRatio(db, wFrom, wTo);
|
|
172
|
+
if (r.ratio !== null)
|
|
173
|
+
priors.push(r.ratio);
|
|
174
|
+
}
|
|
175
|
+
if (priors.length < P1_MIN_PRIOR_WEEKS)
|
|
176
|
+
return "NO_DATA";
|
|
177
|
+
priors.sort((a, b) => a - b);
|
|
178
|
+
const mid = Math.floor(priors.length / 2);
|
|
179
|
+
const median = priors.length % 2 === 0
|
|
180
|
+
? ((priors[mid - 1] ?? 0) + (priors[mid] ?? 0)) / 2
|
|
181
|
+
: (priors[mid] ?? 0);
|
|
182
|
+
const dropPts = (median - current.ratio) * 100;
|
|
183
|
+
return dropPts > P1_ATTENTION_DROP_PTS ? "ATTENTION" : "PASS";
|
|
184
|
+
}
|
|
185
|
+
/** P2: fraction of non-trivial sessions in-window that used more than one model. */
|
|
186
|
+
function computeP2(db, fromIso, toIso) {
|
|
187
|
+
const rows = db
|
|
188
|
+
.prepare(`SELECT session_id, COUNT(DISTINCT model) AS mc, COUNT(*) AS tc
|
|
189
|
+
FROM turns
|
|
190
|
+
WHERE ts >= ? AND ts < ? AND provisional = 0 AND is_sidechain = 0
|
|
191
|
+
GROUP BY session_id
|
|
192
|
+
HAVING tc >= ?`)
|
|
193
|
+
.all(fromIso, toIso, P2_MIN_SESSION_TURNS);
|
|
194
|
+
if (rows.length === 0)
|
|
195
|
+
return "NO_DATA";
|
|
196
|
+
const switched = rows.filter((r) => r.mc > 1).length;
|
|
197
|
+
return switched / rows.length > P2_ATTENTION_FRACTION ? "ATTENTION" : "PASS";
|
|
198
|
+
}
|
|
199
|
+
/** P5: line count of the most-recently-probed CLAUDE.md against the 200-line ceiling. */
|
|
200
|
+
function computeP5(db) {
|
|
201
|
+
const row = db
|
|
202
|
+
.prepare(`SELECT file_ref
|
|
203
|
+
FROM context_inventory
|
|
204
|
+
WHERE component = 'CLAUDE_MD'
|
|
205
|
+
ORDER BY probed_at DESC
|
|
206
|
+
LIMIT 1`)
|
|
207
|
+
.get();
|
|
208
|
+
if (!row)
|
|
209
|
+
return "NO_DATA";
|
|
210
|
+
try {
|
|
211
|
+
// Structural read only: the line count leaves this surface, never the content (SEC-101).
|
|
212
|
+
const text = readFileSync(row.file_ref, "utf8");
|
|
213
|
+
const lines = text.length === 0 ? 0 : text.split("\n").length;
|
|
214
|
+
return lines <= P5_MAX_CLAUDE_MD_LINES ? "PASS" : "ATTENTION";
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
return "NO_DATA";
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
/** P7: offload share is observable → PASS (trend-only), else NO_DATA. No ATTENTION line. */
|
|
221
|
+
function computeP7(db, fromIso, toIso) {
|
|
222
|
+
const share = getOffloadShare(db, { workspaceId: null, from: fromIso, to: toIso }).data
|
|
223
|
+
?.offload_share;
|
|
224
|
+
return share === null || share === undefined ? "NO_DATA" : "PASS";
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Compute the practice scorecard over [from, to). Detector-backed practices
|
|
228
|
+
* (P3/P4/P6/P8) read the live detector statuses anchored at `to`; the rest read
|
|
229
|
+
* existing aggregates directly.
|
|
230
|
+
*/
|
|
231
|
+
export function getPractices(db, opts) {
|
|
232
|
+
const { from, to } = opts;
|
|
233
|
+
const detectorStatus = new Map();
|
|
234
|
+
for (const s of getDetectorStatuses(db, { now: new Date(to) })) {
|
|
235
|
+
detectorStatus.set(s.detector_id, s.status);
|
|
236
|
+
}
|
|
237
|
+
const practices = REGISTRY.map((entry) => {
|
|
238
|
+
let status;
|
|
239
|
+
switch (entry.practice_id) {
|
|
240
|
+
case "P1":
|
|
241
|
+
status = computeP1(db, to);
|
|
242
|
+
break;
|
|
243
|
+
case "P2":
|
|
244
|
+
status = computeP2(db, from, to);
|
|
245
|
+
break;
|
|
246
|
+
case "P5":
|
|
247
|
+
status = computeP5(db);
|
|
248
|
+
break;
|
|
249
|
+
case "P7":
|
|
250
|
+
status = computeP7(db, from, to);
|
|
251
|
+
break;
|
|
252
|
+
default:
|
|
253
|
+
// P3→D8, P4→D2, P6→D4, P8→D10 (signal is the detector id).
|
|
254
|
+
status = detectorPracticeStatus(detectorStatus.get(entry.signal));
|
|
255
|
+
}
|
|
256
|
+
return { ...entry, status };
|
|
257
|
+
});
|
|
258
|
+
return { practices, window: { from, to } };
|
|
259
|
+
}
|