@echomem/mcp 1.4.44 → 1.4.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -25
- package/dist/city/README.md +9 -0
- package/dist/city/echo-ai-city-only.html +2232 -0
- package/dist/city/echo-extraction-plate.html +330 -0
- package/dist/city/echo-face-cutout.png +0 -0
- package/dist/city/personality_stickers/bossy.png +0 -0
- package/dist/city/personality_stickers/ghosty.png +0 -0
- package/dist/city/personality_stickers/loopy.png +0 -0
- package/dist/city/personality_stickers/lusty.png +0 -0
- package/dist/city/personality_stickers/maxxy.png +0 -0
- package/dist/city/personality_stickers/tabby.png +0 -0
- package/dist/city/vendor/OrbitControls.js +1417 -0
- package/dist/city/vendor/RoundedBoxGeometry.js +155 -0
- package/dist/city/vendor/echo_general-file-21.riv +0 -0
- package/dist/city/vendor/rive.js +8139 -0
- package/dist/city/vendor/rive.wasm +0 -0
- package/dist/city/vendor/three.module.min.js +6 -0
- package/dist/context-analysis/claude-native-canonical.js +2 -2
- package/dist/context-analysis/vendored-canonical.js +2 -2
- package/dist/context-analysis/workspace-report.js +3 -3
- package/dist/forensics.js +1531 -0
- package/dist/hud/hooks.js +31 -43
- package/dist/index.js +79 -15
- package/dist/local-data-paths.js +38 -0
- package/dist/migrate.js +140 -70
- package/dist/report.js +721 -0
- package/dist/save-checkpoint-hook.js +1 -1
- package/dist/setup-page/client-core.js +372 -18
- package/dist/setup-page/client-extraction.js +204 -37
- package/dist/setup-page/client-lifecycle.js +101 -31
- package/dist/setup-page/client-report-audit.js +819 -0
- package/dist/setup-page/client-report-city.js +356 -0
- package/dist/setup-page/client-report.js +6 -0
- package/dist/setup-page/client.js +2 -0
- package/dist/setup-page/styles-city-report.js +880 -0
- package/dist/setup-page/styles-context-audit.js +470 -0
- package/dist/setup-page/styles-extraction.js +31 -1
- package/dist/setup-page/styles-foundation.js +89 -0
- package/dist/setup-page/styles-mvp.js +155 -10
- package/dist/setup-page/styles-website-alignment.js +204 -0
- package/dist/setup-page/styles.js +4 -0
- package/dist/setup-page.js +4 -4
- package/dist/setup-preview.js +212 -4
- package/dist/setup.js +702 -321
- package/dist/source-session.js +3 -9
- package/dist/v1-contract.js +8 -0
- package/package.json +7 -9
- package/dist/config-files.js +0 -63
- package/dist/local-jsonl.js +0 -87
- package/dist/onboarding-stats.js +0 -16
|
@@ -0,0 +1,1531 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local "Context Doctor" forensic report — the pre-auth onboarding payload.
|
|
3
|
+
*
|
|
4
|
+
* Scans the user's LOCAL coding-agent transcripts (Claude Code under ~/.claude/projects AND Codex
|
|
5
|
+
* rollouts under ~/.codex/sessions) and turns them into a single MERGED workspace physical-exam:
|
|
6
|
+
* accomplishment (days/repos/tokens/cost/attention), the cost split (cold/cache/output per model),
|
|
7
|
+
* and the waste (token-reread forensics + a Context Cleanliness score). Everything is local; nothing
|
|
8
|
+
* leaves the machine and no model is called.
|
|
9
|
+
*
|
|
10
|
+
* Output shape mirrors the yeahecho `onboarding-two` ContextDoctorTwo report so the same UI renders it.
|
|
11
|
+
* Two source adapters feed ONE source-agnostic engine, so the reread/cleanliness logic is written once.
|
|
12
|
+
*
|
|
13
|
+
* Honesty (the team sanity-checks this): cost is an API-EQUIVALENT estimate at list prices, NOT a
|
|
14
|
+
* subscription bill; `time.*`/avoidableWait are ESTIMATES labeled as such; re-read % is a TOKEN share.
|
|
15
|
+
* Time-of-day buckets use the USER's LOCAL timezone (not a hardcoded one).
|
|
16
|
+
*/
|
|
17
|
+
import fs from "node:fs";
|
|
18
|
+
import os from "node:os";
|
|
19
|
+
import path from "node:path";
|
|
20
|
+
import { execFileSync } from "node:child_process";
|
|
21
|
+
import { calculateContextMetrics } from "./context-metrics/calculate.js";
|
|
22
|
+
import { resolveModelContextLimit } from "./context-metrics/model-limits.js";
|
|
23
|
+
import { buildWorkspaceContextReport } from "./context-analysis/workspace-report.js";
|
|
24
|
+
import { buildCanonicalGoldenReport } from "./context-analysis/canonical-golden.js";
|
|
25
|
+
import { eachLine, walk } from "./report.js";
|
|
26
|
+
import { discoverCodexSessionFiles } from "./codex-session-files.js";
|
|
27
|
+
import { resolveClaudeProjectsDir } from "./local-data-paths.js";
|
|
28
|
+
/** Fast, safe per-repo commit count since a date. Metadata only (no diffs) so it stays cheap;
|
|
29
|
+
* `rev-list --count` is git-indexed. Any failure (not a repo, git missing, timeout) → 0. */
|
|
30
|
+
function gitCommitCount(cwd, sinceIso) {
|
|
31
|
+
if (!cwd)
|
|
32
|
+
return 0;
|
|
33
|
+
try {
|
|
34
|
+
const args = ["-C", cwd, "rev-list", "--count"];
|
|
35
|
+
if (sinceIso)
|
|
36
|
+
args.push(`--since=${sinceIso}`);
|
|
37
|
+
args.push("HEAD");
|
|
38
|
+
const out = execFileSync("git", args, { timeout: 3000, stdio: ["ignore", "pipe", "ignore"] }).toString().trim();
|
|
39
|
+
const n = parseInt(out, 10);
|
|
40
|
+
return Number.isFinite(n) ? n : 0;
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return 0;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// Constants (named on purpose — no magic numbers downstream)
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
const WORK_BLOCK_GAP_MS = 30 * 60 * 1000; // >30min gap between real user prompts => new work block
|
|
50
|
+
const FOCUS_DAY_HOURS = 5; // one "focused work day" == 5h of attention, for day-equivalent math
|
|
51
|
+
const CONTEXT_REACQUISITION_FRACTION = 0.5; // ESTIMATE: share of stale-reread time felt as avoidable wait
|
|
52
|
+
const NIGHT_HOURS = new Set([22, 23, 0, 1, 2, 3]); // late-night grind window (local tz)
|
|
53
|
+
const CHARS_PER_TOKEN = 4; // rough token estimate for read (tool-result) output size
|
|
54
|
+
// Per-million-token USD. Standard API-equivalent list prices, reviewed 2026-07-12.
|
|
55
|
+
// These are deliberately model-specific: using an Opus fallback for every repo was the source of
|
|
56
|
+
// the old impossible city math where two repo labels alone exceeded TOTAL SPENT.
|
|
57
|
+
const CLAUDE_PRICES = {
|
|
58
|
+
"claude-fable-5": { input: 10, cacheWrite: 12.5, cacheRead: 1, output: 50 },
|
|
59
|
+
"claude-opus-4-8": { input: 5, cacheWrite: 6.25, cacheRead: 0.5, output: 25 },
|
|
60
|
+
"claude-opus-4-7": { input: 5, cacheWrite: 6.25, cacheRead: 0.5, output: 25 },
|
|
61
|
+
"claude-opus-4-6": { input: 5, cacheWrite: 6.25, cacheRead: 0.5, output: 25 },
|
|
62
|
+
"claude-opus-4-5": { input: 5, cacheWrite: 6.25, cacheRead: 0.5, output: 25 },
|
|
63
|
+
"claude-sonnet-4-6": { input: 3, cacheWrite: 3.75, cacheRead: 0.3, output: 15 },
|
|
64
|
+
"claude-sonnet-4-5": { input: 3, cacheWrite: 3.75, cacheRead: 0.3, output: 15 },
|
|
65
|
+
"claude-haiku-4-5": { input: 1, cacheWrite: 1.25, cacheRead: 0.1, output: 5 },
|
|
66
|
+
};
|
|
67
|
+
const CLAUDE_SONNET_5_INTRO = { input: 2, cacheWrite: 2.5, cacheRead: 0.2, output: 10 };
|
|
68
|
+
const CLAUDE_SONNET_5_STANDARD = { input: 3, cacheWrite: 3.75, cacheRead: 0.3, output: 15 };
|
|
69
|
+
const CLAUDE_SONNET_5_STANDARD_START_MS = Date.parse("2026-09-01T00:00:00.000Z");
|
|
70
|
+
const CLAUDE_DEFAULT = CLAUDE_PRICES["claude-sonnet-4-6"];
|
|
71
|
+
// OpenAI/Codex list prices. GPT-5.6 adds explicit cache writes; older Codex logs expose only
|
|
72
|
+
// uncached and cached input, so their cacheWrite rate remains zero.
|
|
73
|
+
const OPENAI_PRICES = {
|
|
74
|
+
"gpt-5.6-terra": { input: 2.5, cacheWrite: 3.125, cacheRead: 0.25, output: 15 },
|
|
75
|
+
"gpt-5.6-luna": { input: 1, cacheWrite: 1.25, cacheRead: 0.1, output: 6 },
|
|
76
|
+
"gpt-5.6-sol": { input: 5, cacheWrite: 6.25, cacheRead: 0.5, output: 30 },
|
|
77
|
+
"gpt-5.6": { input: 5, cacheWrite: 6.25, cacheRead: 0.5, output: 30 },
|
|
78
|
+
"gpt-5.4-mini": { input: 0.75, cacheWrite: 0, cacheRead: 0.075, output: 4.5 },
|
|
79
|
+
"gpt-5.4-nano": { input: 0.2, cacheWrite: 0, cacheRead: 0.02, output: 1.25 },
|
|
80
|
+
"gpt-5.5": { input: 5, cacheWrite: 0, cacheRead: 0.5, output: 30 },
|
|
81
|
+
"gpt-5.4": { input: 2.5, cacheWrite: 0, cacheRead: 0.25, output: 15 },
|
|
82
|
+
"gpt-5": { input: 1.25, cacheWrite: 0, cacheRead: 0.125, output: 10 },
|
|
83
|
+
"gpt-5-codex": { input: 1.25, cacheWrite: 0, cacheRead: 0.125, output: 10 },
|
|
84
|
+
};
|
|
85
|
+
const OPENAI_DEFAULT = OPENAI_PRICES["gpt-5"];
|
|
86
|
+
function longestPrefixPrice(table, model) {
|
|
87
|
+
const key = Object.keys(table)
|
|
88
|
+
.filter((candidate) => model.startsWith(candidate))
|
|
89
|
+
.sort((a, b) => b.length - a.length)[0];
|
|
90
|
+
return key ? table[key] : null;
|
|
91
|
+
}
|
|
92
|
+
function priceFor(model, ms) {
|
|
93
|
+
if (model.startsWith("claude-sonnet-5")) {
|
|
94
|
+
const at = typeof ms === "number" && Number.isFinite(ms) ? ms : Date.now();
|
|
95
|
+
return at < CLAUDE_SONNET_5_STANDARD_START_MS ? CLAUDE_SONNET_5_INTRO : CLAUDE_SONNET_5_STANDARD;
|
|
96
|
+
}
|
|
97
|
+
const table = model.startsWith("gpt") || model.startsWith("o1") || model.startsWith("o3") ? OPENAI_PRICES : CLAUDE_PRICES;
|
|
98
|
+
const def = table === OPENAI_PRICES ? OPENAI_DEFAULT : CLAUDE_DEFAULT;
|
|
99
|
+
return table[model] || longestPrefixPrice(table, model) || def;
|
|
100
|
+
}
|
|
101
|
+
function hasOpenAiLongContextPremium(model, inputTokens) {
|
|
102
|
+
if (inputTokens <= 272_000 || model.startsWith("gpt-5.4-mini") || model.startsWith("gpt-5.4-nano"))
|
|
103
|
+
return false;
|
|
104
|
+
return model.startsWith("gpt-5.4") || model.startsWith("gpt-5.5") || model.startsWith("gpt-5.6");
|
|
105
|
+
}
|
|
106
|
+
export function estimateApiEquivalentCost(model, t, ms) {
|
|
107
|
+
const price = priceFor(model, ms);
|
|
108
|
+
const inputTokens = (t.cold || 0) + (t.cacheWrite || 0) + (t.cacheRead || 0);
|
|
109
|
+
const longContextPremiumApplied = hasOpenAiLongContextPremium(model, inputTokens);
|
|
110
|
+
const inputMultiplier = longContextPremiumApplied ? 2 : 1;
|
|
111
|
+
const outputMultiplier = longContextPremiumApplied ? 1.5 : 1;
|
|
112
|
+
const coldInputCost = ((t.cold || 0) * price.input * inputMultiplier) / 1_000_000;
|
|
113
|
+
const cacheWriteCost = ((t.cacheWrite || 0) * price.cacheWrite * inputMultiplier) / 1_000_000;
|
|
114
|
+
const cacheReadCost = ((t.cacheRead || 0) * price.cacheRead * inputMultiplier) / 1_000_000;
|
|
115
|
+
const outputCost = ((t.output || 0) * price.output * outputMultiplier) / 1_000_000;
|
|
116
|
+
return {
|
|
117
|
+
coldInputCost,
|
|
118
|
+
cacheWriteCost,
|
|
119
|
+
cacheReadCost,
|
|
120
|
+
outputCost,
|
|
121
|
+
total: coldInputCost + cacheWriteCost + cacheReadCost + outputCost,
|
|
122
|
+
longContextPremiumApplied,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
function emptyCostAccumulator() {
|
|
126
|
+
return { coldInputCost: 0, cacheWriteCost: 0, cacheReadCost: 0, outputCost: 0, total: 0, longContextRequestCount: 0 };
|
|
127
|
+
}
|
|
128
|
+
function addEstimatedCost(target, value) {
|
|
129
|
+
target.coldInputCost += value.coldInputCost;
|
|
130
|
+
target.cacheWriteCost += value.cacheWriteCost;
|
|
131
|
+
target.cacheReadCost += value.cacheReadCost;
|
|
132
|
+
target.outputCost += value.outputCost;
|
|
133
|
+
target.total += value.total;
|
|
134
|
+
if (value.longContextPremiumApplied)
|
|
135
|
+
target.longContextRequestCount += 1;
|
|
136
|
+
}
|
|
137
|
+
function addAccumulatedCost(target, value) {
|
|
138
|
+
target.coldInputCost += value.coldInputCost;
|
|
139
|
+
target.cacheWriteCost += value.cacheWriteCost;
|
|
140
|
+
target.cacheReadCost += value.cacheReadCost;
|
|
141
|
+
target.outputCost += value.outputCost;
|
|
142
|
+
target.total += value.total;
|
|
143
|
+
target.longContextRequestCount += value.longContextRequestCount;
|
|
144
|
+
}
|
|
145
|
+
// Files that are "project rules / design principles / repo map" — re-reading these unchanged is the
|
|
146
|
+
// most quotable kind of waste.
|
|
147
|
+
const PRINCIPLE_FILE_RE = /(^claude\.md$|^agents\.md$|^tokens\.css$|^readme|cursorrules|^foundations|principle)/i;
|
|
148
|
+
const IMAGE_FILE_RE = /\.(png|jpe?g|gif|webp|svg|ico|avif|bmp|mp4|mov)$/i;
|
|
149
|
+
// Shell read commands whose target file we try to extract for Codex reread forensics.
|
|
150
|
+
const SHELL_READ_BINS = new Set(["cat", "head", "tail", "less", "more", "bat", "nl", "sed", "awk", "od", "strings"]);
|
|
151
|
+
// ---------------------------------------------------------------------------
|
|
152
|
+
// Local-time helpers (USER's tz — never hardcode a zone)
|
|
153
|
+
// ---------------------------------------------------------------------------
|
|
154
|
+
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
|
155
|
+
function localDay(ms) {
|
|
156
|
+
const d = new Date(ms);
|
|
157
|
+
const m = String(d.getMonth() + 1).padStart(2, "0");
|
|
158
|
+
const day = String(d.getDate()).padStart(2, "0");
|
|
159
|
+
return `${d.getFullYear()}-${m}-${day}`;
|
|
160
|
+
}
|
|
161
|
+
const localHour = (ms) => new Date(ms).getHours();
|
|
162
|
+
const localWeekday = (ms) => WEEKDAYS[new Date(ms).getDay()];
|
|
163
|
+
function round(value, digits = 1) {
|
|
164
|
+
const f = 10 ** digits;
|
|
165
|
+
return Math.round((Number(value) || 0) * f) / f;
|
|
166
|
+
}
|
|
167
|
+
function tokensFromChars(chars = 0) {
|
|
168
|
+
return Math.round((chars || 0) / CHARS_PER_TOKEN);
|
|
169
|
+
}
|
|
170
|
+
export function repoLabel(cwd) {
|
|
171
|
+
if (!cwd)
|
|
172
|
+
return "home";
|
|
173
|
+
const m = cwd.match(/worktrees\/[^/]+\/(.+)$/);
|
|
174
|
+
const base = path.basename(m ? m[1] : cwd);
|
|
175
|
+
return base && base !== "." && base !== "/" ? base : "home";
|
|
176
|
+
}
|
|
177
|
+
function fileIdentity(target, cwd) {
|
|
178
|
+
const value = String(target).trim();
|
|
179
|
+
if (!value)
|
|
180
|
+
return "unknown";
|
|
181
|
+
if (value === "~")
|
|
182
|
+
return os.homedir();
|
|
183
|
+
if (value.startsWith(`~${path.sep}`) || value.startsWith("~/")) {
|
|
184
|
+
return path.normalize(path.join(os.homedir(), value.slice(2)));
|
|
185
|
+
}
|
|
186
|
+
if (path.isAbsolute(value))
|
|
187
|
+
return path.normalize(value);
|
|
188
|
+
if (cwd && path.isAbsolute(cwd))
|
|
189
|
+
return path.resolve(cwd, value);
|
|
190
|
+
// An unknown/relative cwd cannot safely be resolved against the MCP process cwd. Namespace it so
|
|
191
|
+
// identical relative paths in unrelated workspaces never become the same forensic file.
|
|
192
|
+
return `${repoLabel(cwd)}::${path.normalize(value)}`;
|
|
193
|
+
}
|
|
194
|
+
function fileLabel(filePath) {
|
|
195
|
+
return path.basename(String(filePath)) || "unknown";
|
|
196
|
+
}
|
|
197
|
+
function contentLength(content) {
|
|
198
|
+
if (content == null)
|
|
199
|
+
return 0;
|
|
200
|
+
if (typeof content === "string")
|
|
201
|
+
return content.length;
|
|
202
|
+
try {
|
|
203
|
+
return JSON.stringify(content).length;
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
return 0;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
function isRealUserText(text) {
|
|
210
|
+
if (typeof text !== "string")
|
|
211
|
+
return false;
|
|
212
|
+
const t = text.trim();
|
|
213
|
+
if (!t)
|
|
214
|
+
return false;
|
|
215
|
+
if (t.startsWith("<"))
|
|
216
|
+
return false; // <system-reminder>, <command-…, <local-command…
|
|
217
|
+
if (t.startsWith("Caveat:"))
|
|
218
|
+
return false;
|
|
219
|
+
if (t.startsWith("[Request interrupted"))
|
|
220
|
+
return false;
|
|
221
|
+
if (t.startsWith("This session is being continued"))
|
|
222
|
+
return false;
|
|
223
|
+
if (t.includes("<command-name>") || t.includes("<local-command"))
|
|
224
|
+
return false;
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
/** Best-effort: pull the file a Codex read-style shell command targets (cat/sed/head FILE). */
|
|
228
|
+
function extractReadPath(cmd) {
|
|
229
|
+
const tokens = String(cmd).trim().split(/\s+/);
|
|
230
|
+
let bin = (tokens[0] || "").split("/").pop() || "";
|
|
231
|
+
if (bin === "sudo")
|
|
232
|
+
bin = (tokens[1] || "").split("/").pop() || "";
|
|
233
|
+
if (!SHELL_READ_BINS.has(bin))
|
|
234
|
+
return null;
|
|
235
|
+
// last token that looks like a path (has a / or a file extension, not a flag/number)
|
|
236
|
+
for (let i = tokens.length - 1; i >= 1; i--) {
|
|
237
|
+
const t = tokens[i].replace(/['"]/g, "");
|
|
238
|
+
if (!t || t.startsWith("-"))
|
|
239
|
+
continue;
|
|
240
|
+
if (t.includes("/") || /\.[A-Za-z0-9]{1,6}$/.test(t))
|
|
241
|
+
return t;
|
|
242
|
+
}
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
245
|
+
class Forensics {
|
|
246
|
+
models = new Map();
|
|
247
|
+
repos = new Map();
|
|
248
|
+
files = new Map();
|
|
249
|
+
fileVersions = new Map();
|
|
250
|
+
lastReadVersion = new Map();
|
|
251
|
+
fileReadSessions = new Map();
|
|
252
|
+
pendingReads = new Map();
|
|
253
|
+
latestContextObservation = null;
|
|
254
|
+
contextObservationSeq = 0;
|
|
255
|
+
hourHistogram = new Array(24).fill(0);
|
|
256
|
+
weekdayCounts = new Map();
|
|
257
|
+
lateNightSessions = new Map();
|
|
258
|
+
allSessionIds = new Set();
|
|
259
|
+
realUserTurns = 0;
|
|
260
|
+
minTs = null;
|
|
261
|
+
maxTs = null;
|
|
262
|
+
nightExample = null;
|
|
263
|
+
modelBucket(model) {
|
|
264
|
+
const key = model || "unknown";
|
|
265
|
+
let m = this.models.get(key);
|
|
266
|
+
if (!m) {
|
|
267
|
+
m = { messages: 0, cold: 0, cacheWrite: 0, cacheRead: 0, output: 0, cost: emptyCostAccumulator() };
|
|
268
|
+
this.models.set(key, m);
|
|
269
|
+
}
|
|
270
|
+
return m;
|
|
271
|
+
}
|
|
272
|
+
repoBucket(cwd) {
|
|
273
|
+
const name = repoLabel(cwd);
|
|
274
|
+
let r = this.repos.get(name);
|
|
275
|
+
if (!r) {
|
|
276
|
+
r = { name, cwd: null, sessions: new Set(), assistantMessages: 0, cold: 0, cacheWrite: 0, cacheRead: 0, output: 0, cost: emptyCostAccumulator(), codexTokens: 0, claudeTokens: 0, reads: 0, rereads: 0, staleRereads: 0, userTimestamps: [], byDay: new Map() };
|
|
277
|
+
this.repos.set(name, r);
|
|
278
|
+
}
|
|
279
|
+
if (cwd && !r.cwd)
|
|
280
|
+
r.cwd = cwd; // remember a real working dir for this repo (for the git pass)
|
|
281
|
+
return r;
|
|
282
|
+
}
|
|
283
|
+
fileRecord(p, repo) {
|
|
284
|
+
let rec = this.files.get(p);
|
|
285
|
+
if (!rec) {
|
|
286
|
+
const label = fileLabel(p);
|
|
287
|
+
rec = {
|
|
288
|
+
label, repo, isPrinciple: PRINCIPLE_FILE_RE.test(label), isImage: IMAGE_FILE_RE.test(label),
|
|
289
|
+
totalReads: 0, rereads: 0, staleRereads: 0, crossSessionRereads: 0, everEdited: false, editCount: 0,
|
|
290
|
+
readTokens: 0, staleReadTokens: 0, perSession: new Map(), days: new Set(), nightReads: 0,
|
|
291
|
+
};
|
|
292
|
+
this.files.set(p, rec);
|
|
293
|
+
}
|
|
294
|
+
return rec;
|
|
295
|
+
}
|
|
296
|
+
noteTs(ms) {
|
|
297
|
+
if (this.minTs == null || ms < this.minTs)
|
|
298
|
+
this.minTs = ms;
|
|
299
|
+
if (this.maxTs == null || ms > this.maxTs)
|
|
300
|
+
this.maxTs = ms;
|
|
301
|
+
}
|
|
302
|
+
noteSession(id) {
|
|
303
|
+
if (id)
|
|
304
|
+
this.allSessionIds.add(id);
|
|
305
|
+
}
|
|
306
|
+
/** assistant-turn token usage (already split into cold/cacheWrite/cacheRead/output). */
|
|
307
|
+
recordUsage(model, cwd, session, u, provider, ms) {
|
|
308
|
+
const m = this.modelBucket(model);
|
|
309
|
+
const estimatedCost = estimateApiEquivalentCost(model, u, ms);
|
|
310
|
+
m.messages += 1;
|
|
311
|
+
m.cold += u.cold;
|
|
312
|
+
m.cacheWrite += u.cacheWrite;
|
|
313
|
+
m.cacheRead += u.cacheRead;
|
|
314
|
+
m.output += u.output;
|
|
315
|
+
addEstimatedCost(m.cost, estimatedCost);
|
|
316
|
+
// Keep usage with missing legacy cwd metadata in an explicit "home" bucket. Silently omitting it
|
|
317
|
+
// makes repo/day timelines fail to reconcile with the provider ledger even though scale includes it.
|
|
318
|
+
const r = this.repoBucket(cwd);
|
|
319
|
+
if (session)
|
|
320
|
+
r.sessions.add(session);
|
|
321
|
+
r.assistantMessages += 1;
|
|
322
|
+
r.cold += u.cold;
|
|
323
|
+
r.cacheWrite += u.cacheWrite;
|
|
324
|
+
r.cacheRead += u.cacheRead;
|
|
325
|
+
r.output += u.output;
|
|
326
|
+
addEstimatedCost(r.cost, estimatedCost);
|
|
327
|
+
// per-repo provider split → drives the city's per-cube colour (Codex green / Claude orange)
|
|
328
|
+
const turnTokens = u.cold + u.cacheWrite + u.cacheRead + u.output;
|
|
329
|
+
if (provider === "codex")
|
|
330
|
+
r.codexTokens += turnTokens;
|
|
331
|
+
else
|
|
332
|
+
r.claudeTokens += turnTokens;
|
|
333
|
+
if (ms != null && turnTokens > 0) {
|
|
334
|
+
const d = localDay(ms);
|
|
335
|
+
r.byDay.set(d, (r.byDay.get(d) || 0) + turnTokens);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
recordContextObservation(input) {
|
|
339
|
+
const inputTokens = Math.max(0, Math.round(input.inputTokens || 0));
|
|
340
|
+
if (inputTokens <= 0)
|
|
341
|
+
return;
|
|
342
|
+
const resolvedLimit = resolveModelContextLimit({
|
|
343
|
+
loggedLimitTokens: input.modelContextLimitTokens ?? null,
|
|
344
|
+
model: input.model,
|
|
345
|
+
});
|
|
346
|
+
const observation = {
|
|
347
|
+
seq: ++this.contextObservationSeq,
|
|
348
|
+
model: input.model || "unknown",
|
|
349
|
+
inputTokens,
|
|
350
|
+
outputTokens: Math.max(0, Math.round(input.outputTokens || 0)),
|
|
351
|
+
modelContextLimitTokens: resolvedLimit.tokens,
|
|
352
|
+
modelContextLimitSource: resolvedLimit.source,
|
|
353
|
+
ms: input.ms ?? null,
|
|
354
|
+
};
|
|
355
|
+
const current = this.latestContextObservation;
|
|
356
|
+
if (!current ||
|
|
357
|
+
(observation.ms != null && current.ms != null && observation.ms >= current.ms) ||
|
|
358
|
+
(observation.ms != null && current.ms == null) ||
|
|
359
|
+
(observation.ms == null && current.ms == null && observation.seq >= current.seq)) {
|
|
360
|
+
this.latestContextObservation = observation;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
recordEdit(target, cwd) {
|
|
364
|
+
if (!target)
|
|
365
|
+
return;
|
|
366
|
+
const identity = fileIdentity(target, cwd);
|
|
367
|
+
this.fileVersions.set(identity, (this.fileVersions.get(identity) || 0) + 1);
|
|
368
|
+
const rec = this.fileRecord(identity, repoLabel(cwd));
|
|
369
|
+
rec.everEdited = true;
|
|
370
|
+
rec.editCount += 1;
|
|
371
|
+
}
|
|
372
|
+
/** A file read; decides stale/cross-session at read time and parks the token settle by toolId. */
|
|
373
|
+
recordRead(target, cwd, session, ms, toolId) {
|
|
374
|
+
if (!target)
|
|
375
|
+
return;
|
|
376
|
+
const repo = repoLabel(cwd);
|
|
377
|
+
const identity = fileIdentity(target, cwd);
|
|
378
|
+
const rec = this.fileRecord(identity, repo);
|
|
379
|
+
const priorReads = rec.totalReads;
|
|
380
|
+
const currentVersion = this.fileVersions.get(identity) || 0;
|
|
381
|
+
const seenSessions = this.fileReadSessions.get(identity) || new Set();
|
|
382
|
+
const isReread = priorReads > 0;
|
|
383
|
+
const crossSession = isReread && !!session && !seenSessions.has(session);
|
|
384
|
+
const changedSinceLastRead = isReread && currentVersion > (this.lastReadVersion.get(identity) || 0);
|
|
385
|
+
const stale = isReread && !changedSinceLastRead;
|
|
386
|
+
rec.totalReads += 1;
|
|
387
|
+
if (cwd)
|
|
388
|
+
this.repoBucket(cwd).reads += 1;
|
|
389
|
+
if (isReread) {
|
|
390
|
+
rec.rereads += 1;
|
|
391
|
+
if (cwd)
|
|
392
|
+
this.repoBucket(cwd).rereads += 1;
|
|
393
|
+
}
|
|
394
|
+
if (crossSession)
|
|
395
|
+
rec.crossSessionRereads += 1;
|
|
396
|
+
if (stale) {
|
|
397
|
+
rec.staleRereads += 1;
|
|
398
|
+
if (cwd)
|
|
399
|
+
this.repoBucket(cwd).staleRereads += 1;
|
|
400
|
+
}
|
|
401
|
+
const day = ms != null ? localDay(ms) : "unknown";
|
|
402
|
+
const hour = ms != null ? localHour(ms) : 12;
|
|
403
|
+
const isNight = NIGHT_HOURS.has(hour);
|
|
404
|
+
rec.days.add(day);
|
|
405
|
+
if (isNight)
|
|
406
|
+
rec.nightReads += 1;
|
|
407
|
+
const ps = rec.perSession.get(session || "") || { count: 0, day, nightCount: 0 };
|
|
408
|
+
ps.count += 1;
|
|
409
|
+
if (isNight)
|
|
410
|
+
ps.nightCount += 1;
|
|
411
|
+
rec.perSession.set(session || "", ps);
|
|
412
|
+
if (stale && isNight) {
|
|
413
|
+
const candidate = { file: rec.label, path: target, repo, day, hour, sessionShort: (session || "").slice(0, 8), priorReads };
|
|
414
|
+
if (!this.nightExample || priorReads > this.nightExample.priorReads)
|
|
415
|
+
this.nightExample = candidate;
|
|
416
|
+
}
|
|
417
|
+
if (!this.fileReadSessions.has(identity))
|
|
418
|
+
this.fileReadSessions.set(identity, new Set());
|
|
419
|
+
if (session)
|
|
420
|
+
this.fileReadSessions.get(identity).add(session);
|
|
421
|
+
this.lastReadVersion.set(identity, currentVersion);
|
|
422
|
+
if (toolId)
|
|
423
|
+
this.pendingReads.set(toolId, { rec, stale });
|
|
424
|
+
}
|
|
425
|
+
recordReadOutput(toolId, chars) {
|
|
426
|
+
const pending = this.pendingReads.get(toolId);
|
|
427
|
+
if (!pending)
|
|
428
|
+
return;
|
|
429
|
+
const tokens = tokensFromChars(chars);
|
|
430
|
+
pending.rec.readTokens += tokens;
|
|
431
|
+
if (pending.stale)
|
|
432
|
+
pending.rec.staleReadTokens += tokens;
|
|
433
|
+
this.pendingReads.delete(toolId);
|
|
434
|
+
}
|
|
435
|
+
recordUserMsg(cwd, session, ms) {
|
|
436
|
+
this.realUserTurns += 1;
|
|
437
|
+
const hour = localHour(ms);
|
|
438
|
+
this.hourHistogram[hour] += 1;
|
|
439
|
+
const wd = localWeekday(ms);
|
|
440
|
+
this.weekdayCounts.set(wd, (this.weekdayCounts.get(wd) || 0) + 1);
|
|
441
|
+
if (cwd)
|
|
442
|
+
this.repoBucket(cwd).userTimestamps.push(ms);
|
|
443
|
+
if (NIGHT_HOURS.has(hour) && session) {
|
|
444
|
+
const ln = this.lateNightSessions.get(session) || { repo: repoLabel(cwd), day: localDay(ms), nightMsgs: 0 };
|
|
445
|
+
ln.nightMsgs += 1;
|
|
446
|
+
this.lateNightSessions.set(session, ln);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
// ----- work blocks (real attention hours) -----
|
|
450
|
+
workBlocks(timestamps) {
|
|
451
|
+
const sorted = [...timestamps].sort((a, b) => a - b);
|
|
452
|
+
const blocks = [];
|
|
453
|
+
let start = null;
|
|
454
|
+
let last = null;
|
|
455
|
+
for (const ms of sorted) {
|
|
456
|
+
if (start == null) {
|
|
457
|
+
start = ms;
|
|
458
|
+
last = ms;
|
|
459
|
+
continue;
|
|
460
|
+
}
|
|
461
|
+
if (ms - last > WORK_BLOCK_GAP_MS) {
|
|
462
|
+
blocks.push({ start, end: last });
|
|
463
|
+
start = ms;
|
|
464
|
+
}
|
|
465
|
+
last = ms;
|
|
466
|
+
}
|
|
467
|
+
if (start != null)
|
|
468
|
+
blocks.push({ start, end: last });
|
|
469
|
+
const attentionHours = blocks.reduce((s, b) => s + (b.end - b.start) / 3_600_000, 0);
|
|
470
|
+
const days = new Set(blocks.map((b) => localDay(b.start)));
|
|
471
|
+
return { blocks, attentionHours, activeDays: days.size };
|
|
472
|
+
}
|
|
473
|
+
build() {
|
|
474
|
+
// ----- tokens + cost -----
|
|
475
|
+
let cold = 0, cacheWrite = 0, cacheRead = 0, output = 0;
|
|
476
|
+
const modelOut = {};
|
|
477
|
+
const accumulatedCost = emptyCostAccumulator();
|
|
478
|
+
for (const [name, m] of this.models) {
|
|
479
|
+
cold += m.cold;
|
|
480
|
+
cacheWrite += m.cacheWrite;
|
|
481
|
+
cacheRead += m.cacheRead;
|
|
482
|
+
output += m.output;
|
|
483
|
+
addAccumulatedCost(accumulatedCost, m.cost);
|
|
484
|
+
if (name === "<synthetic>" || m.cold + m.cacheWrite + m.cacheRead + m.output === 0)
|
|
485
|
+
continue;
|
|
486
|
+
modelOut[name] = {
|
|
487
|
+
messages: m.messages, cold: m.cold, cacheWrite: m.cacheWrite, cacheRead: m.cacheRead, output: m.output,
|
|
488
|
+
total: m.cold + m.cacheWrite + m.cacheRead + m.output,
|
|
489
|
+
coldInputCost: round(m.cost.coldInputCost, 2),
|
|
490
|
+
cacheWriteCost: round(m.cost.cacheWriteCost, 2),
|
|
491
|
+
cacheReadCost: round(m.cost.cacheReadCost, 2),
|
|
492
|
+
outputCost: round(m.cost.outputCost, 2),
|
|
493
|
+
cost: round(m.cost.total, 2),
|
|
494
|
+
longContextRequestCount: m.cost.longContextRequestCount,
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
const totalInput = cold + cacheWrite + cacheRead;
|
|
498
|
+
const totalTokens = totalInput + output;
|
|
499
|
+
const cost = {
|
|
500
|
+
coldInputCost: round(accumulatedCost.coldInputCost, 2),
|
|
501
|
+
cacheWriteCost: round(accumulatedCost.cacheWriteCost, 2),
|
|
502
|
+
cacheReadCost: round(accumulatedCost.cacheReadCost, 2),
|
|
503
|
+
outputCost: round(accumulatedCost.outputCost, 2),
|
|
504
|
+
total: round(accumulatedCost.total, 2),
|
|
505
|
+
longContextRequestCount: accumulatedCost.longContextRequestCount,
|
|
506
|
+
pricingBasis: "standard-api-list-price",
|
|
507
|
+
pricingAsOf: "2026-07-12",
|
|
508
|
+
};
|
|
509
|
+
const rankedModels = Object.entries(modelOut).sort((a, b) => b[1].total - a[1].total);
|
|
510
|
+
const topModel = rankedModels[0]?.[0] || "unknown";
|
|
511
|
+
const topShare = totalTokens ? Math.round(((rankedModels[0]?.[1].total || 0) / totalTokens) * 100) : 0;
|
|
512
|
+
let identityLabel = "workspace power user";
|
|
513
|
+
if (topShare >= 95)
|
|
514
|
+
identityLabel = "all-in frontier-model builder";
|
|
515
|
+
else if (rankedModels.length >= 3)
|
|
516
|
+
identityLabel = "multi-model operator";
|
|
517
|
+
else if (topShare >= 75)
|
|
518
|
+
identityLabel = "primary-model power user";
|
|
519
|
+
// ----- per-repo attention + global -----
|
|
520
|
+
const allUserTimestamps = [];
|
|
521
|
+
const dayAttention = new Map();
|
|
522
|
+
const repoOut = [];
|
|
523
|
+
const sinceIso = this.minTs != null ? new Date(this.minTs).toISOString() : null;
|
|
524
|
+
let totalCommits = 0;
|
|
525
|
+
for (const repo of this.repos.values()) {
|
|
526
|
+
allUserTimestamps.push(...repo.userTimestamps);
|
|
527
|
+
const wb = this.workBlocks(repo.userTimestamps);
|
|
528
|
+
const commits = gitCommitCount(repo.cwd, sinceIso); // fast git pass: commits landed in this repo during the scan window
|
|
529
|
+
totalCommits += commits;
|
|
530
|
+
repoOut.push({
|
|
531
|
+
name: repo.name, sessions: repo.sessions.size, assistantMessages: repo.assistantMessages,
|
|
532
|
+
attentionHours: round(wb.attentionHours, 1), activeAttentionDays: wb.activeDays,
|
|
533
|
+
dailyFocusHours: wb.activeDays ? round(wb.attentionHours / wb.activeDays, 2) : 0,
|
|
534
|
+
equivalentFocusDays: round(wb.attentionHours / FOCUS_DAY_HOURS, 1), workBlocks: wb.blocks.length,
|
|
535
|
+
tokens: repo.cold + repo.cacheWrite + repo.cacheRead + repo.output,
|
|
536
|
+
cost: round(repo.cost.total, 2),
|
|
537
|
+
reads: repo.reads, rereads: repo.rereads, staleRereads: repo.staleRereads,
|
|
538
|
+
dominantProvider: repo.codexTokens >= repo.claudeTokens ? "codex" : "claude",
|
|
539
|
+
codexTokens: repo.codexTokens, claudeTokens: repo.claudeTokens,
|
|
540
|
+
commits,
|
|
541
|
+
daily: [...repo.byDay.entries()].sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)),
|
|
542
|
+
});
|
|
543
|
+
for (const b of wb.blocks) {
|
|
544
|
+
const d = localDay(b.start);
|
|
545
|
+
dayAttention.set(d, (dayAttention.get(d) || 0) + (b.end - b.start) / 3_600_000);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
repoOut.sort((a, b) => b.attentionHours - a.attentionHours || b.tokens - a.tokens);
|
|
549
|
+
const globalBlocks = this.workBlocks(allUserTimestamps);
|
|
550
|
+
const userAttentionHours = round(globalBlocks.attentionHours, 1);
|
|
551
|
+
const activeAttentionDays = globalBlocks.activeDays;
|
|
552
|
+
const dayHoursList = [...dayAttention.entries()].sort((a, b) => b[1] - a[1]);
|
|
553
|
+
const peakDay = dayHoursList[0] ? { date: dayHoursList[0][0], hours: round(dayHoursList[0][1], 1) } : null;
|
|
554
|
+
// ----- rhythm -----
|
|
555
|
+
const sumHours = (hrs) => hrs.reduce((s, h) => s + this.hourHistogram[h], 0);
|
|
556
|
+
const totalMsgForShare = Math.max(this.realUserTurns, 1);
|
|
557
|
+
const busiestWeekday = [...this.weekdayCounts.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] || null;
|
|
558
|
+
const lateNightSessionList = [...this.lateNightSessions.entries()]
|
|
559
|
+
.map(([id, v]) => ({ session: id.slice(0, 8), repo: v.repo, day: v.day, nightMsgs: v.nightMsgs }))
|
|
560
|
+
.sort((a, b) => b.nightMsgs - a.nightMsgs)
|
|
561
|
+
.slice(0, 6);
|
|
562
|
+
// ----- cleanliness (token-weighted; non-image code/text reads only) -----
|
|
563
|
+
const codeFiles = [...this.files.values()].filter((f) => !f.isImage && f.totalReads > 0);
|
|
564
|
+
const codeReadTokens = codeFiles.reduce((s, f) => s + f.readTokens, 0);
|
|
565
|
+
const codeStaleReadTokens = codeFiles.reduce((s, f) => s + f.staleReadTokens, 0);
|
|
566
|
+
const codeReads = codeFiles.reduce((s, f) => s + f.totalReads, 0);
|
|
567
|
+
const codeRereads = codeFiles.reduce((s, f) => s + f.rereads, 0);
|
|
568
|
+
const codeStaleRereads = codeFiles.reduce((s, f) => s + f.staleRereads, 0);
|
|
569
|
+
const codeCrossStale = codeFiles.reduce((s, f) => s + f.crossSessionRereads, 0);
|
|
570
|
+
const staleShare = codeReadTokens ? codeStaleReadTokens / codeReadTokens : 0;
|
|
571
|
+
const cleanlinessScore = Math.max(0, Math.min(100, Math.round(100 * (1 - staleShare))));
|
|
572
|
+
// ----- reread forensics -----
|
|
573
|
+
const ranked = [...this.files.values()]
|
|
574
|
+
.filter((f) => f.totalReads > 0 && !f.isImage)
|
|
575
|
+
.sort((a, b) => b.crossSessionRereads - a.crossSessionRereads || b.perSession.size - a.perSession.size || b.rereads - a.rereads || b.totalReads - a.totalReads);
|
|
576
|
+
const topFiles = ranked.slice(0, 12).map((f) => ({
|
|
577
|
+
file: f.label, repo: f.repo, isPrinciple: f.isPrinciple, totalReads: f.totalReads, rereads: f.rereads,
|
|
578
|
+
staleRereads: f.staleRereads, crossSessionRereads: f.crossSessionRereads, everChanged: f.everEdited,
|
|
579
|
+
editCount: f.editCount, sessionsTouched: f.perSession.size, daysTouched: f.days.size, nightReads: f.nightReads,
|
|
580
|
+
readTokens: f.readTokens, staleReadTokens: f.staleReadTokens,
|
|
581
|
+
perSession: [...f.perSession.entries()].map(([id, v]) => ({ session: (id || "").slice(0, 8), count: v.count, day: v.day, nightCount: v.nightCount })).sort((a, b) => b.count - a.count).slice(0, 6),
|
|
582
|
+
}));
|
|
583
|
+
const neverChangedHeavy = ranked.filter((f) => !f.everEdited && f.rereads >= 2).slice(0, 8).map((f) => ({
|
|
584
|
+
file: f.label, repo: f.repo, isPrinciple: f.isPrinciple, totalReads: f.totalReads,
|
|
585
|
+
sessionsTouched: f.perSession.size, daysTouched: f.days.size, nightReads: f.nightReads, staleReadTokens: f.staleReadTokens,
|
|
586
|
+
}));
|
|
587
|
+
const principleReads = ranked.filter((f) => f.isPrinciple && f.rereads > 0).sort((a, b) => b.staleReadTokens - a.staleReadTokens).slice(0, 6).map((f) => ({
|
|
588
|
+
file: f.label, repo: f.repo, totalReads: f.totalReads, staleReadTokens: f.staleReadTokens, sessionsTouched: f.perSession.size,
|
|
589
|
+
}));
|
|
590
|
+
// ----- avoidable wait + counterfactual (ESTIMATES) -----
|
|
591
|
+
const dailyFocus = activeAttentionDays ? userAttentionHours / activeAttentionDays : repoOut[0]?.dailyFocusHours || 0;
|
|
592
|
+
const avoidableWaitHours = round(userAttentionHours * staleShare * CONTEXT_REACQUISITION_FRACTION, 1);
|
|
593
|
+
const avoidableWaitDays = dailyFocus ? round(avoidableWaitHours / dailyFocus, 1) : 0;
|
|
594
|
+
const daysAgo = Math.max(1, Math.round(avoidableWaitDays));
|
|
595
|
+
const scanEnd = this.maxTs ?? Date.now();
|
|
596
|
+
const cfDate = scanEnd - daysAgo * 86_400_000;
|
|
597
|
+
return {
|
|
598
|
+
schemaVersion: 1,
|
|
599
|
+
dataOrigin: "local-workspace-scan",
|
|
600
|
+
generatedFrom: [],
|
|
601
|
+
llmCallsUsed: 0,
|
|
602
|
+
transcriptsUploaded: false,
|
|
603
|
+
scanStartDate: this.minTs != null ? localDay(this.minTs) : null,
|
|
604
|
+
scanEndDate: this.maxTs != null ? localDay(this.maxTs) : null,
|
|
605
|
+
scale: {
|
|
606
|
+
sessionCount: this.allSessionIds.size,
|
|
607
|
+
repoCount: repoOut.length,
|
|
608
|
+
activeDays: activeAttentionDays,
|
|
609
|
+
totalCommits,
|
|
610
|
+
totalTokens, totalInputTokens: totalInput, coldInputTokens: cold, cacheWriteTokens: cacheWrite,
|
|
611
|
+
cacheReadTokens: cacheRead, outputTokens: output,
|
|
612
|
+
inputPct: totalTokens ? Math.round((totalInput / totalTokens) * 100) : 0,
|
|
613
|
+
},
|
|
614
|
+
cost: { ...cost, byModel: modelOut },
|
|
615
|
+
modelUsage: { topModel, topModelShare: topShare, identityLabel },
|
|
616
|
+
userWorkingHours: {
|
|
617
|
+
realUserTurns: this.realUserTurns, userAttentionHours, activeAttentionDays,
|
|
618
|
+
averageDailyAttentionHours: activeAttentionDays ? round(userAttentionHours / activeAttentionDays, 2) : 0,
|
|
619
|
+
maxDailyAttentionHours: peakDay ? peakDay.hours : 0, peakDay, workBlocks: globalBlocks.blocks.length,
|
|
620
|
+
},
|
|
621
|
+
repos: repoOut,
|
|
622
|
+
rhythm: {
|
|
623
|
+
hourHistogram: this.hourHistogram,
|
|
624
|
+
morningShare: Math.round((sumHours([5, 6, 7, 8, 9, 10, 11]) / totalMsgForShare) * 100),
|
|
625
|
+
eveningShare: Math.round((sumHours([18, 19, 20, 21]) / totalMsgForShare) * 100),
|
|
626
|
+
lateNightShare: Math.round((sumHours([22, 23, 0, 1, 2, 3]) / totalMsgForShare) * 100),
|
|
627
|
+
lateNightSessionCount: this.lateNightSessions.size, lateNightSessions: lateNightSessionList,
|
|
628
|
+
busiestWeekday, peakDay,
|
|
629
|
+
},
|
|
630
|
+
cleanliness: {
|
|
631
|
+
score: cleanlinessScore, totalReads: codeReads, rereads: codeRereads, staleRereads: codeStaleRereads,
|
|
632
|
+
crossSessionStaleReads: codeCrossStale, totalReadTokens: codeReadTokens, staleReadTokens: codeStaleReadTokens,
|
|
633
|
+
staleReadSharePct: Math.round(staleShare * 100),
|
|
634
|
+
},
|
|
635
|
+
contextWindow: this.buildContextWindowSummary(topModel, staleShare),
|
|
636
|
+
rereadForensics: { topFiles, neverChangedHeavy, principleReads, nightExample: this.nightExample },
|
|
637
|
+
avoidableWait: {
|
|
638
|
+
readTokens: codeStaleReadTokens,
|
|
639
|
+
hours: avoidableWaitHours,
|
|
640
|
+
days: avoidableWaitDays,
|
|
641
|
+
cost: round(estimateApiEquivalentCost(topModel, { cold: codeStaleReadTokens }, this.maxTs).total, 2),
|
|
642
|
+
},
|
|
643
|
+
counterfactual: { daysAgo, counterfactualDate: localDay(cfDate), daysGained: avoidableWaitDays, targetScore: Math.min(100, cleanlinessScore + 30) },
|
|
644
|
+
};
|
|
645
|
+
}
|
|
646
|
+
buildContextWindowSummary(topModel, staleShare) {
|
|
647
|
+
const latest = this.latestContextObservation;
|
|
648
|
+
const latestInputTokens = latest?.inputTokens ?? 0;
|
|
649
|
+
const currentResidentWasteTokens = Math.round(latestInputTokens * Math.max(0, staleShare));
|
|
650
|
+
return {
|
|
651
|
+
source: latest ? "latest-observed-input" : "unavailable",
|
|
652
|
+
model: latest?.model || topModel || "unknown",
|
|
653
|
+
modelContextLimitSource: latest?.modelContextLimitSource || "unavailable",
|
|
654
|
+
sampleCount: this.contextObservationSeq,
|
|
655
|
+
staleReadSharePct: Math.round(staleShare * 100),
|
|
656
|
+
metrics: calculateContextMetrics({
|
|
657
|
+
latestInputTokens,
|
|
658
|
+
modelContextLimitTokens: latest?.modelContextLimitTokens ?? null,
|
|
659
|
+
currentResidentWasteTokens,
|
|
660
|
+
newOutputThisTurnTokens: latest?.outputTokens,
|
|
661
|
+
}),
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
// ---------------------------------------------------------------------------
|
|
666
|
+
// Source adapters — translate each log schema into engine calls
|
|
667
|
+
// ---------------------------------------------------------------------------
|
|
668
|
+
function sessionTitleFromText(text) {
|
|
669
|
+
const normalized = text.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
670
|
+
if (!normalized)
|
|
671
|
+
return null;
|
|
672
|
+
const sentence = normalized.split(/(?<=[.!?。!?])\s/)[0] || normalized;
|
|
673
|
+
return sentence.slice(0, 96);
|
|
674
|
+
}
|
|
675
|
+
/** Parse one Claude Code transcript without mutating global state. Replaying every provider on one
|
|
676
|
+
* timestamp-ordered timeline prevents a later-scanned file/source from contaminating reread order. */
|
|
677
|
+
function extractClaude(file) {
|
|
678
|
+
const usageByRequest = new Map();
|
|
679
|
+
const seenToolUses = new Set();
|
|
680
|
+
const context = [];
|
|
681
|
+
const ev = [];
|
|
682
|
+
let cwd = null;
|
|
683
|
+
let session = null;
|
|
684
|
+
let firstTs = null;
|
|
685
|
+
let lastTs = null;
|
|
686
|
+
let fallbackTitle = null;
|
|
687
|
+
let anonymousRequestSeq = 0;
|
|
688
|
+
const requestKey = (row) => {
|
|
689
|
+
const messageId = typeof row.message?.id === "string" ? row.message.id.trim() : "";
|
|
690
|
+
if (messageId)
|
|
691
|
+
return `message:${messageId}`;
|
|
692
|
+
const requestId = typeof row.requestId === "string" ? row.requestId.trim() : "";
|
|
693
|
+
if (requestId)
|
|
694
|
+
return `request:${requestId}`;
|
|
695
|
+
const uuid = typeof row.uuid === "string" ? row.uuid.trim() : "";
|
|
696
|
+
if (uuid)
|
|
697
|
+
return `event:${uuid}`;
|
|
698
|
+
// Without a provider identifier it is unsafe to assume two rows are the same request.
|
|
699
|
+
return `anonymous:${++anonymousRequestSeq}`;
|
|
700
|
+
};
|
|
701
|
+
const usageNumber = (value) => (typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : 0);
|
|
702
|
+
const toolUseKey = (request, block, index) => {
|
|
703
|
+
const id = typeof block?.id === "string" ? block.id.trim() : "";
|
|
704
|
+
if (id)
|
|
705
|
+
return `id:${id}`;
|
|
706
|
+
let input = "";
|
|
707
|
+
try {
|
|
708
|
+
input = JSON.stringify(block?.input ?? null);
|
|
709
|
+
}
|
|
710
|
+
catch {
|
|
711
|
+
input = "[unserializable]";
|
|
712
|
+
}
|
|
713
|
+
// Claude tool_use blocks normally have an id. The positional fingerprint is a conservative
|
|
714
|
+
// fallback that deduplicates repeated snapshots without collapsing two calls in one request.
|
|
715
|
+
return `fallback:${request}:${index}:${String(block?.name || "")}:${input}`;
|
|
716
|
+
};
|
|
717
|
+
eachLine(file, (o) => {
|
|
718
|
+
const ts = typeof o.timestamp === "string" ? Date.parse(o.timestamp) : NaN;
|
|
719
|
+
if (Number.isFinite(ts)) {
|
|
720
|
+
if (firstTs == null || ts < firstTs)
|
|
721
|
+
firstTs = ts;
|
|
722
|
+
if (lastTs == null || ts > lastTs)
|
|
723
|
+
lastTs = ts;
|
|
724
|
+
}
|
|
725
|
+
if (!session && typeof o.sessionId === "string")
|
|
726
|
+
session = o.sessionId;
|
|
727
|
+
if (!cwd && typeof o.cwd === "string")
|
|
728
|
+
cwd = o.cwd;
|
|
729
|
+
if (o.type === "assistant" && o.message) {
|
|
730
|
+
const key = requestKey(o);
|
|
731
|
+
const u = o.message.usage || {};
|
|
732
|
+
const model = typeof o.message.model === "string" && o.message.model ? o.message.model : "unknown";
|
|
733
|
+
const existing = usageByRequest.get(key);
|
|
734
|
+
const merged = existing || {
|
|
735
|
+
model,
|
|
736
|
+
cwd,
|
|
737
|
+
session,
|
|
738
|
+
ms: Number.isFinite(ts) ? ts : null,
|
|
739
|
+
cold: 0,
|
|
740
|
+
cacheWrite: 0,
|
|
741
|
+
cacheRead: 0,
|
|
742
|
+
output: 0,
|
|
743
|
+
};
|
|
744
|
+
const candidateCold = usageNumber(u.input_tokens);
|
|
745
|
+
const candidateCacheWrite = usageNumber(u.cache_creation_input_tokens);
|
|
746
|
+
const candidateCacheRead = usageNumber(u.cache_read_input_tokens);
|
|
747
|
+
const candidateInput = candidateCold + candidateCacheWrite + candidateCacheRead;
|
|
748
|
+
const mergedInput = merged.cold + merged.cacheWrite + merged.cacheRead;
|
|
749
|
+
if (candidateInput > mergedInput) {
|
|
750
|
+
// Keep the three input components from one provider snapshot. Taking their independent
|
|
751
|
+
// maxima can synthesize an official-input total that never appeared in the transcript.
|
|
752
|
+
merged.cold = candidateCold;
|
|
753
|
+
merged.cacheWrite = candidateCacheWrite;
|
|
754
|
+
merged.cacheRead = candidateCacheRead;
|
|
755
|
+
}
|
|
756
|
+
merged.output = Math.max(merged.output, usageNumber(u.output_tokens));
|
|
757
|
+
if (merged.model === "unknown" && model !== "unknown")
|
|
758
|
+
merged.model = model;
|
|
759
|
+
if (!merged.cwd && cwd)
|
|
760
|
+
merged.cwd = cwd;
|
|
761
|
+
if (!merged.session && session)
|
|
762
|
+
merged.session = session;
|
|
763
|
+
if (Number.isFinite(ts) && (merged.ms == null || ts >= merged.ms))
|
|
764
|
+
merged.ms = ts;
|
|
765
|
+
usageByRequest.set(key, merged);
|
|
766
|
+
const blocks = Array.isArray(o.message.content) ? o.message.content : [];
|
|
767
|
+
for (let index = 0; index < blocks.length; index += 1) {
|
|
768
|
+
const b = blocks[index];
|
|
769
|
+
if (b?.type !== "tool_use")
|
|
770
|
+
continue;
|
|
771
|
+
const name = String(b.name || "");
|
|
772
|
+
const toolKey = toolUseKey(key, b, index);
|
|
773
|
+
if (seenToolUses.has(toolKey))
|
|
774
|
+
continue;
|
|
775
|
+
if (name === "Edit" || name === "Write" || name === "MultiEdit") {
|
|
776
|
+
const target = b.input?.file_path || b.input?.path;
|
|
777
|
+
if (target) {
|
|
778
|
+
ev.push({ t: "e", path: target, ms: Number.isFinite(ts) ? ts : null });
|
|
779
|
+
seenToolUses.add(toolKey);
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
else if (name === "Read") {
|
|
783
|
+
const target = b.input?.file_path;
|
|
784
|
+
if (target) {
|
|
785
|
+
ev.push({ t: "r", path: target, ms: Number.isFinite(ts) ? ts : null, id: b.id || null });
|
|
786
|
+
seenToolUses.add(toolKey);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
return;
|
|
791
|
+
}
|
|
792
|
+
if (o.type === "user" && o.message) {
|
|
793
|
+
const content = o.message.content;
|
|
794
|
+
if (typeof content === "string") {
|
|
795
|
+
if (isRealUserText(content) && Number.isFinite(ts) && !o.isSidechain) {
|
|
796
|
+
ev.push({ t: "m", ms: ts });
|
|
797
|
+
if (!fallbackTitle)
|
|
798
|
+
fallbackTitle = sessionTitleFromText(content);
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
else if (Array.isArray(content)) {
|
|
802
|
+
for (const b of content) {
|
|
803
|
+
if (b?.type === "tool_result" && b.tool_use_id) {
|
|
804
|
+
ev.push({ t: "o", id: b.tool_use_id, chars: contentLength(b.content), ms: Number.isFinite(ts) ? ts : null });
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
});
|
|
810
|
+
const usage = [];
|
|
811
|
+
// Claude can persist several snapshots of the same provider message. Account official usage once,
|
|
812
|
+
// retaining the one input tuple with the largest official total.
|
|
813
|
+
for (const request of usageByRequest.values()) {
|
|
814
|
+
usage.push({
|
|
815
|
+
model: request.model,
|
|
816
|
+
cold: request.cold,
|
|
817
|
+
cacheWrite: request.cacheWrite,
|
|
818
|
+
cacheRead: request.cacheRead,
|
|
819
|
+
output: request.output,
|
|
820
|
+
ms: request.ms,
|
|
821
|
+
});
|
|
822
|
+
context.push({
|
|
823
|
+
model: request.model,
|
|
824
|
+
inputTokens: request.cold + request.cacheWrite + request.cacheRead,
|
|
825
|
+
outputTokens: request.output,
|
|
826
|
+
ms: request.ms,
|
|
827
|
+
});
|
|
828
|
+
}
|
|
829
|
+
return { source: "claude-code", session, cwd, firstTs, lastTs, fallbackTitle, usage, context, ev };
|
|
830
|
+
}
|
|
831
|
+
/** Parse one Codex rollout into a compact, cacheable FileEvents. token_count is CUMULATIVE (delta'd);
|
|
832
|
+
* edits arrive via patch_apply_end; reads are exec_command shell commands (path extracted best-effort). */
|
|
833
|
+
function extractCodex(file) {
|
|
834
|
+
let cwd = null;
|
|
835
|
+
let session = null;
|
|
836
|
+
// Default to a Codex/OpenAI model so token counts logged before the model field appears are priced
|
|
837
|
+
// as OpenAI (not the Claude opus default) — the real model overrides this as soon as it's seen.
|
|
838
|
+
let model = "gpt-5-codex";
|
|
839
|
+
let prev = { cold: 0, cacheRead: 0, output: 0 }; // last cumulative seen (for deltas)
|
|
840
|
+
let firstTs = null;
|
|
841
|
+
let lastTs = null;
|
|
842
|
+
let fallbackTitle = null;
|
|
843
|
+
const usage = [];
|
|
844
|
+
const context = [];
|
|
845
|
+
const ev = [];
|
|
846
|
+
eachLine(file, (o) => {
|
|
847
|
+
const ts = typeof o.timestamp === "string" ? Date.parse(o.timestamp) : NaN;
|
|
848
|
+
if (Number.isFinite(ts)) {
|
|
849
|
+
if (firstTs == null || ts < firstTs)
|
|
850
|
+
firstTs = ts;
|
|
851
|
+
if (lastTs == null || ts > lastTs)
|
|
852
|
+
lastTs = ts;
|
|
853
|
+
}
|
|
854
|
+
const p = o && typeof o.payload === "object" && o.payload ? o.payload : o;
|
|
855
|
+
if (!p || typeof p !== "object")
|
|
856
|
+
return;
|
|
857
|
+
if (o.type === "session_meta") {
|
|
858
|
+
if (typeof p.cwd === "string")
|
|
859
|
+
cwd = p.cwd;
|
|
860
|
+
if (typeof p.id === "string" && !session)
|
|
861
|
+
session = p.id;
|
|
862
|
+
// Codex records the actual launch time in session metadata. It can precede the first persisted
|
|
863
|
+
// JSONL event by several seconds, so use it for the semantic start of the timeline when present.
|
|
864
|
+
const startedAt = typeof p.timestamp === "string" ? Date.parse(p.timestamp) : NaN;
|
|
865
|
+
if (Number.isFinite(startedAt) && (firstTs == null || startedAt < firstTs))
|
|
866
|
+
firstTs = startedAt;
|
|
867
|
+
return;
|
|
868
|
+
}
|
|
869
|
+
if (typeof p.model === "string")
|
|
870
|
+
model = p.model;
|
|
871
|
+
if (p.type === "token_count") {
|
|
872
|
+
const u = p.info?.total_token_usage;
|
|
873
|
+
if (u) {
|
|
874
|
+
const inputTotal = u.input_tokens || 0;
|
|
875
|
+
const cacheRead = u.cached_input_tokens || 0;
|
|
876
|
+
const cold = Math.max(0, inputTotal - cacheRead);
|
|
877
|
+
const out = (u.output_tokens || 0) + (u.reasoning_output_tokens || 0);
|
|
878
|
+
const dCold = Math.max(0, cold - prev.cold);
|
|
879
|
+
const dCr = Math.max(0, cacheRead - prev.cacheRead);
|
|
880
|
+
const dOut = Math.max(0, out - prev.output);
|
|
881
|
+
const last = p.info?.last_token_usage || {};
|
|
882
|
+
const lastInput = last.input_tokens || inputTotal;
|
|
883
|
+
const lastOutput = (last.output_tokens || 0) + (last.reasoning_output_tokens || 0) || dOut;
|
|
884
|
+
context.push({
|
|
885
|
+
model,
|
|
886
|
+
inputTokens: lastInput,
|
|
887
|
+
outputTokens: lastOutput,
|
|
888
|
+
modelContextLimitTokens: p.info?.model_context_window || p.model_context_window || null,
|
|
889
|
+
ms: Number.isFinite(ts) ? ts : null,
|
|
890
|
+
});
|
|
891
|
+
if (dCold || dCr || dOut) {
|
|
892
|
+
usage.push({
|
|
893
|
+
model,
|
|
894
|
+
cold: dCold,
|
|
895
|
+
cacheWrite: 0,
|
|
896
|
+
cacheRead: dCr,
|
|
897
|
+
output: dOut,
|
|
898
|
+
ms: Number.isFinite(ts) ? ts : null,
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
prev = { cold, cacheRead, output: out };
|
|
902
|
+
}
|
|
903
|
+
return;
|
|
904
|
+
}
|
|
905
|
+
if (p.type === "user_message") {
|
|
906
|
+
if (Number.isFinite(ts))
|
|
907
|
+
ev.push({ t: "m", ms: ts });
|
|
908
|
+
if (!fallbackTitle) {
|
|
909
|
+
const text = typeof p.message === "string" ? p.message : typeof p.content === "string" ? p.content : "";
|
|
910
|
+
if (text)
|
|
911
|
+
fallbackTitle = sessionTitleFromText(text);
|
|
912
|
+
}
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
if (p.type === "patch_apply_end" && p.changes && typeof p.changes === "object") {
|
|
916
|
+
for (const target of Object.keys(p.changes))
|
|
917
|
+
ev.push({ t: "e", path: target, ms: Number.isFinite(ts) ? ts : null });
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
// older format: exec_command_end with parsed_cmd[].type==='read'
|
|
921
|
+
if (p.type === "exec_command_end" && Array.isArray(p.parsed_cmd)) {
|
|
922
|
+
for (const c of p.parsed_cmd) {
|
|
923
|
+
if (c?.type === "read" && typeof c.path === "string")
|
|
924
|
+
ev.push({ t: "r", path: c.path, ms: Number.isFinite(ts) ? ts : null, id: p.call_id || null });
|
|
925
|
+
}
|
|
926
|
+
return;
|
|
927
|
+
}
|
|
928
|
+
// newer format: reads are exec_command function_calls; extract the path best-effort
|
|
929
|
+
if (p.type === "function_call" && (p.name === "exec_command" || p.name === "shell")) {
|
|
930
|
+
try {
|
|
931
|
+
const args = JSON.parse(typeof p.arguments === "string" ? p.arguments : "{}");
|
|
932
|
+
const target = typeof args.cmd === "string" ? extractReadPath(args.cmd) : null;
|
|
933
|
+
if (target)
|
|
934
|
+
ev.push({ t: "r", path: target, ms: Number.isFinite(ts) ? ts : null, id: p.call_id || null });
|
|
935
|
+
}
|
|
936
|
+
catch {
|
|
937
|
+
/* skip */
|
|
938
|
+
}
|
|
939
|
+
return;
|
|
940
|
+
}
|
|
941
|
+
if (p.type === "function_call_output" && p.call_id) {
|
|
942
|
+
ev.push({ t: "o", id: p.call_id, chars: contentLength(p.output), ms: Number.isFinite(ts) ? ts : null });
|
|
943
|
+
}
|
|
944
|
+
});
|
|
945
|
+
return { source: "codex", session, cwd, firstTs, lastTs, fallbackTitle, usage, context, ev };
|
|
946
|
+
}
|
|
947
|
+
/** Replay every provider on one semantic timeline. Provider timestamps, not source/path order, drive
|
|
948
|
+
* daily token buckets and order-dependent edit/reread state across overlapping sessions. */
|
|
949
|
+
function replayTimelineFiles(eng, files) {
|
|
950
|
+
const orderedEvents = [];
|
|
951
|
+
const orderedUsage = [];
|
|
952
|
+
files.forEach(({ fe }, fileOrder) => {
|
|
953
|
+
if (fe.firstTs != null)
|
|
954
|
+
eng.noteTs(fe.firstTs);
|
|
955
|
+
if (fe.lastTs != null)
|
|
956
|
+
eng.noteTs(fe.lastTs);
|
|
957
|
+
eng.noteSession(fe.session);
|
|
958
|
+
(fe.usage || []).forEach((u, usageOrder) => orderedUsage.push({ fe, u, fileOrder, usageOrder }));
|
|
959
|
+
for (const c of fe.context || [])
|
|
960
|
+
eng.recordContextObservation(c);
|
|
961
|
+
(fe.ev || []).forEach((e, eventOrder) => orderedEvents.push({ fe, e, fileOrder, eventOrder }));
|
|
962
|
+
});
|
|
963
|
+
orderedUsage.sort((a, b) => (a.u.ms ?? a.fe.firstTs ?? Number.POSITIVE_INFINITY) - (b.u.ms ?? b.fe.firstTs ?? Number.POSITIVE_INFINITY) ||
|
|
964
|
+
a.fileOrder - b.fileOrder ||
|
|
965
|
+
a.usageOrder - b.usageOrder);
|
|
966
|
+
for (const { fe, u } of orderedUsage) {
|
|
967
|
+
const provider = fe.source === "codex" ? "codex" : "claude";
|
|
968
|
+
eng.recordUsage(u.model, fe.cwd, fe.session, { cold: u.cold, cacheWrite: u.cacheWrite, cacheRead: u.cacheRead, output: u.output }, provider, u.ms ?? undefined);
|
|
969
|
+
}
|
|
970
|
+
const eventMs = (item) => item.e.ms ?? item.fe.firstTs ?? Number.POSITIVE_INFINITY;
|
|
971
|
+
orderedEvents.sort((a, b) => eventMs(a) - eventMs(b) ||
|
|
972
|
+
a.fileOrder - b.fileOrder ||
|
|
973
|
+
a.eventOrder - b.eventOrder);
|
|
974
|
+
for (const { fe, e } of orderedEvents) {
|
|
975
|
+
const callId = (id) => id ? `${fe.source}:${fe.session || "unknown"}:${id}` : null;
|
|
976
|
+
if (e.t === "m")
|
|
977
|
+
eng.recordUserMsg(fe.cwd, fe.session, e.ms);
|
|
978
|
+
else if (e.t === "e")
|
|
979
|
+
eng.recordEdit(e.path, fe.cwd);
|
|
980
|
+
else if (e.t === "r")
|
|
981
|
+
eng.recordRead(e.path, fe.cwd, fe.session, e.ms, callId(e.id));
|
|
982
|
+
else if (e.t === "o")
|
|
983
|
+
eng.recordReadOutput(callId(e.id) || e.id, e.chars);
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
// ---------------------------------------------------------------------------
|
|
987
|
+
// Per-file parse cache — keyed by path+mtime+size so re-runs only reparse changed/new Codex files.
|
|
988
|
+
// ---------------------------------------------------------------------------
|
|
989
|
+
const CACHE_VERSION = 6;
|
|
990
|
+
function forensicCachePath() {
|
|
991
|
+
return path.join(os.homedir(), ".echomem", "forensic-cache.json");
|
|
992
|
+
}
|
|
993
|
+
function loadForensicCache() {
|
|
994
|
+
try {
|
|
995
|
+
const parsed = JSON.parse(fs.readFileSync(forensicCachePath(), "utf8"));
|
|
996
|
+
if (parsed && parsed.v === CACHE_VERSION && parsed.files && typeof parsed.files === "object")
|
|
997
|
+
return parsed.files;
|
|
998
|
+
}
|
|
999
|
+
catch {
|
|
1000
|
+
/* no cache yet, or a stale/corrupt one — rebuild */
|
|
1001
|
+
}
|
|
1002
|
+
return {};
|
|
1003
|
+
}
|
|
1004
|
+
function saveForensicCache(files) {
|
|
1005
|
+
try {
|
|
1006
|
+
const p = forensicCachePath();
|
|
1007
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
1008
|
+
fs.writeFileSync(p, JSON.stringify({ v: CACHE_VERSION, files }));
|
|
1009
|
+
}
|
|
1010
|
+
catch {
|
|
1011
|
+
/* best effort: a cache write failure must never break the scan */
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
function loadCodexSessionTitles() {
|
|
1015
|
+
const titles = new Map();
|
|
1016
|
+
try {
|
|
1017
|
+
const indexPath = path.join(os.homedir(), ".codex", "session_index.jsonl");
|
|
1018
|
+
for (const line of fs.readFileSync(indexPath, "utf8").split("\n")) {
|
|
1019
|
+
if (!line.trim())
|
|
1020
|
+
continue;
|
|
1021
|
+
try {
|
|
1022
|
+
const row = JSON.parse(line);
|
|
1023
|
+
if (typeof row.id === "string" && typeof row.thread_name === "string" && row.thread_name.trim()) {
|
|
1024
|
+
titles.set(row.id, row.thread_name.trim());
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
catch {
|
|
1028
|
+
/* Ignore incomplete index rows. */
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
catch {
|
|
1033
|
+
/* Index is optional; transcript-derived titles remain available. */
|
|
1034
|
+
}
|
|
1035
|
+
return titles;
|
|
1036
|
+
}
|
|
1037
|
+
function loadClaudeSessionTitles() {
|
|
1038
|
+
const titles = new Map();
|
|
1039
|
+
const roots = [
|
|
1040
|
+
path.join(os.homedir(), "Library", "Application Support", "Claude", "claude-code-sessions"),
|
|
1041
|
+
path.join(os.homedir(), "Library", "Application Support", "Claude", "local-agent-mode-sessions"),
|
|
1042
|
+
];
|
|
1043
|
+
for (const root of roots) {
|
|
1044
|
+
const files = walk(root, (p) => /^local_[0-9a-f-]+\.json$/i.test(path.basename(p)), () => false);
|
|
1045
|
+
for (const file of files) {
|
|
1046
|
+
try {
|
|
1047
|
+
const row = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
1048
|
+
if (typeof row.cliSessionId === "string" && typeof row.title === "string" && row.title.trim()) {
|
|
1049
|
+
titles.set(row.cliSessionId, row.title.trim());
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
catch {
|
|
1053
|
+
/* Ignore incomplete live metadata. */
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
return titles;
|
|
1058
|
+
}
|
|
1059
|
+
function enrichCanonicalSessionExamples(canonical, sessionRecords) {
|
|
1060
|
+
if (!canonical || "error" in canonical)
|
|
1061
|
+
return;
|
|
1062
|
+
const codexTitles = loadCodexSessionTitles();
|
|
1063
|
+
const claudeTitles = loadClaudeSessionTitles();
|
|
1064
|
+
const records = sessionRecords
|
|
1065
|
+
.filter((record) => Boolean(record.session))
|
|
1066
|
+
.sort((a, b) => (a.firstTs ?? Number.POSITIVE_INFINITY) - (b.firstTs ?? Number.POSITIVE_INFINITY));
|
|
1067
|
+
const sourceTotals = {
|
|
1068
|
+
codex: records.filter((record) => record.source === "codex").length,
|
|
1069
|
+
"claude-code": records.filter((record) => record.source === "claude-code").length,
|
|
1070
|
+
};
|
|
1071
|
+
const sourceOrdinals = { codex: 0, "claude-code": 0 };
|
|
1072
|
+
const sessionMeta = new Map();
|
|
1073
|
+
for (const record of records) {
|
|
1074
|
+
if (!record.session)
|
|
1075
|
+
continue;
|
|
1076
|
+
sourceOrdinals[record.source] += 1;
|
|
1077
|
+
const indexedTitle = record.source === "codex" ? codexTitles.get(record.session) : claudeTitles.get(record.session);
|
|
1078
|
+
sessionMeta.set(record.session, {
|
|
1079
|
+
id: record.session,
|
|
1080
|
+
title: indexedTitle || record.fallbackTitle || `${record.cwd ? path.basename(record.cwd) : "Local"} session`,
|
|
1081
|
+
ordinal: sourceOrdinals[record.source],
|
|
1082
|
+
total: sourceTotals[record.source],
|
|
1083
|
+
});
|
|
1084
|
+
}
|
|
1085
|
+
const reports = [canonical];
|
|
1086
|
+
if (canonical.sourceReports?.codex)
|
|
1087
|
+
reports.push(canonical.sourceReports.codex);
|
|
1088
|
+
if (canonical.sourceReports?.claudeCode)
|
|
1089
|
+
reports.push(canonical.sourceReports.claudeCode);
|
|
1090
|
+
for (const contextReport of reports) {
|
|
1091
|
+
for (const problem of contextReport.problems || []) {
|
|
1092
|
+
for (const example of problem.examples || []) {
|
|
1093
|
+
const exact = sessionMeta.get(example.session);
|
|
1094
|
+
const matched = exact || [...sessionMeta.entries()].find(([id]) => id.startsWith(example.session))?.[1];
|
|
1095
|
+
if (!matched)
|
|
1096
|
+
continue;
|
|
1097
|
+
example.session = matched.id;
|
|
1098
|
+
example.sessionTitle = matched.title;
|
|
1099
|
+
example.sessionOrdinal = matched.ordinal;
|
|
1100
|
+
example.sourceSessionCount = matched.total;
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
export function projectCanonicalWasteToBilledInput(billedInputTokens, summary) {
|
|
1106
|
+
if (!Number.isFinite(billedInputTokens) ||
|
|
1107
|
+
!Number.isFinite(summary.officialInputTokens) ||
|
|
1108
|
+
!Number.isFinite(summary.wasteTokens))
|
|
1109
|
+
return null;
|
|
1110
|
+
const billedInput = Math.round(billedInputTokens);
|
|
1111
|
+
const canonicalInput = Math.round(summary.officialInputTokens);
|
|
1112
|
+
const canonicalWaste = Math.round(summary.wasteTokens);
|
|
1113
|
+
// With aligned source scope, all provider requests must contain at least as many input tokens as
|
|
1114
|
+
// one selected request per human turn. Refuse to manufacture a projection from inverted ledgers.
|
|
1115
|
+
if (billedInput <= 0 ||
|
|
1116
|
+
canonicalInput <= 0 ||
|
|
1117
|
+
canonicalWaste < 0 ||
|
|
1118
|
+
canonicalWaste > canonicalInput ||
|
|
1119
|
+
billedInput < canonicalInput)
|
|
1120
|
+
return null;
|
|
1121
|
+
const billingMultiplier = billedInput / canonicalInput;
|
|
1122
|
+
const projectedWaste = Math.max(0, Math.min(billedInput, Math.round(canonicalWaste * billingMultiplier)));
|
|
1123
|
+
const wastePct = (canonicalWaste / canonicalInput) * 100;
|
|
1124
|
+
return {
|
|
1125
|
+
method: "canonical-window-share-over-provider-input",
|
|
1126
|
+
billedInputTokens: billedInput,
|
|
1127
|
+
canonicalInputTokens: canonicalInput,
|
|
1128
|
+
canonicalUsefulTokens: canonicalInput - canonicalWaste,
|
|
1129
|
+
canonicalWasteTokens: canonicalWaste,
|
|
1130
|
+
billingMultiplier,
|
|
1131
|
+
projectedUsefulTokens: billedInput - projectedWaste,
|
|
1132
|
+
projectedWasteTokens: projectedWaste,
|
|
1133
|
+
usefulPct: 100 - wastePct,
|
|
1134
|
+
wastePct,
|
|
1135
|
+
};
|
|
1136
|
+
}
|
|
1137
|
+
function recordValue(value) {
|
|
1138
|
+
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
1139
|
+
? value
|
|
1140
|
+
: null;
|
|
1141
|
+
}
|
|
1142
|
+
function nonNegativeInteger(value) {
|
|
1143
|
+
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value) && value >= 0
|
|
1144
|
+
? value
|
|
1145
|
+
: null;
|
|
1146
|
+
}
|
|
1147
|
+
function nonNegativeNumber(value) {
|
|
1148
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
|
|
1149
|
+
}
|
|
1150
|
+
function approximatelyEqual(actual, expected, relativeTolerance = 1e-9) {
|
|
1151
|
+
return typeof actual === "number" &&
|
|
1152
|
+
Number.isFinite(actual) &&
|
|
1153
|
+
Math.abs(actual - expected) <= Math.max(1, Math.abs(expected)) * relativeTolerance;
|
|
1154
|
+
}
|
|
1155
|
+
function invalidSetupReport(code, message) {
|
|
1156
|
+
return { ok: false, code, message };
|
|
1157
|
+
}
|
|
1158
|
+
/**
|
|
1159
|
+
* Fail-closed boundary between the local scanner and setup UI.
|
|
1160
|
+
*
|
|
1161
|
+
* The setup page must never infer that a fixture, unreconciled scan, or internally inconsistent
|
|
1162
|
+
* token ledger is the user's report. This validator checks provenance and the cross-ledger
|
|
1163
|
+
* identities the UI relies on before any absolute token number is rendered.
|
|
1164
|
+
*/
|
|
1165
|
+
export function validateForensicReportForSetup(value) {
|
|
1166
|
+
const report = recordValue(value);
|
|
1167
|
+
if (!report)
|
|
1168
|
+
return invalidSetupReport("REPORT_MALFORMED", "The local report payload is not an object.");
|
|
1169
|
+
if (report.schemaVersion !== 1) {
|
|
1170
|
+
return invalidSetupReport("REPORT_SCHEMA_UNSUPPORTED", "The local report schema is missing or unsupported.");
|
|
1171
|
+
}
|
|
1172
|
+
if (report.dataOrigin !== "local-workspace-scan") {
|
|
1173
|
+
return invalidSetupReport("REPORT_NOT_LOCAL", "The report did not come from this local workspace scan.");
|
|
1174
|
+
}
|
|
1175
|
+
const scale = recordValue(report.scale);
|
|
1176
|
+
const canonical = recordValue(report.canonicalGoldenStandard);
|
|
1177
|
+
const requiredObjects = [
|
|
1178
|
+
report.cost,
|
|
1179
|
+
report.modelUsage,
|
|
1180
|
+
report.userWorkingHours,
|
|
1181
|
+
report.rhythm,
|
|
1182
|
+
report.cleanliness,
|
|
1183
|
+
report.contextWindow,
|
|
1184
|
+
report.rereadForensics,
|
|
1185
|
+
report.avoidableWait,
|
|
1186
|
+
report.counterfactual,
|
|
1187
|
+
];
|
|
1188
|
+
if (!scale ||
|
|
1189
|
+
!canonical ||
|
|
1190
|
+
!Array.isArray(report.generatedFrom) ||
|
|
1191
|
+
!Array.isArray(report.repos) ||
|
|
1192
|
+
requiredObjects.some((entry) => !recordValue(entry)) ||
|
|
1193
|
+
typeof report.llmCallsUsed !== "number" ||
|
|
1194
|
+
!Number.isFinite(report.llmCallsUsed) ||
|
|
1195
|
+
typeof report.transcriptsUploaded !== "boolean" ||
|
|
1196
|
+
!(report.scanStartDate === null || typeof report.scanStartDate === "string") ||
|
|
1197
|
+
!(report.scanEndDate === null || typeof report.scanEndDate === "string")) {
|
|
1198
|
+
return invalidSetupReport("REPORT_MALFORMED", "The local report is missing required setup fields.");
|
|
1199
|
+
}
|
|
1200
|
+
if (Object.prototype.hasOwnProperty.call(canonical, "error")) {
|
|
1201
|
+
return invalidSetupReport("REPORT_CANONICAL_FAILED", "Canonical context analysis did not complete.");
|
|
1202
|
+
}
|
|
1203
|
+
const summary = recordValue(canonical.summary);
|
|
1204
|
+
if (!summary)
|
|
1205
|
+
return invalidSetupReport("REPORT_MALFORMED", "Canonical context summary is missing.");
|
|
1206
|
+
const sessionCount = nonNegativeInteger(scale.sessionCount);
|
|
1207
|
+
const canonicalSessionCount = nonNegativeInteger(summary.sessionsAnalyzed);
|
|
1208
|
+
const billedInput = nonNegativeInteger(scale.totalInputTokens);
|
|
1209
|
+
const totalTokens = nonNegativeInteger(scale.totalTokens);
|
|
1210
|
+
const coldInput = nonNegativeInteger(scale.coldInputTokens);
|
|
1211
|
+
const cacheWrite = nonNegativeInteger(scale.cacheWriteTokens);
|
|
1212
|
+
const cacheRead = nonNegativeInteger(scale.cacheReadTokens);
|
|
1213
|
+
const output = nonNegativeInteger(scale.outputTokens);
|
|
1214
|
+
const canonicalInput = nonNegativeInteger(summary.officialInputTokens);
|
|
1215
|
+
const canonicalUseful = nonNegativeInteger(summary.usefulTokens);
|
|
1216
|
+
const canonicalWaste = nonNegativeInteger(summary.wasteTokens);
|
|
1217
|
+
if (sessionCount === null ||
|
|
1218
|
+
canonicalSessionCount === null ||
|
|
1219
|
+
billedInput === null ||
|
|
1220
|
+
totalTokens === null ||
|
|
1221
|
+
coldInput === null ||
|
|
1222
|
+
cacheWrite === null ||
|
|
1223
|
+
cacheRead === null ||
|
|
1224
|
+
output === null ||
|
|
1225
|
+
canonicalInput === null ||
|
|
1226
|
+
canonicalUseful === null ||
|
|
1227
|
+
canonicalWaste === null) {
|
|
1228
|
+
return invalidSetupReport("REPORT_USAGE_INVALID", "The report contains invalid token or session totals.");
|
|
1229
|
+
}
|
|
1230
|
+
if (coldInput + cacheWrite + cacheRead !== billedInput || billedInput + output !== totalTokens) {
|
|
1231
|
+
return invalidSetupReport("REPORT_USAGE_INVALID", "Provider token totals do not reconcile.");
|
|
1232
|
+
}
|
|
1233
|
+
const reportCost = recordValue(report.cost);
|
|
1234
|
+
const costByModel = reportCost ? recordValue(reportCost.byModel) : null;
|
|
1235
|
+
const coldInputCost = reportCost ? nonNegativeNumber(reportCost.coldInputCost) : null;
|
|
1236
|
+
const cacheWriteCost = reportCost ? nonNegativeNumber(reportCost.cacheWriteCost) : null;
|
|
1237
|
+
const cacheReadCost = reportCost ? nonNegativeNumber(reportCost.cacheReadCost) : null;
|
|
1238
|
+
const outputCost = reportCost ? nonNegativeNumber(reportCost.outputCost) : null;
|
|
1239
|
+
const totalCost = reportCost ? nonNegativeNumber(reportCost.total) : null;
|
|
1240
|
+
if (!reportCost || !costByModel || coldInputCost === null || cacheWriteCost === null ||
|
|
1241
|
+
cacheReadCost === null || outputCost === null || totalCost === null) {
|
|
1242
|
+
return invalidSetupReport("REPORT_COST_INVALID", "The API-equivalent cost ledger is missing or malformed.");
|
|
1243
|
+
}
|
|
1244
|
+
const componentCost = coldInputCost + cacheWriteCost + cacheReadCost + outputCost;
|
|
1245
|
+
if (Math.abs(componentCost - totalCost) > 0.05) {
|
|
1246
|
+
return invalidSetupReport("REPORT_COST_INVALID", "Cost components do not reconcile with total spend.");
|
|
1247
|
+
}
|
|
1248
|
+
const modelCosts = Object.values(costByModel).map((value) => {
|
|
1249
|
+
const model = recordValue(value);
|
|
1250
|
+
return model ? nonNegativeNumber(model.cost) : null;
|
|
1251
|
+
});
|
|
1252
|
+
if (modelCosts.some((value) => value === null)) {
|
|
1253
|
+
return invalidSetupReport("REPORT_COST_INVALID", "A model cost entry is malformed.");
|
|
1254
|
+
}
|
|
1255
|
+
const modelCostSum = modelCosts.reduce((sum, value) => sum + (value ?? 0), 0);
|
|
1256
|
+
if (Math.abs(modelCostSum - totalCost) > Math.max(0.05, modelCosts.length * 0.011)) {
|
|
1257
|
+
return invalidSetupReport("REPORT_COST_INVALID", "Per-model costs do not reconcile with total spend.");
|
|
1258
|
+
}
|
|
1259
|
+
const repoCosts = report.repos.map((value) => {
|
|
1260
|
+
const repo = recordValue(value);
|
|
1261
|
+
return repo ? nonNegativeNumber(repo.cost) : null;
|
|
1262
|
+
});
|
|
1263
|
+
if (repoCosts.some((value) => value === null)) {
|
|
1264
|
+
return invalidSetupReport("REPORT_COST_INVALID", "A workspace cost entry is malformed.");
|
|
1265
|
+
}
|
|
1266
|
+
const repoCostSum = repoCosts.reduce((sum, value) => sum + (value ?? 0), 0);
|
|
1267
|
+
if (Math.abs(repoCostSum - totalCost) > Math.max(0.05, repoCosts.length * 0.011)) {
|
|
1268
|
+
return invalidSetupReport("REPORT_COST_INVALID", "Per-workspace costs do not reconcile with total spend.");
|
|
1269
|
+
}
|
|
1270
|
+
const diagnostics = canonical.diagnostics === undefined ? null : recordValue(canonical.diagnostics);
|
|
1271
|
+
let skippedCanonicalSessions = 0;
|
|
1272
|
+
if (canonical.diagnostics !== undefined && !diagnostics) {
|
|
1273
|
+
return invalidSetupReport("REPORT_MALFORMED", "Canonical scan diagnostics are malformed.");
|
|
1274
|
+
}
|
|
1275
|
+
if (diagnostics) {
|
|
1276
|
+
const skippedSessions = nonNegativeInteger(diagnostics.skippedSessions);
|
|
1277
|
+
const errors = diagnostics.errors;
|
|
1278
|
+
if (skippedSessions === null || !Array.isArray(errors)) {
|
|
1279
|
+
return invalidSetupReport("REPORT_MALFORMED", "Canonical scan diagnostics are malformed.");
|
|
1280
|
+
}
|
|
1281
|
+
if (skippedSessions !== errors.length) {
|
|
1282
|
+
return invalidSetupReport("REPORT_MALFORMED", "Canonical scan diagnostics do not reconcile.");
|
|
1283
|
+
}
|
|
1284
|
+
skippedCanonicalSessions = skippedSessions;
|
|
1285
|
+
}
|
|
1286
|
+
if (canonicalSessionCount > sessionCount ||
|
|
1287
|
+
canonicalSessionCount + skippedCanonicalSessions !== sessionCount ||
|
|
1288
|
+
(sessionCount > 0 && canonicalSessionCount === 0)) {
|
|
1289
|
+
return invalidSetupReport("REPORT_COHORT_MISMATCH", "Canonical analysis did not retain a usable local-session cohort.");
|
|
1290
|
+
}
|
|
1291
|
+
if (canonicalUseful + canonicalWaste !== canonicalInput) {
|
|
1292
|
+
return invalidSetupReport("REPORT_CANONICAL_INVALID", "Canonical useful and waste totals do not reconcile.");
|
|
1293
|
+
}
|
|
1294
|
+
const canonicalWastePct = canonicalInput > 0 ? (canonicalWaste / canonicalInput) * 100 : 0;
|
|
1295
|
+
const canonicalUsefulPct = canonicalInput > 0 ? (canonicalUseful / canonicalInput) * 100 : 0;
|
|
1296
|
+
if (!approximatelyEqual(summary.wastePct, canonicalWastePct) ||
|
|
1297
|
+
!approximatelyEqual(summary.usefulPct, canonicalUsefulPct)) {
|
|
1298
|
+
return invalidSetupReport("REPORT_CANONICAL_INVALID", "Canonical token percentages do not reconcile with their totals.");
|
|
1299
|
+
}
|
|
1300
|
+
if (sessionCount === 0) {
|
|
1301
|
+
if (totalTokens !== 0 ||
|
|
1302
|
+
billedInput !== 0 ||
|
|
1303
|
+
coldInput !== 0 ||
|
|
1304
|
+
cacheWrite !== 0 ||
|
|
1305
|
+
cacheRead !== 0 ||
|
|
1306
|
+
output !== 0 ||
|
|
1307
|
+
canonicalInput !== 0 ||
|
|
1308
|
+
canonicalUseful !== 0 ||
|
|
1309
|
+
canonicalWaste !== 0) {
|
|
1310
|
+
return invalidSetupReport("REPORT_EMPTY_USAGE_MISMATCH", "An empty scan contains unexpected token usage.");
|
|
1311
|
+
}
|
|
1312
|
+
if (report.billedWasteProjection !== undefined) {
|
|
1313
|
+
return invalidSetupReport("REPORT_PROJECTION_INVALID", "An empty scan must not contain a billed waste projection.");
|
|
1314
|
+
}
|
|
1315
|
+
return { ok: true, kind: "empty", report: report };
|
|
1316
|
+
}
|
|
1317
|
+
const hasRenderableRepo = report.repos.some((value) => {
|
|
1318
|
+
const repo = recordValue(value);
|
|
1319
|
+
return !!repo && typeof repo.name === "string" && repo.name.trim().length > 0 &&
|
|
1320
|
+
nonNegativeInteger(repo.tokens) !== null && Number(repo.tokens) > 0;
|
|
1321
|
+
});
|
|
1322
|
+
if (!hasRenderableRepo) {
|
|
1323
|
+
return invalidSetupReport("REPORT_REPOS_EMPTY", "A non-empty scan has no tokenized local workspace to render.");
|
|
1324
|
+
}
|
|
1325
|
+
if (billedInput <= 0 || canonicalInput <= 0) {
|
|
1326
|
+
return invalidSetupReport("REPORT_USAGE_INVALID", "A non-empty scan must contain positive provider and canonical input totals.");
|
|
1327
|
+
}
|
|
1328
|
+
if (canonicalInput > billedInput) {
|
|
1329
|
+
return invalidSetupReport("REPORT_LEDGER_INVERTED", "Canonical input exceeds provider-billed input for the same sessions.");
|
|
1330
|
+
}
|
|
1331
|
+
const expectedProjection = projectCanonicalWasteToBilledInput(billedInput, {
|
|
1332
|
+
officialInputTokens: canonicalInput,
|
|
1333
|
+
wasteTokens: canonicalWaste,
|
|
1334
|
+
});
|
|
1335
|
+
const projection = recordValue(report.billedWasteProjection);
|
|
1336
|
+
if (!expectedProjection || !projection) {
|
|
1337
|
+
return invalidSetupReport("REPORT_PROJECTION_MISSING", "The billed waste projection is missing for a non-empty scan.");
|
|
1338
|
+
}
|
|
1339
|
+
if (projection.method !== expectedProjection.method ||
|
|
1340
|
+
nonNegativeInteger(projection.billedInputTokens) !== expectedProjection.billedInputTokens ||
|
|
1341
|
+
nonNegativeInteger(projection.canonicalInputTokens) !== expectedProjection.canonicalInputTokens ||
|
|
1342
|
+
nonNegativeInteger(projection.canonicalUsefulTokens) !== expectedProjection.canonicalUsefulTokens ||
|
|
1343
|
+
nonNegativeInteger(projection.canonicalWasteTokens) !== expectedProjection.canonicalWasteTokens ||
|
|
1344
|
+
nonNegativeInteger(projection.projectedUsefulTokens) !== expectedProjection.projectedUsefulTokens ||
|
|
1345
|
+
nonNegativeInteger(projection.projectedWasteTokens) !== expectedProjection.projectedWasteTokens ||
|
|
1346
|
+
!approximatelyEqual(projection.billingMultiplier, expectedProjection.billingMultiplier) ||
|
|
1347
|
+
!approximatelyEqual(projection.usefulPct, expectedProjection.usefulPct) ||
|
|
1348
|
+
!approximatelyEqual(projection.wastePct, expectedProjection.wastePct)) {
|
|
1349
|
+
return invalidSetupReport("REPORT_PROJECTION_INVALID", "The billed waste projection does not reconcile with its source ledgers.");
|
|
1350
|
+
}
|
|
1351
|
+
if (expectedProjection.projectedUsefulTokens + expectedProjection.projectedWasteTokens !== billedInput ||
|
|
1352
|
+
!approximatelyEqual(expectedProjection.projectedWasteTokens / billedInput, canonicalWaste / canonicalInput, 1 / Math.max(1, billedInput))) {
|
|
1353
|
+
return invalidSetupReport("REPORT_PROJECTION_INVALID", "The billed waste projection violates its token-share identity.");
|
|
1354
|
+
}
|
|
1355
|
+
return { ok: true, kind: "ready", report: report };
|
|
1356
|
+
}
|
|
1357
|
+
/** Scan both sources and build the merged forensic report. Local-only, $0, never throws on bad files. */
|
|
1358
|
+
export async function buildForensicReport(opts) {
|
|
1359
|
+
const sources = opts?.sources ?? ["codex", "claude"];
|
|
1360
|
+
let eng = new Forensics();
|
|
1361
|
+
const codexDiscovery = sources.includes("codex")
|
|
1362
|
+
? discoverCodexSessionFiles({ includeArchived: opts?.includeArchivedCodex !== false })
|
|
1363
|
+
: null;
|
|
1364
|
+
const claudeRoot = sources.includes("claude") ? resolveClaudeProjectsDir() : null;
|
|
1365
|
+
// Discovery combines active + archived stores, de-duplicates by stable session identity, and orders
|
|
1366
|
+
// sessions by semantic start time. Canonical human-turn analysis intentionally excludes subagents.
|
|
1367
|
+
const reportCodexSessions = codexDiscovery?.files.filter((file) => file.userInitiated !== false) ?? [];
|
|
1368
|
+
// Keep the canonical denominator on the same top-level-session population as the billed ledger.
|
|
1369
|
+
// Legacy sessions without provenance are already admitted to reportCodexSessions; dropping them
|
|
1370
|
+
// only here creates a small but real numerator/denominator scope mismatch.
|
|
1371
|
+
const canonicalCodexFiles = reportCodexSessions.map((file) => file.path);
|
|
1372
|
+
const codexFiles = reportCodexSessions.map((file) => file.path);
|
|
1373
|
+
// Claude stores subagent/workflow implementation logs separately. They can share a parent
|
|
1374
|
+
// session ID and must not be counted as additional user conversations.
|
|
1375
|
+
const claudeFiles = claudeRoot
|
|
1376
|
+
? walk(claudeRoot, (p) => p.endsWith(".jsonl"), (name) => name === "subagents" || name === "workflows").sort()
|
|
1377
|
+
: [];
|
|
1378
|
+
const total = codexFiles.length + claudeFiles.length;
|
|
1379
|
+
let done = 0;
|
|
1380
|
+
// Every stage reports the same file counters as scanned/total, so the visible "N of M sessions
|
|
1381
|
+
// scanned" only ever climbs. A stage's own counters (the canonical pass counts a different set,
|
|
1382
|
+
// on a different scale) drive nothing but its slice of the overall bar — feeding them straight
|
|
1383
|
+
// into scanned/total is what made the bar and the caption fall back partway through the scan.
|
|
1384
|
+
const STAGE_BANDS = {
|
|
1385
|
+
"reading-transcripts": [0, 0.7],
|
|
1386
|
+
"building-summary": [0.7, 0.75],
|
|
1387
|
+
"classifying-repeated-context": [0.75, 0.95],
|
|
1388
|
+
"finalizing-report": [0.95, 1],
|
|
1389
|
+
};
|
|
1390
|
+
const overallFor = (stage, stageDone, stageTotal) => {
|
|
1391
|
+
const band = STAGE_BANDS[stage];
|
|
1392
|
+
if (!band)
|
|
1393
|
+
return 0;
|
|
1394
|
+
const ratio = stageTotal > 0 ? Math.min(1, Math.max(0, stageDone / stageTotal)) : 0;
|
|
1395
|
+
return band[0] + (band[1] - band[0]) * ratio;
|
|
1396
|
+
};
|
|
1397
|
+
const progress = (stage, detail, stageDone = done, stageTotal = total) => {
|
|
1398
|
+
opts?.onProgress?.(done, total, stage, detail, overallFor(stage, stageDone, stageTotal), stageDone, stageTotal);
|
|
1399
|
+
};
|
|
1400
|
+
progress("reading-transcripts", "finding local Codex and Claude transcript files");
|
|
1401
|
+
const tick = (stage = "reading-transcripts") => {
|
|
1402
|
+
done++;
|
|
1403
|
+
if (opts?.onProgress && (done % 8 === 0 || done === total))
|
|
1404
|
+
progress(stage);
|
|
1405
|
+
};
|
|
1406
|
+
// Codex is the slow source (GB-scale rollouts). Cache each file's parse by mtime+size so only
|
|
1407
|
+
// changed/new sessions reparse; the cross-session state is rebuilt later on one provider timeline.
|
|
1408
|
+
const cache = loadForensicCache();
|
|
1409
|
+
const nextCache = {};
|
|
1410
|
+
const parsedCodexFiles = [];
|
|
1411
|
+
const discoveredCodexByPath = new Map(codexDiscovery?.files.map((file) => [file.path, file]) ?? []);
|
|
1412
|
+
for (const f of codexFiles) {
|
|
1413
|
+
let st;
|
|
1414
|
+
try {
|
|
1415
|
+
st = fs.statSync(f);
|
|
1416
|
+
}
|
|
1417
|
+
catch {
|
|
1418
|
+
tick("reading-transcripts");
|
|
1419
|
+
continue;
|
|
1420
|
+
}
|
|
1421
|
+
const hit = cache[f];
|
|
1422
|
+
const fe = hit && hit.mtimeMs === st.mtimeMs && hit.size === st.size ? hit.fe : extractCodex(f);
|
|
1423
|
+
const discovered = discoveredCodexByPath.get(f);
|
|
1424
|
+
if (!fe.session && discovered)
|
|
1425
|
+
fe.session = discovered.sessionKey;
|
|
1426
|
+
nextCache[f] = { mtimeMs: st.mtimeMs, size: st.size, fe };
|
|
1427
|
+
parsedCodexFiles.push({ path: f, fe });
|
|
1428
|
+
tick("reading-transcripts");
|
|
1429
|
+
}
|
|
1430
|
+
if (codexFiles.length)
|
|
1431
|
+
saveForensicCache(nextCache);
|
|
1432
|
+
// Claude is fast (~0.6s) — parse live, then merge its events with Codex before replay.
|
|
1433
|
+
const timelineFiles = [...parsedCodexFiles];
|
|
1434
|
+
for (const f of claudeFiles) {
|
|
1435
|
+
timelineFiles.push({ path: f, fe: extractClaude(f) });
|
|
1436
|
+
tick("reading-transcripts");
|
|
1437
|
+
}
|
|
1438
|
+
timelineFiles.sort((a, b) => (a.fe.firstTs ?? Number.POSITIVE_INFINITY) - (b.fe.firstTs ?? Number.POSITIVE_INFINITY) ||
|
|
1439
|
+
a.path.localeCompare(b.path));
|
|
1440
|
+
// The title/enrichment pass only needs five scalar fields per session. Capture those before
|
|
1441
|
+
// releasing the much larger usage/context/event arrays ahead of canonical classification.
|
|
1442
|
+
const canonicalSessionRecords = timelineFiles.map(({ fe }) => ({
|
|
1443
|
+
source: fe.source,
|
|
1444
|
+
session: fe.session,
|
|
1445
|
+
cwd: fe.cwd,
|
|
1446
|
+
firstTs: fe.firstTs,
|
|
1447
|
+
fallbackTitle: fe.fallbackTitle,
|
|
1448
|
+
}));
|
|
1449
|
+
replayTimelineFiles(eng, timelineFiles);
|
|
1450
|
+
// Entering a stage means it has done none of its own work yet, so it opens its band rather than
|
|
1451
|
+
// inheriting the finished file counters — otherwise a stage announces itself at its band ceiling
|
|
1452
|
+
// and its real sub-progress then drags the bar back down.
|
|
1453
|
+
progress("building-summary", "aggregating rereads, model usage, cost, and local context signals", 0, 1);
|
|
1454
|
+
const report = eng.build();
|
|
1455
|
+
eng = null;
|
|
1456
|
+
const firstTimelineSession = timelineFiles.find(({ fe }) => fe.firstTs != null);
|
|
1457
|
+
report.firstSession = firstTimelineSession ? {
|
|
1458
|
+
date: firstTimelineSession.fe.firstTs != null ? new Date(firstTimelineSession.fe.firstTs).toISOString() : null,
|
|
1459
|
+
source: firstTimelineSession.fe.source,
|
|
1460
|
+
repo: firstTimelineSession.fe.cwd ? path.basename(firstTimelineSession.fe.cwd) : "workspace",
|
|
1461
|
+
} : null;
|
|
1462
|
+
const activeCodexCount = reportCodexSessions.filter((file) => file.rootKind === "active").length;
|
|
1463
|
+
report.activeSessionCoverage = {
|
|
1464
|
+
codex: activeCodexCount,
|
|
1465
|
+
claudeCode: claudeFiles.length,
|
|
1466
|
+
total: activeCodexCount + claudeFiles.length,
|
|
1467
|
+
};
|
|
1468
|
+
report.generatedFrom = [
|
|
1469
|
+
...(codexDiscovery?.roots.map((root) => root.kind === "active" ? "$CODEX_HOME/sessions" : "$CODEX_HOME/archived_sessions") ?? []),
|
|
1470
|
+
...(claudeRoot ? ["$CLAUDE_CONFIG_DIR/projects"] : []),
|
|
1471
|
+
];
|
|
1472
|
+
if (codexDiscovery) {
|
|
1473
|
+
report.sourceCoverage = {
|
|
1474
|
+
codex: {
|
|
1475
|
+
activeSessions: reportCodexSessions.filter((file) => file.rootKind === "active").length,
|
|
1476
|
+
archivedSessions: reportCodexSessions.filter((file) => file.rootKind === "archived").length,
|
|
1477
|
+
unclassifiedSessions: reportCodexSessions.filter((file) => file.userInitiated == null).length,
|
|
1478
|
+
duplicateSessionsDiscarded: codexDiscovery.diagnostics.conflicts.length,
|
|
1479
|
+
identityFallbacks: codexDiscovery.diagnostics.filenameIdentityFallbacks + codexDiscovery.diagnostics.pathIdentityFallbacks,
|
|
1480
|
+
timelineBasis: "provider-event-timestamps",
|
|
1481
|
+
},
|
|
1482
|
+
};
|
|
1483
|
+
}
|
|
1484
|
+
// The forensic cache and timeline can be hundreds of MB for a small number of very long sessions.
|
|
1485
|
+
// They are no longer needed after replay/build, and retaining them while loading the canonical
|
|
1486
|
+
// cache was the peak-memory overlap that terminated the setup worker on large histories.
|
|
1487
|
+
parsedCodexFiles.length = 0;
|
|
1488
|
+
timelineFiles.length = 0;
|
|
1489
|
+
for (const key of Object.keys(cache))
|
|
1490
|
+
delete cache[key];
|
|
1491
|
+
for (const key of Object.keys(nextCache))
|
|
1492
|
+
delete nextCache[key];
|
|
1493
|
+
if (opts?.includeLegacyGoldenStandard !== false) {
|
|
1494
|
+
try {
|
|
1495
|
+
report.goldenStandard = buildWorkspaceContextReport();
|
|
1496
|
+
}
|
|
1497
|
+
catch (error) {
|
|
1498
|
+
report.goldenStandard = { error: error instanceof Error ? error.message : String(error) };
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
try {
|
|
1502
|
+
const canonicalSources = [];
|
|
1503
|
+
if (sources.includes("codex"))
|
|
1504
|
+
canonicalSources.push("codex");
|
|
1505
|
+
if (sources.includes("claude"))
|
|
1506
|
+
canonicalSources.push("claude-code");
|
|
1507
|
+
progress("classifying-repeated-context", "reconstructing context windows and attributing P01/P03/P08/P10/P13 waste", 0, 1);
|
|
1508
|
+
report.canonicalGoldenStandard = await buildCanonicalGoldenReport({
|
|
1509
|
+
sources: canonicalSources,
|
|
1510
|
+
codexSessionPaths: canonicalCodexFiles,
|
|
1511
|
+
claudeSessionPaths: claudeFiles,
|
|
1512
|
+
// These paths were discovered from the user's local roots. One malformed/unsupported session
|
|
1513
|
+
// must become a diagnostic, not erase the canonical report for every other valid session.
|
|
1514
|
+
strictSessionErrors: false,
|
|
1515
|
+
onProgress: (canonicalDone, canonicalTotal) => progress("classifying-repeated-context", "reconstructing context windows and attributing P01/P03/P08/P10/P13 waste", canonicalDone, canonicalTotal),
|
|
1516
|
+
});
|
|
1517
|
+
enrichCanonicalSessionExamples(report.canonicalGoldenStandard, canonicalSessionRecords);
|
|
1518
|
+
}
|
|
1519
|
+
catch (error) {
|
|
1520
|
+
report.canonicalGoldenStandard = { error: error instanceof Error ? error.message : String(error) };
|
|
1521
|
+
}
|
|
1522
|
+
if (report.canonicalGoldenStandard &&
|
|
1523
|
+
!("error" in report.canonicalGoldenStandard) &&
|
|
1524
|
+
report.canonicalGoldenStandard.summary.sessionsAnalyzed > 0) {
|
|
1525
|
+
const projection = projectCanonicalWasteToBilledInput(Number(report.scale.totalInputTokens || 0), report.canonicalGoldenStandard.summary);
|
|
1526
|
+
if (projection)
|
|
1527
|
+
report.billedWasteProjection = projection;
|
|
1528
|
+
}
|
|
1529
|
+
progress("finalizing-report", "preparing setup page data", total, total);
|
|
1530
|
+
return report;
|
|
1531
|
+
}
|