@echomem/mcp 1.4.7 → 1.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -9
- package/assets/canonical-scorer/README.md +18 -0
- package/assets/canonical-scorer/analyze-10-problems.mjs +857 -0
- package/assets/canonical-scorer/build-session-waste-dashboard.mjs +1628 -0
- package/assets/canonical-scorer/golden_anchors.mjs +83 -0
- package/assets/canonical-scorer/optimizable_detail.mjs +633 -0
- package/assets/hud/claude.svg +1 -0
- package/assets/hud/codex.svg +1 -0
- package/assets/hud/session-viewer.html +35 -0
- package/dist/city/chaos-to-clarity-pencil.html +582 -0
- package/dist/city/echo-ai-city-only.html +1126 -109
- package/dist/city/echo-ai-city-only.template.html +1126 -109
- package/dist/city/echo-face-cutout.png +0 -0
- package/dist/city/pencil-pie-generator.html +883 -0
- package/dist/city/pencil-webgl-landscape.html +1239 -0
- package/dist/city/spatial-fan-story.html +479 -0
- package/dist/codex-session-files.js +283 -0
- package/dist/codex-sync.js +7 -2
- package/dist/context-analysis/canonical-golden.js +47 -0
- package/dist/context-analysis/claude-native-canonical.js +1193 -0
- package/dist/context-analysis/vendored-canonical.js +793 -0
- package/dist/context-analysis/workspace-report.js +1838 -0
- package/dist/context-metrics/calculate.js +56 -0
- package/dist/context-metrics/model-limits.js +26 -0
- package/dist/context-metrics/types.js +1 -0
- package/dist/forensics-10-problems.js +7 -6
- package/dist/forensics.js +863 -132
- package/dist/hud/adapters.js +8 -4
- package/dist/hud/autostart.js +66 -0
- package/dist/hud/cli.js +31 -0
- package/dist/hud/electron-main.js +182 -19
- package/dist/hud/metric.js +13 -4
- package/dist/hud/monitor.js +171 -84
- package/dist/hud/preload.cjs +3 -0
- package/dist/hud/server.js +321 -4
- package/dist/hud/web.js +880 -270
- package/dist/index.js +122 -24
- package/dist/local-data-paths.js +87 -0
- package/dist/migrate.js +55 -29
- package/dist/report.js +101 -40
- package/dist/setup-page.js +4257 -245
- package/dist/setup-preview.js +245 -0
- package/dist/setup.js +786 -75
- package/dist/v1-contract.js +20 -2
- package/package.json +6 -4
- package/templates/echomem-recall.md +2 -2
|
@@ -0,0 +1,1838 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { discoverCodexSessionFiles } from "../codex-session-files.js";
|
|
5
|
+
import { resolveClaudeProjectsDir } from "../local-data-paths.js";
|
|
6
|
+
import { eachLine, walk } from "../report.js";
|
|
7
|
+
const PROBLEMS = {
|
|
8
|
+
P01: { id: "P01", label: "Outdated Images & Screenshots", bucket: "dead", category: "Runtime Bug", confidence: "high" },
|
|
9
|
+
P02: { id: "P02", label: "Ignored User Instructions", bucket: "refind", category: "Model Behavior", confidence: "medium" },
|
|
10
|
+
P03: { id: "P03", label: "Old Files", bucket: "duplicate", category: "Structural Accumulation", confidence: "high" },
|
|
11
|
+
P04: { id: "P04", label: "Repeated Setup After Compaction", bucket: "refind", category: "Structural Accumulation", confidence: "high" },
|
|
12
|
+
P05: { id: "P05", label: "Premature Completion Fixes", bucket: "refind", category: "Human Cost", confidence: "medium" },
|
|
13
|
+
P06: { id: "P06", label: "Session Re-heat", bucket: "refind", category: "Human Cost", confidence: "medium" },
|
|
14
|
+
P07: { id: "P07", label: "Failed Turn Leftovers", bucket: "dead", category: "Runtime Bug", confidence: "medium" },
|
|
15
|
+
P08: { id: "P08", label: "Repeated Git Check Logs", bucket: "refind", category: "Model Behavior", confidence: "high" },
|
|
16
|
+
P09: { id: "P09", label: "Repeated Fix Attempts", bucket: "refind", category: "Human Cost", confidence: "medium" },
|
|
17
|
+
P10: { id: "P10", label: "Repeated Search", bucket: "refind", category: "Model Behavior", confidence: "high" },
|
|
18
|
+
P11: { id: "P11", label: "Visual Debug Logs", bucket: "dead", category: "Context Hygiene", confidence: "fallback" },
|
|
19
|
+
P12: { id: "P12", label: "Tool Call Logs", bucket: "dead", category: "Context Hygiene", confidence: "fallback" },
|
|
20
|
+
P13: { id: "P13", label: "Agent's Reasoning Notes", bucket: "dead", category: "Context Hygiene", confidence: "fallback" },
|
|
21
|
+
};
|
|
22
|
+
const READ_BINS = new Set(["cat", "head", "tail", "less", "more", "bat", "nl", "sed", "awk", "strings", "view"]);
|
|
23
|
+
const SEARCH_BINS = new Set(["grep", "egrep", "fgrep", "rg", "ag", "ack", "find", "fd"]);
|
|
24
|
+
const GIT_MEMORY_SUBS = new Set(["log", "show", "diff", "blame", "grep", "status", "ls-files", "cat-file"]);
|
|
25
|
+
const RUN_BINS = new Set(["npm", "pnpm", "yarn", "bun", "node", "npx", "tsx", "vitest", "jest", "pytest", "cargo", "go", "make"]);
|
|
26
|
+
const CONTENT_KINDS = new Set(["read", "search", "command", "image", "written", "conv_user", "conv_agent"]);
|
|
27
|
+
const R_CODE = 3.3;
|
|
28
|
+
const R_TEXT = 4;
|
|
29
|
+
const OUTPUT_TOKEN_CAP = 12_000;
|
|
30
|
+
const IMAGE_TOKENS = 4_000;
|
|
31
|
+
const SERIALIZATION_FRACTION = 0.03;
|
|
32
|
+
const GAP_MS = 30 * 60 * 1000;
|
|
33
|
+
const ORIGINAL_TOKEN_RE = /Original token count:\s*(\d+)/;
|
|
34
|
+
const CORRECTION_RE = /(?<!对)不对|并不是|不是这样|还是[不没]|错了|搞错|理解错|重做|重新[做写改]|没(?:有)?(?:生效|起作用|变化)|不应该|不是让你|revert|undo|roll\s*back|\bthat'?s not (?:right|what|it)\b|\bnot what i\b|\bstill (?:broken|wrong|failing|not working|the same)\b|\byou (?:should have|were supposed)\b|\bredo\b/i;
|
|
35
|
+
const CONSTRAINT_RE = /must|must not|should not|never|always|required|constraint|requirement|important|do not|don't|only|exactly|instead|避免|必须|不要|不能|只要|约束|protect|private|signed|preserve/i;
|
|
36
|
+
const VIOLATION_RE = /violat|broke|ignored|missed|didn'?t follow|not follow|against|违背|违反|没按|不符合/i;
|
|
37
|
+
const COMPLETION_RE = /done|complete|fixed|implemented|finished|ready|all set|works now|完成|修好了|搞定/i;
|
|
38
|
+
const REORIENT_RE = /where were we|continue|pick up|resume|recap|remind|what did we|state|status|context|继续|接着|恢复|进展|状态/i;
|
|
39
|
+
const MANUAL_REPAIR_RE = /try again|rerun|retry|fix it|still broken|again|重新|再试|再跑|继续修|还是不对/i;
|
|
40
|
+
export function buildWorkspaceContextReport(opts) {
|
|
41
|
+
const scoringMode = opts?.scoringMode ?? "local-live";
|
|
42
|
+
const sources = opts?.sources ?? ["codex", "claude-code"];
|
|
43
|
+
const claudeRoot = sources.includes("claude-code") && !opts?.claudeSessionPaths ? resolveClaudeProjectsDir() : null;
|
|
44
|
+
const discoveredCodexFiles = sources.includes("codex")
|
|
45
|
+
? (opts?.codexSessionPaths
|
|
46
|
+
?? discoverCodexSessionFiles({
|
|
47
|
+
includeArchived: true,
|
|
48
|
+
userInitiatedOnly: opts?.userInitiatedCodexOnly === true,
|
|
49
|
+
}).files.map((file) => file.path))
|
|
50
|
+
: [];
|
|
51
|
+
const codexFiles = opts?.userInitiatedCodexOnly
|
|
52
|
+
? discoveredCodexFiles.filter(isUserInitiatedCodexSession)
|
|
53
|
+
: discoveredCodexFiles;
|
|
54
|
+
const claudeFiles = sources.includes("claude-code")
|
|
55
|
+
? (opts?.claudeSessionPaths ?? (claudeRoot ? walk(claudeRoot, (p) => p.endsWith(".jsonl") && !p.includes(`${path.sep}subagents${path.sep}`) && !p.includes(`${path.sep}workflows${path.sep}`), () => false) : [])).sort()
|
|
56
|
+
: [];
|
|
57
|
+
const selectedCodex = typeof opts?.limitFiles === "number" ? codexFiles.slice(-Math.max(0, opts.limitFiles)) : codexFiles;
|
|
58
|
+
const selectedClaude = typeof opts?.limitFiles === "number" ? claudeFiles.slice(-Math.max(0, opts.limitFiles)) : claudeFiles;
|
|
59
|
+
const useCache = scoringMode === "local-live";
|
|
60
|
+
const cache = useCache ? loadCache() : {};
|
|
61
|
+
const nextCache = {};
|
|
62
|
+
const codexFindings = [];
|
|
63
|
+
const claudeFindings = [];
|
|
64
|
+
const total = selectedCodex.length + selectedClaude.length;
|
|
65
|
+
let done = 0;
|
|
66
|
+
for (const file of selectedCodex) {
|
|
67
|
+
let st;
|
|
68
|
+
try {
|
|
69
|
+
st = fs.statSync(file);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
done += 1;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
const hit = cache[file];
|
|
76
|
+
const cacheHit = useCache && hit && hit.mtimeMs === st.mtimeMs && hit.size === st.size;
|
|
77
|
+
const finding = cacheHit ? hit.finding : analyzeCodexSession(file, scoringMode);
|
|
78
|
+
if (useCache)
|
|
79
|
+
nextCache[file] = { mtimeMs: st.mtimeMs, size: st.size, finding };
|
|
80
|
+
codexFindings.push(finding);
|
|
81
|
+
done += 1;
|
|
82
|
+
if (opts?.onProgress && (done % 8 === 0 || done === total))
|
|
83
|
+
opts.onProgress(done, total);
|
|
84
|
+
}
|
|
85
|
+
for (const file of selectedClaude) {
|
|
86
|
+
try {
|
|
87
|
+
claudeFindings.push(analyzeClaudeSession(file, scoringMode));
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
/* Ignore malformed/partial Claude Code sessions; the setup report should still render. */
|
|
91
|
+
}
|
|
92
|
+
done += 1;
|
|
93
|
+
if (opts?.onProgress && (done % 8 === 0 || done === total))
|
|
94
|
+
opts.onProgress(done, total);
|
|
95
|
+
}
|
|
96
|
+
if (useCache)
|
|
97
|
+
saveCache(nextCache);
|
|
98
|
+
const codexReport = aggregate(codexFindings, {
|
|
99
|
+
scoringMode,
|
|
100
|
+
generatedFrom: ["~/.codex/sessions"],
|
|
101
|
+
note: "Each Codex JSONL session is analyzed independently. Every token-counted turn is reconstructed as a retained context window that reconciles to C_t, partitioned into keep_oh/keep_prod/opt_dup/opt_refind/opt_dead, then opt_* waste is attributed to P01-P13 before workspace aggregation.",
|
|
102
|
+
});
|
|
103
|
+
const claudeReport = aggregate(claudeFindings, {
|
|
104
|
+
scoringMode,
|
|
105
|
+
generatedFrom: ["~/.claude/projects"],
|
|
106
|
+
note: "Each Claude Code JSONL session is analyzed independently with the golden-standard Claude adapter. C_t is input_tokens + cache_read_input_tokens + cache_creation_input_tokens. Retained-window useful/waste is estimated from Claude tool effort and on-target efficiency, then waste is attributed across P01-P13. This is comparable for dashboarding but not the same raw Codex payload reconstruction.",
|
|
107
|
+
});
|
|
108
|
+
const combined = aggregate([...codexFindings, ...claudeFindings], {
|
|
109
|
+
scoringMode,
|
|
110
|
+
generatedFrom: ["~/.codex/sessions", "~/.claude/projects"],
|
|
111
|
+
note: "Combined Codex + Claude Code workspace rollup. Codex uses retained-window reconstruction against official C_t. Claude Code uses the golden-standard Claude adapter over per-request usage and tool results. Source tabs expose each basis separately.",
|
|
112
|
+
});
|
|
113
|
+
combined.sourceReports = {
|
|
114
|
+
codex: codexReport,
|
|
115
|
+
claudeCode: claudeReport,
|
|
116
|
+
};
|
|
117
|
+
return combined;
|
|
118
|
+
}
|
|
119
|
+
function isUserInitiatedCodexSession(file) {
|
|
120
|
+
try {
|
|
121
|
+
const fd = fs.openSync(file, "r");
|
|
122
|
+
try {
|
|
123
|
+
const buffer = Buffer.alloc(64 * 1024);
|
|
124
|
+
const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, 0);
|
|
125
|
+
const firstLine = buffer.toString("utf8", 0, bytesRead).split("\n", 1)[0];
|
|
126
|
+
const top = JSON.parse(firstLine);
|
|
127
|
+
const record = isRecord(top) ? top : {};
|
|
128
|
+
const payload = isRecord(record.payload) ? record.payload : record;
|
|
129
|
+
const source = payload.source;
|
|
130
|
+
const spawnedByCodex = isRecord(source) && Object.prototype.hasOwnProperty.call(source, "subagent");
|
|
131
|
+
const hasParentThread = typeof payload.parent_thread_id === "string" && payload.parent_thread_id.length > 0;
|
|
132
|
+
return !spawnedByCodex && !hasParentThread;
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
fs.closeSync(fd);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
// Older user sessions may not carry provenance metadata; keep them unless they explicitly identify as spawned.
|
|
140
|
+
return true;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
function analyzeCodexSession(file, scoringMode) {
|
|
144
|
+
const parsed = parseCodexSession(file);
|
|
145
|
+
assignEpisodes(parsed.turns);
|
|
146
|
+
qualifyProblems(parsed.turns);
|
|
147
|
+
return buildSessionFinding(parsed, scoringMode);
|
|
148
|
+
}
|
|
149
|
+
function analyzeClaudeSession(file, scoringMode) {
|
|
150
|
+
const parsed = parseClaudeSession(file);
|
|
151
|
+
applyClaudeOnTarget(parsed.turns);
|
|
152
|
+
applyClaudeProblemAttribution(parsed.turns);
|
|
153
|
+
return buildClaudeSessionFinding(parsed, scoringMode);
|
|
154
|
+
}
|
|
155
|
+
function parseClaudeSession(file) {
|
|
156
|
+
const turns = [];
|
|
157
|
+
let current = null;
|
|
158
|
+
let session = path.basename(file).replace(/\.jsonl$/, "");
|
|
159
|
+
let cwd = null;
|
|
160
|
+
let firstTs = null;
|
|
161
|
+
let lastTs = null;
|
|
162
|
+
let lastUserTs = null;
|
|
163
|
+
const callMeta = new Map();
|
|
164
|
+
const lines = fs.readFileSync(file, "utf8").split(/\n/);
|
|
165
|
+
for (const raw of lines) {
|
|
166
|
+
if (!raw.trim())
|
|
167
|
+
continue;
|
|
168
|
+
let record;
|
|
169
|
+
try {
|
|
170
|
+
record = JSON.parse(raw);
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
const top = isRecord(record) ? record : {};
|
|
176
|
+
if (typeof top.sessionId === "string")
|
|
177
|
+
session = top.sessionId;
|
|
178
|
+
if (typeof top.cwd === "string") {
|
|
179
|
+
cwd ||= top.cwd;
|
|
180
|
+
}
|
|
181
|
+
const ts = typeof top.timestamp === "string" ? Date.parse(top.timestamp) : NaN;
|
|
182
|
+
const ms = Number.isFinite(ts) ? ts : null;
|
|
183
|
+
if (ms != null) {
|
|
184
|
+
if (firstTs == null)
|
|
185
|
+
firstTs = ms;
|
|
186
|
+
lastTs = ms;
|
|
187
|
+
}
|
|
188
|
+
const type = typeof top.type === "string" ? top.type : "";
|
|
189
|
+
if (type === "user" && !isClaudeToolResultUser(top)) {
|
|
190
|
+
const text = extractClaudeText(isRecord(top.message) ? top.message.content : null);
|
|
191
|
+
current = {
|
|
192
|
+
number: turns.length + 1,
|
|
193
|
+
timestampMs: ms,
|
|
194
|
+
userMessage: text,
|
|
195
|
+
inputTokens: 0,
|
|
196
|
+
gapHours: lastUserTs != null && ms != null ? Math.max(0, (ms - lastUserTs) / 36e5) : 0,
|
|
197
|
+
effortTokens: 0,
|
|
198
|
+
onTargetTokens: 0,
|
|
199
|
+
markers: { commit: false, compaction: false, abort: false, rollback: false },
|
|
200
|
+
items: [],
|
|
201
|
+
};
|
|
202
|
+
turns.push(current);
|
|
203
|
+
if (ms != null)
|
|
204
|
+
lastUserTs = ms;
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
if (!current)
|
|
208
|
+
continue;
|
|
209
|
+
if (type === "assistant") {
|
|
210
|
+
const message = isRecord(top.message) ? top.message : {};
|
|
211
|
+
const usage = isRecord(message.usage) ? message.usage : {};
|
|
212
|
+
current.inputTokens = Math.max(current.inputTokens, claudeInputTokens(usage));
|
|
213
|
+
const content = Array.isArray(message.content) ? message.content : [];
|
|
214
|
+
for (const block of content) {
|
|
215
|
+
if (!isRecord(block) || block.type !== "tool_use")
|
|
216
|
+
continue;
|
|
217
|
+
const toolUse = claudeItemFromToolUse(block, current.number);
|
|
218
|
+
if (!toolUse)
|
|
219
|
+
continue;
|
|
220
|
+
addClaudeItem(current, toolUse);
|
|
221
|
+
if (typeof block.id === "string") {
|
|
222
|
+
callMeta.set(block.id, {
|
|
223
|
+
kind: toolUse.kind,
|
|
224
|
+
file: toolUse.file,
|
|
225
|
+
command: toolUse.command,
|
|
226
|
+
imageProducer: toolUse.imageOutput,
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
if (type === "user" && isClaudeToolResultUser(top)) {
|
|
233
|
+
const content = isRecord(top.message) && Array.isArray(top.message.content) ? top.message.content : [];
|
|
234
|
+
for (const block of content) {
|
|
235
|
+
if (!isRecord(block) || block.type !== "tool_result")
|
|
236
|
+
continue;
|
|
237
|
+
const toolId = typeof block.tool_use_id === "string" ? block.tool_use_id : "";
|
|
238
|
+
const meta = callMeta.get(toolId);
|
|
239
|
+
const text = extractClaudeText(block.content);
|
|
240
|
+
const imageCount = countClaudeImages(block.content);
|
|
241
|
+
const imageOutput = imageCount > 0 || Boolean(isRecord(top.toolUseResult) && top.toolUseResult.isImage);
|
|
242
|
+
const tokens = Math.min(OUTPUT_TOKEN_CAP, codeTokens(text) + (imageOutput ? IMAGE_TOKENS : 0));
|
|
243
|
+
if (tokens <= 0)
|
|
244
|
+
continue;
|
|
245
|
+
addClaudeItem(current, {
|
|
246
|
+
kind: meta?.kind || "command",
|
|
247
|
+
tokens,
|
|
248
|
+
turn: current.number,
|
|
249
|
+
file: meta?.file || null,
|
|
250
|
+
command: meta?.command || "tool_result",
|
|
251
|
+
imageOutput,
|
|
252
|
+
claim: null,
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
return {
|
|
258
|
+
session,
|
|
259
|
+
cwd,
|
|
260
|
+
repo: repoLabel(cwd),
|
|
261
|
+
firstTs,
|
|
262
|
+
lastTs,
|
|
263
|
+
turns,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
function claudeItemFromToolUse(block, turn) {
|
|
267
|
+
const name = typeof block.name === "string" ? block.name : "";
|
|
268
|
+
const input = isRecord(block.input) ? block.input : {};
|
|
269
|
+
const command = String(input.command || input.pattern || input.query || name || "");
|
|
270
|
+
const file = typeof input.file_path === "string" ? input.file_path : typeof input.path === "string" ? input.path : "";
|
|
271
|
+
let kind = "command";
|
|
272
|
+
if (/^(Read|Glob|LS|NotebookRead)$/i.test(name))
|
|
273
|
+
kind = "read";
|
|
274
|
+
if (/^(Grep|WebSearch|WebFetch)$/i.test(name) || isSearchCommand(command))
|
|
275
|
+
kind = "search";
|
|
276
|
+
if (/^(Write|Edit|MultiEdit|NotebookEdit)$/i.test(name))
|
|
277
|
+
kind = "write";
|
|
278
|
+
if (name === "Bash" && /\bgit\s+/.test(command))
|
|
279
|
+
kind = "git";
|
|
280
|
+
const tokens = Math.max(1, Math.min(OUTPUT_TOKEN_CAP, codeTokens(safeJson(input))));
|
|
281
|
+
return {
|
|
282
|
+
kind,
|
|
283
|
+
tokens,
|
|
284
|
+
turn,
|
|
285
|
+
file: file ? normalizeFile(file) : null,
|
|
286
|
+
command: command || name,
|
|
287
|
+
imageOutput: isVisualToolName(name) || isVisualProbeCommand(command),
|
|
288
|
+
claim: null,
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
function addClaudeItem(turn, item) {
|
|
292
|
+
turn.items.push(item);
|
|
293
|
+
turn.effortTokens += item.tokens;
|
|
294
|
+
if (item.kind === "git" && /\bgit\s+commit\b/.test(item.command))
|
|
295
|
+
turn.markers.commit = true;
|
|
296
|
+
}
|
|
297
|
+
function applyClaudeOnTarget(turns) {
|
|
298
|
+
const scores = new Map();
|
|
299
|
+
for (const turn of turns) {
|
|
300
|
+
for (const item of turn.items) {
|
|
301
|
+
if (!item.file)
|
|
302
|
+
continue;
|
|
303
|
+
scores.set(item.file, (scores.get(item.file) || 0) + (item.kind === "write" ? 5 : 1));
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
const targetFiles = new Set([...scores.entries()].sort((a, b) => b[1] - a[1]).map(([file]) => file));
|
|
307
|
+
for (const turn of turns) {
|
|
308
|
+
let onTarget = 0;
|
|
309
|
+
for (const item of turn.items) {
|
|
310
|
+
if (item.kind === "write")
|
|
311
|
+
onTarget += item.tokens;
|
|
312
|
+
else if (item.kind === "read" && item.file && targetFiles.has(item.file))
|
|
313
|
+
onTarget += item.tokens;
|
|
314
|
+
}
|
|
315
|
+
turn.onTargetTokens = onTarget;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
function applyClaudeProblemAttribution(turns) {
|
|
319
|
+
const seenSearch = new Map();
|
|
320
|
+
const seenRead = new Map();
|
|
321
|
+
const seenGit = new Map();
|
|
322
|
+
let lastEditTurn = 0;
|
|
323
|
+
for (const turn of turns) {
|
|
324
|
+
if (turn.gapHours >= 1) {
|
|
325
|
+
claimFirstClaudeItem(turn, (item) => item.kind === "read" || item.kind === "search" || item.kind === "command", "P06", "temporal gap followed by re-anchor/rediscovery");
|
|
326
|
+
}
|
|
327
|
+
const localSearches = new Map();
|
|
328
|
+
for (const item of turn.items) {
|
|
329
|
+
if (item.kind === "write")
|
|
330
|
+
lastEditTurn = turn.number;
|
|
331
|
+
if (item.claim)
|
|
332
|
+
continue;
|
|
333
|
+
if (item.imageOutput) {
|
|
334
|
+
item.claim = { problemId: "P01", label: "image-bearing tool output" };
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
if (item.kind === "git") {
|
|
338
|
+
const key = normalizeCommand(item.command).slice(0, 140);
|
|
339
|
+
const previous = seenGit.get(key);
|
|
340
|
+
if (previous && turn.number - previous <= 5)
|
|
341
|
+
item.claim = { problemId: "P08", label: "repeated git command" };
|
|
342
|
+
seenGit.set(key, turn.number);
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
if (item.kind === "search") {
|
|
346
|
+
const key = canonicalClaudeSearch(item.command);
|
|
347
|
+
const localCount = localSearches.get(key) || 0;
|
|
348
|
+
localSearches.set(key, localCount + 1);
|
|
349
|
+
const previous = seenSearch.get(key);
|
|
350
|
+
if (localCount > 0)
|
|
351
|
+
item.claim = { problemId: "P10", label: `same search pattern repeated in T${turn.number}` };
|
|
352
|
+
else if (previous && previous > lastEditTurn)
|
|
353
|
+
item.claim = { problemId: "P10", label: "same search pattern replayed with no intervening edit" };
|
|
354
|
+
seenSearch.set(key, turn.number);
|
|
355
|
+
continue;
|
|
356
|
+
}
|
|
357
|
+
if (item.kind === "read" && item.file) {
|
|
358
|
+
const previous = seenRead.get(item.file);
|
|
359
|
+
if (previous && turn.number - previous <= 8)
|
|
360
|
+
item.claim = { problemId: "P03", label: "same file read again within nearby turns" };
|
|
361
|
+
seenRead.set(item.file, turn.number);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
if (MANUAL_REPAIR_RE.test(turn.userMessage) || CORRECTION_RE.test(turn.userMessage)) {
|
|
365
|
+
claimFirstClaudeItem(turn, (item) => !item.claim && item.kind !== "write", "P05", "repair/rerun language in user turn");
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
function claimFirstClaudeItem(turn, predicate, problemId, label) {
|
|
370
|
+
const item = turn.items.find(predicate);
|
|
371
|
+
if (item && !item.claim)
|
|
372
|
+
item.claim = { problemId, label };
|
|
373
|
+
}
|
|
374
|
+
function buildClaudeSessionFinding(parsed, scoringMode) {
|
|
375
|
+
void scoringMode;
|
|
376
|
+
const buckets = { duplicate: 0, refind: 0, dead: 0, unattributed: 0 };
|
|
377
|
+
const items = [];
|
|
378
|
+
const qualifiedTokenPressure = {};
|
|
379
|
+
let officialInputTokens = 0;
|
|
380
|
+
let usefulTokens = 0;
|
|
381
|
+
let wasteTokens = 0;
|
|
382
|
+
for (const turn of parsed.turns) {
|
|
383
|
+
if (turn.inputTokens <= 0)
|
|
384
|
+
continue;
|
|
385
|
+
const efficiency = Math.max(0, Math.min(0.92, turn.effortTokens > 0 ? turn.onTargetTokens / turn.effortTokens : 0));
|
|
386
|
+
const useful = Math.round(turn.inputTokens * efficiency);
|
|
387
|
+
const waste = Math.max(0, turn.inputTokens - useful);
|
|
388
|
+
officialInputTokens += turn.inputTokens;
|
|
389
|
+
usefulTokens += useful;
|
|
390
|
+
wasteTokens += waste;
|
|
391
|
+
const contributions = turn.items.filter((item) => item.claim);
|
|
392
|
+
const weighted = contributions.length
|
|
393
|
+
? contributions.map((item) => ({ item, weight: Math.max(1, item.tokens) }))
|
|
394
|
+
: [{ item: null, weight: 1 }];
|
|
395
|
+
const totalWeight = weighted.reduce((sum, entry) => sum + entry.weight, 0) || 1;
|
|
396
|
+
let allocated = 0;
|
|
397
|
+
weighted.forEach((entry, index) => {
|
|
398
|
+
const tokens = index === weighted.length - 1 ? waste - allocated : Math.round((waste * entry.weight) / totalWeight);
|
|
399
|
+
allocated += tokens;
|
|
400
|
+
if (tokens <= 0)
|
|
401
|
+
return;
|
|
402
|
+
const problemId = entry.item?.claim?.problemId || "P13";
|
|
403
|
+
const bucket = publicBucket(bucketForClaudeProblem(problemId));
|
|
404
|
+
buckets[bucket] += tokens;
|
|
405
|
+
qualifiedTokenPressure[problemId] = (qualifiedTokenPressure[problemId] || 0) + turn.inputTokens;
|
|
406
|
+
items.push({
|
|
407
|
+
problemId,
|
|
408
|
+
bucket,
|
|
409
|
+
tokens,
|
|
410
|
+
turn: turn.number,
|
|
411
|
+
sourceTurn: turn.number,
|
|
412
|
+
evidence: entry.item?.claim?.label || "claude adapter estimated residual context",
|
|
413
|
+
});
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
return {
|
|
417
|
+
source: "claude-code",
|
|
418
|
+
session: parsed.session,
|
|
419
|
+
cwd: parsed.cwd,
|
|
420
|
+
repo: parsed.repo,
|
|
421
|
+
firstTs: parsed.firstTs,
|
|
422
|
+
lastTs: parsed.lastTs,
|
|
423
|
+
turnCount: parsed.turns.length,
|
|
424
|
+
officialInputTokens,
|
|
425
|
+
usefulTokens: Math.round(usefulTokens),
|
|
426
|
+
wasteTokens: Math.round(wasteTokens),
|
|
427
|
+
rawUsefulTokens: Math.round(usefulTokens),
|
|
428
|
+
rawOutcomeResidueTokens: Math.round(wasteTokens),
|
|
429
|
+
excludedUnlabeledTokens: 0,
|
|
430
|
+
buckets: mapBucketValues(buckets, Math.round),
|
|
431
|
+
rawBuckets: {
|
|
432
|
+
keep_oh: Math.round(usefulTokens),
|
|
433
|
+
keep_prod: 0,
|
|
434
|
+
opt_dup: Math.round(buckets.duplicate),
|
|
435
|
+
opt_refind: Math.round(buckets.refind),
|
|
436
|
+
opt_dead: Math.round(buckets.dead),
|
|
437
|
+
},
|
|
438
|
+
qualifiedTokenPressure,
|
|
439
|
+
items: items.map((item) => ({ ...item, tokens: Math.round(item.tokens) })),
|
|
440
|
+
episodes: buildClaudeEpisodes(parsed.turns, items),
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
function buildClaudeEpisodes(turns, items) {
|
|
444
|
+
const byProblem = new Map();
|
|
445
|
+
let wasteTokens = 0;
|
|
446
|
+
for (const item of items) {
|
|
447
|
+
wasteTokens += item.tokens;
|
|
448
|
+
if (item.problemId)
|
|
449
|
+
byProblem.set(item.problemId, (byProblem.get(item.problemId) || 0) + item.tokens);
|
|
450
|
+
}
|
|
451
|
+
const top = [...byProblem.entries()].sort((a, b) => b[1] - a[1])[0] || null;
|
|
452
|
+
return [{
|
|
453
|
+
id: "E1",
|
|
454
|
+
turnStart: turns[0]?.number || 1,
|
|
455
|
+
turnEnd: turns[turns.length - 1]?.number || turns.length,
|
|
456
|
+
wasteTokens: Math.round(wasteTokens),
|
|
457
|
+
dominantProblemId: top ? top[0] : null,
|
|
458
|
+
}];
|
|
459
|
+
}
|
|
460
|
+
function parseCodexSession(file) {
|
|
461
|
+
let session = "";
|
|
462
|
+
let cwd = null;
|
|
463
|
+
let repo = "home";
|
|
464
|
+
let firstTs = null;
|
|
465
|
+
let lastTs = null;
|
|
466
|
+
let current = null;
|
|
467
|
+
let itemId = 0;
|
|
468
|
+
let imageId = 0;
|
|
469
|
+
let baseTokens = 0;
|
|
470
|
+
let toolTokens = 0;
|
|
471
|
+
let turnContextTokens = 0;
|
|
472
|
+
const turns = [];
|
|
473
|
+
const ledgerItems = [];
|
|
474
|
+
const callMeta = new Map();
|
|
475
|
+
eachLine(file, (raw) => {
|
|
476
|
+
const top = isRecord(raw) ? raw : {};
|
|
477
|
+
const payload = isRecord(top.payload) ? top.payload : top;
|
|
478
|
+
const type = typeof payload.type === "string" ? payload.type : typeof top.type === "string" ? top.type : "";
|
|
479
|
+
const ts = typeof top.timestamp === "string" ? Date.parse(top.timestamp) : typeof payload.timestamp === "string" ? Date.parse(payload.timestamp) : NaN;
|
|
480
|
+
const ms = Number.isFinite(ts) ? ts : null;
|
|
481
|
+
if (ms != null) {
|
|
482
|
+
if (firstTs == null)
|
|
483
|
+
firstTs = ms;
|
|
484
|
+
lastTs = ms;
|
|
485
|
+
}
|
|
486
|
+
if (top.type === "session_meta" || type === "session_meta") {
|
|
487
|
+
if (typeof payload.id === "string")
|
|
488
|
+
session = payload.id;
|
|
489
|
+
if (typeof payload.cwd === "string") {
|
|
490
|
+
cwd = payload.cwd;
|
|
491
|
+
repo = repoLabel(cwd);
|
|
492
|
+
}
|
|
493
|
+
baseTokens = Math.max(baseTokens, textTokens(readNestedText(payload.base_instructions)));
|
|
494
|
+
toolTokens = Math.max(toolTokens, codeTokens(JSON.stringify(payload.dynamic_tools || [])));
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
497
|
+
if (top.type === "turn_context" || isRecord(payload.collaboration_mode)) {
|
|
498
|
+
turnContextTokens = Math.max(turnContextTokens, textTokens(JSON.stringify(payload)));
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
if (type === "user_message") {
|
|
502
|
+
const text = userTextFromPayload(payload);
|
|
503
|
+
current = {
|
|
504
|
+
number: turns.length + 1,
|
|
505
|
+
timestampMs: ms,
|
|
506
|
+
userMessage: text,
|
|
507
|
+
inputTokens: 0,
|
|
508
|
+
cachedInputTokens: 0,
|
|
509
|
+
outputTokens: 0,
|
|
510
|
+
reasoningTokens: 0,
|
|
511
|
+
compaction: false,
|
|
512
|
+
abort: false,
|
|
513
|
+
rollback: false,
|
|
514
|
+
commit: false,
|
|
515
|
+
patchFiles: [],
|
|
516
|
+
commands: [],
|
|
517
|
+
assistantMessages: [],
|
|
518
|
+
episode: 1,
|
|
519
|
+
episodeStart: 1,
|
|
520
|
+
anchorKind: null,
|
|
521
|
+
qualifiedProblems: new Set(),
|
|
522
|
+
};
|
|
523
|
+
turns.push(current);
|
|
524
|
+
addLedger(ledgerItems, {
|
|
525
|
+
id: ++itemId,
|
|
526
|
+
turn: current.number,
|
|
527
|
+
kind: "conv_user",
|
|
528
|
+
tokens: textTokens(text),
|
|
529
|
+
label: "user_message",
|
|
530
|
+
});
|
|
531
|
+
for (let i = 0; i < countUserImages(payload); i += 1) {
|
|
532
|
+
addLedger(ledgerItems, {
|
|
533
|
+
id: ++itemId,
|
|
534
|
+
turn: current.number,
|
|
535
|
+
kind: "image",
|
|
536
|
+
tokens: IMAGE_TOKENS,
|
|
537
|
+
label: "user_reference",
|
|
538
|
+
source: "user_reference",
|
|
539
|
+
target: "user_reference",
|
|
540
|
+
imageId: ++imageId,
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
545
|
+
if (!current)
|
|
546
|
+
return;
|
|
547
|
+
if (type === "token_count") {
|
|
548
|
+
const info = isRecord(payload.info) ? payload.info : {};
|
|
549
|
+
const last = isRecord(info.last_token_usage) ? info.last_token_usage : {};
|
|
550
|
+
const input = readNumber(last.input_tokens);
|
|
551
|
+
if (input > current.inputTokens) {
|
|
552
|
+
current.inputTokens = input;
|
|
553
|
+
current.cachedInputTokens = readNumber(last.cached_input_tokens);
|
|
554
|
+
current.outputTokens = readNumber(last.output_tokens);
|
|
555
|
+
}
|
|
556
|
+
current.reasoningTokens = Math.max(current.reasoningTokens, readNumber(last.reasoning_output_tokens));
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
if (type === "context_compacted" || top.type === "compacted" || type === "compacted") {
|
|
560
|
+
current.compaction = true;
|
|
561
|
+
return;
|
|
562
|
+
}
|
|
563
|
+
if (type === "turn_aborted") {
|
|
564
|
+
current.abort = true;
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
if (type === "thread_rolled_back") {
|
|
568
|
+
current.rollback = true;
|
|
569
|
+
return;
|
|
570
|
+
}
|
|
571
|
+
if (type === "agent_message" || type === "assistant_message") {
|
|
572
|
+
const text = assistantTextFromPayload(payload);
|
|
573
|
+
if (text) {
|
|
574
|
+
current.assistantMessages.push(text);
|
|
575
|
+
addLedger(ledgerItems, {
|
|
576
|
+
id: ++itemId,
|
|
577
|
+
turn: current.number,
|
|
578
|
+
kind: "conv_agent",
|
|
579
|
+
tokens: textTokens(text),
|
|
580
|
+
label: "agent_message",
|
|
581
|
+
});
|
|
582
|
+
}
|
|
583
|
+
return;
|
|
584
|
+
}
|
|
585
|
+
if (type === "reasoning") {
|
|
586
|
+
addLedger(ledgerItems, {
|
|
587
|
+
id: ++itemId,
|
|
588
|
+
turn: current.number,
|
|
589
|
+
kind: "reasoning",
|
|
590
|
+
tokens: 450,
|
|
591
|
+
label: "reasoning",
|
|
592
|
+
});
|
|
593
|
+
return;
|
|
594
|
+
}
|
|
595
|
+
if (type === "patch_apply_end" && isRecord(payload.changes)) {
|
|
596
|
+
const files = Object.keys(payload.changes);
|
|
597
|
+
current.patchFiles.push(...files);
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
if ((type === "custom_tool_call" || type === "tool_call") && typeof payload.input === "string") {
|
|
601
|
+
const files = extractPatchFiles(payload.input);
|
|
602
|
+
addLedger(ledgerItems, {
|
|
603
|
+
id: ++itemId,
|
|
604
|
+
turn: current.number,
|
|
605
|
+
kind: "written",
|
|
606
|
+
tokens: codeTokens(payload.input),
|
|
607
|
+
file: files.length === 1 ? files[0] : null,
|
|
608
|
+
files,
|
|
609
|
+
label: String(payload.name || "tool_call"),
|
|
610
|
+
text: payload.input.slice(0, 400_000),
|
|
611
|
+
});
|
|
612
|
+
return;
|
|
613
|
+
}
|
|
614
|
+
if (type === "function_call") {
|
|
615
|
+
const name = typeof payload.name === "string" ? payload.name : "";
|
|
616
|
+
const callId = typeof payload.call_id === "string" ? payload.call_id : "";
|
|
617
|
+
const args = parseArguments(payload.arguments);
|
|
618
|
+
if (name === "exec_command" || name === "shell") {
|
|
619
|
+
const cmd = isRecord(args) && typeof args.cmd === "string" ? args.cmd : isRecord(args) && typeof args.command === "string" ? args.command : "";
|
|
620
|
+
const normalizedCommand = normalizeCommand(cmd);
|
|
621
|
+
const commandKind = classifyCommand(cmd);
|
|
622
|
+
const read = shellRead(cmd);
|
|
623
|
+
const kind = commandKind === "read" ? "read" : commandKind === "search" ? "search" : "command";
|
|
624
|
+
current.commands.push({
|
|
625
|
+
kind: commandKind,
|
|
626
|
+
command: cmd,
|
|
627
|
+
normalized: normalizedCommand,
|
|
628
|
+
file: read?.file || null,
|
|
629
|
+
});
|
|
630
|
+
if (/git\s+(?:-[^\s]+\s+)*commit\b/.test(cmd))
|
|
631
|
+
current.commit = true;
|
|
632
|
+
if (callId) {
|
|
633
|
+
callMeta.set(callId, {
|
|
634
|
+
kind,
|
|
635
|
+
file: read?.file || null,
|
|
636
|
+
range: read ? { start: read.start, end: read.end } : null,
|
|
637
|
+
command: cmd,
|
|
638
|
+
normalizedCommand,
|
|
639
|
+
imageProducer: isVisualToolName(name) || isVisualProbeCommand(cmd),
|
|
640
|
+
target: isVisualProbeCommand(cmd) ? normalizeCommand(cmd) : undefined,
|
|
641
|
+
});
|
|
642
|
+
}
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
645
|
+
if (callId) {
|
|
646
|
+
callMeta.set(callId, {
|
|
647
|
+
kind: "command",
|
|
648
|
+
command: name,
|
|
649
|
+
normalizedCommand: normalizeCommand(name),
|
|
650
|
+
imageProducer: name === "view_image" || isVisualToolName(name),
|
|
651
|
+
target: name || "tool_image",
|
|
652
|
+
});
|
|
653
|
+
}
|
|
654
|
+
return;
|
|
655
|
+
}
|
|
656
|
+
if (type === "function_call_output") {
|
|
657
|
+
const callId = typeof payload.call_id === "string" ? payload.call_id : "";
|
|
658
|
+
const meta = callMeta.get(callId);
|
|
659
|
+
const output = payload.output;
|
|
660
|
+
const image = outputLooksLikeImage(output) || Boolean(meta?.imageProducer && typeof output !== "string");
|
|
661
|
+
if (image) {
|
|
662
|
+
addLedger(ledgerItems, {
|
|
663
|
+
id: ++itemId,
|
|
664
|
+
turn: current.number,
|
|
665
|
+
kind: "image",
|
|
666
|
+
tokens: IMAGE_TOKENS,
|
|
667
|
+
command: meta?.command || null,
|
|
668
|
+
normalizedCommand: meta?.normalizedCommand || null,
|
|
669
|
+
label: meta?.command || "image",
|
|
670
|
+
source: "tool_screenshot",
|
|
671
|
+
target: meta?.target || meta?.command || "tool_image",
|
|
672
|
+
imageId: ++imageId,
|
|
673
|
+
imageOutput: true,
|
|
674
|
+
});
|
|
675
|
+
return;
|
|
676
|
+
}
|
|
677
|
+
const tokens = tokensFromOutput(output);
|
|
678
|
+
if (tokens <= 0)
|
|
679
|
+
return;
|
|
680
|
+
const outputText = typeof output === "string" ? output : safeJson(output);
|
|
681
|
+
addLedger(ledgerItems, {
|
|
682
|
+
id: ++itemId,
|
|
683
|
+
turn: current.number,
|
|
684
|
+
kind: meta?.kind || "command",
|
|
685
|
+
tokens,
|
|
686
|
+
file: meta?.file || null,
|
|
687
|
+
range: meta?.range || null,
|
|
688
|
+
command: meta?.command || null,
|
|
689
|
+
normalizedCommand: meta?.normalizedCommand || null,
|
|
690
|
+
label: meta?.file || meta?.normalizedCommand || meta?.command || "tool_output",
|
|
691
|
+
text: meta?.kind === "read" ? outputText.slice(0, 60_000) : undefined,
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
});
|
|
695
|
+
if (!session)
|
|
696
|
+
session = path.basename(file).replace(/^rollout-/, "").replace(/\.jsonl$/, "");
|
|
697
|
+
const overheadTokens = Math.max(0, baseTokens + toolTokens + turnContextTokens);
|
|
698
|
+
return { session, cwd, repo, firstTs, lastTs, overheadTokens, turns, ledgerItems };
|
|
699
|
+
}
|
|
700
|
+
function assignEpisodes(turns) {
|
|
701
|
+
const anchors = chooseCodexEpisodeAnchors(turns);
|
|
702
|
+
let episode = 1;
|
|
703
|
+
let start = 1;
|
|
704
|
+
let anchorIndex = 0;
|
|
705
|
+
for (const turn of turns) {
|
|
706
|
+
while (anchorIndex < anchors.length && turn.number > anchors[anchorIndex].turn) {
|
|
707
|
+
episode += 1;
|
|
708
|
+
start = anchors[anchorIndex].turn + 1;
|
|
709
|
+
anchorIndex += 1;
|
|
710
|
+
}
|
|
711
|
+
turn.episode = episode;
|
|
712
|
+
turn.episodeStart = start;
|
|
713
|
+
const anchor = anchors.find((item) => item.turn === turn.number);
|
|
714
|
+
turn.anchorKind = anchor?.kind || null;
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
export function chooseCodexEpisodeAnchors(turns) {
|
|
718
|
+
const commits = turns.filter((turn) => turn.commit).map((turn) => ({ turn: turn.number, kind: "commit" }));
|
|
719
|
+
if (commits.length)
|
|
720
|
+
return commits;
|
|
721
|
+
const userTurns = turns.filter((turn) => turn.inputTokens > 0);
|
|
722
|
+
const positive = turns
|
|
723
|
+
.filter((turn) => turn.inputTokens > 0 && isStrongPositiveFeedback(turn.userMessage))
|
|
724
|
+
.map((turn) => ({ turn: turn.number, kind: "positive-user-feedback" }));
|
|
725
|
+
if (positive.length)
|
|
726
|
+
return positive;
|
|
727
|
+
const last = userTurns[userTurns.length - 1];
|
|
728
|
+
return last ? [{ turn: last.number, kind: "final-fallback" }] : [];
|
|
729
|
+
}
|
|
730
|
+
export function isStrongPositiveFeedback(message) {
|
|
731
|
+
const text = String(message || "")
|
|
732
|
+
.toLowerCase()
|
|
733
|
+
.replace(/[’]/g, "'")
|
|
734
|
+
.replace(/\s+/g, " ")
|
|
735
|
+
.trim();
|
|
736
|
+
if (!text)
|
|
737
|
+
return false;
|
|
738
|
+
if (/\b(no|not|wrong|incorrect|fix|bad|worse|broken|laggy|shaking|remove|revert|don't|do not|doesn't|does not|didn't|did not|isn't|is not)\b/.test(text)) {
|
|
739
|
+
if (!/\b(save this|looks good|perfect|exactly what i want|that's what i want|that is what i want)\b/.test(text))
|
|
740
|
+
return false;
|
|
741
|
+
}
|
|
742
|
+
return [
|
|
743
|
+
/\blooks? good\b/,
|
|
744
|
+
/\bthis is good\b/,
|
|
745
|
+
/\bcaveat font is good\b/,
|
|
746
|
+
/\bperfect\b/,
|
|
747
|
+
/\bthis is exactly\b/,
|
|
748
|
+
/\bthat's exactly\b/,
|
|
749
|
+
/\bthat is exactly\b/,
|
|
750
|
+
/\bexactly what i want\b/,
|
|
751
|
+
/\bthat's what i want\b/,
|
|
752
|
+
/\bthat is what i want\b/,
|
|
753
|
+
/\bwhat i want\b/,
|
|
754
|
+
/\byes[,.\s].*\blet'?s do it\b/,
|
|
755
|
+
/\bok[,.\s].*\blet'?s do it\b/,
|
|
756
|
+
/\blet'?s do it\b/,
|
|
757
|
+
/\bsave this\b/,
|
|
758
|
+
/\bship it\b/,
|
|
759
|
+
/\bapproved\b/,
|
|
760
|
+
].some((pattern) => pattern.test(text));
|
|
761
|
+
}
|
|
762
|
+
function qualifyProblems(turns) {
|
|
763
|
+
for (const turn of turns) {
|
|
764
|
+
const previous = turns[turn.number - 2];
|
|
765
|
+
const previousText = [previous?.userMessage || "", ...(previous?.assistantMessages || [])].join("\n");
|
|
766
|
+
if (CORRECTION_RE.test(turn.userMessage)) {
|
|
767
|
+
if (CONSTRAINT_RE.test(previousText) || VIOLATION_RE.test(turn.userMessage))
|
|
768
|
+
turn.qualifiedProblems.add("P02");
|
|
769
|
+
if ((previous?.patchFiles.length || 0) > 0 || COMPLETION_RE.test(previousText))
|
|
770
|
+
turn.qualifiedProblems.add("P05");
|
|
771
|
+
}
|
|
772
|
+
if (MANUAL_REPAIR_RE.test(turn.userMessage))
|
|
773
|
+
turn.qualifiedProblems.add("P09");
|
|
774
|
+
if (previous?.timestampMs != null && turn.timestampMs != null && turn.timestampMs - previous.timestampMs >= GAP_MS) {
|
|
775
|
+
if (REORIENT_RE.test(turn.userMessage) || turn.commands.some((cmd) => ["read", "git", "search"].includes(cmd.kind))) {
|
|
776
|
+
turn.qualifiedProblems.add("P06");
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
if (turn.abort)
|
|
780
|
+
turn.qualifiedProblems.add("P07");
|
|
781
|
+
}
|
|
782
|
+
for (const compactionTurn of turns.filter((turn) => turn.compaction)) {
|
|
783
|
+
const knownFiles = new Set(turns
|
|
784
|
+
.filter((turn) => turn.number <= compactionTurn.number)
|
|
785
|
+
.flatMap((turn) => turn.commands.filter((command) => command.kind === "read" && command.file).map((command) => command.file)));
|
|
786
|
+
const knownCommands = new Set(turns
|
|
787
|
+
.filter((turn) => turn.number <= compactionTurn.number)
|
|
788
|
+
.flatMap((turn) => turn.commands.filter((command) => ["git", "search"].includes(command.kind)).map((command) => command.normalized)));
|
|
789
|
+
for (const turn of turns.filter((turn) => turn.number > compactionTurn.number && turn.number <= compactionTurn.number + 3)) {
|
|
790
|
+
const rediscovered = turn.commands.some((command) => command.kind === "read" && Boolean(command.file && knownFiles.has(command.file))
|
|
791
|
+
|| ["git", "search"].includes(command.kind) && knownCommands.has(command.normalized));
|
|
792
|
+
if (rediscovered)
|
|
793
|
+
turn.qualifiedProblems.add("P04");
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
const seenGit = new Map();
|
|
797
|
+
const seenSearch = new Map();
|
|
798
|
+
const seenRun = new Map();
|
|
799
|
+
for (const turn of turns) {
|
|
800
|
+
for (const command of turn.commands) {
|
|
801
|
+
const key = `${turn.episode}:${command.normalized}`;
|
|
802
|
+
if (command.kind === "git") {
|
|
803
|
+
if (seenGit.has(key))
|
|
804
|
+
turn.qualifiedProblems.add("P08");
|
|
805
|
+
else
|
|
806
|
+
seenGit.set(key, turn.number);
|
|
807
|
+
}
|
|
808
|
+
else if (command.kind === "search") {
|
|
809
|
+
if (seenSearch.has(key))
|
|
810
|
+
turn.qualifiedProblems.add("P10");
|
|
811
|
+
else
|
|
812
|
+
seenSearch.set(key, turn.number);
|
|
813
|
+
}
|
|
814
|
+
else if (command.kind === "run" || command.kind === "command") {
|
|
815
|
+
const prior = seenRun.get(key);
|
|
816
|
+
if (prior != null && turn.number - prior <= 8)
|
|
817
|
+
turn.qualifiedProblems.add("P09");
|
|
818
|
+
else if (prior == null)
|
|
819
|
+
seenRun.set(key, turn.number);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
function buildSessionFinding(parsed, scoringMode) {
|
|
825
|
+
const turnByNumber = new Map(parsed.turns.map((turn) => [turn.number, turn]));
|
|
826
|
+
const compactionTurns = parsed.turns.filter((turn) => turn.compaction).map((turn) => turn.number);
|
|
827
|
+
const eligibleTurns = parsed.turns.filter((turn) => turn.inputTokens > 0);
|
|
828
|
+
const episodeOutcomes = buildEpisodeOutcomeIndex(parsed);
|
|
829
|
+
const buckets = { duplicate: 0, refind: 0, dead: 0, unattributed: 0 };
|
|
830
|
+
const rawBuckets = { keep_oh: 0, keep_prod: 0, opt_dup: 0, opt_refind: 0, opt_dead: 0 };
|
|
831
|
+
const items = [];
|
|
832
|
+
let officialInputTokens = 0;
|
|
833
|
+
let usefulTokens = 0;
|
|
834
|
+
let wasteTokens = 0;
|
|
835
|
+
let rawUsefulTokens = 0;
|
|
836
|
+
let rawOutcomeResidueTokens = 0;
|
|
837
|
+
let excludedUnlabeledTokens = 0;
|
|
838
|
+
for (const turn of eligibleTurns) {
|
|
839
|
+
const row = buildTurnAccounting({
|
|
840
|
+
turn,
|
|
841
|
+
turns: parsed.turns,
|
|
842
|
+
turnByNumber,
|
|
843
|
+
ledgerItems: parsed.ledgerItems,
|
|
844
|
+
compactionTurns,
|
|
845
|
+
overheadTokens: parsed.overheadTokens,
|
|
846
|
+
scoringMode,
|
|
847
|
+
episodeOutcomes,
|
|
848
|
+
});
|
|
849
|
+
for (const item of row.wasteItems)
|
|
850
|
+
item.problemId = problemForWasteItem(item, turn, turnByNumber);
|
|
851
|
+
const rawUseful = row.keep_oh + row.keep_prod;
|
|
852
|
+
const rawResidue = row.opt_dup + row.opt_refind + row.opt_dead;
|
|
853
|
+
rawBuckets.keep_oh += row.keep_oh;
|
|
854
|
+
rawBuckets.keep_prod += row.keep_prod;
|
|
855
|
+
rawBuckets.opt_dup += row.opt_dup;
|
|
856
|
+
rawBuckets.opt_refind += row.opt_refind;
|
|
857
|
+
rawBuckets.opt_dead += row.opt_dead;
|
|
858
|
+
const labeledItems = row.wasteItems.filter((item) => item.problemId);
|
|
859
|
+
const labeledWaste = labeledItems.reduce((total, item) => total + item.tokens, 0);
|
|
860
|
+
officialInputTokens += row.C_t;
|
|
861
|
+
rawUsefulTokens += rawUseful;
|
|
862
|
+
rawOutcomeResidueTokens += rawResidue;
|
|
863
|
+
usefulTokens += rawUseful;
|
|
864
|
+
wasteTokens += rawResidue;
|
|
865
|
+
excludedUnlabeledTokens += Math.max(0, rawResidue - labeledWaste);
|
|
866
|
+
buckets.duplicate += row.opt_dup;
|
|
867
|
+
buckets.refind += row.opt_refind;
|
|
868
|
+
buckets.dead += row.opt_dead;
|
|
869
|
+
for (const item of row.wasteItems) {
|
|
870
|
+
items.push({
|
|
871
|
+
...item,
|
|
872
|
+
timestampMs: turn.timestampMs,
|
|
873
|
+
turnInputTokens: row.C_t,
|
|
874
|
+
turnUsefulTokens: Math.round(rawUseful),
|
|
875
|
+
turnWasteTokens: Math.round(rawResidue),
|
|
876
|
+
});
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
const qualifiedTokenPressure = {};
|
|
880
|
+
const qualifiedProblemTurns = new Set();
|
|
881
|
+
for (const turn of eligibleTurns) {
|
|
882
|
+
for (const problemId of turn.qualifiedProblems) {
|
|
883
|
+
qualifiedProblemTurns.add(`${problemId}:${turn.number}`);
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
for (const item of items) {
|
|
887
|
+
if (item.problemId)
|
|
888
|
+
qualifiedProblemTurns.add(`${item.problemId}:${item.turn}`);
|
|
889
|
+
}
|
|
890
|
+
for (const key of qualifiedProblemTurns) {
|
|
891
|
+
const [problemId, turnRaw] = key.split(":");
|
|
892
|
+
const turn = turnByNumber.get(Number(turnRaw));
|
|
893
|
+
if (turn?.inputTokens)
|
|
894
|
+
qualifiedTokenPressure[problemId] = (qualifiedTokenPressure[problemId] || 0) + turn.inputTokens;
|
|
895
|
+
}
|
|
896
|
+
return {
|
|
897
|
+
source: "codex",
|
|
898
|
+
session: parsed.session,
|
|
899
|
+
cwd: parsed.cwd,
|
|
900
|
+
repo: parsed.repo,
|
|
901
|
+
firstTs: parsed.firstTs,
|
|
902
|
+
lastTs: parsed.lastTs,
|
|
903
|
+
turnCount: parsed.turns.length,
|
|
904
|
+
officialInputTokens,
|
|
905
|
+
usefulTokens: Math.round(usefulTokens),
|
|
906
|
+
wasteTokens: Math.round(wasteTokens),
|
|
907
|
+
rawUsefulTokens: Math.round(rawUsefulTokens),
|
|
908
|
+
rawOutcomeResidueTokens: Math.round(rawOutcomeResidueTokens),
|
|
909
|
+
excludedUnlabeledTokens: Math.round(excludedUnlabeledTokens),
|
|
910
|
+
buckets: mapBucketValues(buckets, Math.round),
|
|
911
|
+
rawBuckets: mapBucketValues(rawBuckets, Math.round),
|
|
912
|
+
qualifiedTokenPressure,
|
|
913
|
+
items: items.map((item) => ({ ...item, tokens: Math.round(item.tokens) })),
|
|
914
|
+
episodes: buildSessionEpisodes(parsed, items.filter((item) => item.problemId)),
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
function buildTurnAccounting(args) {
|
|
918
|
+
const { turn, turns, ledgerItems, compactionTurns, overheadTokens, scoringMode, episodeOutcomes } = args;
|
|
919
|
+
const C_t = turn.inputTokens;
|
|
920
|
+
const lastCompaction = lastBefore(compactionTurns, turn.number);
|
|
921
|
+
const reasoningSegments = reasoningSegmentsSinceCompaction(turns, lastCompaction, turn.number);
|
|
922
|
+
const reasoningTokens = reasoningSegments.reduce((total, segment) => total + segment.tokens, 0);
|
|
923
|
+
const structureFloor = Math.round(SERIALIZATION_FRACTION * C_t);
|
|
924
|
+
const fixed = Math.min(C_t, overheadTokens + structureFloor + reasoningTokens);
|
|
925
|
+
const budget = Math.max(0, C_t - fixed);
|
|
926
|
+
const carried = ledgerItems.filter((item) => item.turn <= turn.number && CONTENT_KINDS.has(item.kind));
|
|
927
|
+
const retained = retainMostRecentFirst(carried, budget);
|
|
928
|
+
const row = { C_t, keep_oh: overheadTokens + structureFloor, keep_prod: 0, opt_dup: 0, opt_refind: 0, opt_dead: 0, wasteItems: [] };
|
|
929
|
+
const outcome = episodeOutcomes.get(turn.episode);
|
|
930
|
+
for (const segment of reasoningSegments) {
|
|
931
|
+
const bucket = scoringMode === "episode-outcome"
|
|
932
|
+
? segment.turn === turn.number && turn.number === outcome?.end ? "keep_prod" : "opt_refind"
|
|
933
|
+
: segment.turn === turn.number ? "keep_prod" : "opt_refind";
|
|
934
|
+
row[bucket] += segment.tokens;
|
|
935
|
+
if (bucket === "opt_refind") {
|
|
936
|
+
row.wasteItems.push({
|
|
937
|
+
problemId: scoringMode === "local-live" ? "P13" : null,
|
|
938
|
+
bucket: "refind",
|
|
939
|
+
tokens: segment.tokens,
|
|
940
|
+
turn: turn.number,
|
|
941
|
+
sourceTurn: segment.turn,
|
|
942
|
+
kind: "reasoning",
|
|
943
|
+
evidence: scoringMode === "episode-outcome" ? "reasoning did not survive as episode outcome context" : "resident reasoning/self-talk in retained context",
|
|
944
|
+
prompt: compactPreview(turn.userMessage),
|
|
945
|
+
output: compactBlockPreview(turn.assistantMessages[turn.assistantMessages.length - 1] || ""),
|
|
946
|
+
});
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
const localState = { seenWrite: false, seenSameTurnTransient: new Set() };
|
|
950
|
+
for (const item of retained) {
|
|
951
|
+
const classified = scoringMode === "episode-outcome"
|
|
952
|
+
? classifyEpisodeOutcomeItem({ item, turn, turns, ledgerItems, outcome })
|
|
953
|
+
: classifyLocalLiveItem({ item, turn, turns, ledgerItems, outcome, state: localState });
|
|
954
|
+
row[classified.bucket] += item.retainedTokens;
|
|
955
|
+
if (classified.bucket.startsWith("opt_")) {
|
|
956
|
+
const bucket = publicBucket(classified.bucket);
|
|
957
|
+
row.wasteItems.push({
|
|
958
|
+
problemId: classified.problemId,
|
|
959
|
+
bucket,
|
|
960
|
+
tokens: item.retainedTokens,
|
|
961
|
+
turn: turn.number,
|
|
962
|
+
sourceTurn: item.turn,
|
|
963
|
+
kind: item.kind,
|
|
964
|
+
file: item.file,
|
|
965
|
+
files: item.files,
|
|
966
|
+
command: item.command,
|
|
967
|
+
normalizedCommand: item.normalizedCommand,
|
|
968
|
+
source: item.source,
|
|
969
|
+
imageOutput: item.imageOutput,
|
|
970
|
+
evidence: classified.evidence,
|
|
971
|
+
prompt: compactPreview(turn.userMessage),
|
|
972
|
+
output: compactBlockPreview(turn.assistantMessages[turn.assistantMessages.length - 1] || ""),
|
|
973
|
+
});
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
const sum = row.keep_oh + row.keep_prod + row.opt_dup + row.opt_refind + row.opt_dead;
|
|
977
|
+
const residual = C_t - sum;
|
|
978
|
+
if (residual >= 0) {
|
|
979
|
+
row.opt_refind += residual;
|
|
980
|
+
if (residual > 0) {
|
|
981
|
+
row.wasteItems.push({
|
|
982
|
+
problemId: null,
|
|
983
|
+
bucket: "refind",
|
|
984
|
+
tokens: residual,
|
|
985
|
+
turn: turn.number,
|
|
986
|
+
sourceTurn: turn.number,
|
|
987
|
+
kind: "residual",
|
|
988
|
+
evidence: "unrepresented serialization/reconciliation residue",
|
|
989
|
+
prompt: compactPreview(turn.userMessage),
|
|
990
|
+
output: compactBlockPreview(turn.assistantMessages[turn.assistantMessages.length - 1] || ""),
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
else {
|
|
995
|
+
const scale = C_t / Math.max(1, sum);
|
|
996
|
+
row.keep_oh *= scale;
|
|
997
|
+
row.keep_prod *= scale;
|
|
998
|
+
row.opt_dup *= scale;
|
|
999
|
+
row.opt_refind *= scale;
|
|
1000
|
+
row.opt_dead *= scale;
|
|
1001
|
+
for (const item of row.wasteItems)
|
|
1002
|
+
item.tokens *= scale;
|
|
1003
|
+
}
|
|
1004
|
+
return row;
|
|
1005
|
+
}
|
|
1006
|
+
function reasoningSegmentsSinceCompaction(turns, lastCompaction, currentTurn) {
|
|
1007
|
+
const segments = [];
|
|
1008
|
+
let previousCumulative = 0;
|
|
1009
|
+
for (const turn of turns.filter((item) => item.number > lastCompaction && item.number <= currentTurn)) {
|
|
1010
|
+
const cumulative = Math.max(previousCumulative, turn.reasoningTokens || previousCumulative);
|
|
1011
|
+
const tokens = cumulative - previousCumulative;
|
|
1012
|
+
if (tokens > 0)
|
|
1013
|
+
segments.push({ turn: turn.number, tokens });
|
|
1014
|
+
previousCumulative = cumulative;
|
|
1015
|
+
}
|
|
1016
|
+
return segments;
|
|
1017
|
+
}
|
|
1018
|
+
function buildEpisodeOutcomeIndex(parsed) {
|
|
1019
|
+
const outcomes = new Map();
|
|
1020
|
+
const supportingFiles = buildSupportingFileSet(parsed);
|
|
1021
|
+
const episodeNumbers = [...new Set(parsed.turns.map((turn) => turn.episode))].sort((a, b) => a - b);
|
|
1022
|
+
for (const episode of episodeNumbers) {
|
|
1023
|
+
const episodeTurns = parsed.turns.filter((turn) => turn.episode === episode);
|
|
1024
|
+
if (!episodeTurns.length)
|
|
1025
|
+
continue;
|
|
1026
|
+
const start = episodeTurns[0].number;
|
|
1027
|
+
const end = episodeTurns[episodeTurns.length - 1].number;
|
|
1028
|
+
const editedFiles = new Set(episodeTurns.flatMap((turn) => turn.patchFiles.map(normalizeFile)));
|
|
1029
|
+
const writtenItems = parsed.ledgerItems
|
|
1030
|
+
.filter((item) => item.kind === "written" && item.turn >= start && item.turn <= end)
|
|
1031
|
+
.sort((a, b) => b.id - a.id);
|
|
1032
|
+
const usefulWrittenIds = new Set();
|
|
1033
|
+
const latestByFile = new Map();
|
|
1034
|
+
const unfiled = [];
|
|
1035
|
+
for (const item of writtenItems) {
|
|
1036
|
+
const files = item.files?.length ? item.files : item.file ? [item.file] : [];
|
|
1037
|
+
if (!files.length) {
|
|
1038
|
+
unfiled.push(item);
|
|
1039
|
+
continue;
|
|
1040
|
+
}
|
|
1041
|
+
for (const file of files) {
|
|
1042
|
+
const versions = latestByFile.get(file) || [];
|
|
1043
|
+
if (versions.length >= 2)
|
|
1044
|
+
continue;
|
|
1045
|
+
versions.push(item);
|
|
1046
|
+
usefulWrittenIds.add(item.id);
|
|
1047
|
+
latestByFile.set(file, versions);
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
for (const item of unfiled.slice(0, 2))
|
|
1051
|
+
usefulWrittenIds.add(item.id);
|
|
1052
|
+
const latestTransientByKey = new Map();
|
|
1053
|
+
for (const item of parsed.ledgerItems) {
|
|
1054
|
+
if (item.turn < start || item.turn > end || (item.kind !== "search" && item.kind !== "command"))
|
|
1055
|
+
continue;
|
|
1056
|
+
const key = transientKey(item);
|
|
1057
|
+
if (!key)
|
|
1058
|
+
continue;
|
|
1059
|
+
const previous = latestTransientByKey.get(key);
|
|
1060
|
+
if (!previous || item.id > previous.id)
|
|
1061
|
+
latestTransientByKey.set(key, item);
|
|
1062
|
+
}
|
|
1063
|
+
outcomes.set(episode, { episode, start, end, editedFiles, supportingFiles, usefulWrittenIds, latestTransientByKey });
|
|
1064
|
+
}
|
|
1065
|
+
return outcomes;
|
|
1066
|
+
}
|
|
1067
|
+
function buildSupportingFileSet(parsed) {
|
|
1068
|
+
const supporting = new Set(parsed.turns.flatMap((turn) => turn.patchFiles.map(normalizeFile)));
|
|
1069
|
+
const writeSignals = contentSignals(parsed.ledgerItems.filter((item) => item.kind === "written").map((item) => item.text || "").join("\n"));
|
|
1070
|
+
const readTextByFile = new Map();
|
|
1071
|
+
for (const item of parsed.ledgerItems) {
|
|
1072
|
+
if (item.kind !== "read" || !item.file || !item.text)
|
|
1073
|
+
continue;
|
|
1074
|
+
const previous = readTextByFile.get(item.file) || "";
|
|
1075
|
+
if (previous.length < 60_000)
|
|
1076
|
+
readTextByFile.set(item.file, `${previous}\n${item.text}`.slice(0, 60_000));
|
|
1077
|
+
}
|
|
1078
|
+
for (const [file, text] of readTextByFile) {
|
|
1079
|
+
const readSignals = contentSignals(text);
|
|
1080
|
+
let matches = 0;
|
|
1081
|
+
for (const signal of readSignals)
|
|
1082
|
+
if (writeSignals.has(signal))
|
|
1083
|
+
matches += 1;
|
|
1084
|
+
if (matches >= 4 || (readSignals.size >= 8 && matches / readSignals.size >= 0.15))
|
|
1085
|
+
supporting.add(file);
|
|
1086
|
+
}
|
|
1087
|
+
return supporting;
|
|
1088
|
+
}
|
|
1089
|
+
function contentSignals(text) {
|
|
1090
|
+
const signals = new Set();
|
|
1091
|
+
for (const match of String(text).matchAll(/#[0-9a-fA-F]{3,8}\b/g))
|
|
1092
|
+
signals.add(`c:${match[0].toLowerCase()}`);
|
|
1093
|
+
for (const match of String(text).matchAll(/--[a-zA-Z][\w-]{2,}/g))
|
|
1094
|
+
signals.add(`v:${match[0].toLowerCase()}`);
|
|
1095
|
+
for (const match of String(text).matchAll(/\b\d{1,4}(?:\.\d+)?(?:px|rem|em|vh|vw|ms)\b/g))
|
|
1096
|
+
signals.add(`u:${match[0].toLowerCase()}`);
|
|
1097
|
+
return signals;
|
|
1098
|
+
}
|
|
1099
|
+
function transientKey(item) {
|
|
1100
|
+
if (item.kind === "search")
|
|
1101
|
+
return `search:${item.normalizedCommand || item.command || ""}`;
|
|
1102
|
+
if (item.kind === "command")
|
|
1103
|
+
return `command:${item.normalizedCommand || item.command || ""}`;
|
|
1104
|
+
return "";
|
|
1105
|
+
}
|
|
1106
|
+
function classifyLocalLiveItem(args) {
|
|
1107
|
+
const { item, turn, turns, ledgerItems, outcome, state } = args;
|
|
1108
|
+
if (item.kind === "conv_user")
|
|
1109
|
+
return { bucket: "keep_prod", problemId: null, evidence: "current user request/product instruction" };
|
|
1110
|
+
if (item.kind === "conv_agent") {
|
|
1111
|
+
return item.turn === turn.number
|
|
1112
|
+
? { bucket: "keep_prod", problemId: null, evidence: "current assistant output" }
|
|
1113
|
+
: { bucket: "opt_refind", problemId: null, evidence: "carried assistant explanation residue" };
|
|
1114
|
+
}
|
|
1115
|
+
if (item.kind === "search" || item.kind === "command") {
|
|
1116
|
+
if (item.turn !== turn.number)
|
|
1117
|
+
return { bucket: "opt_refind", problemId: null, evidence: "carried transient tool output" };
|
|
1118
|
+
const key = transientKey(item);
|
|
1119
|
+
if (key && state.seenSameTurnTransient.has(key)) {
|
|
1120
|
+
return { bucket: "opt_refind", problemId: null, evidence: "same-turn transient output superseded by a later duplicate" };
|
|
1121
|
+
}
|
|
1122
|
+
if (key)
|
|
1123
|
+
state.seenSameTurnTransient.add(key);
|
|
1124
|
+
return { bucket: "keep_prod", problemId: null, evidence: "active same-turn discovery/support" };
|
|
1125
|
+
}
|
|
1126
|
+
if (item.kind === "image") {
|
|
1127
|
+
const currentOutcome = outcome
|
|
1128
|
+
? { ...outcome, end: turn.number }
|
|
1129
|
+
: { episode: turn.episode, start: turn.episodeStart, end: turn.number, editedFiles: new Set(), supportingFiles: new Set(), usefulWrittenIds: new Set(), latestTransientByKey: new Map() };
|
|
1130
|
+
return imageActiveAtEpisodeOutcome(item, currentOutcome, turns, ledgerItems)
|
|
1131
|
+
? { bucket: "keep_prod", problemId: null, evidence: "current visual/reference evidence" }
|
|
1132
|
+
: { bucket: "opt_dead", problemId: null, evidence: "expired or superseded visual context" };
|
|
1133
|
+
}
|
|
1134
|
+
if (item.kind === "written") {
|
|
1135
|
+
if (!state.seenWrite) {
|
|
1136
|
+
state.seenWrite = true;
|
|
1137
|
+
return { bucket: "keep_prod", problemId: null, evidence: "latest written work" };
|
|
1138
|
+
}
|
|
1139
|
+
return { bucket: "opt_dup", problemId: null, evidence: "superseded earlier written version" };
|
|
1140
|
+
}
|
|
1141
|
+
if (item.kind === "read") {
|
|
1142
|
+
if (hasPriorUnchangedRead(item, turns, ledgerItems)) {
|
|
1143
|
+
return { bucket: "opt_dup", problemId: null, evidence: "same file/range was read earlier without an intervening edit" };
|
|
1144
|
+
}
|
|
1145
|
+
if (item.turn === turn.number)
|
|
1146
|
+
return { bucket: "keep_prod", problemId: null, evidence: "active same-turn file context" };
|
|
1147
|
+
return item.file && outcome?.supportingFiles.has(item.file)
|
|
1148
|
+
? { bucket: "keep_prod", problemId: null, evidence: "carried read supports edited product files" }
|
|
1149
|
+
: { bucket: "opt_dead", problemId: null, evidence: "carried read has no outcome connection" };
|
|
1150
|
+
}
|
|
1151
|
+
return { bucket: "opt_dead", problemId: null, evidence: "context item is no longer active" };
|
|
1152
|
+
}
|
|
1153
|
+
function hasPriorUnchangedRead(item, turns, ledgerItems) {
|
|
1154
|
+
if (item.kind !== "read" || !item.file)
|
|
1155
|
+
return false;
|
|
1156
|
+
return ledgerItems.some((prior) => prior.kind === "read"
|
|
1157
|
+
&& prior.file === item.file
|
|
1158
|
+
&& prior.turn < item.turn
|
|
1159
|
+
&& rangesOverlap(prior.range, item.range)
|
|
1160
|
+
&& !fileEditedBetween(turns, item.file, prior.turn, item.turn));
|
|
1161
|
+
}
|
|
1162
|
+
function classifyEpisodeOutcomeItem(args) {
|
|
1163
|
+
const { item, turn, turns, ledgerItems } = args;
|
|
1164
|
+
const outcome = args.outcome || {
|
|
1165
|
+
episode: turn.episode,
|
|
1166
|
+
start: turn.episodeStart,
|
|
1167
|
+
end: turn.number,
|
|
1168
|
+
editedFiles: new Set(),
|
|
1169
|
+
supportingFiles: new Set(),
|
|
1170
|
+
usefulWrittenIds: new Set(),
|
|
1171
|
+
latestTransientByKey: new Map(),
|
|
1172
|
+
};
|
|
1173
|
+
const sourceTurn = turns[item.turn - 1];
|
|
1174
|
+
if (item.kind === "conv_user") {
|
|
1175
|
+
return sourceTurn?.episode === turn.episode
|
|
1176
|
+
? { bucket: "keep_prod", problemId: null, evidence: "same-episode user/product instruction" }
|
|
1177
|
+
: { bucket: "opt_dead", problemId: null, evidence: "user/spec context from an older episode" };
|
|
1178
|
+
}
|
|
1179
|
+
if (item.kind === "conv_agent") {
|
|
1180
|
+
return item.turn === outcome.end && turn.number === outcome.end
|
|
1181
|
+
? { bucket: "keep_prod", problemId: null, evidence: "assistant output active at episode outcome" }
|
|
1182
|
+
: { bucket: "opt_refind", problemId: null, evidence: "assistant explanation did not survive as episode outcome context" };
|
|
1183
|
+
}
|
|
1184
|
+
if (item.kind === "search" || item.kind === "command") {
|
|
1185
|
+
const key = transientKey(item);
|
|
1186
|
+
const latest = key ? outcome.latestTransientByKey.get(key) : null;
|
|
1187
|
+
if (latest && latest.id !== item.id) {
|
|
1188
|
+
return { bucket: "opt_refind", problemId: null, evidence: "transient output superseded before episode outcome" };
|
|
1189
|
+
}
|
|
1190
|
+
return item.turn === outcome.end && turn.number === outcome.end
|
|
1191
|
+
? { bucket: "keep_prod", problemId: null, evidence: "transient output active at episode outcome" }
|
|
1192
|
+
: { bucket: "opt_refind", problemId: null, evidence: "transient output did not survive as episode outcome context" };
|
|
1193
|
+
}
|
|
1194
|
+
if (item.kind === "image") {
|
|
1195
|
+
return imageActiveAtEpisodeOutcome(item, outcome, turns, ledgerItems)
|
|
1196
|
+
? { bucket: "keep_prod", problemId: null, evidence: "visual/reference context active at episode outcome" }
|
|
1197
|
+
: { bucket: "opt_dead", problemId: null, evidence: "visual/reference context expired before episode outcome" };
|
|
1198
|
+
}
|
|
1199
|
+
if (item.kind === "written") {
|
|
1200
|
+
return outcome.usefulWrittenIds.has(item.id)
|
|
1201
|
+
? { bucket: "keep_prod", problemId: null, evidence: "latest two written versions by episode outcome" }
|
|
1202
|
+
: { bucket: "opt_dup", problemId: null, evidence: "written version older than latest two before episode outcome" };
|
|
1203
|
+
}
|
|
1204
|
+
if (item.kind === "read") {
|
|
1205
|
+
if (hasLaterDuplicateReadBeforeOutcome(item, outcome, turns, ledgerItems)) {
|
|
1206
|
+
return { bucket: "opt_dup", problemId: null, evidence: "same file/range was re-read before episode outcome" };
|
|
1207
|
+
}
|
|
1208
|
+
const connects = item.file && (outcome.editedFiles.size ? outcome.editedFiles.has(item.file) : outcome.supportingFiles.has(item.file));
|
|
1209
|
+
return connects
|
|
1210
|
+
? { bucket: "keep_prod", problemId: null, evidence: "read connects to episode outcome files" }
|
|
1211
|
+
: { bucket: "opt_dead", problemId: null, evidence: "read does not connect to episode outcome files" };
|
|
1212
|
+
}
|
|
1213
|
+
return { bucket: "opt_dead", problemId: null, evidence: "item does not connect to episode outcome" };
|
|
1214
|
+
}
|
|
1215
|
+
function hasLaterDuplicateReadBeforeOutcome(item, outcome, turns, ledgerItems) {
|
|
1216
|
+
if (item.kind !== "read" || !item.file)
|
|
1217
|
+
return false;
|
|
1218
|
+
return ledgerItems.some((later) => later.kind === "read"
|
|
1219
|
+
&& later.file === item.file
|
|
1220
|
+
&& later.turn > item.turn
|
|
1221
|
+
&& later.turn <= outcome.end
|
|
1222
|
+
&& rangesOverlap(later.range, item.range)
|
|
1223
|
+
&& !fileEditedBetween(turns, item.file, item.turn, later.turn));
|
|
1224
|
+
}
|
|
1225
|
+
function imageActiveAtEpisodeOutcome(item, outcome, turns, ledgerItems) {
|
|
1226
|
+
if (item.source === "user_reference") {
|
|
1227
|
+
if (turns[item.turn - 1]?.episode !== outcome.episode)
|
|
1228
|
+
return false;
|
|
1229
|
+
return !ledgerItems.some((later) => later.kind === "image"
|
|
1230
|
+
&& later.source === "user_reference"
|
|
1231
|
+
&& later.turn > item.turn
|
|
1232
|
+
&& later.turn <= outcome.end
|
|
1233
|
+
&& userReferenceReplacement(turns[later.turn - 1]?.userMessage || ""));
|
|
1234
|
+
}
|
|
1235
|
+
if (item.source === "tool_screenshot") {
|
|
1236
|
+
const laterEdit = turns.some((turn) => turn.number > item.turn && turn.number <= outcome.end && turn.patchFiles.length > 0);
|
|
1237
|
+
const laterScreenshot = ledgerItems.some((later) => later.kind === "image"
|
|
1238
|
+
&& later.source === "tool_screenshot"
|
|
1239
|
+
&& later.turn > item.turn
|
|
1240
|
+
&& later.turn <= outcome.end
|
|
1241
|
+
&& (later.target || "tool_image") === (item.target || "tool_image"));
|
|
1242
|
+
return !laterEdit && !laterScreenshot;
|
|
1243
|
+
}
|
|
1244
|
+
return false;
|
|
1245
|
+
}
|
|
1246
|
+
function userReferenceReplacement(message) {
|
|
1247
|
+
return /\b(use this instead|instead use|replace|updated|new mockup|new reference|ignore (?:the )?(?:previous|old)|actually use|latest version|use this version)\b/i.test(message);
|
|
1248
|
+
}
|
|
1249
|
+
function fileEditedBetween(turns, file, startExclusive, endInclusive) {
|
|
1250
|
+
return turns.some((turn) => turn.number > startExclusive
|
|
1251
|
+
&& turn.number <= endInclusive
|
|
1252
|
+
&& turn.patchFiles.some((edited) => normalizeFile(edited) === file));
|
|
1253
|
+
}
|
|
1254
|
+
function problemForWasteItem(item, currentTurn, turnByNumber) {
|
|
1255
|
+
const kind = item.kind || "";
|
|
1256
|
+
const command = String(item.command || item.normalizedCommand || "");
|
|
1257
|
+
const age = Math.max(0, currentTurn.number - item.sourceTurn);
|
|
1258
|
+
const currentProblems = currentTurn.qualifiedProblems;
|
|
1259
|
+
const sourceProblems = turnByNumber.get(item.sourceTurn)?.qualifiedProblems || new Set();
|
|
1260
|
+
const rediscovery = ["read", "search", "command", "residual", "conv_agent"].includes(kind);
|
|
1261
|
+
const repairLocal = age <= 1;
|
|
1262
|
+
if (currentProblems.has("P07") && rediscovery)
|
|
1263
|
+
return "P07";
|
|
1264
|
+
if (currentProblems.has("P02") && rediscovery && repairLocal)
|
|
1265
|
+
return "P02";
|
|
1266
|
+
if (kind === "command" && isGitCommand(command) && (currentProblems.has("P08") || sourceProblems.has("P08")))
|
|
1267
|
+
return "P08";
|
|
1268
|
+
if (kind === "search" && (currentProblems.has("P10") || sourceProblems.has("P10")))
|
|
1269
|
+
return "P10";
|
|
1270
|
+
if (currentProblems.has("P06") && rediscovery)
|
|
1271
|
+
return "P06";
|
|
1272
|
+
if ((currentProblems.has("P04") || sourceProblems.has("P04")) && isConcreteRediscoveryItem(item) && repairLocal)
|
|
1273
|
+
return "P04";
|
|
1274
|
+
if (currentProblems.has("P05") && rediscovery && repairLocal)
|
|
1275
|
+
return "P05";
|
|
1276
|
+
if (item.bucket === "duplicate" && (kind === "read" || kind === "written"))
|
|
1277
|
+
return "P03";
|
|
1278
|
+
if (currentProblems.has("P09") && rediscovery && repairLocal)
|
|
1279
|
+
return "P09";
|
|
1280
|
+
if ((kind === "image" && item.bucket === "dead") || item.imageOutput)
|
|
1281
|
+
return "P01";
|
|
1282
|
+
if (isVisualProbeCommand(command))
|
|
1283
|
+
return "P11";
|
|
1284
|
+
if (kind === "command" || kind === "search" || (kind === "read" && item.bucket === "dead") || isOrientationCommand(command))
|
|
1285
|
+
return "P12";
|
|
1286
|
+
if (kind === "reasoning" || kind === "residual" || kind === "conv_agent" || kind === "conv_user")
|
|
1287
|
+
return "P13";
|
|
1288
|
+
return null;
|
|
1289
|
+
}
|
|
1290
|
+
function isConcreteRediscoveryItem(item) {
|
|
1291
|
+
if (item.kind === "read" || item.kind === "search")
|
|
1292
|
+
return true;
|
|
1293
|
+
return item.kind === "command" && isGitCommand(String(item.command || item.normalizedCommand || ""));
|
|
1294
|
+
}
|
|
1295
|
+
function isOrientationCommand(command) {
|
|
1296
|
+
const normalized = command.trim();
|
|
1297
|
+
return normalized === "search_memories"
|
|
1298
|
+
|| normalized === "search_memories_by_keywords"
|
|
1299
|
+
|| isGitCommand(normalized)
|
|
1300
|
+
|| /^ls(?:\s|$)/.test(normalized);
|
|
1301
|
+
}
|
|
1302
|
+
function aggregate(sessions, opts) {
|
|
1303
|
+
const problemStats = new Map();
|
|
1304
|
+
const exampleCandidates = new Map();
|
|
1305
|
+
const repos = new Map();
|
|
1306
|
+
const sessionRows = [];
|
|
1307
|
+
const episodes = [];
|
|
1308
|
+
const buckets = { duplicate: 0, refind: 0, dead: 0, unattributed: 0 };
|
|
1309
|
+
const rawBuckets = { keep_oh: 0, keep_prod: 0, opt_dup: 0, opt_refind: 0, opt_dead: 0 };
|
|
1310
|
+
let officialInputTokens = 0;
|
|
1311
|
+
let usefulTokens = 0;
|
|
1312
|
+
let wasteTokens = 0;
|
|
1313
|
+
let rawUsefulTokens = 0;
|
|
1314
|
+
let rawOutcomeResidueTokens = 0;
|
|
1315
|
+
let excludedUnlabeledTokens = 0;
|
|
1316
|
+
let turnsAnalyzed = 0;
|
|
1317
|
+
for (const session of sessions) {
|
|
1318
|
+
officialInputTokens += session.officialInputTokens;
|
|
1319
|
+
usefulTokens += session.usefulTokens;
|
|
1320
|
+
wasteTokens += session.wasteTokens;
|
|
1321
|
+
rawUsefulTokens += session.rawUsefulTokens;
|
|
1322
|
+
rawOutcomeResidueTokens += session.rawOutcomeResidueTokens;
|
|
1323
|
+
excludedUnlabeledTokens += session.excludedUnlabeledTokens;
|
|
1324
|
+
turnsAnalyzed += session.turnCount;
|
|
1325
|
+
for (const bucket of Object.keys(buckets))
|
|
1326
|
+
buckets[bucket] += session.buckets[bucket] || 0;
|
|
1327
|
+
for (const bucket of Object.keys(rawBuckets))
|
|
1328
|
+
rawBuckets[bucket] += session.rawBuckets[bucket] || 0;
|
|
1329
|
+
const repo = repos.get(session.repo) || { sessions: new Set(), officialInputTokens: 0, wasteTokens: 0 };
|
|
1330
|
+
repo.sessions.add(session.session);
|
|
1331
|
+
repo.officialInputTokens += session.officialInputTokens;
|
|
1332
|
+
repo.wasteTokens += session.wasteTokens;
|
|
1333
|
+
repos.set(session.repo, repo);
|
|
1334
|
+
const byProblem = new Map();
|
|
1335
|
+
for (const item of session.items) {
|
|
1336
|
+
if (!item.problemId) {
|
|
1337
|
+
continue;
|
|
1338
|
+
}
|
|
1339
|
+
byProblem.set(item.problemId, (byProblem.get(item.problemId) || 0) + item.tokens);
|
|
1340
|
+
const stat = problemStats.get(item.problemId) || { count: 0, tokens: 0, pressure: 0, sessions: new Set(), examples: [] };
|
|
1341
|
+
stat.count += 1;
|
|
1342
|
+
stat.tokens += item.tokens;
|
|
1343
|
+
stat.sessions.add(session.session);
|
|
1344
|
+
problemStats.set(item.problemId, stat);
|
|
1345
|
+
const candidateKey = `${item.problemId}:${session.session}:${item.turn}`;
|
|
1346
|
+
const candidate = exampleCandidates.get(candidateKey) || {
|
|
1347
|
+
// Full id: the setup page's "open original session" button needs it to resume the session.
|
|
1348
|
+
session: session.session,
|
|
1349
|
+
repo: session.repo,
|
|
1350
|
+
agentSource: session.source,
|
|
1351
|
+
turn: item.turn,
|
|
1352
|
+
tokens: 0,
|
|
1353
|
+
evidence: item.evidence,
|
|
1354
|
+
command: item.command || item.normalizedCommand || null,
|
|
1355
|
+
file: item.file || null,
|
|
1356
|
+
prompt: item.prompt,
|
|
1357
|
+
output: item.output,
|
|
1358
|
+
timestampMs: item.timestampMs ?? null,
|
|
1359
|
+
sessionStartedAt: session.firstTs,
|
|
1360
|
+
sessionTurns: session.turnCount,
|
|
1361
|
+
turnInputTokens: item.turnInputTokens,
|
|
1362
|
+
turnUsefulTokens: item.turnUsefulTokens,
|
|
1363
|
+
turnWasteTokens: item.turnWasteTokens,
|
|
1364
|
+
};
|
|
1365
|
+
candidate.tokens += item.tokens;
|
|
1366
|
+
if (item.tokens > 0 && item.evidence)
|
|
1367
|
+
candidate.evidence = item.evidence;
|
|
1368
|
+
exampleCandidates.set(candidateKey, candidate);
|
|
1369
|
+
}
|
|
1370
|
+
for (const [problemId, pressure] of Object.entries(session.qualifiedTokenPressure)) {
|
|
1371
|
+
const stat = problemStats.get(problemId) || { count: 0, tokens: 0, pressure: 0, sessions: new Set(), examples: [] };
|
|
1372
|
+
stat.pressure += pressure;
|
|
1373
|
+
stat.sessions.add(session.session);
|
|
1374
|
+
problemStats.set(problemId, stat);
|
|
1375
|
+
}
|
|
1376
|
+
const topProblem = [...byProblem.entries()].sort((a, b) => b[1] - a[1])[0] || null;
|
|
1377
|
+
sessionRows.push({
|
|
1378
|
+
session: session.session.slice(0, 8),
|
|
1379
|
+
repo: session.repo,
|
|
1380
|
+
turns: session.turnCount,
|
|
1381
|
+
officialInputTokens: session.officialInputTokens,
|
|
1382
|
+
wasteTokens: session.wasteTokens,
|
|
1383
|
+
topProblemId: topProblem ? topProblem[0] : null,
|
|
1384
|
+
});
|
|
1385
|
+
for (const episode of session.episodes) {
|
|
1386
|
+
if (!episode.dominantProblemId)
|
|
1387
|
+
continue;
|
|
1388
|
+
episodes.push({
|
|
1389
|
+
id: `${session.session.slice(0, 8)}-${episode.id}`,
|
|
1390
|
+
session: session.session.slice(0, 8),
|
|
1391
|
+
repo: session.repo,
|
|
1392
|
+
turnStart: episode.turnStart,
|
|
1393
|
+
turnEnd: episode.turnEnd,
|
|
1394
|
+
wasteTokens: episode.wasteTokens,
|
|
1395
|
+
dominantProblemId: episode.dominantProblemId,
|
|
1396
|
+
});
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
for (const [key, candidate] of exampleCandidates) {
|
|
1400
|
+
const problemId = key.split(":")[0];
|
|
1401
|
+
const stat = problemStats.get(problemId);
|
|
1402
|
+
if (!stat)
|
|
1403
|
+
continue;
|
|
1404
|
+
upsertWorstExample(stat.examples, { ...candidate, tokens: Math.round(candidate.tokens) });
|
|
1405
|
+
}
|
|
1406
|
+
const problems = Object.keys(PROBLEMS)
|
|
1407
|
+
.map((id) => {
|
|
1408
|
+
const base = PROBLEMS[id];
|
|
1409
|
+
const stat = problemStats.get(id) || { count: 0, tokens: 0, pressure: 0, sessions: new Set(), examples: [] };
|
|
1410
|
+
return {
|
|
1411
|
+
...base,
|
|
1412
|
+
count: stat.count,
|
|
1413
|
+
sessions: stat.sessions.size,
|
|
1414
|
+
allocatedWasteTokens: Math.round(stat.tokens),
|
|
1415
|
+
qualifiedTokenPressure: Math.round(stat.pressure),
|
|
1416
|
+
severity: severity(stat.tokens, stat.count, sessions.length),
|
|
1417
|
+
examples: stat.examples,
|
|
1418
|
+
};
|
|
1419
|
+
})
|
|
1420
|
+
.sort((a, b) => b.allocatedWasteTokens - a.allocatedWasteTokens || b.count - a.count);
|
|
1421
|
+
return {
|
|
1422
|
+
version: 1,
|
|
1423
|
+
generatedFrom: opts?.generatedFrom || ["~/.codex/sessions"],
|
|
1424
|
+
methodology: {
|
|
1425
|
+
name: "Context Golden Standard workspace rollup",
|
|
1426
|
+
status: "deterministic-plus-heuristic",
|
|
1427
|
+
scoringMode: opts?.scoringMode || "local-live",
|
|
1428
|
+
note: opts?.note || "Each session is analyzed independently, partitioned into episodes where the source supports it, and waste is attributed to P01-P13 before workspace aggregation.",
|
|
1429
|
+
},
|
|
1430
|
+
summary: {
|
|
1431
|
+
sessionsAnalyzed: sessions.length,
|
|
1432
|
+
reposAnalyzed: repos.size,
|
|
1433
|
+
turnsAnalyzed,
|
|
1434
|
+
officialInputTokens: Math.round(officialInputTokens),
|
|
1435
|
+
usefulTokens: Math.round(usefulTokens),
|
|
1436
|
+
wasteTokens: Math.round(wasteTokens),
|
|
1437
|
+
usefulPct: pct(usefulTokens, officialInputTokens),
|
|
1438
|
+
wastePct: pct(wasteTokens, officialInputTokens),
|
|
1439
|
+
attributedWasteTokens: Math.round(Math.max(0, wasteTokens - excludedUnlabeledTokens)),
|
|
1440
|
+
unattributedWasteTokens: Math.round(excludedUnlabeledTokens),
|
|
1441
|
+
rawUsefulTokens: Math.round(rawUsefulTokens),
|
|
1442
|
+
rawOutcomeResidueTokens: Math.round(rawOutcomeResidueTokens),
|
|
1443
|
+
excludedUnlabeledTokens: 0,
|
|
1444
|
+
},
|
|
1445
|
+
buckets: mapBucketValues(buckets, Math.round),
|
|
1446
|
+
rawBuckets: mapBucketValues(rawBuckets, Math.round),
|
|
1447
|
+
problems,
|
|
1448
|
+
sessions: sessionRows.sort((a, b) => b.wasteTokens - a.wasteTokens).slice(0, 8),
|
|
1449
|
+
repos: [...repos.entries()]
|
|
1450
|
+
.map(([repoName, value]) => ({
|
|
1451
|
+
repo: repoName,
|
|
1452
|
+
sessions: value.sessions.size,
|
|
1453
|
+
officialInputTokens: Math.round(value.officialInputTokens),
|
|
1454
|
+
wasteTokens: Math.round(value.wasteTokens),
|
|
1455
|
+
}))
|
|
1456
|
+
.sort((a, b) => b.wasteTokens - a.wasteTokens)
|
|
1457
|
+
.slice(0, 8),
|
|
1458
|
+
episodes: episodes.sort((a, b) => b.wasteTokens - a.wasteTokens).slice(0, 8),
|
|
1459
|
+
};
|
|
1460
|
+
}
|
|
1461
|
+
function buildSessionEpisodes(parsed, items) {
|
|
1462
|
+
const groups = new Map();
|
|
1463
|
+
for (const turn of parsed.turns) {
|
|
1464
|
+
const group = groups.get(turn.episode) || { start: turn.number, end: turn.number, waste: 0, byProblem: new Map() };
|
|
1465
|
+
group.start = Math.min(group.start, turn.number);
|
|
1466
|
+
group.end = Math.max(group.end, turn.number);
|
|
1467
|
+
groups.set(turn.episode, group);
|
|
1468
|
+
}
|
|
1469
|
+
for (const item of items) {
|
|
1470
|
+
const turn = parsed.turns[item.turn - 1];
|
|
1471
|
+
if (!turn)
|
|
1472
|
+
continue;
|
|
1473
|
+
const group = groups.get(turn.episode);
|
|
1474
|
+
if (!group)
|
|
1475
|
+
continue;
|
|
1476
|
+
group.waste += item.tokens;
|
|
1477
|
+
if (item.problemId)
|
|
1478
|
+
group.byProblem.set(item.problemId, (group.byProblem.get(item.problemId) || 0) + item.tokens);
|
|
1479
|
+
}
|
|
1480
|
+
return [...groups.entries()].map(([episode, group]) => {
|
|
1481
|
+
const top = [...group.byProblem.entries()].sort((a, b) => b[1] - a[1])[0] || null;
|
|
1482
|
+
return {
|
|
1483
|
+
id: `E${episode}`,
|
|
1484
|
+
turnStart: group.start,
|
|
1485
|
+
turnEnd: group.end,
|
|
1486
|
+
wasteTokens: Math.round(group.waste),
|
|
1487
|
+
dominantProblemId: top ? top[0] : null,
|
|
1488
|
+
};
|
|
1489
|
+
});
|
|
1490
|
+
}
|
|
1491
|
+
function retainMostRecentFirst(items, budget) {
|
|
1492
|
+
const retained = [];
|
|
1493
|
+
let running = 0;
|
|
1494
|
+
for (const item of [...items].reverse()) {
|
|
1495
|
+
if (running >= budget)
|
|
1496
|
+
break;
|
|
1497
|
+
const tokens = Math.min(item.tokens, budget - running);
|
|
1498
|
+
if (tokens <= 0)
|
|
1499
|
+
continue;
|
|
1500
|
+
retained.push({ ...item, retainedTokens: tokens, partial: tokens < item.tokens });
|
|
1501
|
+
running += tokens;
|
|
1502
|
+
}
|
|
1503
|
+
return retained;
|
|
1504
|
+
}
|
|
1505
|
+
function addLedger(items, item) {
|
|
1506
|
+
if (item.tokens > 0)
|
|
1507
|
+
items.push(item);
|
|
1508
|
+
}
|
|
1509
|
+
function classifyCommand(cmd) {
|
|
1510
|
+
const first = firstCommand(cmd);
|
|
1511
|
+
const bin = first.split(/\s+/)[0] || "";
|
|
1512
|
+
if (bin === "git") {
|
|
1513
|
+
const sub = first.split(/\s+/).find((part, index) => index > 0 && !part.startsWith("-")) || "";
|
|
1514
|
+
return GIT_MEMORY_SUBS.has(sub) ? "git" : "command";
|
|
1515
|
+
}
|
|
1516
|
+
if (READ_BINS.has(bin))
|
|
1517
|
+
return "read";
|
|
1518
|
+
if (SEARCH_BINS.has(bin))
|
|
1519
|
+
return "search";
|
|
1520
|
+
if (RUN_BINS.has(bin))
|
|
1521
|
+
return "run";
|
|
1522
|
+
return "command";
|
|
1523
|
+
}
|
|
1524
|
+
function isClaudeToolResultUser(record) {
|
|
1525
|
+
const message = isRecord(record.message) ? record.message : {};
|
|
1526
|
+
const content = message.content;
|
|
1527
|
+
return Array.isArray(content) && content.some((block) => isRecord(block) && block.type === "tool_result");
|
|
1528
|
+
}
|
|
1529
|
+
function extractClaudeText(value) {
|
|
1530
|
+
if (!value)
|
|
1531
|
+
return "";
|
|
1532
|
+
if (typeof value === "string")
|
|
1533
|
+
return value;
|
|
1534
|
+
if (Array.isArray(value))
|
|
1535
|
+
return value.map(extractClaudeText).filter(Boolean).join("\n");
|
|
1536
|
+
if (isRecord(value)) {
|
|
1537
|
+
if (typeof value.text === "string")
|
|
1538
|
+
return value.text;
|
|
1539
|
+
if (typeof value.content === "string")
|
|
1540
|
+
return value.content;
|
|
1541
|
+
if (Array.isArray(value.content))
|
|
1542
|
+
return value.content.map(extractClaudeText).filter(Boolean).join("\n");
|
|
1543
|
+
}
|
|
1544
|
+
return "";
|
|
1545
|
+
}
|
|
1546
|
+
function countClaudeImages(value) {
|
|
1547
|
+
const text = extractClaudeText(value);
|
|
1548
|
+
return (text.match(/data:image\/|codex-clipboard-|\.png|\.jpe?g|\.webp/gi) || []).length;
|
|
1549
|
+
}
|
|
1550
|
+
function claudeInputTokens(usage) {
|
|
1551
|
+
return readNumber(usage.input_tokens) + readNumber(usage.cache_read_input_tokens) + readNumber(usage.cache_creation_input_tokens);
|
|
1552
|
+
}
|
|
1553
|
+
function isSearchCommand(command) {
|
|
1554
|
+
const first = firstCommand(command);
|
|
1555
|
+
const bin = first.split(/\s+/)[0] || "";
|
|
1556
|
+
return SEARCH_BINS.has(bin) || bin === "ls";
|
|
1557
|
+
}
|
|
1558
|
+
function canonicalClaudeSearch(command) {
|
|
1559
|
+
return normalizeCommand(command)
|
|
1560
|
+
.replace(/\/private\/tmp\/[^/\s]+\/[^\s]+/g, "/tmp/...")
|
|
1561
|
+
.replace(/\d{4}-\d{2}-\d{2}T[^\s]+/g, "<timestamp>")
|
|
1562
|
+
.replace(/\b[0-9a-f]{8}-[0-9a-f-]{27,36}\b/g, "<uuid>")
|
|
1563
|
+
.slice(0, 180);
|
|
1564
|
+
}
|
|
1565
|
+
function bucketForClaudeProblem(problemId) {
|
|
1566
|
+
if (problemId === "P03")
|
|
1567
|
+
return "opt_dup";
|
|
1568
|
+
if (problemId === "P01" || problemId === "P02" || problemId === "P05" || problemId === "P07" || problemId === "P09" || problemId === "P13")
|
|
1569
|
+
return "opt_dead";
|
|
1570
|
+
return "opt_refind";
|
|
1571
|
+
}
|
|
1572
|
+
function compactPreview(text, limit = 360) {
|
|
1573
|
+
const compacted = text.replace(/\s+/g, " ").trim();
|
|
1574
|
+
if (!compacted)
|
|
1575
|
+
return undefined;
|
|
1576
|
+
return compacted.length > limit ? `${compacted.slice(0, limit - 1)}…` : compacted;
|
|
1577
|
+
}
|
|
1578
|
+
function compactBlockPreview(text, limit = 900) {
|
|
1579
|
+
const compacted = text
|
|
1580
|
+
.replace(/\r\n/g, "\n")
|
|
1581
|
+
.replace(/[ \t]+\n/g, "\n")
|
|
1582
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
1583
|
+
.trim();
|
|
1584
|
+
if (!compacted)
|
|
1585
|
+
return undefined;
|
|
1586
|
+
return compacted.length > limit ? `${compacted.slice(0, limit - 1)}…` : compacted;
|
|
1587
|
+
}
|
|
1588
|
+
function upsertWorstExample(examples, example) {
|
|
1589
|
+
examples.push(example);
|
|
1590
|
+
examples.sort((left, right) => (right.tokens || 0) - (left.tokens || 0));
|
|
1591
|
+
examples.splice(1);
|
|
1592
|
+
}
|
|
1593
|
+
function shellRead(cmd) {
|
|
1594
|
+
const first = firstCommand(cmd);
|
|
1595
|
+
const parts = first.split(/\s+/).map((part) => part.replace(/^['"]|['"]$/g, ""));
|
|
1596
|
+
const bin = parts[0] || "";
|
|
1597
|
+
if (!READ_BINS.has(bin))
|
|
1598
|
+
return null;
|
|
1599
|
+
let start = 1;
|
|
1600
|
+
let end = Infinity;
|
|
1601
|
+
const sedRange = first.match(/(\d+),(\d+)p/);
|
|
1602
|
+
if (sedRange) {
|
|
1603
|
+
start = Number(sedRange[1]);
|
|
1604
|
+
end = Number(sedRange[2]);
|
|
1605
|
+
}
|
|
1606
|
+
for (let i = parts.length - 1; i >= 1; i -= 1) {
|
|
1607
|
+
const token = parts[i];
|
|
1608
|
+
if (!token || token.startsWith("-") || /^\d+,?\d*p?$/.test(token))
|
|
1609
|
+
continue;
|
|
1610
|
+
if (token.includes("/") || /\.[A-Za-z0-9]{1,8}$/.test(token))
|
|
1611
|
+
return { file: normalizeFile(token), start, end };
|
|
1612
|
+
}
|
|
1613
|
+
return null;
|
|
1614
|
+
}
|
|
1615
|
+
function firstCommand(cmd) {
|
|
1616
|
+
return String(cmd).split(/&&|\|\||[;|\n]/)[0]?.split("|")[0]?.trim() || "";
|
|
1617
|
+
}
|
|
1618
|
+
function normalizeCommand(cmd) {
|
|
1619
|
+
return String(cmd)
|
|
1620
|
+
.replace(/\s+/g, " ")
|
|
1621
|
+
.replace(/(['"])([^'"]{160,})\1/g, "$1...$1")
|
|
1622
|
+
.trim()
|
|
1623
|
+
.slice(0, 260);
|
|
1624
|
+
}
|
|
1625
|
+
function isGitCommand(cmd) {
|
|
1626
|
+
return /^\s*git(?:\s|$)/.test(firstCommand(cmd));
|
|
1627
|
+
}
|
|
1628
|
+
function isRunCommand(cmd) {
|
|
1629
|
+
const bin = firstCommand(cmd).split(/\s+/)[0] || "";
|
|
1630
|
+
return RUN_BINS.has(bin);
|
|
1631
|
+
}
|
|
1632
|
+
function isVisualToolName(name) {
|
|
1633
|
+
return /browser|screenshot|playwright|chrome|view_image|computer/i.test(name);
|
|
1634
|
+
}
|
|
1635
|
+
function isVisualProbeCommand(cmd) {
|
|
1636
|
+
const normalized = String(cmd).trim();
|
|
1637
|
+
return normalized === "js" || /^curl\s+-I\s+https?:\/\/(?:127\.0\.0\.1|localhost|\[?::1\]?)(?::|\b|\/)/.test(normalized);
|
|
1638
|
+
}
|
|
1639
|
+
function outputLooksLikeImage(output) {
|
|
1640
|
+
if (Array.isArray(output))
|
|
1641
|
+
return output.some((item) => isRecord(item) && (item.type === "input_image" || item.image_url));
|
|
1642
|
+
if (!isRecord(output))
|
|
1643
|
+
return false;
|
|
1644
|
+
return output.type === "input_image" || typeof output.image_url === "string";
|
|
1645
|
+
}
|
|
1646
|
+
function countImagesDeep(value) {
|
|
1647
|
+
if (!value)
|
|
1648
|
+
return 0;
|
|
1649
|
+
if (typeof value === "string") {
|
|
1650
|
+
return (value.match(/"type"\s*:\s*"input_image"/g) || []).length + (value.match(/data:image\//g) || []).length;
|
|
1651
|
+
}
|
|
1652
|
+
if (Array.isArray(value))
|
|
1653
|
+
return value.reduce((total, item) => total + countImagesDeep(item), 0);
|
|
1654
|
+
if (isRecord(value)) {
|
|
1655
|
+
return (value.type === "input_image" ? 1 : 0) + Object.values(value).reduce((total, item) => total + countImagesDeep(item), 0);
|
|
1656
|
+
}
|
|
1657
|
+
return 0;
|
|
1658
|
+
}
|
|
1659
|
+
function countUserImages(payload) {
|
|
1660
|
+
const images = Array.isArray(payload.images) ? payload.images.length : 0;
|
|
1661
|
+
const localImages = Array.isArray(payload.local_images) ? payload.local_images.length : 0;
|
|
1662
|
+
return images + localImages + countImagesDeep(payload.message || "");
|
|
1663
|
+
}
|
|
1664
|
+
function extractPatchFiles(input) {
|
|
1665
|
+
const files = new Set();
|
|
1666
|
+
for (const line of String(input || "").split("\n")) {
|
|
1667
|
+
const match = line.match(/^\*\*\* (?:Add|Update|Delete) File: (.+)$/) || line.match(/^\*\*\* Move to: (.+)$/);
|
|
1668
|
+
if (match?.[1])
|
|
1669
|
+
files.add(normalizeFile(match[1]));
|
|
1670
|
+
}
|
|
1671
|
+
return [...files];
|
|
1672
|
+
}
|
|
1673
|
+
function tokensFromOutput(output) {
|
|
1674
|
+
const text = typeof output === "string" ? output : safeJson(output);
|
|
1675
|
+
if (!text)
|
|
1676
|
+
return 0;
|
|
1677
|
+
const original = text.match(ORIGINAL_TOKEN_RE);
|
|
1678
|
+
const tokens = original ? Number(original[1]) : codeTokens(text);
|
|
1679
|
+
return Math.min(OUTPUT_TOKEN_CAP, Math.max(0, tokens));
|
|
1680
|
+
}
|
|
1681
|
+
function textTokens(text) {
|
|
1682
|
+
return Math.round(String(text || "").length / R_TEXT);
|
|
1683
|
+
}
|
|
1684
|
+
function codeTokens(text) {
|
|
1685
|
+
return Math.round(String(text || "").length / R_CODE);
|
|
1686
|
+
}
|
|
1687
|
+
function readNestedText(value) {
|
|
1688
|
+
if (typeof value === "string")
|
|
1689
|
+
return value;
|
|
1690
|
+
if (!isRecord(value))
|
|
1691
|
+
return "";
|
|
1692
|
+
if (typeof value.text === "string")
|
|
1693
|
+
return value.text;
|
|
1694
|
+
return safeJson(value);
|
|
1695
|
+
}
|
|
1696
|
+
function userTextFromPayload(payload) {
|
|
1697
|
+
const message = payload.message;
|
|
1698
|
+
if (typeof message === "string")
|
|
1699
|
+
return message.trim();
|
|
1700
|
+
if (Array.isArray(message))
|
|
1701
|
+
return message.map(textFromContentBlock).filter(Boolean).join("\n").trim();
|
|
1702
|
+
if (isRecord(message)) {
|
|
1703
|
+
const content = message.content;
|
|
1704
|
+
if (typeof content === "string")
|
|
1705
|
+
return content.trim();
|
|
1706
|
+
if (Array.isArray(content))
|
|
1707
|
+
return content.map(textFromContentBlock).filter(Boolean).join("\n").trim();
|
|
1708
|
+
}
|
|
1709
|
+
return "";
|
|
1710
|
+
}
|
|
1711
|
+
function assistantTextFromPayload(payload) {
|
|
1712
|
+
const message = payload.message;
|
|
1713
|
+
if (typeof message === "string")
|
|
1714
|
+
return message.trim();
|
|
1715
|
+
if (isRecord(message)) {
|
|
1716
|
+
const content = message.content;
|
|
1717
|
+
if (typeof content === "string")
|
|
1718
|
+
return content.trim();
|
|
1719
|
+
if (Array.isArray(content))
|
|
1720
|
+
return content.map(textFromContentBlock).filter(Boolean).join("\n").trim();
|
|
1721
|
+
}
|
|
1722
|
+
const content = payload.content;
|
|
1723
|
+
if (typeof content === "string")
|
|
1724
|
+
return content.trim();
|
|
1725
|
+
if (Array.isArray(content))
|
|
1726
|
+
return content.map(textFromContentBlock).filter(Boolean).join("\n").trim();
|
|
1727
|
+
return "";
|
|
1728
|
+
}
|
|
1729
|
+
function textFromContentBlock(block) {
|
|
1730
|
+
if (typeof block === "string")
|
|
1731
|
+
return block;
|
|
1732
|
+
if (!isRecord(block))
|
|
1733
|
+
return "";
|
|
1734
|
+
const text = block.text;
|
|
1735
|
+
if (typeof text === "string")
|
|
1736
|
+
return text;
|
|
1737
|
+
const content = block.content;
|
|
1738
|
+
return typeof content === "string" ? content : "";
|
|
1739
|
+
}
|
|
1740
|
+
function parseArguments(value) {
|
|
1741
|
+
if (isRecord(value))
|
|
1742
|
+
return value;
|
|
1743
|
+
if (typeof value !== "string")
|
|
1744
|
+
return {};
|
|
1745
|
+
try {
|
|
1746
|
+
return JSON.parse(value);
|
|
1747
|
+
}
|
|
1748
|
+
catch {
|
|
1749
|
+
return {};
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
function readNumber(value) {
|
|
1753
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
1754
|
+
}
|
|
1755
|
+
function normalizeFile(filePath) {
|
|
1756
|
+
return String(filePath).replace(/^['"]|['"]$/g, "").replace(/^\.\//, "").toLowerCase();
|
|
1757
|
+
}
|
|
1758
|
+
function repoLabel(cwd) {
|
|
1759
|
+
if (!cwd)
|
|
1760
|
+
return "home";
|
|
1761
|
+
const m = cwd.match(/worktrees\/[^/]+\/(.+)$/);
|
|
1762
|
+
const base = path.basename(m ? m[1] : cwd);
|
|
1763
|
+
return base && base !== "." && base !== "/" ? base : "home";
|
|
1764
|
+
}
|
|
1765
|
+
function lastBefore(values, turn) {
|
|
1766
|
+
let out = 0;
|
|
1767
|
+
for (const value of values)
|
|
1768
|
+
if (value < turn)
|
|
1769
|
+
out = Math.max(out, value);
|
|
1770
|
+
return out;
|
|
1771
|
+
}
|
|
1772
|
+
function rangesOverlap(a, b) {
|
|
1773
|
+
if (!a || !b)
|
|
1774
|
+
return true;
|
|
1775
|
+
return a.start <= b.end && b.start <= a.end;
|
|
1776
|
+
}
|
|
1777
|
+
function publicBucket(bucket) {
|
|
1778
|
+
if (bucket === "opt_dup")
|
|
1779
|
+
return "duplicate";
|
|
1780
|
+
if (bucket === "opt_refind")
|
|
1781
|
+
return "refind";
|
|
1782
|
+
if (bucket === "opt_dead")
|
|
1783
|
+
return "dead";
|
|
1784
|
+
return "unattributed";
|
|
1785
|
+
}
|
|
1786
|
+
function itemEvidence(item) {
|
|
1787
|
+
return (item.label || item.file || item.normalizedCommand || item.command || item.kind || "item").replace(/\s+/g, " ").slice(0, 180);
|
|
1788
|
+
}
|
|
1789
|
+
function mapBucketValues(values, fn) {
|
|
1790
|
+
const out = { ...values };
|
|
1791
|
+
for (const key of Object.keys(out))
|
|
1792
|
+
out[key] = fn(out[key]);
|
|
1793
|
+
return out;
|
|
1794
|
+
}
|
|
1795
|
+
function safeJson(value) {
|
|
1796
|
+
try {
|
|
1797
|
+
return JSON.stringify(value || "");
|
|
1798
|
+
}
|
|
1799
|
+
catch {
|
|
1800
|
+
return "";
|
|
1801
|
+
}
|
|
1802
|
+
}
|
|
1803
|
+
function isRecord(value) {
|
|
1804
|
+
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
1805
|
+
}
|
|
1806
|
+
function pct(part, total) {
|
|
1807
|
+
return total > 0 ? Math.round((part / total) * 1000) / 10 : 0;
|
|
1808
|
+
}
|
|
1809
|
+
function severity(tokens, count, sessionCount) {
|
|
1810
|
+
if (tokens <= 0 && count <= 0)
|
|
1811
|
+
return 0;
|
|
1812
|
+
const tokenScore = Math.log10(Math.max(10, tokens)) * 1.15;
|
|
1813
|
+
const frequencyScore = Math.log10(Math.max(1, count) + 1) * 1.4;
|
|
1814
|
+
const spreadScore = sessionCount ? Math.min(2, count / Math.max(1, sessionCount)) : 0;
|
|
1815
|
+
return Math.round(Math.min(10, tokenScore + frequencyScore + spreadScore) * 10) / 10;
|
|
1816
|
+
}
|
|
1817
|
+
function loadCache() {
|
|
1818
|
+
try {
|
|
1819
|
+
const parsed = JSON.parse(fs.readFileSync(cachePath(), "utf8"));
|
|
1820
|
+
return parsed && parsed.v === 7 && parsed.files && typeof parsed.files === "object" ? parsed.files : {};
|
|
1821
|
+
}
|
|
1822
|
+
catch {
|
|
1823
|
+
return {};
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
function saveCache(files) {
|
|
1827
|
+
try {
|
|
1828
|
+
const target = cachePath();
|
|
1829
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
1830
|
+
fs.writeFileSync(target, JSON.stringify({ v: 7, files }));
|
|
1831
|
+
}
|
|
1832
|
+
catch {
|
|
1833
|
+
/* best effort */
|
|
1834
|
+
}
|
|
1835
|
+
}
|
|
1836
|
+
function cachePath() {
|
|
1837
|
+
return path.join(os.homedir(), ".echomem", "context-analysis-cache.json");
|
|
1838
|
+
}
|