@tangle-network/agent-eval 0.120.4 → 0.122.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -16
- package/dist/analyst/index.d.ts +1 -0
- package/dist/analyst/index.js +4 -4
- package/dist/belief-state/index.d.ts +43 -0
- package/dist/belief-state/index.js +48 -326
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +5 -1
- package/dist/benchmarks/index.js +6 -6
- package/dist/campaign/index.d.ts +16 -18
- package/dist/campaign/index.js +7 -5
- package/dist/{chunk-SQQED7ZH.js → chunk-3A246TSA.js} +2 -2
- package/dist/{chunk-NMN4WGSJ.js → chunk-4I2E3LLO.js} +2 -2
- package/dist/chunk-7VYEGHWF.js +1425 -0
- package/dist/chunk-7VYEGHWF.js.map +1 -0
- package/dist/{chunk-ZMXDQ4K7.js → chunk-JHOJHHU7.js} +5 -8
- package/dist/{chunk-ZMXDQ4K7.js.map → chunk-JHOJHHU7.js.map} +1 -1
- package/dist/{chunk-3XH4Y2SS.js → chunk-P62I523A.js} +5 -3
- package/dist/{chunk-3XH4Y2SS.js.map → chunk-P62I523A.js.map} +1 -1
- package/dist/{chunk-CVJP5TMD.js → chunk-PICTDURQ.js} +2 -2
- package/dist/{chunk-5BYTIDZ7.js → chunk-QWMPPZ3X.js} +2 -2
- package/dist/{chunk-5CVUPHJ4.js → chunk-S5TT5R3L.js} +2 -2
- package/dist/{chunk-ZTJPIIVI.js → chunk-TDHX3BVA.js} +160 -7
- package/dist/chunk-TDHX3BVA.js.map +1 -0
- package/dist/{chunk-7QBFOJ3E.js → chunk-VZ6VKOJT.js} +32 -211
- package/dist/chunk-VZ6VKOJT.js.map +1 -0
- package/dist/{chunk-SYV364BL.js → chunk-XDIRG3TO.js} +2 -2
- package/dist/cli.js +2 -2
- package/dist/contract/index.d.ts +118 -29
- package/dist/contract/index.js +552 -502
- package/dist/contract/index.js.map +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/index.d.ts +7 -19
- package/dist/index.js +13 -11
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/rl.d.ts +5 -1
- package/dist/{run-campaign-75RTPVV5.js → run-campaign-HNFPJET4.js} +3 -3
- package/dist/wire/index.js +2 -2
- package/docs/eval-surface-map.md +1 -1
- package/package.json +2 -2
- package/dist/chunk-7QBFOJ3E.js.map +0 -1
- package/dist/chunk-HKUCJ437.js +0 -787
- package/dist/chunk-HKUCJ437.js.map +0 -1
- package/dist/chunk-ZTJPIIVI.js.map +0 -1
- /package/dist/{chunk-SQQED7ZH.js.map → chunk-3A246TSA.js.map} +0 -0
- /package/dist/{chunk-NMN4WGSJ.js.map → chunk-4I2E3LLO.js.map} +0 -0
- /package/dist/{chunk-CVJP5TMD.js.map → chunk-PICTDURQ.js.map} +0 -0
- /package/dist/{chunk-5BYTIDZ7.js.map → chunk-QWMPPZ3X.js.map} +0 -0
- /package/dist/{chunk-5CVUPHJ4.js.map → chunk-S5TT5R3L.js.map} +0 -0
- /package/dist/{chunk-SYV364BL.js.map → chunk-XDIRG3TO.js.map} +0 -0
- /package/dist/{run-campaign-75RTPVV5.js.map → run-campaign-HNFPJET4.js.map} +0 -0
|
@@ -0,0 +1,1425 @@
|
|
|
1
|
+
import {
|
|
2
|
+
estimateCost,
|
|
3
|
+
isModelPriced
|
|
4
|
+
} from "./chunk-VI2UW6B6.js";
|
|
5
|
+
import {
|
|
6
|
+
extractUsage
|
|
7
|
+
} from "./chunk-PXE2VKMX.js";
|
|
8
|
+
|
|
9
|
+
// src/contract/intake/code-agent-observation.ts
|
|
10
|
+
function observeCodeAgentSession(options) {
|
|
11
|
+
const entries = options.entries.filter(isRecord);
|
|
12
|
+
const projection = projectionFor(options.source, entries);
|
|
13
|
+
const terminal = terminalWithExecution(projection, options.execution);
|
|
14
|
+
return {
|
|
15
|
+
source: options.source,
|
|
16
|
+
sessionId: sessionIdFromEntries(options.source, entries) ?? fallbackSessionId(options.source, options.sourcePath),
|
|
17
|
+
finalText: projection.finalText,
|
|
18
|
+
terminal,
|
|
19
|
+
actions: projection.actions
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
function projectionFor(source, entries) {
|
|
23
|
+
switch (source) {
|
|
24
|
+
case "codex":
|
|
25
|
+
return codexProjection(entries);
|
|
26
|
+
case "claude-code":
|
|
27
|
+
return claudeProjection(entries);
|
|
28
|
+
case "opencode":
|
|
29
|
+
return openCodeProjection(entries);
|
|
30
|
+
case "kimi-code":
|
|
31
|
+
return kimiProjection(entries);
|
|
32
|
+
case "pi":
|
|
33
|
+
return piProjection(entries);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
function codexProjection(entries) {
|
|
37
|
+
const actions = [];
|
|
38
|
+
const calls = /* @__PURE__ */ new Map();
|
|
39
|
+
let finalText = null;
|
|
40
|
+
let terminal = "unknown";
|
|
41
|
+
let explicitTerminal = false;
|
|
42
|
+
for (const entry of entries) {
|
|
43
|
+
const entryType = stringField(entry, "type");
|
|
44
|
+
const payload = record(entry.payload) ?? {};
|
|
45
|
+
const payloadType = stringField(payload, "type");
|
|
46
|
+
const timestampMs2 = timestamp(entry.timestamp);
|
|
47
|
+
const item = record(entry.item);
|
|
48
|
+
if (entryType === "item.started" && item) {
|
|
49
|
+
const itemType = stringField(item, "type");
|
|
50
|
+
if (isCodexActionItem(itemType)) {
|
|
51
|
+
const id = stringField(item, "id") ?? `item-${actions.length}`;
|
|
52
|
+
const action = actionFor({
|
|
53
|
+
id,
|
|
54
|
+
stepIndex: actions.length,
|
|
55
|
+
kind: itemType === "file_change" ? "patch" : "tool",
|
|
56
|
+
surface: codexItemSurface(itemType, item),
|
|
57
|
+
name: codexItemName(itemType, item),
|
|
58
|
+
status: "started",
|
|
59
|
+
timestampMs: timestampMs2,
|
|
60
|
+
metadata: compactMetadata({ sourceEventType: entryType, itemType })
|
|
61
|
+
});
|
|
62
|
+
calls.set(id, action);
|
|
63
|
+
actions.push(action);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
if (entryType === "item.completed" && item) {
|
|
67
|
+
const itemType = stringField(item, "type");
|
|
68
|
+
if (itemType === "agent_message" || itemType === "message") {
|
|
69
|
+
finalText = nonEmpty(stringField(item, "text")) ?? finalText;
|
|
70
|
+
}
|
|
71
|
+
if (isCodexActionItem(itemType)) {
|
|
72
|
+
const id = stringField(item, "id") ?? `item-${actions.length}`;
|
|
73
|
+
const status = codexItemStatus(itemType, item);
|
|
74
|
+
const existing = calls.get(id);
|
|
75
|
+
if (existing) {
|
|
76
|
+
existing.status = status;
|
|
77
|
+
} else {
|
|
78
|
+
const action = actionFor({
|
|
79
|
+
id,
|
|
80
|
+
stepIndex: actions.length,
|
|
81
|
+
kind: itemType === "file_change" ? "patch" : "tool",
|
|
82
|
+
surface: codexItemSurface(itemType, item),
|
|
83
|
+
name: codexItemName(itemType, item),
|
|
84
|
+
status,
|
|
85
|
+
timestampMs: timestampMs2,
|
|
86
|
+
metadata: compactMetadata({ sourceEventType: entryType, itemType })
|
|
87
|
+
});
|
|
88
|
+
calls.set(id, action);
|
|
89
|
+
actions.push(action);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (entryType === "response_item") {
|
|
94
|
+
if (payloadType === "message" || payloadType === "agent_message") {
|
|
95
|
+
finalText = textFromMessagePayload(payload) ?? finalText;
|
|
96
|
+
}
|
|
97
|
+
if (payloadType === "function_call" || payloadType === "custom_tool_call") {
|
|
98
|
+
const id = stringField(payload, "call_id") ?? stringField(payload, "id") ?? `call-${actions.length}`;
|
|
99
|
+
const name = stringField(payload, "name") ?? payloadType;
|
|
100
|
+
const action = actionFor({
|
|
101
|
+
id,
|
|
102
|
+
stepIndex: actions.length,
|
|
103
|
+
kind: "tool",
|
|
104
|
+
surface: surfaceForTool(name),
|
|
105
|
+
name,
|
|
106
|
+
status: "started",
|
|
107
|
+
timestampMs: timestampMs2,
|
|
108
|
+
metadata: compactMetadata({ sourceEventType: entryType, payloadType })
|
|
109
|
+
});
|
|
110
|
+
calls.set(id, action);
|
|
111
|
+
actions.push(action);
|
|
112
|
+
}
|
|
113
|
+
if (payloadType === "function_call_output" || payloadType === "custom_tool_call_output") {
|
|
114
|
+
const id = stringField(payload, "call_id") ?? stringField(payload, "id");
|
|
115
|
+
const existing = id ? calls.get(id) : void 0;
|
|
116
|
+
if (existing) existing.status = looksLikeError(payload.output) ? "failed" : "completed";
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (entryType === "event_msg") {
|
|
120
|
+
if (payloadType === "agent_message") {
|
|
121
|
+
finalText = nonEmpty(stringField(payload, "message")) ?? nonEmpty(stringField(payload, "text")) ?? finalText;
|
|
122
|
+
}
|
|
123
|
+
if (payloadType === "patch_apply_end") {
|
|
124
|
+
const patchStatus = typeof payload.success === "boolean" ? payload.success ? "completed" : "failed" : "unknown";
|
|
125
|
+
actions.push(
|
|
126
|
+
actionFor({
|
|
127
|
+
id: stringField(payload, "call_id") ?? `patch-${actions.length}`,
|
|
128
|
+
stepIndex: actions.length,
|
|
129
|
+
kind: "patch",
|
|
130
|
+
surface: "code",
|
|
131
|
+
name: "patch",
|
|
132
|
+
status: patchStatus,
|
|
133
|
+
timestampMs: timestampMs2,
|
|
134
|
+
metadata: compactMetadata({ sourceEventType: entryType, payloadType })
|
|
135
|
+
})
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
if (payloadType === "sub_agent_activity") {
|
|
139
|
+
const name = stringField(payload, "kind") ?? "subagent";
|
|
140
|
+
actions.push(
|
|
141
|
+
actionFor({
|
|
142
|
+
id: stringField(payload, "event_id") ?? `subagent-${actions.length}`,
|
|
143
|
+
stepIndex: actions.length,
|
|
144
|
+
kind: "tool",
|
|
145
|
+
surface: "subagent",
|
|
146
|
+
name,
|
|
147
|
+
status: statusFrom(payload),
|
|
148
|
+
timestampMs: timestampMs2,
|
|
149
|
+
metadata: compactMetadata({ sourceEventType: entryType, payloadType })
|
|
150
|
+
})
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
if (payloadType === "task_complete" || payloadType === "turn_aborted") {
|
|
154
|
+
const completed = payloadType === "task_complete";
|
|
155
|
+
terminal = completed ? "completed" : "failed";
|
|
156
|
+
explicitTerminal = true;
|
|
157
|
+
actions.push(terminalAction(actions.length, payloadType, completed, timestampMs2));
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (entryType === "turn.completed" || entryType === "exec_done") {
|
|
161
|
+
const completed = entry.success !== false;
|
|
162
|
+
terminal = completed ? "completed" : "failed";
|
|
163
|
+
explicitTerminal = true;
|
|
164
|
+
finalText = nonEmpty(stringField(entry, "message")) ?? finalText;
|
|
165
|
+
actions.push(terminalAction(actions.length, entryType, completed, timestampMs2));
|
|
166
|
+
}
|
|
167
|
+
if (entryType === "turn.failed" || entryType === "error") {
|
|
168
|
+
terminal = "failed";
|
|
169
|
+
explicitTerminal = true;
|
|
170
|
+
actions.push(terminalAction(actions.length, entryType, false, timestampMs2));
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return { finalText, terminal, explicitTerminal, actions };
|
|
174
|
+
}
|
|
175
|
+
function claudeProjection(entries) {
|
|
176
|
+
const actions = [];
|
|
177
|
+
const calls = /* @__PURE__ */ new Map();
|
|
178
|
+
let finalText = null;
|
|
179
|
+
let resultText = null;
|
|
180
|
+
let terminal = "unknown";
|
|
181
|
+
let explicitTerminal = false;
|
|
182
|
+
for (const entry of entries) {
|
|
183
|
+
const entryType = stringField(entry, "type");
|
|
184
|
+
const timestampMs2 = timestamp(entry.timestamp);
|
|
185
|
+
const message = record(entry.message);
|
|
186
|
+
const content = Array.isArray(message?.content) ? message.content : [];
|
|
187
|
+
if (entryType === "assistant") {
|
|
188
|
+
const text = textBlocks(content);
|
|
189
|
+
if (text !== null) finalText = text;
|
|
190
|
+
for (const value of content) {
|
|
191
|
+
const part = record(value);
|
|
192
|
+
if (!part) continue;
|
|
193
|
+
const partType = stringField(part, "type");
|
|
194
|
+
if (partType === "tool_use") {
|
|
195
|
+
const id = stringField(part, "id") ?? `tool-${actions.length}`;
|
|
196
|
+
const name = stringField(part, "name") ?? "tool";
|
|
197
|
+
const action = actionFor({
|
|
198
|
+
id,
|
|
199
|
+
stepIndex: actions.length,
|
|
200
|
+
kind: "tool",
|
|
201
|
+
surface: surfaceForTool(name),
|
|
202
|
+
name,
|
|
203
|
+
status: "started",
|
|
204
|
+
timestampMs: timestampMs2,
|
|
205
|
+
metadata: compactMetadata({ sourceEventType: entryType, partType })
|
|
206
|
+
});
|
|
207
|
+
calls.set(id, action);
|
|
208
|
+
actions.push(action);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
completeClaudeTools(content, calls);
|
|
212
|
+
const stopReason = stringField(message ?? {}, "stop_reason");
|
|
213
|
+
if (stopReason === "end_turn") {
|
|
214
|
+
terminal = "completed";
|
|
215
|
+
explicitTerminal = true;
|
|
216
|
+
actions.push(terminalAction(actions.length, stopReason, true, timestampMs2));
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (entryType === "user") {
|
|
220
|
+
completeClaudeTools(content, calls);
|
|
221
|
+
}
|
|
222
|
+
if (entryType === "tool_use") {
|
|
223
|
+
const id = stringField(entry, "id") ?? `tool-${actions.length}`;
|
|
224
|
+
const name = stringField(entry, "name") ?? "tool";
|
|
225
|
+
const action = actionFor({
|
|
226
|
+
id,
|
|
227
|
+
stepIndex: actions.length,
|
|
228
|
+
kind: "tool",
|
|
229
|
+
surface: surfaceForTool(name),
|
|
230
|
+
name,
|
|
231
|
+
status: "started",
|
|
232
|
+
timestampMs: timestampMs2,
|
|
233
|
+
metadata: { sourceEventType: entryType }
|
|
234
|
+
});
|
|
235
|
+
calls.set(id, action);
|
|
236
|
+
actions.push(action);
|
|
237
|
+
}
|
|
238
|
+
if (entryType === "tool_result") {
|
|
239
|
+
const id = stringField(entry, "tool_use_id");
|
|
240
|
+
const action = id ? calls.get(id) : void 0;
|
|
241
|
+
if (action) action.status = entry.is_error === true ? "failed" : "completed";
|
|
242
|
+
}
|
|
243
|
+
if (entryType === "system") {
|
|
244
|
+
const subtype = stringField(entry, "subtype");
|
|
245
|
+
const hookName = stringField(entry, "hook_name") ?? stringField(entry, "hookName");
|
|
246
|
+
if (subtype?.toLowerCase().includes("hook") || hookName) {
|
|
247
|
+
actions.push(
|
|
248
|
+
actionFor({
|
|
249
|
+
id: stringField(entry, "uuid") ?? `hook-${actions.length}`,
|
|
250
|
+
stepIndex: actions.length,
|
|
251
|
+
kind: "tool",
|
|
252
|
+
surface: "hook",
|
|
253
|
+
name: hookName ?? subtype ?? "hook",
|
|
254
|
+
status: statusFrom(entry),
|
|
255
|
+
timestampMs: timestampMs2,
|
|
256
|
+
metadata: compactMetadata({ sourceEventType: entryType, subtype })
|
|
257
|
+
})
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
if (entryType === "pr-link") {
|
|
262
|
+
actions.push(terminalAction(actions.length, entryType, true, timestampMs2));
|
|
263
|
+
}
|
|
264
|
+
if (entryType === "result") {
|
|
265
|
+
resultText = nonEmpty(stringField(entry, "result")) ?? resultText;
|
|
266
|
+
const subtype = stringField(entry, "subtype");
|
|
267
|
+
const completed = entry.is_error !== true && !subtype?.toLowerCase().includes("error");
|
|
268
|
+
terminal = completed ? "completed" : "failed";
|
|
269
|
+
explicitTerminal = true;
|
|
270
|
+
actions.push(terminalAction(actions.length, subtype ?? entryType, completed, timestampMs2));
|
|
271
|
+
}
|
|
272
|
+
if (entryType === "error") {
|
|
273
|
+
terminal = "failed";
|
|
274
|
+
explicitTerminal = true;
|
|
275
|
+
actions.push(terminalAction(actions.length, entryType, false, timestampMs2));
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
return {
|
|
279
|
+
finalText: resultText ?? finalText,
|
|
280
|
+
terminal,
|
|
281
|
+
explicitTerminal,
|
|
282
|
+
actions
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
function openCodeProjection(entries) {
|
|
286
|
+
const actions = [];
|
|
287
|
+
const assistantMessageIds = new Set(
|
|
288
|
+
entries.filter((entry) => stringField(entry, "role") === "assistant").map((entry) => stringField(entry, "id")).filter((id) => id !== void 0)
|
|
289
|
+
);
|
|
290
|
+
let finalText = null;
|
|
291
|
+
let terminal = "unknown";
|
|
292
|
+
let explicitTerminal = false;
|
|
293
|
+
for (const entry of entries) {
|
|
294
|
+
const entryType = stringField(entry, "type");
|
|
295
|
+
const part = record(entry.part) ?? entry;
|
|
296
|
+
const partType = stringField(part, "type");
|
|
297
|
+
const timestampMs2 = timestamp(entry.timestamp) ?? timestamp(record(entry.time)?.created);
|
|
298
|
+
if (entryType === "text" && record(entry.part)) {
|
|
299
|
+
finalText = nonEmpty(stringField(part, "text")) ?? finalText;
|
|
300
|
+
} else if (partType === "text" && (assistantMessageIds.size === 0 || assistantMessageIds.has(stringField(part, "messageID") ?? ""))) {
|
|
301
|
+
finalText = nonEmpty(stringField(part, "text")) ?? finalText;
|
|
302
|
+
}
|
|
303
|
+
if (entryType === "tool_use" || partType === "tool") {
|
|
304
|
+
const state = record(part.state) ?? {};
|
|
305
|
+
const name = stringField(part, "tool") ?? "tool";
|
|
306
|
+
actions.push(
|
|
307
|
+
actionFor({
|
|
308
|
+
id: stringField(part, "callID") ?? stringField(part, "id") ?? `tool-${actions.length}`,
|
|
309
|
+
stepIndex: actions.length,
|
|
310
|
+
kind: "tool",
|
|
311
|
+
surface: surfaceForTool(name),
|
|
312
|
+
name,
|
|
313
|
+
status: statusFrom(state),
|
|
314
|
+
timestampMs: timestampMs2,
|
|
315
|
+
metadata: compactMetadata({ sourceEventType: entryType, partType })
|
|
316
|
+
})
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
if (partType === "patch") {
|
|
320
|
+
actions.push(
|
|
321
|
+
actionFor({
|
|
322
|
+
id: stringField(part, "id") ?? `patch-${actions.length}`,
|
|
323
|
+
stepIndex: actions.length,
|
|
324
|
+
kind: "patch",
|
|
325
|
+
surface: "code",
|
|
326
|
+
name: "patch",
|
|
327
|
+
status: "completed",
|
|
328
|
+
timestampMs: timestampMs2,
|
|
329
|
+
metadata: compactMetadata({ sourceEventType: entryType, partType })
|
|
330
|
+
})
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
if (stringField(entry, "role") === "assistant") {
|
|
334
|
+
const finish = stringField(entry, "finish");
|
|
335
|
+
if (finish === "stop" || finish === "error") {
|
|
336
|
+
const completed = finish === "stop";
|
|
337
|
+
terminal = completed ? "completed" : "failed";
|
|
338
|
+
explicitTerminal = true;
|
|
339
|
+
actions.push(terminalAction(actions.length, finish, completed, timestampMs2));
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
if (entryType === "error") {
|
|
343
|
+
terminal = "failed";
|
|
344
|
+
explicitTerminal = true;
|
|
345
|
+
actions.push(terminalAction(actions.length, entryType, false, timestampMs2));
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return { finalText, terminal, explicitTerminal, actions };
|
|
349
|
+
}
|
|
350
|
+
function kimiProjection(entries) {
|
|
351
|
+
const actions = [];
|
|
352
|
+
const calls = /* @__PURE__ */ new Map();
|
|
353
|
+
let finalText = null;
|
|
354
|
+
let terminal = "unknown";
|
|
355
|
+
let explicitTerminal = false;
|
|
356
|
+
for (const entry of entries) {
|
|
357
|
+
const timestampMs2 = timestamp(entry.timestamp);
|
|
358
|
+
const message = record(entry.message);
|
|
359
|
+
const messageType = stringField(message ?? {}, "type");
|
|
360
|
+
const payload = record(message?.payload) ?? {};
|
|
361
|
+
if (messageType === "ContentPart" && stringField(payload, "type") === "text") {
|
|
362
|
+
finalText = nonEmpty(stringField(payload, "text")) ?? nonEmpty(stringField(payload, "content")) ?? finalText;
|
|
363
|
+
}
|
|
364
|
+
if (messageType === "ToolCall") {
|
|
365
|
+
const fn = record(payload.function) ?? {};
|
|
366
|
+
const id = stringField(payload, "id") ?? `tool-${actions.length}`;
|
|
367
|
+
const name = stringField(fn, "name") ?? "tool";
|
|
368
|
+
const action = actionFor({
|
|
369
|
+
id,
|
|
370
|
+
stepIndex: actions.length,
|
|
371
|
+
kind: "tool",
|
|
372
|
+
surface: surfaceForTool(name),
|
|
373
|
+
name,
|
|
374
|
+
status: "started",
|
|
375
|
+
timestampMs: timestampMs2,
|
|
376
|
+
metadata: { sourceEventType: messageType }
|
|
377
|
+
});
|
|
378
|
+
calls.set(id, action);
|
|
379
|
+
actions.push(action);
|
|
380
|
+
}
|
|
381
|
+
if (messageType === "ToolResult") {
|
|
382
|
+
const id = stringField(payload, "tool_call_id");
|
|
383
|
+
const action = id ? calls.get(id) : void 0;
|
|
384
|
+
if (action)
|
|
385
|
+
action.status = record(payload.return_value)?.is_error === true ? "failed" : "completed";
|
|
386
|
+
}
|
|
387
|
+
if (messageType === "TurnEnd" || messageType === "StepInterrupted") {
|
|
388
|
+
const completed = messageType === "TurnEnd";
|
|
389
|
+
terminal = completed ? "completed" : "failed";
|
|
390
|
+
explicitTerminal = true;
|
|
391
|
+
actions.push(terminalAction(actions.length, messageType, completed, timestampMs2));
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
return { finalText, terminal, explicitTerminal, actions };
|
|
395
|
+
}
|
|
396
|
+
function piProjection(entries) {
|
|
397
|
+
const actions = [];
|
|
398
|
+
let terminal = "unknown";
|
|
399
|
+
let explicitTerminal = false;
|
|
400
|
+
for (const entry of entries) {
|
|
401
|
+
const nodes = Array.isArray(entry.nodes) ? entry.nodes : [];
|
|
402
|
+
for (const value of nodes) {
|
|
403
|
+
const node = record(value);
|
|
404
|
+
const ir = record(node?.ir) ?? node;
|
|
405
|
+
const kind = stringField(ir ?? {}, "kind");
|
|
406
|
+
if (kind === "ToolInvocation") {
|
|
407
|
+
actions.push(
|
|
408
|
+
actionFor({
|
|
409
|
+
id: stringField(ir ?? {}, "id") ?? stringField(node ?? {}, "id") ?? `tool-${actions.length}`,
|
|
410
|
+
stepIndex: actions.length,
|
|
411
|
+
kind: "tool",
|
|
412
|
+
surface: "tool",
|
|
413
|
+
name: stringField(ir ?? {}, "name") ?? "graph-tool",
|
|
414
|
+
status: "started",
|
|
415
|
+
timestampMs: timestamp(ir?.createdAt),
|
|
416
|
+
metadata: { sourceEventType: "graph-node", graphKind: kind }
|
|
417
|
+
})
|
|
418
|
+
);
|
|
419
|
+
}
|
|
420
|
+
if (kind === "ToolResult") {
|
|
421
|
+
const prior = [...actions].reverse().find((action) => action.kind === "tool");
|
|
422
|
+
if (prior) prior.status = "completed";
|
|
423
|
+
}
|
|
424
|
+
if (kind === "CompletionDecision") {
|
|
425
|
+
terminal = "completed";
|
|
426
|
+
explicitTerminal = true;
|
|
427
|
+
actions.push(
|
|
428
|
+
actionFor({
|
|
429
|
+
id: stringField(ir ?? {}, "id") ?? stringField(node ?? {}, "id") ?? `completion-${actions.length}`,
|
|
430
|
+
stepIndex: actions.length,
|
|
431
|
+
kind: "graph-completion",
|
|
432
|
+
surface: "tool",
|
|
433
|
+
name: "complete",
|
|
434
|
+
status: "completed",
|
|
435
|
+
timestampMs: timestamp(ir?.createdAt),
|
|
436
|
+
metadata: { sourceEventType: "graph-node", graphKind: kind }
|
|
437
|
+
})
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
return { finalText: null, terminal, explicitTerminal, actions };
|
|
443
|
+
}
|
|
444
|
+
function completeClaudeTools(content, calls) {
|
|
445
|
+
for (const value of content) {
|
|
446
|
+
const part = record(value);
|
|
447
|
+
if (!part || stringField(part, "type") !== "tool_result") continue;
|
|
448
|
+
const id = stringField(part, "tool_use_id");
|
|
449
|
+
const action = id ? calls.get(id) : void 0;
|
|
450
|
+
if (action) action.status = part.is_error === true ? "failed" : "completed";
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
function isCodexActionItem(itemType) {
|
|
454
|
+
return itemType === "command_execution" || itemType === "mcp_tool_call" || itemType === "collab_tool_call" || itemType === "web_search" || itemType === "file_change";
|
|
455
|
+
}
|
|
456
|
+
function codexItemSurface(itemType, item) {
|
|
457
|
+
if (itemType === "mcp_tool_call") return "mcp";
|
|
458
|
+
if (itemType === "collab_tool_call") return "subagent";
|
|
459
|
+
if (itemType === "web_search") return "web";
|
|
460
|
+
if (itemType === "file_change") return "code";
|
|
461
|
+
return surfaceForTool(stringField(item, "name") ?? itemType ?? "tool");
|
|
462
|
+
}
|
|
463
|
+
function codexItemName(itemType, item) {
|
|
464
|
+
if (itemType === "mcp_tool_call") {
|
|
465
|
+
const server = stringField(item, "server");
|
|
466
|
+
const tool = stringField(item, "tool");
|
|
467
|
+
return [server, tool].filter(Boolean).join("/") || "mcp";
|
|
468
|
+
}
|
|
469
|
+
if (itemType === "collab_tool_call") return stringField(item, "tool") ?? "subagent";
|
|
470
|
+
if (itemType === "web_search") return "web_search";
|
|
471
|
+
if (itemType === "file_change") return "file_change";
|
|
472
|
+
return stringField(item, "name") ?? itemType ?? "tool";
|
|
473
|
+
}
|
|
474
|
+
function codexItemStatus(itemType, item) {
|
|
475
|
+
const status = statusFrom(item);
|
|
476
|
+
if (status === "failed") return status;
|
|
477
|
+
if (itemType === "command_execution") {
|
|
478
|
+
const exitCode = numberField(item, "exit_code");
|
|
479
|
+
if (exitCode !== void 0) return exitCode === 0 ? "completed" : "failed";
|
|
480
|
+
}
|
|
481
|
+
if (itemType === "mcp_tool_call" && record(item.error)) return "failed";
|
|
482
|
+
return status === "unknown" || status === "started" ? "completed" : status;
|
|
483
|
+
}
|
|
484
|
+
function textFromMessagePayload(payload) {
|
|
485
|
+
const direct = nonEmpty(stringField(payload, "text"));
|
|
486
|
+
if (direct) return direct;
|
|
487
|
+
const content = payload.content;
|
|
488
|
+
if (typeof content === "string") return nonEmpty(content);
|
|
489
|
+
if (Array.isArray(content)) return textBlocks(content);
|
|
490
|
+
return null;
|
|
491
|
+
}
|
|
492
|
+
function textBlocks(values) {
|
|
493
|
+
const text = values.map((value) => record(value)).filter((value) => value !== null).filter((value) => stringField(value, "type") === "text").map((value) => stringField(value, "text")).filter((value) => value !== void 0).join("\n");
|
|
494
|
+
return nonEmpty(text);
|
|
495
|
+
}
|
|
496
|
+
function terminalWithExecution(projection, execution) {
|
|
497
|
+
if (!execution) {
|
|
498
|
+
return { status: projection.terminal, explicit: projection.explicitTerminal };
|
|
499
|
+
}
|
|
500
|
+
assertExecutionReceipt(execution);
|
|
501
|
+
return {
|
|
502
|
+
status: execution.exitCode !== 0 || projection.terminal === "failed" ? "failed" : "completed",
|
|
503
|
+
explicit: true
|
|
504
|
+
};
|
|
505
|
+
}
|
|
506
|
+
function assertExecutionReceipt(receipt) {
|
|
507
|
+
if (!Number.isSafeInteger(receipt.exitCode)) {
|
|
508
|
+
throw new Error("code-agent execution exitCode must be a safe integer");
|
|
509
|
+
}
|
|
510
|
+
for (const [name, value] of [
|
|
511
|
+
["startedAtMs", receipt.startedAtMs],
|
|
512
|
+
["completedAtMs", receipt.completedAtMs]
|
|
513
|
+
]) {
|
|
514
|
+
if (value !== void 0 && (!Number.isFinite(value) || value < 0)) {
|
|
515
|
+
throw new Error(`code-agent execution ${name} must be a non-negative finite number`);
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
if (receipt.startedAtMs !== void 0 && receipt.completedAtMs !== void 0 && receipt.completedAtMs < receipt.startedAtMs) {
|
|
519
|
+
throw new Error("code-agent execution completedAtMs precedes startedAtMs");
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
function terminalAction(stepIndex, name, completed, timestampMs2) {
|
|
523
|
+
return actionFor({
|
|
524
|
+
id: `terminal-${stepIndex}`,
|
|
525
|
+
stepIndex,
|
|
526
|
+
kind: "terminal",
|
|
527
|
+
surface: "tool",
|
|
528
|
+
name,
|
|
529
|
+
status: completed ? "completed" : "failed",
|
|
530
|
+
timestampMs: timestampMs2,
|
|
531
|
+
metadata: { sourceEventType: name }
|
|
532
|
+
});
|
|
533
|
+
}
|
|
534
|
+
function actionFor(input) {
|
|
535
|
+
return input;
|
|
536
|
+
}
|
|
537
|
+
function surfaceForTool(name) {
|
|
538
|
+
const normalized = name.toLowerCase();
|
|
539
|
+
if (normalized.startsWith("mcp__") || normalized.includes("mcp_tool")) return "mcp";
|
|
540
|
+
if (normalized === "task" || normalized === "agent" || normalized.includes("subagent") || normalized.includes("spawn_agent") || normalized.includes("collab") || normalized.startsWith("multi_agent")) {
|
|
541
|
+
return "subagent";
|
|
542
|
+
}
|
|
543
|
+
if (normalized === "skill" || normalized.includes("skill")) return "skill";
|
|
544
|
+
if (normalized.includes("hook")) return "hook";
|
|
545
|
+
if (normalized.includes("web_search") || normalized.includes("webfetch") || normalized === "web") {
|
|
546
|
+
return "web";
|
|
547
|
+
}
|
|
548
|
+
if (normalized === "edit" || normalized === "write" || normalized.includes("patch") || normalized.includes("file_change")) {
|
|
549
|
+
return "code";
|
|
550
|
+
}
|
|
551
|
+
return "tool";
|
|
552
|
+
}
|
|
553
|
+
function statusFrom(value) {
|
|
554
|
+
const status = (stringField(value, "status") ?? "").toLowerCase();
|
|
555
|
+
if (status === "completed" || status === "success" || status === "succeeded") return "completed";
|
|
556
|
+
if (status === "error" || status === "failed" || status === "declined") return "failed";
|
|
557
|
+
if (status === "running" || status === "started" || status === "in_progress") return "started";
|
|
558
|
+
return "unknown";
|
|
559
|
+
}
|
|
560
|
+
function compactMetadata(input) {
|
|
561
|
+
return Object.fromEntries(Object.entries(input).filter((entry) => entry[1] !== void 0));
|
|
562
|
+
}
|
|
563
|
+
function sessionIdFromEntries(source, entries) {
|
|
564
|
+
for (const entry of entries) {
|
|
565
|
+
if (source === "codex") {
|
|
566
|
+
const threadId = stringField(entry, "thread_id");
|
|
567
|
+
if (threadId) return threadId;
|
|
568
|
+
const payload2 = record(entry.payload);
|
|
569
|
+
const id = payload2 ? stringField(payload2, "id") : void 0;
|
|
570
|
+
if (id) return id;
|
|
571
|
+
}
|
|
572
|
+
const message = record(entry.message);
|
|
573
|
+
const payload = record(message?.payload);
|
|
574
|
+
const sessionId = stringField(entry, "sessionID") ?? stringField(entry, "sessionId") ?? stringField(entry, "session_id") ?? stringField(payload ?? {}, "session_id");
|
|
575
|
+
if (sessionId) return sessionId;
|
|
576
|
+
}
|
|
577
|
+
return void 0;
|
|
578
|
+
}
|
|
579
|
+
function fallbackSessionId(source, sourcePath) {
|
|
580
|
+
return `${source}:${sourcePath ?? "unknown-session"}`;
|
|
581
|
+
}
|
|
582
|
+
function looksLikeError(value) {
|
|
583
|
+
if (isRecord(value)) {
|
|
584
|
+
if (value.success === false || value.is_error === true) return true;
|
|
585
|
+
const exitCode = numberField(value, "exit_code");
|
|
586
|
+
if (exitCode !== void 0 && exitCode !== 0) return true;
|
|
587
|
+
}
|
|
588
|
+
const text = typeof value === "string" ? value : JSON.stringify(value ?? "");
|
|
589
|
+
return /\b(?:error|failed|failure)\b|exit(?:_| )code["\s:]+[1-9]/i.test(text);
|
|
590
|
+
}
|
|
591
|
+
function nonEmpty(value) {
|
|
592
|
+
return value !== void 0 && value.length > 0 ? value : null;
|
|
593
|
+
}
|
|
594
|
+
function isRecord(value) {
|
|
595
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
596
|
+
}
|
|
597
|
+
function record(value) {
|
|
598
|
+
return isRecord(value) ? value : null;
|
|
599
|
+
}
|
|
600
|
+
function stringField(value, key) {
|
|
601
|
+
const field = value[key];
|
|
602
|
+
return typeof field === "string" && field.length > 0 ? field : void 0;
|
|
603
|
+
}
|
|
604
|
+
function numberField(value, key) {
|
|
605
|
+
const field = value[key];
|
|
606
|
+
return typeof field === "number" && Number.isFinite(field) ? field : void 0;
|
|
607
|
+
}
|
|
608
|
+
function timestamp(value) {
|
|
609
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
610
|
+
return value > 1e12 ? value : value * 1e3;
|
|
611
|
+
}
|
|
612
|
+
if (typeof value === "string" && value.length > 0) {
|
|
613
|
+
const parsed = Date.parse(value);
|
|
614
|
+
return Number.isFinite(parsed) ? parsed : void 0;
|
|
615
|
+
}
|
|
616
|
+
return void 0;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
// src/contract/intake/code-agent-session.ts
|
|
620
|
+
import { createHash } from "crypto";
|
|
621
|
+
function parseCodeAgentJsonl(jsonl) {
|
|
622
|
+
const entries = [];
|
|
623
|
+
let malformedLines = 0;
|
|
624
|
+
for (const line of jsonl.split("\n")) {
|
|
625
|
+
const trimmed = line.trim();
|
|
626
|
+
if (!trimmed) continue;
|
|
627
|
+
try {
|
|
628
|
+
entries.push(JSON.parse(trimmed));
|
|
629
|
+
} catch {
|
|
630
|
+
malformedLines += 1;
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
return { entries, malformedLines };
|
|
634
|
+
}
|
|
635
|
+
function fromCodexSession(options) {
|
|
636
|
+
return fromCodeAgentSession("codex", options);
|
|
637
|
+
}
|
|
638
|
+
function fromClaudeCodeSession(options) {
|
|
639
|
+
return fromCodeAgentSession("claude-code", options);
|
|
640
|
+
}
|
|
641
|
+
function fromOpenCodeSession(options) {
|
|
642
|
+
return fromCodeAgentSession("opencode", options);
|
|
643
|
+
}
|
|
644
|
+
function fromKimiCodeSession(options) {
|
|
645
|
+
return fromCodeAgentSession("kimi-code", options);
|
|
646
|
+
}
|
|
647
|
+
function fromPiSession(options) {
|
|
648
|
+
return fromCodeAgentSession("pi", options);
|
|
649
|
+
}
|
|
650
|
+
var fromPigraphSession = fromPiSession;
|
|
651
|
+
function fromCodeAgentSession(source, options) {
|
|
652
|
+
const entries = options.entries.filter(isRecord2);
|
|
653
|
+
if (entries.length === 0) {
|
|
654
|
+
return {
|
|
655
|
+
runs: [],
|
|
656
|
+
diagnostics: [
|
|
657
|
+
{
|
|
658
|
+
source,
|
|
659
|
+
sessionId: fallbackSessionId2(source, options.sourcePath),
|
|
660
|
+
sourcePath: options.sourcePath,
|
|
661
|
+
entries: 0,
|
|
662
|
+
malformedLines: options.malformedLines ?? 0,
|
|
663
|
+
inferredScore: true,
|
|
664
|
+
hasExplicitTerminalSignal: false,
|
|
665
|
+
hasFinalOutput: false,
|
|
666
|
+
hasQualityLabel: false,
|
|
667
|
+
hasTokenUsage: false,
|
|
668
|
+
hasCost: false,
|
|
669
|
+
costKind: "uncaptured",
|
|
670
|
+
warnings: ["no parseable session entries"]
|
|
671
|
+
}
|
|
672
|
+
],
|
|
673
|
+
metrics: [],
|
|
674
|
+
observations: []
|
|
675
|
+
};
|
|
676
|
+
}
|
|
677
|
+
const metrics = metricsFor(source, entries);
|
|
678
|
+
const sessionId = sessionIdFromEntries2(source, entries) ?? fallbackSessionId2(source, options.sourcePath);
|
|
679
|
+
const observation = observeCodeAgentSession({
|
|
680
|
+
source,
|
|
681
|
+
entries,
|
|
682
|
+
sourcePath: options.sourcePath,
|
|
683
|
+
execution: options.execution
|
|
684
|
+
});
|
|
685
|
+
applyObservedSurfaceMetrics(metrics, observation);
|
|
686
|
+
const model = withSnapshot(options.model ?? modelFromEntries(source, entries) ?? source);
|
|
687
|
+
const tokenUsage = {
|
|
688
|
+
input: metrics.inputTokens,
|
|
689
|
+
output: metrics.outputTokens,
|
|
690
|
+
...metrics.reasoningTokens > 0 ? { reasoning: metrics.reasoningTokens } : {},
|
|
691
|
+
...metrics.cachedTokens > 0 ? { cached: metrics.cachedTokens } : {},
|
|
692
|
+
...metrics.cacheWriteTokens > 0 ? { cacheWrite: metrics.cacheWriteTokens } : {}
|
|
693
|
+
};
|
|
694
|
+
const { costUsd, costProvenance } = resolveSessionCost(options.costProvenance, metrics, model);
|
|
695
|
+
const score = clamp01(options.score ?? metrics.processScore);
|
|
696
|
+
const promptHash = options.promptHash ?? hashString(`prompt:${source}:${firstUserText(entries) ?? ""}`);
|
|
697
|
+
const configHash = options.configHash ?? hashJson({
|
|
698
|
+
source,
|
|
699
|
+
model,
|
|
700
|
+
sourcePath: options.sourcePath,
|
|
701
|
+
cwd: cwdFromEntries(entries),
|
|
702
|
+
entryCount: entries.length
|
|
703
|
+
});
|
|
704
|
+
const explicitTerminal = observation.terminal.explicit;
|
|
705
|
+
const warnings = diagnosticsFor(metrics, {
|
|
706
|
+
model,
|
|
707
|
+
explicitTerminal,
|
|
708
|
+
malformedLines: options.malformedLines ?? 0,
|
|
709
|
+
scoreOverridden: options.score !== void 0,
|
|
710
|
+
costKind: costProvenance.kind
|
|
711
|
+
});
|
|
712
|
+
const run = {
|
|
713
|
+
runId: `${source}:${sessionId}`,
|
|
714
|
+
experimentId: options.experimentId ?? `${source}-local-sessions`,
|
|
715
|
+
candidateId: options.candidateId ?? model,
|
|
716
|
+
seed: options.seed ?? stableSeed(sessionId),
|
|
717
|
+
model,
|
|
718
|
+
promptHash,
|
|
719
|
+
configHash,
|
|
720
|
+
commitSha: options.commitSha ?? "local-session",
|
|
721
|
+
wallMs: metrics.wallMs,
|
|
722
|
+
costUsd,
|
|
723
|
+
costProvenance,
|
|
724
|
+
tokenUsage,
|
|
725
|
+
outcome: {
|
|
726
|
+
holdoutScore: score,
|
|
727
|
+
raw: {
|
|
728
|
+
entries: metrics.entries,
|
|
729
|
+
user_messages: metrics.userMessages,
|
|
730
|
+
assistant_messages: metrics.assistantMessages,
|
|
731
|
+
reasoning_items: metrics.reasoningItems,
|
|
732
|
+
tool_calls: metrics.toolCalls,
|
|
733
|
+
tool_outputs: metrics.toolOutputs,
|
|
734
|
+
tool_errors: metrics.toolErrors,
|
|
735
|
+
patch_attempts: metrics.patchAttempts,
|
|
736
|
+
patch_successes: metrics.patchSuccesses,
|
|
737
|
+
patch_failures: metrics.patchFailures,
|
|
738
|
+
turns_started: metrics.turnsStarted,
|
|
739
|
+
turns_completed: metrics.turnsCompleted,
|
|
740
|
+
turns_aborted: metrics.turnsAborted,
|
|
741
|
+
context_compactions: metrics.contextCompactions,
|
|
742
|
+
mcp_calls: metrics.mcpCalls,
|
|
743
|
+
subagent_calls: metrics.subagentCalls,
|
|
744
|
+
skill_calls: metrics.skillCalls,
|
|
745
|
+
hook_calls: metrics.hookCalls,
|
|
746
|
+
web_calls: metrics.webCalls,
|
|
747
|
+
code_actions: metrics.codeActions,
|
|
748
|
+
pr_links: metrics.prLinks,
|
|
749
|
+
file_snapshots: metrics.fileSnapshots,
|
|
750
|
+
graph_nodes: metrics.graphNodes,
|
|
751
|
+
graph_edges: metrics.graphEdges,
|
|
752
|
+
action_candidates: metrics.actionCandidates,
|
|
753
|
+
verification_reports: metrics.verificationReports,
|
|
754
|
+
completion_decisions: metrics.completionDecisions,
|
|
755
|
+
reliability_rows: metrics.reliabilityRows,
|
|
756
|
+
reliability_lift: metrics.reliabilityLift,
|
|
757
|
+
input_tokens: metrics.inputTokens,
|
|
758
|
+
output_tokens: metrics.outputTokens,
|
|
759
|
+
reasoning_tokens: metrics.reasoningTokens,
|
|
760
|
+
cached_tokens: metrics.cachedTokens,
|
|
761
|
+
cache_write_tokens: metrics.cacheWriteTokens,
|
|
762
|
+
observed_cost_usd: metrics.observedCostUsd,
|
|
763
|
+
observed_cost_captured: metrics.observedCostCaptured ? 1 : 0,
|
|
764
|
+
estimated_cost_usd: costProvenance.kind === "estimated" ? costUsd : 0,
|
|
765
|
+
process_score: score,
|
|
766
|
+
inferred_score: options.score === void 0 ? 1 : 0,
|
|
767
|
+
explicit_terminal_signal: explicitTerminal ? 1 : 0,
|
|
768
|
+
quality_label_present: options.score !== void 0 ? 1 : 0,
|
|
769
|
+
cost_observed: costProvenance.kind === "observed" ? 1 : 0,
|
|
770
|
+
cost_estimated: costProvenance.kind === "estimated" ? 1 : 0,
|
|
771
|
+
cost_uncaptured: costProvenance.kind === "uncaptured" ? 1 : 0,
|
|
772
|
+
cost_unknown: costProvenance.kind === "uncaptured" ? 1 : 0
|
|
773
|
+
}
|
|
774
|
+
},
|
|
775
|
+
splitTag: options.splitTag ?? "holdout",
|
|
776
|
+
...options.scenarioId ? { scenarioId: options.scenarioId } : {},
|
|
777
|
+
...metrics.toolErrors > 0 || metrics.turnsAborted > 0 ? { failureMode: metrics.turnsAborted > 0 ? "turn_aborted" : "tool_error" } : {}
|
|
778
|
+
};
|
|
779
|
+
return {
|
|
780
|
+
runs: [run],
|
|
781
|
+
diagnostics: [
|
|
782
|
+
{
|
|
783
|
+
source,
|
|
784
|
+
sessionId,
|
|
785
|
+
sourcePath: options.sourcePath,
|
|
786
|
+
entries: entries.length,
|
|
787
|
+
malformedLines: options.malformedLines ?? 0,
|
|
788
|
+
inferredScore: options.score === void 0,
|
|
789
|
+
hasExplicitTerminalSignal: explicitTerminal,
|
|
790
|
+
hasFinalOutput: observation.finalText !== null,
|
|
791
|
+
hasQualityLabel: options.score !== void 0,
|
|
792
|
+
hasTokenUsage: metrics.inputTokens > 0 || metrics.outputTokens > 0 || metrics.reasoningTokens > 0 || metrics.cachedTokens > 0 || metrics.cacheWriteTokens > 0,
|
|
793
|
+
hasCost: costProvenance.kind !== "uncaptured",
|
|
794
|
+
costKind: costProvenance.kind,
|
|
795
|
+
warnings
|
|
796
|
+
}
|
|
797
|
+
],
|
|
798
|
+
metrics: [metrics],
|
|
799
|
+
observations: [{ ...observation, sessionId }]
|
|
800
|
+
};
|
|
801
|
+
}
|
|
802
|
+
function metricsFor(source, entries) {
|
|
803
|
+
switch (source) {
|
|
804
|
+
case "codex":
|
|
805
|
+
return codexMetrics(entries);
|
|
806
|
+
case "claude-code":
|
|
807
|
+
return claudeCodeMetrics(entries);
|
|
808
|
+
case "opencode":
|
|
809
|
+
return openCodeMetrics(entries);
|
|
810
|
+
case "kimi-code":
|
|
811
|
+
return kimiCodeMetrics(entries);
|
|
812
|
+
case "pi":
|
|
813
|
+
return piMetrics(entries);
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
function codexMetrics(entries) {
|
|
817
|
+
const metrics = emptyMetrics(entries.length);
|
|
818
|
+
const startedToolIds = /* @__PURE__ */ new Set();
|
|
819
|
+
let startedAt;
|
|
820
|
+
let completedAt;
|
|
821
|
+
for (const entry of entries) {
|
|
822
|
+
const payload = record2(entry.payload) ?? {};
|
|
823
|
+
const entryType = stringField2(entry, "type");
|
|
824
|
+
const payloadType = stringField2(payload, "type");
|
|
825
|
+
const timestamp2 = timestampMs(entry.timestamp);
|
|
826
|
+
if (timestamp2 !== void 0) {
|
|
827
|
+
startedAt = startedAt === void 0 ? timestamp2 : Math.min(startedAt, timestamp2);
|
|
828
|
+
completedAt = completedAt === void 0 ? timestamp2 : Math.max(completedAt, timestamp2);
|
|
829
|
+
}
|
|
830
|
+
if (entryType === "turn.started") metrics.turnsStarted += 1;
|
|
831
|
+
if (entryType === "turn.completed") {
|
|
832
|
+
metrics.turnsCompleted += 1;
|
|
833
|
+
addCodexExecUsage(metrics, record2(entry.usage));
|
|
834
|
+
}
|
|
835
|
+
if (entryType === "turn.failed") metrics.turnsAborted += 1;
|
|
836
|
+
if (entryType === "error") metrics.toolErrors += 1;
|
|
837
|
+
const item = record2(entry.item);
|
|
838
|
+
if (item && (entryType === "item.started" || entryType === "item.completed")) {
|
|
839
|
+
addCodexExecItem(metrics, startedToolIds, entryType, item);
|
|
840
|
+
}
|
|
841
|
+
if (entryType === "response_item") {
|
|
842
|
+
if (payloadType === "function_call" || payloadType === "custom_tool_call")
|
|
843
|
+
metrics.toolCalls += 1;
|
|
844
|
+
if (payloadType === "function_call_output" || payloadType === "custom_tool_call_output")
|
|
845
|
+
metrics.toolOutputs += 1;
|
|
846
|
+
if (payloadType === "reasoning") metrics.reasoningItems += 1;
|
|
847
|
+
if (payloadType === "message" && stringField2(payload, "role") === "assistant")
|
|
848
|
+
metrics.assistantMessages += 1;
|
|
849
|
+
}
|
|
850
|
+
if (entryType === "event_msg") {
|
|
851
|
+
if (payloadType === "user_message") metrics.userMessages += 1;
|
|
852
|
+
if (payloadType === "agent_message") metrics.assistantMessages += 1;
|
|
853
|
+
if (payloadType === "task_started") {
|
|
854
|
+
metrics.turnsStarted += 1;
|
|
855
|
+
const started = timestampMs(payload.started_at);
|
|
856
|
+
if (started !== void 0) {
|
|
857
|
+
startedAt = startedAt === void 0 ? started : Math.min(startedAt, started);
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
if (payloadType === "task_complete") {
|
|
861
|
+
metrics.turnsCompleted += 1;
|
|
862
|
+
const completed = timestampMs(payload.completed_at);
|
|
863
|
+
if (completed !== void 0) {
|
|
864
|
+
completedAt = completedAt === void 0 ? completed : Math.max(completedAt, completed);
|
|
865
|
+
}
|
|
866
|
+
const duration = numberField2(payload, "duration_ms");
|
|
867
|
+
if (duration !== void 0) metrics.wallMs = Math.max(metrics.wallMs, duration);
|
|
868
|
+
}
|
|
869
|
+
if (payloadType === "turn_aborted") {
|
|
870
|
+
metrics.turnsAborted += 1;
|
|
871
|
+
const completed = timestampMs(payload.completed_at);
|
|
872
|
+
if (completed !== void 0) {
|
|
873
|
+
completedAt = completedAt === void 0 ? completed : Math.max(completedAt, completed);
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
if (payloadType === "context_compacted") metrics.contextCompactions += 1;
|
|
877
|
+
if (payloadType === "patch_apply_end") {
|
|
878
|
+
metrics.patchAttempts += 1;
|
|
879
|
+
if (payload.success === true) metrics.patchSuccesses += 1;
|
|
880
|
+
else {
|
|
881
|
+
metrics.patchFailures += 1;
|
|
882
|
+
metrics.toolErrors += 1;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
if (payloadType === "token_count")
|
|
886
|
+
setCodexCumulativeUsage(metrics, record2(record2(payload.info)?.total_token_usage));
|
|
887
|
+
const result = record2(payload.result);
|
|
888
|
+
if (result && "Err" in result) metrics.toolErrors += 1;
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
if (metrics.wallMs === 0 && startedAt !== void 0 && completedAt !== void 0) {
|
|
892
|
+
metrics.wallMs = Math.max(0, completedAt - startedAt);
|
|
893
|
+
}
|
|
894
|
+
metrics.processScore = codexProcessScore(metrics);
|
|
895
|
+
return metrics;
|
|
896
|
+
}
|
|
897
|
+
function addCodexExecItem(metrics, startedToolIds, eventType, item) {
|
|
898
|
+
const itemType = stringField2(item, "type");
|
|
899
|
+
const itemId = stringField2(item, "id");
|
|
900
|
+
const isTool = itemType === "command_execution" || itemType === "mcp_tool_call" || itemType === "collab_tool_call" || itemType === "web_search";
|
|
901
|
+
if (eventType === "item.started" && isTool) {
|
|
902
|
+
metrics.toolCalls += 1;
|
|
903
|
+
if (itemId) startedToolIds.add(itemId);
|
|
904
|
+
}
|
|
905
|
+
if (eventType === "item.completed" && isTool) {
|
|
906
|
+
if (!itemId || !startedToolIds.has(itemId)) metrics.toolCalls += 1;
|
|
907
|
+
metrics.toolOutputs += 1;
|
|
908
|
+
}
|
|
909
|
+
if (eventType !== "item.completed") return;
|
|
910
|
+
if (itemType === "agent_message" || itemType === "message") metrics.assistantMessages += 1;
|
|
911
|
+
if (itemType === "reasoning") metrics.reasoningItems += 1;
|
|
912
|
+
if (itemType === "file_change") {
|
|
913
|
+
metrics.patchAttempts += 1;
|
|
914
|
+
const status = stringField2(item, "status");
|
|
915
|
+
if (status === "completed") metrics.patchSuccesses += 1;
|
|
916
|
+
if (status === "failed") {
|
|
917
|
+
metrics.patchFailures += 1;
|
|
918
|
+
metrics.toolErrors += 1;
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
if (itemType === "error" || codexExecToolFailed(itemType, item)) metrics.toolErrors += 1;
|
|
922
|
+
}
|
|
923
|
+
function codexExecToolFailed(itemType, item) {
|
|
924
|
+
const status = stringField2(item, "status");
|
|
925
|
+
if (itemType === "command_execution") {
|
|
926
|
+
const exitCode = numberField2(item, "exit_code");
|
|
927
|
+
return status === "failed" || status === "declined" || exitCode !== void 0 && exitCode !== 0;
|
|
928
|
+
}
|
|
929
|
+
if (itemType === "mcp_tool_call") return status === "failed" || record2(item.error) !== null;
|
|
930
|
+
return itemType === "collab_tool_call" && status === "failed";
|
|
931
|
+
}
|
|
932
|
+
function addCodexExecUsage(metrics, usage) {
|
|
933
|
+
const parsed = readUsage(usage);
|
|
934
|
+
metrics.inputTokens += parsed.input;
|
|
935
|
+
metrics.outputTokens += parsed.output;
|
|
936
|
+
metrics.reasoningTokens += parsed.reasoning;
|
|
937
|
+
metrics.cachedTokens += parsed.cached;
|
|
938
|
+
metrics.cacheWriteTokens += parsed.cacheWrite;
|
|
939
|
+
}
|
|
940
|
+
function setCodexCumulativeUsage(metrics, usage) {
|
|
941
|
+
const parsed = readUsage(usage);
|
|
942
|
+
metrics.inputTokens = Math.max(metrics.inputTokens, parsed.input);
|
|
943
|
+
metrics.outputTokens = Math.max(metrics.outputTokens, parsed.output);
|
|
944
|
+
metrics.reasoningTokens = Math.max(metrics.reasoningTokens, parsed.reasoning);
|
|
945
|
+
metrics.cachedTokens = Math.max(metrics.cachedTokens, parsed.cached);
|
|
946
|
+
metrics.cacheWriteTokens = Math.max(metrics.cacheWriteTokens, parsed.cacheWrite);
|
|
947
|
+
}
|
|
948
|
+
function claudeCodeMetrics(entries) {
|
|
949
|
+
const metrics = emptyMetrics(entries.length);
|
|
950
|
+
let startedAt;
|
|
951
|
+
let completedAt;
|
|
952
|
+
for (const entry of entries) {
|
|
953
|
+
const type = stringField2(entry, "type");
|
|
954
|
+
const timestamp2 = timestampMs(entry.timestamp);
|
|
955
|
+
if (timestamp2 !== void 0) {
|
|
956
|
+
startedAt = startedAt === void 0 ? timestamp2 : Math.min(startedAt, timestamp2);
|
|
957
|
+
completedAt = completedAt === void 0 ? timestamp2 : Math.max(completedAt, timestamp2);
|
|
958
|
+
}
|
|
959
|
+
if (type === "user") metrics.userMessages += 1;
|
|
960
|
+
if (type === "assistant") metrics.assistantMessages += 1;
|
|
961
|
+
if (type === "pr-link") metrics.prLinks += 1;
|
|
962
|
+
if (type === "file-history-snapshot") metrics.fileSnapshots += 1;
|
|
963
|
+
const message = record2(entry.message);
|
|
964
|
+
if (message) addUsage(metrics, record2(message.usage));
|
|
965
|
+
const content = Array.isArray(message?.content) ? message.content : [];
|
|
966
|
+
for (const item of content) {
|
|
967
|
+
const part = record2(item);
|
|
968
|
+
if (!part) continue;
|
|
969
|
+
const partType = stringField2(part, "type");
|
|
970
|
+
if (partType === "thinking") metrics.reasoningItems += 1;
|
|
971
|
+
if (partType === "tool_use") metrics.toolCalls += 1;
|
|
972
|
+
if (partType === "tool_result") {
|
|
973
|
+
metrics.toolOutputs += 1;
|
|
974
|
+
if (part.is_error === true) metrics.toolErrors += 1;
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
if (startedAt !== void 0 && completedAt !== void 0)
|
|
979
|
+
metrics.wallMs = Math.max(0, completedAt - startedAt);
|
|
980
|
+
metrics.processScore = claudeProcessScore(metrics);
|
|
981
|
+
return metrics;
|
|
982
|
+
}
|
|
983
|
+
function openCodeMetrics(entries) {
|
|
984
|
+
const metrics = emptyMetrics(entries.length);
|
|
985
|
+
const messageUsage = emptyTokenTotals();
|
|
986
|
+
const partUsage = emptyTokenTotals();
|
|
987
|
+
let messageCost = 0;
|
|
988
|
+
let partCost = 0;
|
|
989
|
+
let messageCostCaptured = false;
|
|
990
|
+
let partCostCaptured = false;
|
|
991
|
+
let startedAt;
|
|
992
|
+
let completedAt;
|
|
993
|
+
for (const entry of entries) {
|
|
994
|
+
const time = record2(entry.time);
|
|
995
|
+
const created = timestampMs(time?.created);
|
|
996
|
+
const completed = timestampMs(time?.completed);
|
|
997
|
+
if (created !== void 0)
|
|
998
|
+
startedAt = startedAt === void 0 ? created : Math.min(startedAt, created);
|
|
999
|
+
if (completed !== void 0)
|
|
1000
|
+
completedAt = completedAt === void 0 ? completed : Math.max(completedAt, completed);
|
|
1001
|
+
const role = stringField2(entry, "role");
|
|
1002
|
+
if (role === "user") metrics.userMessages += 1;
|
|
1003
|
+
if (role === "assistant") {
|
|
1004
|
+
metrics.assistantMessages += 1;
|
|
1005
|
+
const finish = stringField2(entry, "finish");
|
|
1006
|
+
if (finish === "stop") metrics.turnsCompleted += 1;
|
|
1007
|
+
if (finish === "error") metrics.turnsAborted += 1;
|
|
1008
|
+
}
|
|
1009
|
+
const type = stringField2(entry, "type");
|
|
1010
|
+
if (type === "reasoning") metrics.reasoningItems += 1;
|
|
1011
|
+
if (type === "tool") {
|
|
1012
|
+
metrics.toolCalls += 1;
|
|
1013
|
+
const status = stringField2(record2(entry.state) ?? {}, "status");
|
|
1014
|
+
if (status === "completed") metrics.toolOutputs += 1;
|
|
1015
|
+
if (status === "error") {
|
|
1016
|
+
metrics.toolOutputs += 1;
|
|
1017
|
+
metrics.toolErrors += 1;
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
if (type === "patch") {
|
|
1021
|
+
metrics.patchAttempts += 1;
|
|
1022
|
+
metrics.patchSuccesses += 1;
|
|
1023
|
+
}
|
|
1024
|
+
const cost = numberField2(entry, "cost");
|
|
1025
|
+
if (record2(entry.tokens) && role) {
|
|
1026
|
+
addUsageTo(messageUsage, record2(entry.tokens));
|
|
1027
|
+
if (cost !== void 0) {
|
|
1028
|
+
messageCost += cost;
|
|
1029
|
+
messageCostCaptured = true;
|
|
1030
|
+
}
|
|
1031
|
+
} else if (record2(entry.tokens)) {
|
|
1032
|
+
addUsageTo(partUsage, record2(entry.tokens));
|
|
1033
|
+
if (cost !== void 0) {
|
|
1034
|
+
partCost += cost;
|
|
1035
|
+
partCostCaptured = true;
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
const usage = hasTokenTotals(messageUsage) ? messageUsage : partUsage;
|
|
1040
|
+
metrics.inputTokens = usage.input;
|
|
1041
|
+
metrics.outputTokens = usage.output;
|
|
1042
|
+
metrics.reasoningTokens = usage.reasoning;
|
|
1043
|
+
metrics.cachedTokens = usage.cached;
|
|
1044
|
+
metrics.cacheWriteTokens = usage.cacheWrite;
|
|
1045
|
+
metrics.observedCostUsd = messageCostCaptured ? messageCost : partCost;
|
|
1046
|
+
metrics.observedCostCaptured = messageCostCaptured || partCostCaptured;
|
|
1047
|
+
if (metrics.wallMs === 0 && startedAt !== void 0 && completedAt !== void 0)
|
|
1048
|
+
metrics.wallMs = Math.max(0, completedAt - startedAt);
|
|
1049
|
+
metrics.processScore = terminalProcessScore(metrics);
|
|
1050
|
+
return metrics;
|
|
1051
|
+
}
|
|
1052
|
+
function kimiCodeMetrics(entries) {
|
|
1053
|
+
const metrics = emptyMetrics(entries.length);
|
|
1054
|
+
let startedAt;
|
|
1055
|
+
let completedAt;
|
|
1056
|
+
for (const entry of entries) {
|
|
1057
|
+
const timestamp2 = timestampMs(entry.timestamp);
|
|
1058
|
+
if (timestamp2 !== void 0) {
|
|
1059
|
+
startedAt = startedAt === void 0 ? timestamp2 : Math.min(startedAt, timestamp2);
|
|
1060
|
+
completedAt = completedAt === void 0 ? timestamp2 : Math.max(completedAt, timestamp2);
|
|
1061
|
+
}
|
|
1062
|
+
const role = stringField2(entry, "role");
|
|
1063
|
+
if (role === "user") metrics.userMessages += 1;
|
|
1064
|
+
if (role === "assistant") metrics.assistantMessages += 1;
|
|
1065
|
+
if (role === "_usage") addUsage(metrics, entry);
|
|
1066
|
+
const message = record2(entry.message);
|
|
1067
|
+
const messageType = message ? stringField2(message, "type") : void 0;
|
|
1068
|
+
const payload = record2(message?.payload) ?? {};
|
|
1069
|
+
if (messageType === "TurnBegin") {
|
|
1070
|
+
metrics.userMessages += 1;
|
|
1071
|
+
metrics.turnsStarted += 1;
|
|
1072
|
+
}
|
|
1073
|
+
if (messageType === "TurnEnd") metrics.turnsCompleted += 1;
|
|
1074
|
+
if (messageType === "StepInterrupted") metrics.turnsAborted += 1;
|
|
1075
|
+
if (messageType === "StepBegin") metrics.reasoningItems += 1;
|
|
1076
|
+
if (messageType === "ContentPart") {
|
|
1077
|
+
const payloadType = stringField2(payload, "type");
|
|
1078
|
+
if (payloadType === "think") metrics.reasoningItems += 1;
|
|
1079
|
+
if (payloadType === "text") metrics.assistantMessages += 1;
|
|
1080
|
+
}
|
|
1081
|
+
if (messageType === "ToolCall") metrics.toolCalls += 1;
|
|
1082
|
+
if (messageType === "ToolResult") {
|
|
1083
|
+
metrics.toolOutputs += 1;
|
|
1084
|
+
if (record2(payload.return_value)?.is_error === true) metrics.toolErrors += 1;
|
|
1085
|
+
}
|
|
1086
|
+
if (messageType === "StatusUpdate") addUsage(metrics, record2(payload.token_usage));
|
|
1087
|
+
if (messageType === "Notification") {
|
|
1088
|
+
const notification = record2(payload.payload);
|
|
1089
|
+
const exitCode = numberField2(notification ?? {}, "exit_code");
|
|
1090
|
+
if (notification?.timed_out === true || exitCode !== void 0 && exitCode !== 0) {
|
|
1091
|
+
metrics.toolErrors += 1;
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
if (startedAt !== void 0 && completedAt !== void 0)
|
|
1096
|
+
metrics.wallMs = Math.max(0, completedAt - startedAt);
|
|
1097
|
+
metrics.processScore = terminalProcessScore(metrics);
|
|
1098
|
+
return metrics;
|
|
1099
|
+
}
|
|
1100
|
+
function piMetrics(entries) {
|
|
1101
|
+
const metrics = emptyMetrics(entries.length);
|
|
1102
|
+
let bestReliabilityScore;
|
|
1103
|
+
for (const entry of entries) {
|
|
1104
|
+
const nodes = Array.isArray(entry.nodes) ? entry.nodes : [];
|
|
1105
|
+
const edges = Array.isArray(entry.edges) ? entry.edges : [];
|
|
1106
|
+
metrics.graphNodes += nodes.length;
|
|
1107
|
+
metrics.graphEdges += edges.length;
|
|
1108
|
+
for (const node of nodes) {
|
|
1109
|
+
const obj = record2(node);
|
|
1110
|
+
const ir = record2(obj?.ir) ?? obj;
|
|
1111
|
+
const kind = stringField2(ir ?? {}, "kind");
|
|
1112
|
+
if (kind === "ActionCandidate") metrics.actionCandidates += 1;
|
|
1113
|
+
if (kind === "ToolInvocation") metrics.toolCalls += 1;
|
|
1114
|
+
if (kind === "ToolResult") metrics.toolOutputs += 1;
|
|
1115
|
+
if (kind === "VerificationReport") metrics.verificationReports += 1;
|
|
1116
|
+
if (kind === "CompletionDecision") {
|
|
1117
|
+
metrics.completionDecisions += 1;
|
|
1118
|
+
metrics.turnsCompleted += 1;
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
const averageReliability = numberField2(entry, "averageNodeReliability");
|
|
1122
|
+
if (averageReliability !== void 0)
|
|
1123
|
+
bestReliabilityScore = maxOptional(bestReliabilityScore, averageReliability);
|
|
1124
|
+
const pessimisticPath = numberField2(entry, "pessimisticPathEstimate");
|
|
1125
|
+
if (pessimisticPath !== void 0)
|
|
1126
|
+
bestReliabilityScore = maxOptional(bestReliabilityScore, pessimisticPath);
|
|
1127
|
+
const rows = Array.isArray(entry.rows) ? entry.rows : [];
|
|
1128
|
+
metrics.reliabilityRows += rows.length;
|
|
1129
|
+
for (const row of rows) {
|
|
1130
|
+
const obj = record2(row);
|
|
1131
|
+
if (!obj) continue;
|
|
1132
|
+
const lift = numberField2(obj, "lift");
|
|
1133
|
+
if (lift !== void 0) metrics.reliabilityLift = Math.max(metrics.reliabilityLift, lift);
|
|
1134
|
+
const validated = numberField2(obj, "validatedSuccessEstimate");
|
|
1135
|
+
if (validated !== void 0)
|
|
1136
|
+
bestReliabilityScore = maxOptional(bestReliabilityScore, validated);
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
metrics.processScore = bestReliabilityScore !== void 0 ? clamp01(bestReliabilityScore) : metrics.completionDecisions > 0 ? 1 : 0;
|
|
1140
|
+
return metrics;
|
|
1141
|
+
}
|
|
1142
|
+
function emptyMetrics(entries) {
|
|
1143
|
+
return {
|
|
1144
|
+
entries,
|
|
1145
|
+
userMessages: 0,
|
|
1146
|
+
assistantMessages: 0,
|
|
1147
|
+
reasoningItems: 0,
|
|
1148
|
+
toolCalls: 0,
|
|
1149
|
+
toolOutputs: 0,
|
|
1150
|
+
toolErrors: 0,
|
|
1151
|
+
patchAttempts: 0,
|
|
1152
|
+
patchSuccesses: 0,
|
|
1153
|
+
patchFailures: 0,
|
|
1154
|
+
turnsStarted: 0,
|
|
1155
|
+
turnsCompleted: 0,
|
|
1156
|
+
turnsAborted: 0,
|
|
1157
|
+
contextCompactions: 0,
|
|
1158
|
+
mcpCalls: 0,
|
|
1159
|
+
subagentCalls: 0,
|
|
1160
|
+
skillCalls: 0,
|
|
1161
|
+
hookCalls: 0,
|
|
1162
|
+
webCalls: 0,
|
|
1163
|
+
codeActions: 0,
|
|
1164
|
+
prLinks: 0,
|
|
1165
|
+
fileSnapshots: 0,
|
|
1166
|
+
graphNodes: 0,
|
|
1167
|
+
graphEdges: 0,
|
|
1168
|
+
actionCandidates: 0,
|
|
1169
|
+
verificationReports: 0,
|
|
1170
|
+
completionDecisions: 0,
|
|
1171
|
+
reliabilityRows: 0,
|
|
1172
|
+
reliabilityLift: 0,
|
|
1173
|
+
inputTokens: 0,
|
|
1174
|
+
outputTokens: 0,
|
|
1175
|
+
reasoningTokens: 0,
|
|
1176
|
+
cachedTokens: 0,
|
|
1177
|
+
cacheWriteTokens: 0,
|
|
1178
|
+
observedCostUsd: 0,
|
|
1179
|
+
observedCostCaptured: false,
|
|
1180
|
+
wallMs: 0,
|
|
1181
|
+
processScore: 0
|
|
1182
|
+
};
|
|
1183
|
+
}
|
|
1184
|
+
function codexProcessScore(metrics) {
|
|
1185
|
+
const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
|
|
1186
|
+
const base = terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? 0.5 : 0;
|
|
1187
|
+
return penalizeErrors(base, metrics);
|
|
1188
|
+
}
|
|
1189
|
+
function claudeProcessScore(metrics) {
|
|
1190
|
+
const base = metrics.prLinks > 0 ? 1 : metrics.assistantMessages > 0 ? 0.75 : 0;
|
|
1191
|
+
return penalizeErrors(base, metrics);
|
|
1192
|
+
}
|
|
1193
|
+
function terminalProcessScore(metrics) {
|
|
1194
|
+
const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
|
|
1195
|
+
const base = terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? 0.75 : metrics.toolCalls > 0 ? 0.5 : 0;
|
|
1196
|
+
return penalizeErrors(base, metrics);
|
|
1197
|
+
}
|
|
1198
|
+
function penalizeErrors(base, metrics) {
|
|
1199
|
+
const operations = Math.max(1, metrics.toolCalls + metrics.patchAttempts);
|
|
1200
|
+
const errorRate = Math.min(1, (metrics.toolErrors + metrics.patchFailures) / operations);
|
|
1201
|
+
return clamp01(base * (1 - 0.5 * errorRate));
|
|
1202
|
+
}
|
|
1203
|
+
function emptyTokenTotals() {
|
|
1204
|
+
return { input: 0, output: 0, reasoning: 0, cached: 0, cacheWrite: 0 };
|
|
1205
|
+
}
|
|
1206
|
+
function hasTokenTotals(usage) {
|
|
1207
|
+
return usage.input > 0 || usage.output > 0 || usage.reasoning > 0 || usage.cached > 0 || usage.cacheWrite > 0;
|
|
1208
|
+
}
|
|
1209
|
+
function addUsage(metrics, usage) {
|
|
1210
|
+
const parsed = readUsage(usage);
|
|
1211
|
+
metrics.inputTokens += parsed.input;
|
|
1212
|
+
metrics.outputTokens += parsed.output;
|
|
1213
|
+
metrics.reasoningTokens += parsed.reasoning;
|
|
1214
|
+
metrics.cachedTokens += parsed.cached;
|
|
1215
|
+
metrics.cacheWriteTokens += parsed.cacheWrite;
|
|
1216
|
+
}
|
|
1217
|
+
function addUsageTo(totals, usage) {
|
|
1218
|
+
const parsed = readUsage(usage);
|
|
1219
|
+
totals.input += parsed.input;
|
|
1220
|
+
totals.output += parsed.output;
|
|
1221
|
+
totals.reasoning += parsed.reasoning;
|
|
1222
|
+
totals.cached += parsed.cached;
|
|
1223
|
+
totals.cacheWrite += parsed.cacheWrite;
|
|
1224
|
+
}
|
|
1225
|
+
function readUsage(usage) {
|
|
1226
|
+
const parsed = extractUsage(usage);
|
|
1227
|
+
return {
|
|
1228
|
+
input: parsed?.input ?? 0,
|
|
1229
|
+
output: parsed?.output ?? 0,
|
|
1230
|
+
reasoning: parsed?.reasoning ?? 0,
|
|
1231
|
+
cached: parsed?.cached ?? 0,
|
|
1232
|
+
cacheWrite: parsed?.cacheWrite ?? 0
|
|
1233
|
+
};
|
|
1234
|
+
}
|
|
1235
|
+
function resolveSessionCost(explicit, metrics, model) {
|
|
1236
|
+
if (explicit) {
|
|
1237
|
+
if (explicit.kind === "uncaptured") {
|
|
1238
|
+
if (explicit.usd !== null) throw new Error("uncaptured cost must have usd: null");
|
|
1239
|
+
return { costUsd: 0, costProvenance: explicit };
|
|
1240
|
+
}
|
|
1241
|
+
if (!Number.isFinite(explicit.usd) || explicit.usd < 0) {
|
|
1242
|
+
throw new Error(`${explicit.kind} cost must be a finite, non-negative USD amount`);
|
|
1243
|
+
}
|
|
1244
|
+
return { costUsd: explicit.usd, costProvenance: explicit };
|
|
1245
|
+
}
|
|
1246
|
+
if (metrics.observedCostCaptured) {
|
|
1247
|
+
const costProvenance = { kind: "observed", usd: metrics.observedCostUsd };
|
|
1248
|
+
return { costUsd: metrics.observedCostUsd, costProvenance };
|
|
1249
|
+
}
|
|
1250
|
+
const hasUsage = metrics.inputTokens > 0 || metrics.outputTokens > 0;
|
|
1251
|
+
if (hasUsage && isModelPriced(model)) {
|
|
1252
|
+
const costUsd = estimateCost(metrics.inputTokens, metrics.outputTokens, model);
|
|
1253
|
+
return { costUsd, costProvenance: { kind: "estimated", usd: costUsd } };
|
|
1254
|
+
}
|
|
1255
|
+
return { costUsd: 0, costProvenance: { kind: "uncaptured", usd: null } };
|
|
1256
|
+
}
|
|
1257
|
+
function diagnosticsFor(metrics, options) {
|
|
1258
|
+
const warnings = [];
|
|
1259
|
+
if (!options.scoreOverridden) warnings.push("outcome score is inferred from process telemetry");
|
|
1260
|
+
if (!options.explicitTerminal) warnings.push("no explicit terminal success/failure signal");
|
|
1261
|
+
if (metrics.inputTokens === 0 && metrics.outputTokens === 0 && metrics.reasoningTokens === 0 && metrics.cachedTokens === 0 && metrics.cacheWriteTokens === 0) {
|
|
1262
|
+
warnings.push("missing token usage");
|
|
1263
|
+
}
|
|
1264
|
+
if (options.costKind === "estimated")
|
|
1265
|
+
warnings.push("USD cost estimated from token usage and model pricing");
|
|
1266
|
+
if (options.costKind === "uncaptured") warnings.push("USD cost uncaptured");
|
|
1267
|
+
if (options.costKind === "uncaptured" && !isModelPriced(options.model))
|
|
1268
|
+
warnings.push("model pricing unknown");
|
|
1269
|
+
if (options.malformedLines > 0)
|
|
1270
|
+
warnings.push(`${options.malformedLines} malformed JSONL lines skipped`);
|
|
1271
|
+
return warnings;
|
|
1272
|
+
}
|
|
1273
|
+
function applyObservedSurfaceMetrics(metrics, observation) {
|
|
1274
|
+
const actions = observation.actions.filter(
|
|
1275
|
+
(action) => action.kind === "tool" || action.kind === "patch"
|
|
1276
|
+
);
|
|
1277
|
+
metrics.mcpCalls = actions.filter((action) => action.surface === "mcp").length;
|
|
1278
|
+
metrics.subagentCalls = actions.filter((action) => action.surface === "subagent").length;
|
|
1279
|
+
metrics.skillCalls = actions.filter((action) => action.surface === "skill").length;
|
|
1280
|
+
metrics.hookCalls = actions.filter((action) => action.surface === "hook").length;
|
|
1281
|
+
metrics.webCalls = actions.filter((action) => action.surface === "web").length;
|
|
1282
|
+
metrics.codeActions = actions.filter((action) => action.surface === "code").length;
|
|
1283
|
+
}
|
|
1284
|
+
function sessionIdFromEntries2(source, entries) {
|
|
1285
|
+
for (const entry of entries) {
|
|
1286
|
+
if (source === "codex") {
|
|
1287
|
+
const threadId = stringField2(entry, "thread_id");
|
|
1288
|
+
if (threadId) return threadId;
|
|
1289
|
+
const payload2 = record2(entry.payload);
|
|
1290
|
+
const id = payload2 ? stringField2(payload2, "id") : void 0;
|
|
1291
|
+
if (id) return id;
|
|
1292
|
+
}
|
|
1293
|
+
const message = record2(entry.message);
|
|
1294
|
+
const payload = record2(message?.payload);
|
|
1295
|
+
const sessionId = stringField2(entry, "sessionID") ?? stringField2(entry, "sessionId") ?? stringField2(entry, "session_id") ?? stringField2(payload ?? {}, "session_id");
|
|
1296
|
+
if (sessionId) return sessionId;
|
|
1297
|
+
}
|
|
1298
|
+
return void 0;
|
|
1299
|
+
}
|
|
1300
|
+
function modelFromEntries(source, entries) {
|
|
1301
|
+
let providerModel;
|
|
1302
|
+
for (const entry of entries) {
|
|
1303
|
+
if (source === "codex") {
|
|
1304
|
+
const payload = record2(entry.payload);
|
|
1305
|
+
const model2 = payload ? stringField2(payload, "model") : void 0;
|
|
1306
|
+
if (model2) return model2;
|
|
1307
|
+
providerModel = providerModel ?? (payload ? stringField2(payload, "model_provider") : void 0);
|
|
1308
|
+
}
|
|
1309
|
+
const openCodeModel = stringField2(entry, "modelID");
|
|
1310
|
+
if (openCodeModel) {
|
|
1311
|
+
const provider = stringField2(entry, "providerID");
|
|
1312
|
+
return provider ? `${provider}/${openCodeModel}` : openCodeModel;
|
|
1313
|
+
}
|
|
1314
|
+
const modelObject = record2(entry.model);
|
|
1315
|
+
const nestedModel = stringField2(modelObject ?? {}, "modelID");
|
|
1316
|
+
if (nestedModel) {
|
|
1317
|
+
const provider = stringField2(modelObject ?? {}, "providerID");
|
|
1318
|
+
return provider ? `${provider}/${nestedModel}` : nestedModel;
|
|
1319
|
+
}
|
|
1320
|
+
const message = record2(entry.message);
|
|
1321
|
+
const model = message ? stringField2(message, "model") : void 0;
|
|
1322
|
+
if (model) return model;
|
|
1323
|
+
}
|
|
1324
|
+
return providerModel;
|
|
1325
|
+
}
|
|
1326
|
+
function cwdFromEntries(entries) {
|
|
1327
|
+
for (const entry of entries) {
|
|
1328
|
+
const path = record2(entry.path);
|
|
1329
|
+
const cwd = stringField2(entry, "cwd") ?? stringField2(path ?? {}, "cwd") ?? stringField2(path ?? {}, "root") ?? (record2(entry.payload) ? stringField2(record2(entry.payload), "cwd") : void 0);
|
|
1330
|
+
if (cwd) return cwd;
|
|
1331
|
+
}
|
|
1332
|
+
return void 0;
|
|
1333
|
+
}
|
|
1334
|
+
function firstUserText(entries) {
|
|
1335
|
+
for (const entry of entries) {
|
|
1336
|
+
const payload = record2(entry.payload);
|
|
1337
|
+
const payloadType = payload ? stringField2(payload, "type") : void 0;
|
|
1338
|
+
if (payloadType === "user_message") return stringField2(payload, "message");
|
|
1339
|
+
if (stringField2(entry, "role") === "user") {
|
|
1340
|
+
const content = entry.content;
|
|
1341
|
+
if (typeof content === "string") return content;
|
|
1342
|
+
}
|
|
1343
|
+
const wirePayload = record2(record2(entry.message)?.payload);
|
|
1344
|
+
const userInput = wirePayload ? stringField2(wirePayload, "user_input") : void 0;
|
|
1345
|
+
if (userInput) return userInput;
|
|
1346
|
+
const message = record2(entry.message);
|
|
1347
|
+
if (message && stringField2(message, "role") === "user") {
|
|
1348
|
+
const content = message.content;
|
|
1349
|
+
if (typeof content === "string") return content;
|
|
1350
|
+
if (Array.isArray(content)) {
|
|
1351
|
+
const text = content.map((part) => {
|
|
1352
|
+
const obj = record2(part);
|
|
1353
|
+
return obj ? stringField2(obj, "text") : void 0;
|
|
1354
|
+
}).filter((part) => part !== void 0).join("\n");
|
|
1355
|
+
if (text) return text;
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
return void 0;
|
|
1360
|
+
}
|
|
1361
|
+
function fallbackSessionId2(source, sourcePath) {
|
|
1362
|
+
return hashString(`${source}:${sourcePath ?? "unknown-session"}`).slice(
|
|
1363
|
+
"sha256:".length,
|
|
1364
|
+
"sha256:".length + 20
|
|
1365
|
+
);
|
|
1366
|
+
}
|
|
1367
|
+
function withSnapshot(model) {
|
|
1368
|
+
const trimmed = model.trim() || "unknown";
|
|
1369
|
+
if (trimmed.includes("@") || /-\d{8}$/.test(trimmed) || /-\d{4}-\d{2}-\d{2}$/.test(trimmed)) {
|
|
1370
|
+
return trimmed;
|
|
1371
|
+
}
|
|
1372
|
+
return `${trimmed}@observed-local`;
|
|
1373
|
+
}
|
|
1374
|
+
function hashJson(value) {
|
|
1375
|
+
return hashString(JSON.stringify(value));
|
|
1376
|
+
}
|
|
1377
|
+
function hashString(value) {
|
|
1378
|
+
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
1379
|
+
}
|
|
1380
|
+
function stableSeed(value) {
|
|
1381
|
+
return createHash("sha256").update(value).digest().readUInt32BE(0);
|
|
1382
|
+
}
|
|
1383
|
+
function maxOptional(current, next) {
|
|
1384
|
+
return current === void 0 ? next : Math.max(current, next);
|
|
1385
|
+
}
|
|
1386
|
+
function clamp01(value) {
|
|
1387
|
+
if (!Number.isFinite(value)) return 0;
|
|
1388
|
+
return Math.max(0, Math.min(1, value));
|
|
1389
|
+
}
|
|
1390
|
+
function isRecord2(value) {
|
|
1391
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1392
|
+
}
|
|
1393
|
+
function record2(value) {
|
|
1394
|
+
return isRecord2(value) ? value : null;
|
|
1395
|
+
}
|
|
1396
|
+
function stringField2(obj, key) {
|
|
1397
|
+
const value = obj[key];
|
|
1398
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
1399
|
+
}
|
|
1400
|
+
function numberField2(obj, key) {
|
|
1401
|
+
const value = obj[key];
|
|
1402
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
1403
|
+
}
|
|
1404
|
+
function timestampMs(value) {
|
|
1405
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
1406
|
+
return value > 1e12 ? value : value * 1e3;
|
|
1407
|
+
}
|
|
1408
|
+
if (typeof value === "string" && value.length > 0) {
|
|
1409
|
+
const parsed = Date.parse(value);
|
|
1410
|
+
return Number.isFinite(parsed) ? parsed : void 0;
|
|
1411
|
+
}
|
|
1412
|
+
return void 0;
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
export {
|
|
1416
|
+
observeCodeAgentSession,
|
|
1417
|
+
parseCodeAgentJsonl,
|
|
1418
|
+
fromCodexSession,
|
|
1419
|
+
fromClaudeCodeSession,
|
|
1420
|
+
fromOpenCodeSession,
|
|
1421
|
+
fromKimiCodeSession,
|
|
1422
|
+
fromPiSession,
|
|
1423
|
+
fromPigraphSession
|
|
1424
|
+
};
|
|
1425
|
+
//# sourceMappingURL=chunk-7VYEGHWF.js.map
|