@tracemarketplace/shared 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extractors/claude-code.d.ts +3 -0
- package/dist/extractors/claude-code.d.ts.map +1 -0
- package/dist/extractors/claude-code.js +158 -0
- package/dist/extractors/claude-code.js.map +1 -0
- package/dist/extractors/codex.d.ts +3 -0
- package/dist/extractors/codex.d.ts.map +1 -0
- package/dist/extractors/codex.js +192 -0
- package/dist/extractors/codex.js.map +1 -0
- package/dist/extractors/cursor.d.ts +3 -0
- package/dist/extractors/cursor.d.ts.map +1 -0
- package/dist/extractors/cursor.js +99 -0
- package/dist/extractors/cursor.js.map +1 -0
- package/dist/hash.d.ts +4 -0
- package/dist/hash.d.ts.map +1 -0
- package/dist/hash.js +13 -0
- package/dist/hash.js.map +1 -0
- package/dist/hash.test.d.ts +2 -0
- package/dist/hash.test.d.ts.map +1 -0
- package/dist/hash.test.js +67 -0
- package/dist/hash.test.js.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -0
- package/dist/index.js.map +1 -0
- package/dist/scoring.d.ts +5 -0
- package/dist/scoring.d.ts.map +1 -0
- package/dist/scoring.js +114 -0
- package/dist/scoring.js.map +1 -0
- package/dist/scoring.test.d.ts +2 -0
- package/dist/scoring.test.d.ts.map +1 -0
- package/dist/scoring.test.js +157 -0
- package/dist/scoring.test.js.map +1 -0
- package/dist/types.d.ts +98 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/utils.d.ts +3 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +11 -0
- package/dist/utils.js.map +1 -0
- package/dist/validators.d.ts +247 -0
- package/dist/validators.d.ts.map +1 -0
- package/dist/validators.js +36 -0
- package/dist/validators.js.map +1 -0
- package/dist/validators.test.d.ts +2 -0
- package/dist/validators.test.d.ts.map +1 -0
- package/dist/validators.test.js +52 -0
- package/dist/validators.test.js.map +1 -0
- package/package.json +42 -0
- package/src/extractors/claude-code.ts +178 -0
- package/src/extractors/codex.ts +208 -0
- package/src/extractors/cursor.ts +118 -0
- package/src/hash.test.ts +72 -0
- package/src/hash.ts +15 -0
- package/src/index.ts +8 -0
- package/src/scoring.test.ts +173 -0
- package/src/scoring.ts +149 -0
- package/src/types.ts +96 -0
- package/src/utils.ts +9 -0
- package/src/validators.test.ts +61 -0
- package/src/validators.ts +41 -0
- package/tsconfig.json +8 -0
- package/vitest.config.ts +8 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { NormalizedTrace, TraceScore, FailureMode } from "./types.js";
|
|
2
|
+
export declare function detectFailureModes(trace: NormalizedTrace): FailureMode[];
|
|
3
|
+
export declare function checkCompleteness(trace: NormalizedTrace): "complete" | "incomplete" | "malformed";
|
|
4
|
+
export declare function scoreTrace(trace: NormalizedTrace): TraceScore;
|
|
5
|
+
//# sourceMappingURL=scoring.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../src/scoring.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE3E,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,eAAe,GAAG,WAAW,EAAE,CA6ExE;AAED,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,eAAe,GACrB,UAAU,GAAG,YAAY,GAAG,WAAW,CAoBzC;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,UAAU,CA2C7D"}
|
package/dist/scoring.js
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
export function detectFailureModes(trace) {
|
|
2
|
+
const modes = new Set();
|
|
3
|
+
const allBlocks = trace.turns.flatMap((t) => t.content);
|
|
4
|
+
// tool_call_failure: any tool_result with is_error=true
|
|
5
|
+
const hasToolError = allBlocks.some((b) => b.type === "tool_result" && b.is_error);
|
|
6
|
+
if (hasToolError)
|
|
7
|
+
modes.add("tool_call_failure");
|
|
8
|
+
// repeated_tool_calls: same tool_name 3+ times in a row
|
|
9
|
+
const toolUses = allBlocks.filter((b) => b.type === "tool_use");
|
|
10
|
+
let streak = 1;
|
|
11
|
+
for (let i = 1; i < toolUses.length; i++) {
|
|
12
|
+
if (toolUses[i].tool_name === toolUses[i - 1].tool_name) {
|
|
13
|
+
streak++;
|
|
14
|
+
if (streak >= 3) {
|
|
15
|
+
modes.add("repeated_tool_calls");
|
|
16
|
+
break;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
else {
|
|
20
|
+
streak = 1;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
// context_limit_approached: text mentioning context/limit
|
|
24
|
+
const contextLimitRegex = /context.*(limit|window|maximum)|context limit/i;
|
|
25
|
+
const hasContextLimit = trace.turns.some((t) => t.content.some((b) => b.type === "text" && contextLimitRegex.test(b.text)));
|
|
26
|
+
if (hasContextLimit)
|
|
27
|
+
modes.add("context_limit_approached");
|
|
28
|
+
// graceful_recovery: tool errors followed by recovery text
|
|
29
|
+
if (hasToolError) {
|
|
30
|
+
const recoveryRegex = /let me try|instead|alternative|another approach|different way/i;
|
|
31
|
+
const laterTurns = trace.turns.slice(Math.floor(trace.turns.length / 2));
|
|
32
|
+
const hasRecovery = laterTurns.some((t) => t.content.some((b) => b.type === "text" && recoveryRegex.test(b.text)));
|
|
33
|
+
if (hasRecovery)
|
|
34
|
+
modes.add("graceful_recovery");
|
|
35
|
+
}
|
|
36
|
+
// repeated_tool_calls → graceful_recovery if later success
|
|
37
|
+
if (modes.has("repeated_tool_calls")) {
|
|
38
|
+
const lastTurn = trace.turns[trace.turns.length - 1];
|
|
39
|
+
if (lastTurn?.role === "assistant") {
|
|
40
|
+
const hasSuccessText = lastTurn.content.some((b) => b.type === "text" && b.text.length > 50);
|
|
41
|
+
if (hasSuccessText)
|
|
42
|
+
modes.add("graceful_recovery");
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
// catastrophic_failure: last 3+ turns are all errors with no recovery
|
|
46
|
+
const lastTurns = trace.turns.slice(-3);
|
|
47
|
+
const allLastAreErrors = lastTurns.length >= 2 &&
|
|
48
|
+
lastTurns.every((t) => t.content.some((b) => b.type === "tool_result" && b.is_error));
|
|
49
|
+
if (allLastAreErrors && !modes.has("graceful_recovery")) {
|
|
50
|
+
modes.add("catastrophic_failure");
|
|
51
|
+
}
|
|
52
|
+
if (modes.size === 0)
|
|
53
|
+
modes.add("no_failure");
|
|
54
|
+
return Array.from(modes);
|
|
55
|
+
}
|
|
56
|
+
export function checkCompleteness(trace) {
|
|
57
|
+
if (trace.turns.length === 0)
|
|
58
|
+
return "malformed";
|
|
59
|
+
// malformed: any turn with empty content
|
|
60
|
+
if (trace.turns.some((t) => t.content.length === 0))
|
|
61
|
+
return "malformed";
|
|
62
|
+
const lastTurn = trace.turns[trace.turns.length - 1];
|
|
63
|
+
if (!lastTurn)
|
|
64
|
+
return "malformed";
|
|
65
|
+
// complete: last assistant turn ends with text
|
|
66
|
+
if (lastTurn.role === "assistant") {
|
|
67
|
+
const hasText = lastTurn.content.some((b) => b.type === "text");
|
|
68
|
+
if (hasText)
|
|
69
|
+
return "complete";
|
|
70
|
+
// ends with tool_use but no following result → incomplete
|
|
71
|
+
const lastBlock = lastTurn.content[lastTurn.content.length - 1];
|
|
72
|
+
if (lastBlock?.type === "tool_use")
|
|
73
|
+
return "incomplete";
|
|
74
|
+
}
|
|
75
|
+
// last turn is user (e.g., tool_result with no following assistant) → incomplete
|
|
76
|
+
return "incomplete";
|
|
77
|
+
}
|
|
78
|
+
export function scoreTrace(trace) {
|
|
79
|
+
const completeness = checkCompleteness(trace);
|
|
80
|
+
const failureModes = detectFailureModes(trace);
|
|
81
|
+
const fidelityBase = trace.content_fidelity === "full" ? 0.4 : 0.15;
|
|
82
|
+
let interestBonus = 0;
|
|
83
|
+
if (failureModes.includes("graceful_recovery"))
|
|
84
|
+
interestBonus += 0.3;
|
|
85
|
+
if (failureModes.includes("repeated_tool_calls"))
|
|
86
|
+
interestBonus += 0.2;
|
|
87
|
+
if (failureModes.includes("catastrophic_failure"))
|
|
88
|
+
interestBonus += 0.15;
|
|
89
|
+
if (failureModes.includes("tool_call_failure") &&
|
|
90
|
+
!failureModes.includes("graceful_recovery"))
|
|
91
|
+
interestBonus += 0.1;
|
|
92
|
+
const totalTokens = (trace.total_input_tokens ?? 0) + (trace.total_output_tokens ?? 0);
|
|
93
|
+
const lengthBonus = Math.min(0.15, Math.log10(Math.max(1, totalTokens / 1000)) * 0.05);
|
|
94
|
+
const total = Math.min(1.0, fidelityBase + interestBonus + lengthBonus);
|
|
95
|
+
const payoutCents = Math.min(500, Math.round(total * 500));
|
|
96
|
+
return {
|
|
97
|
+
completeness,
|
|
98
|
+
failure_modes: failureModes,
|
|
99
|
+
has_error_recovery: failureModes.includes("graceful_recovery"),
|
|
100
|
+
has_repeated_calls: failureModes.includes("repeated_tool_calls"),
|
|
101
|
+
content_fidelity: trace.content_fidelity,
|
|
102
|
+
total,
|
|
103
|
+
payout_cents: payoutCents,
|
|
104
|
+
failure_taxonomy_label: null,
|
|
105
|
+
failure_taxonomy_explanation: null,
|
|
106
|
+
rarity_score: null,
|
|
107
|
+
cluster_id: null,
|
|
108
|
+
is_duplicate: false,
|
|
109
|
+
duplicate_of: null,
|
|
110
|
+
scored_at: new Date().toISOString(),
|
|
111
|
+
scorer_version: "v0-heuristic",
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=scoring.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scoring.js","sourceRoot":"","sources":["../src/scoring.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,kBAAkB,CAAC,KAAsB;IACvD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAe,CAAC;IACrC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAExD,wDAAwD;IACxD,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CACjC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,IAAI,CAAC,CAAC,QAAQ,CAC9C,CAAC;IACF,IAAI,YAAY;QAAE,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IAEjD,wDAAwD;IACxD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAK5D,CAAC;IACH,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;YACxD,MAAM,EAAE,CAAC;YACT,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;gBAChB,KAAK,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;gBACjC,MAAM;YACR,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IAED,0DAA0D;IAC1D,MAAM,iBAAiB,GAAG,gDAAgD,CAAC;IAC3E,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7C,CAAC,CAAC,OAAO,CAAC,IAAI,CACZ,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CACtD,CACF,CAAC;IACF,IAAI,eAAe;QAAE,KAAK,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAE3D,2DAA2D;IAC3D,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,aAAa,GAAG,gEAAgE,CAAC;QACvF,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QACzE,MAAM,WAAW,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CACxC,CAAC,CAAC,OAAO,CAAC,IAAI,CACZ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CACvD,CACF,CAAC;QACF,IAAI,WAAW;YAAE,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IAClD,CAAC;IAED,2DAA2D;IAC3D,IAAI,KAAK,CAAC,GAAG,CAAC,qBAAqB,CAAC,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,IAAI,QAAQ,EAAE,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAC1C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAC/C,CAAC;YACF,IAAI,cAAc;gBAAE,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,sEAAsE;IACtE,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,gBAAgB,GACpB,SAAS,CAAC,MAAM,IAAI,CAAC;QACrB,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CACpB,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,IAAI,CAAC,CAAC,QAAQ,CAAC,CAC9D,CAAC;IACJ,IAAI,gBAAgB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,EAAE,CAAC;QACxD,KAAK,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;QAAE,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAE9C,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,KAAsB;IAEtB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,WAAW,CAAC;IAEjD,yCAAyC;IACzC,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;QAAE,OAAO,WAAW,CAAC;IAExE,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrD,IAAI,CAAC,QAAQ;QAAE,OAAO,WAAW,CAAC;IAElC,+CAA+C;IAC/C,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QAChE,IAAI,OAAO;YAAE,OAAO,UAAU,CAAC;QAC/B,0DAA0D;QAC1D,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChE,IAAI,SAAS,EAAE,IAAI,KAAK,UAAU;YAAE,OAAO,YAAY,CAAC;IAC1D,CAAC;IAED,iFAAiF;IACjF,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAsB;IAC/C,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,YAAY,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAE/C,MAAM,YAAY,GAAG,KAAK,CAAC,gBAAgB,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAEpE,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QAAE,aAAa,IAAI,GAAG,CAAC;IACrE,IAAI,YAAY,CAAC,QAAQ,CAAC,qBAAqB,CAAC;QAAE,aAAa,IAAI,GAAG,CAAC;IACvE,IAAI,YAAY,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QAAE,aAAa,IAAI,IAAI,CAAC;IACzE,IACE,YAAY,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QAC1C,CAAC,YAAY,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QAE3C,aAAa,IAAI,GAAG,CAAC;IAEvB,MAAM,WAAW,GACf,CAAC,KAAK,CAAC,kBAAkB,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,CAAC,CAAC;IACrE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,IAAI,EACJ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CACnD,CAAC;IAEF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,GAAG,aAAa,GAAG,WAAW,CAAC,CAAC;IACxE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC;IAE3D,OAAO;QACL,YAAY;QACZ,aAAa,EAAE,YAAY;QAC3B,kBAAkB,EAAE,YAAY,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QAC9D,kBAAkB,EAAE,YAAY,CAAC,QAAQ,CAAC,qBAAqB,CAAC;QAChE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,KAAK;QACL,YAAY,EAAE,WAAW;QACzB,sBAAsB,EAAE,IAAI;QAC5B,4BAA4B,EAAE,IAAI;QAClC,YAAY,EAAE,IAAI;QAClB,UAAU,EAAE,IAAI;QAChB,YAAY,EAAE,KAAK;QACnB,YAAY,EAAE,IAAI;QAClB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,cAAc,EAAE,cAAc;KAC/B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scoring.test.d.ts","sourceRoot":"","sources":["../src/scoring.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { detectFailureModes, checkCompleteness, scoreTrace } from "./scoring.js";
|
|
3
|
+
function makeTrace(overrides = {}) {
|
|
4
|
+
return {
|
|
5
|
+
trace_id: "test-id",
|
|
6
|
+
schema_version: "1.0",
|
|
7
|
+
source_tool: "claude_code",
|
|
8
|
+
source_session_id: "session-abc",
|
|
9
|
+
source_version: null,
|
|
10
|
+
submitted_by: "user1",
|
|
11
|
+
submitted_at: "2024-01-01T00:00:00Z",
|
|
12
|
+
extracted_at: "2024-01-01T00:00:00Z",
|
|
13
|
+
git_branch: null,
|
|
14
|
+
cwd_hash: null,
|
|
15
|
+
working_language: null,
|
|
16
|
+
started_at: "2024-01-01T00:00:00Z",
|
|
17
|
+
ended_at: "2024-01-01T00:01:00Z",
|
|
18
|
+
turns: [],
|
|
19
|
+
turn_count: 0,
|
|
20
|
+
tool_call_count: 0,
|
|
21
|
+
has_tool_calls: false,
|
|
22
|
+
has_thinking_blocks: false,
|
|
23
|
+
has_file_changes: false,
|
|
24
|
+
has_shell_commands: false,
|
|
25
|
+
total_input_tokens: null,
|
|
26
|
+
total_output_tokens: null,
|
|
27
|
+
total_cache_read_tokens: null,
|
|
28
|
+
content_fidelity: "full",
|
|
29
|
+
env_state: null,
|
|
30
|
+
score: null,
|
|
31
|
+
raw_r2_key: "",
|
|
32
|
+
normalized_r2_key: "",
|
|
33
|
+
...overrides,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
function makeTurn(role, content) {
|
|
37
|
+
return {
|
|
38
|
+
turn_id: Math.random().toString(36).slice(2),
|
|
39
|
+
parent_turn_id: null,
|
|
40
|
+
role,
|
|
41
|
+
timestamp: null,
|
|
42
|
+
content,
|
|
43
|
+
model: null,
|
|
44
|
+
usage: null,
|
|
45
|
+
source_metadata: {},
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
describe("detectFailureModes", () => {
|
|
49
|
+
it("empty turns → no_failure", () => {
|
|
50
|
+
const result = detectFailureModes(makeTrace({ turns: [] }));
|
|
51
|
+
expect(result).toEqual(["no_failure"]);
|
|
52
|
+
});
|
|
53
|
+
it("tool_result with is_error → tool_call_failure", () => {
|
|
54
|
+
const turns = [
|
|
55
|
+
makeTurn("user", [{ type: "tool_result", tool_call_id: "t1", is_error: true, result_content: "err", exit_code: 1 }]),
|
|
56
|
+
];
|
|
57
|
+
const result = detectFailureModes(makeTrace({ turns }));
|
|
58
|
+
expect(result).toContain("tool_call_failure");
|
|
59
|
+
});
|
|
60
|
+
it("same tool 3× consecutive → repeated_tool_calls", () => {
|
|
61
|
+
const toolUse = (n) => ({
|
|
62
|
+
type: "tool_use",
|
|
63
|
+
tool_call_id: `t${n}`,
|
|
64
|
+
tool_name: "bash",
|
|
65
|
+
tool_input: {},
|
|
66
|
+
});
|
|
67
|
+
const turns = [
|
|
68
|
+
makeTurn("assistant", [toolUse(1), toolUse(2), toolUse(3)]),
|
|
69
|
+
];
|
|
70
|
+
const result = detectFailureModes(makeTrace({ turns }));
|
|
71
|
+
expect(result).toContain("repeated_tool_calls");
|
|
72
|
+
});
|
|
73
|
+
it("context window text → context_limit_approached", () => {
|
|
74
|
+
const turns = [
|
|
75
|
+
makeTurn("assistant", [{ type: "text", text: "You have reached the context limit of this session." }]),
|
|
76
|
+
];
|
|
77
|
+
const result = detectFailureModes(makeTrace({ turns }));
|
|
78
|
+
expect(result).toContain("context_limit_approached");
|
|
79
|
+
});
|
|
80
|
+
it("final turns all errors → catastrophic_failure", () => {
|
|
81
|
+
const errResult = { type: "tool_result", tool_call_id: "t1", is_error: true, result_content: "fail", exit_code: 1 };
|
|
82
|
+
const turns = [
|
|
83
|
+
makeTurn("user", [errResult]),
|
|
84
|
+
makeTurn("user", [errResult]),
|
|
85
|
+
];
|
|
86
|
+
const result = detectFailureModes(makeTrace({ turns }));
|
|
87
|
+
expect(result).toContain("catastrophic_failure");
|
|
88
|
+
});
|
|
89
|
+
it("tool errors + later recovery text → graceful_recovery", () => {
|
|
90
|
+
const errResult = { type: "tool_result", tool_call_id: "t1", is_error: true, result_content: "fail", exit_code: 1 };
|
|
91
|
+
const turns = [
|
|
92
|
+
makeTurn("user", [errResult]),
|
|
93
|
+
makeTurn("assistant", [{ type: "text", text: "Let me try a different approach instead." }]),
|
|
94
|
+
];
|
|
95
|
+
const result = detectFailureModes(makeTrace({ turns }));
|
|
96
|
+
expect(result).toContain("graceful_recovery");
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
describe("checkCompleteness", () => {
|
|
100
|
+
it("no turns → malformed", () => {
|
|
101
|
+
expect(checkCompleteness(makeTrace({ turns: [] }))).toBe("malformed");
|
|
102
|
+
});
|
|
103
|
+
it("turn with empty content → malformed", () => {
|
|
104
|
+
const turns = [makeTurn("assistant", [])];
|
|
105
|
+
expect(checkCompleteness(makeTrace({ turns }))).toBe("malformed");
|
|
106
|
+
});
|
|
107
|
+
it("1 user turn → incomplete", () => {
|
|
108
|
+
const turns = [makeTurn("user", [{ type: "text", text: "hello" }])];
|
|
109
|
+
expect(checkCompleteness(makeTrace({ turns }))).toBe("incomplete");
|
|
110
|
+
});
|
|
111
|
+
it("2+ turns ending with assistant text → complete", () => {
|
|
112
|
+
const turns = [
|
|
113
|
+
makeTurn("user", [{ type: "text", text: "hello" }]),
|
|
114
|
+
makeTurn("assistant", [{ type: "text", text: "world" }]),
|
|
115
|
+
];
|
|
116
|
+
expect(checkCompleteness(makeTrace({ turns }))).toBe("complete");
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
describe("scoreTrace", () => {
|
|
120
|
+
it("malformed trace → near-zero payout", () => {
|
|
121
|
+
const score = scoreTrace(makeTrace({ turns: [], content_fidelity: "chat_only" }));
|
|
122
|
+
expect(score.completeness).toBe("malformed");
|
|
123
|
+
expect(score.payout_cents).toBeLessThan(100);
|
|
124
|
+
});
|
|
125
|
+
it("graceful_recovery + tool_call_failure → bonuses stack", () => {
|
|
126
|
+
const baseScore = scoreTrace(makeTrace({ turns: [], content_fidelity: "full" }));
|
|
127
|
+
const errResult = { type: "tool_result", tool_call_id: "t1", is_error: true, result_content: "fail", exit_code: 1 };
|
|
128
|
+
const turns = [
|
|
129
|
+
makeTurn("user", [errResult]),
|
|
130
|
+
makeTurn("assistant", [{ type: "text", text: "Let me try a different approach instead." }]),
|
|
131
|
+
];
|
|
132
|
+
const bonusScore = scoreTrace(makeTrace({ turns, content_fidelity: "full" }));
|
|
133
|
+
expect(bonusScore.payout_cents).toBeGreaterThan(baseScore.payout_cents);
|
|
134
|
+
expect(bonusScore.failure_modes).toContain("graceful_recovery");
|
|
135
|
+
expect(bonusScore.failure_modes).toContain("tool_call_failure");
|
|
136
|
+
});
|
|
137
|
+
it("total clamps to [0, 1]", () => {
|
|
138
|
+
const errResult = { type: "tool_result", tool_call_id: "t1", is_error: true, result_content: "fail", exit_code: 1 };
|
|
139
|
+
const turns = [
|
|
140
|
+
makeTurn("user", [errResult]),
|
|
141
|
+
makeTurn("assistant", [{ type: "text", text: "Let me try a different approach instead." }]),
|
|
142
|
+
];
|
|
143
|
+
const score = scoreTrace(makeTrace({ turns, content_fidelity: "full", total_input_tokens: 1000000, total_output_tokens: 1000000 }));
|
|
144
|
+
expect(score.total).toBeGreaterThanOrEqual(0);
|
|
145
|
+
expect(score.total).toBeLessThanOrEqual(1);
|
|
146
|
+
});
|
|
147
|
+
it("payout_cents = round(total * 500) clamped to 500", () => {
|
|
148
|
+
const turns = [
|
|
149
|
+
makeTurn("user", [{ type: "text", text: "hello" }]),
|
|
150
|
+
makeTurn("assistant", [{ type: "text", text: "world" }]),
|
|
151
|
+
];
|
|
152
|
+
const score = scoreTrace(makeTrace({ turns, content_fidelity: "full" }));
|
|
153
|
+
const expected = Math.min(500, Math.round(score.total * 500));
|
|
154
|
+
expect(score.payout_cents).toBe(expected);
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
//# sourceMappingURL=scoring.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scoring.test.js","sourceRoot":"","sources":["../src/scoring.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAGjF,SAAS,SAAS,CAAC,YAAsC,EAAE;IACzD,OAAO;QACL,QAAQ,EAAE,SAAS;QACnB,cAAc,EAAE,KAAK;QACrB,WAAW,EAAE,aAAa;QAC1B,iBAAiB,EAAE,aAAa;QAChC,cAAc,EAAE,IAAI;QACpB,YAAY,EAAE,OAAO;QACrB,YAAY,EAAE,sBAAsB;QACpC,YAAY,EAAE,sBAAsB;QACpC,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,IAAI;QACd,gBAAgB,EAAE,IAAI;QACtB,UAAU,EAAE,sBAAsB;QAClC,QAAQ,EAAE,sBAAsB;QAChC,KAAK,EAAE,EAAE;QACT,UAAU,EAAE,CAAC;QACb,eAAe,EAAE,CAAC;QAClB,cAAc,EAAE,KAAK;QACrB,mBAAmB,EAAE,KAAK;QAC1B,gBAAgB,EAAE,KAAK;QACvB,kBAAkB,EAAE,KAAK;QACzB,kBAAkB,EAAE,IAAI;QACxB,mBAAmB,EAAE,IAAI;QACzB,uBAAuB,EAAE,IAAI;QAC7B,gBAAgB,EAAE,MAAM;QACxB,SAAS,EAAE,IAAI;QACf,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,EAAE;QACd,iBAAiB,EAAE,EAAE;QACrB,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,IAA0B,EAAE,OAAuB;IACnE,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC5C,cAAc,EAAE,IAAI;QACpB,IAAI;QACJ,SAAS,EAAE,IAAI;QACf,OAAO;QACP,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,IAAI;QACX,eAAe,EAAE,EAAE;KACpB,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QAClC,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,KAAK,GAAG;YACZ,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;SACrH,CAAC;QACF,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,OAAO,GAAG,CAAC,CAAS,EAAgB,EAAE,CAAC,CAAC;YAC5C,IAAI,EAAE,UAAU;YAChB,YAAY,EAAE,IAAI,CAAC,EAAE;YACrB,SAAS,EAAE,MAAM;YACjB,UAAU,EAAE,EAAE;SACf,CAAC,CAAC;QACH,MAAM,KAAK,GAAG;YACZ,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;SAC5D,CAAC;QACF,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,KAAK,GAAG;YACZ,QAAQ,CAAC,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qDAAqD,EAAE,CAAC,CAAC;SACvG,CAAC;QACF,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,SAAS,GAAiB,EAAE,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;QAClI,MAAM,KAAK,GAAG;YACZ,QAAQ,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC;YAC7B,QAAQ,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC;SAC9B,CAAC;QACF,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,SAAS,GAAiB,EAAE,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;QAClI,MAAM,KAAK,GAAG;YACZ,QAAQ,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC;YAC7B,QAAQ,CAAC,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,0CAA0C,EAAE,CAAC,CAAC;SAC5F,CAAC;QACF,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QAClC,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,KAAK,GAAG;YACZ,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACnD,QAAQ,CAAC,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;SACzD,CAAC;QACF,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;QAClF,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7C,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACjF,MAAM,SAAS,GAAiB,EAAE,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;QAClI,MAAM,KAAK,GAAG;YACZ,QAAQ,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC;YAC7B,QAAQ,CAAC,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,0CAA0C,EAAE,CAAC,CAAC;SAC5F,CAAC;QACF,MAAM,UAAU,GAAG,UAAU,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC9E,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACxE,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAChE,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;QAChC,MAAM,SAAS,GAAiB,EAAE,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;QAClI,MAAM,KAAK,GAAG;YACZ,QAAQ,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC;YAC7B,QAAQ,CAAC,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,0CAA0C,EAAE,CAAC,CAAC;SAC5F,CAAC;QACF,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,kBAAkB,EAAE,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACpI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,KAAK,GAAG;YACZ,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACnD,QAAQ,CAAC,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;SACzD,CAAC;QACF,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC;QAC9D,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
export type SourceTool = "claude_code" | "codex_cli" | "cursor";
|
|
2
|
+
export type FailureMode = "tool_call_failure" | "repeated_tool_calls" | "context_limit_approached" | "catastrophic_failure" | "graceful_recovery" | "unexpected_capability" | "wrong_tool_sequence" | "no_failure";
|
|
3
|
+
export interface TokenUsage {
|
|
4
|
+
input_tokens: number;
|
|
5
|
+
output_tokens: number;
|
|
6
|
+
cache_read_input_tokens: number | null;
|
|
7
|
+
cache_creation_input_tokens: number | null;
|
|
8
|
+
reasoning_tokens: number | null;
|
|
9
|
+
}
|
|
10
|
+
export type ContentBlock = {
|
|
11
|
+
type: "text";
|
|
12
|
+
text: string;
|
|
13
|
+
} | {
|
|
14
|
+
type: "thinking";
|
|
15
|
+
text: string;
|
|
16
|
+
} | {
|
|
17
|
+
type: "tool_use";
|
|
18
|
+
tool_call_id: string;
|
|
19
|
+
tool_name: string;
|
|
20
|
+
tool_input: Record<string, unknown>;
|
|
21
|
+
} | {
|
|
22
|
+
type: "tool_result";
|
|
23
|
+
tool_call_id: string;
|
|
24
|
+
is_error: boolean;
|
|
25
|
+
result_content: string | null;
|
|
26
|
+
exit_code: number | null;
|
|
27
|
+
} | {
|
|
28
|
+
type: "image";
|
|
29
|
+
media_type: string;
|
|
30
|
+
data_r2_key: string;
|
|
31
|
+
};
|
|
32
|
+
export interface Turn {
|
|
33
|
+
turn_id: string;
|
|
34
|
+
parent_turn_id: string | null;
|
|
35
|
+
role: "user" | "assistant";
|
|
36
|
+
timestamp: string | null;
|
|
37
|
+
content: ContentBlock[];
|
|
38
|
+
model: string | null;
|
|
39
|
+
usage: TokenUsage | null;
|
|
40
|
+
source_metadata: Record<string, unknown>;
|
|
41
|
+
}
|
|
42
|
+
export interface EnvState {
|
|
43
|
+
git_branch: string | null;
|
|
44
|
+
inferred_file_tree: string[] | null;
|
|
45
|
+
inferred_changed_files: string[] | null;
|
|
46
|
+
inferred_error_files: string[] | null;
|
|
47
|
+
shell_exit_codes: number[] | null;
|
|
48
|
+
open_files_in_editor: string[] | null;
|
|
49
|
+
extraction_method: "passive" | "active";
|
|
50
|
+
}
|
|
51
|
+
export interface TraceScore {
|
|
52
|
+
completeness: "complete" | "incomplete" | "malformed";
|
|
53
|
+
failure_modes: FailureMode[];
|
|
54
|
+
has_error_recovery: boolean;
|
|
55
|
+
has_repeated_calls: boolean;
|
|
56
|
+
content_fidelity: "full" | "chat_only";
|
|
57
|
+
total: number;
|
|
58
|
+
payout_cents: number;
|
|
59
|
+
failure_taxonomy_label: string | null;
|
|
60
|
+
failure_taxonomy_explanation: string | null;
|
|
61
|
+
rarity_score: number | null;
|
|
62
|
+
cluster_id: string | null;
|
|
63
|
+
is_duplicate: boolean;
|
|
64
|
+
duplicate_of: string | null;
|
|
65
|
+
scored_at: string;
|
|
66
|
+
scorer_version: string;
|
|
67
|
+
}
|
|
68
|
+
export interface NormalizedTrace {
|
|
69
|
+
trace_id: string;
|
|
70
|
+
schema_version: string;
|
|
71
|
+
source_tool: SourceTool;
|
|
72
|
+
source_session_id: string;
|
|
73
|
+
source_version: string | null;
|
|
74
|
+
submitted_by: string;
|
|
75
|
+
submitted_at: string;
|
|
76
|
+
extracted_at: string;
|
|
77
|
+
git_branch: string | null;
|
|
78
|
+
cwd_hash: string | null;
|
|
79
|
+
working_language: string | null;
|
|
80
|
+
started_at: string;
|
|
81
|
+
ended_at: string;
|
|
82
|
+
turns: Turn[];
|
|
83
|
+
turn_count: number;
|
|
84
|
+
tool_call_count: number;
|
|
85
|
+
has_tool_calls: boolean;
|
|
86
|
+
has_thinking_blocks: boolean;
|
|
87
|
+
has_file_changes: boolean;
|
|
88
|
+
has_shell_commands: boolean;
|
|
89
|
+
total_input_tokens: number | null;
|
|
90
|
+
total_output_tokens: number | null;
|
|
91
|
+
total_cache_read_tokens: number | null;
|
|
92
|
+
content_fidelity: "full" | "chat_only";
|
|
93
|
+
env_state: EnvState | null;
|
|
94
|
+
score: TraceScore | null;
|
|
95
|
+
raw_r2_key: string;
|
|
96
|
+
normalized_r2_key: string;
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC;AAEhE,MAAM,MAAM,WAAW,GACnB,mBAAmB,GACnB,qBAAqB,GACrB,0BAA0B,GAC1B,sBAAsB,GACtB,mBAAmB,GACnB,uBAAuB,GACvB,qBAAqB,GACrB,YAAY,CAAC;AAEjB,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AAED,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClC;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAClG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GACzH;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC;AAE/D,MAAM,WAAW,IAAI;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC1C;AAED,MAAM,WAAW,QAAQ;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,kBAAkB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACpC,sBAAsB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACxC,oBAAoB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACtC,gBAAgB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAClC,oBAAoB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACtC,iBAAiB,EAAE,SAAS,GAAG,QAAQ,CAAC;CACzC;AAED,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,UAAU,GAAG,YAAY,GAAG,WAAW,CAAC;IACtD,aAAa,EAAE,WAAW,EAAE,CAAC;IAC7B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,gBAAgB,EAAE,MAAM,GAAG,WAAW,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,4BAA4B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,OAAO,CAAC;IACtB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,UAAU,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,OAAO,CAAC;IACxB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,gBAAgB,EAAE,MAAM,GAAG,WAAW,CAAC;IACvC,SAAS,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;CAC3B"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
package/dist/utils.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAGxD;AAED,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAGzD"}
|
package/dist/utils.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export function formatCents(cents) {
|
|
2
|
+
if (cents === null)
|
|
3
|
+
return "—";
|
|
4
|
+
return `$${(cents / 100).toFixed(2)}`;
|
|
5
|
+
}
|
|
6
|
+
export function formatTimestamp(ts) {
|
|
7
|
+
if (!ts)
|
|
8
|
+
return "—";
|
|
9
|
+
return new Date(ts * 1000).toLocaleString();
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,WAAW,CAAC,KAAoB;IAC9C,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC;IAC/B,OAAO,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAiB;IAC/C,IAAI,CAAC,EAAE;QAAE,OAAO,GAAG,CAAC;IACpB,OAAO,IAAI,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;AAC9C,CAAC"}
|