@tracemarketplace/shared 0.0.10 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extractor-claude-code.test.js +53 -0
- package/dist/extractor-claude-code.test.js.map +1 -1
- package/dist/extractor-codex.test.js +5 -0
- package/dist/extractor-codex.test.js.map +1 -1
- package/dist/extractors/claude-code.d.ts.map +1 -1
- package/dist/extractors/claude-code.js +4 -4
- package/dist/extractors/claude-code.js.map +1 -1
- package/dist/extractors/codex.d.ts.map +1 -1
- package/dist/extractors/codex.js +2 -0
- package/dist/extractors/codex.js.map +1 -1
- package/dist/extractors/common.d.ts +1 -2
- package/dist/extractors/common.d.ts.map +1 -1
- package/dist/extractors/common.js +2 -37
- package/dist/extractors/common.js.map +1 -1
- package/dist/extractors/common.test.d.ts +2 -0
- package/dist/extractors/common.test.d.ts.map +1 -0
- package/dist/extractors/common.test.js +17 -0
- package/dist/extractors/common.test.js.map +1 -0
- package/dist/extractors/cursor.d.ts.map +1 -1
- package/dist/extractors/cursor.js +8 -0
- package/dist/extractors/cursor.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/redact.d.ts.map +1 -1
- package/dist/redact.js +3 -1
- package/dist/redact.js.map +1 -1
- package/dist/redact.test.js +9 -0
- package/dist/redact.test.js.map +1 -1
- package/dist/scoring.d.ts +5 -3
- package/dist/scoring.d.ts.map +1 -1
- package/dist/scoring.fixtures.test.d.ts +2 -0
- package/dist/scoring.fixtures.test.d.ts.map +1 -0
- package/dist/scoring.fixtures.test.js +47 -0
- package/dist/scoring.fixtures.test.js.map +1 -0
- package/dist/scoring.js +381 -62
- package/dist/scoring.js.map +1 -1
- package/dist/scoring.test.js +125 -26
- package/dist/scoring.test.js.map +1 -1
- package/dist/tool-normalization.d.ts +66 -0
- package/dist/tool-normalization.d.ts.map +1 -0
- package/dist/tool-normalization.generated.d.ts +181 -0
- package/dist/tool-normalization.generated.d.ts.map +1 -0
- package/dist/tool-normalization.generated.js +261 -0
- package/dist/tool-normalization.generated.js.map +1 -0
- package/dist/tool-normalization.js +463 -0
- package/dist/tool-normalization.js.map +1 -0
- package/dist/tool-normalization.test.d.ts +2 -0
- package/dist/tool-normalization.test.d.ts.map +1 -0
- package/dist/tool-normalization.test.js +188 -0
- package/dist/tool-normalization.test.js.map +1 -0
- package/dist/types.d.ts +38 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/validators.d.ts +23 -6
- package/dist/validators.d.ts.map +1 -1
- package/dist/validators.js +4 -0
- package/dist/validators.js.map +1 -1
- package/dist/validators.test.js +7 -0
- package/dist/validators.test.js.map +1 -1
- package/package.json +5 -5
- package/scripts/generate-tool-normalization.mjs +16 -0
- package/src/extractor-claude-code.test.ts +59 -0
- package/src/extractor-codex.test.ts +5 -0
- package/src/extractors/claude-code.ts +8 -4
- package/src/extractors/codex.ts +2 -0
- package/src/extractors/common.test.ts +21 -0
- package/src/extractors/common.ts +15 -49
- package/src/extractors/cursor.ts +9 -0
- package/src/index.ts +1 -0
- package/src/redact.test.ts +9 -0
- package/src/redact.ts +3 -1
- package/src/scoring.fixtures.test.ts +71 -0
- package/src/scoring.test.ts +151 -26
- package/src/scoring.ts +582 -84
- package/src/tool-normalization.generated.ts +262 -0
- package/src/tool-normalization.spec.json +205 -0
- package/src/tool-normalization.test.ts +221 -0
- package/src/tool-normalization.ts +670 -0
- package/src/types.ts +50 -0
- package/src/validators.test.ts +8 -0
- package/src/validators.ts +8 -0
package/src/types.ts
CHANGED
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
export type SourceTool = "claude_code" | "codex_cli" | "cursor";
|
|
2
2
|
|
|
3
|
+
export type JsonPrimitive = string | number | boolean | null;
|
|
4
|
+
export type JsonValue = JsonPrimitive | JsonObject | JsonValue[];
|
|
5
|
+
|
|
6
|
+
export interface JsonObject {
|
|
7
|
+
[key: string]: JsonValue;
|
|
8
|
+
}
|
|
9
|
+
|
|
3
10
|
export type FailureMode =
|
|
4
11
|
| "tool_call_failure"
|
|
12
|
+
| "repeated_failing_root_cause"
|
|
5
13
|
| "repeated_tool_calls"
|
|
6
14
|
| "context_limit_approached"
|
|
7
15
|
| "catastrophic_failure"
|
|
@@ -10,6 +18,8 @@ export type FailureMode =
|
|
|
10
18
|
| "wrong_tool_sequence"
|
|
11
19
|
| "no_failure";
|
|
12
20
|
|
|
21
|
+
export type QualityTier = "bronze" | "silver" | "gold";
|
|
22
|
+
|
|
13
23
|
export interface TokenUsage {
|
|
14
24
|
input_tokens: number;
|
|
15
25
|
output_tokens: number;
|
|
@@ -49,6 +59,43 @@ export interface EnvState {
|
|
|
49
59
|
extraction_method: "passive" | "active";
|
|
50
60
|
}
|
|
51
61
|
|
|
62
|
+
export type TraceScoreComponentSource =
|
|
63
|
+
| "trace"
|
|
64
|
+
| "normalization"
|
|
65
|
+
| "label"
|
|
66
|
+
| "judge"
|
|
67
|
+
| "corpus";
|
|
68
|
+
|
|
69
|
+
export interface TraceScoreComponent {
|
|
70
|
+
key: string;
|
|
71
|
+
label: string;
|
|
72
|
+
score: number;
|
|
73
|
+
weight: number;
|
|
74
|
+
source: TraceScoreComponentSource;
|
|
75
|
+
available: boolean;
|
|
76
|
+
reason: string;
|
|
77
|
+
evidence: JsonValue | null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface TraceScoreBreakdown {
|
|
81
|
+
aggregation: "weighted_average";
|
|
82
|
+
component_count: number;
|
|
83
|
+
available_weight: number;
|
|
84
|
+
total_weight: number;
|
|
85
|
+
components: TraceScoreComponent[];
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export interface TraceScoreContext {
|
|
89
|
+
failure_modes_override?: FailureMode[];
|
|
90
|
+
anomaly_score?: number | null;
|
|
91
|
+
workflow_shape?: string | null;
|
|
92
|
+
length_bucket?: string | null;
|
|
93
|
+
tool_density?: string | null;
|
|
94
|
+
failure_judge_verdict?: string | null;
|
|
95
|
+
failure_judge_agreement?: string | null;
|
|
96
|
+
failure_judge_confidence?: number | null;
|
|
97
|
+
}
|
|
98
|
+
|
|
52
99
|
export interface TraceScore {
|
|
53
100
|
completeness: "complete" | "incomplete" | "malformed";
|
|
54
101
|
failure_modes: FailureMode[];
|
|
@@ -57,6 +104,7 @@ export interface TraceScore {
|
|
|
57
104
|
content_fidelity: "full" | "chat_only";
|
|
58
105
|
total: number;
|
|
59
106
|
payout_cents: number;
|
|
107
|
+
breakdown: TraceScoreBreakdown;
|
|
60
108
|
failure_taxonomy_label: string | null;
|
|
61
109
|
failure_taxonomy_explanation: string | null;
|
|
62
110
|
rarity_score: number | null;
|
|
@@ -99,6 +147,8 @@ export interface NormalizedTrace {
|
|
|
99
147
|
content_fidelity: "full" | "chat_only";
|
|
100
148
|
env_state: EnvState | null;
|
|
101
149
|
score: TraceScore | null;
|
|
150
|
+
raw_json?: JsonValue | null;
|
|
151
|
+
raw_json_format?: string | null;
|
|
102
152
|
raw_r2_key: string;
|
|
103
153
|
normalized_r2_key: string;
|
|
104
154
|
}
|
package/src/validators.test.ts
CHANGED
|
@@ -34,6 +34,14 @@ describe("NormalizedTraceSchema", () => {
|
|
|
34
34
|
const result = NormalizedTraceSchema.safeParse(makeTrace({ turn_count: -1 }));
|
|
35
35
|
expect(result.success).toBe(false);
|
|
36
36
|
});
|
|
37
|
+
|
|
38
|
+
it("accepts raw_json payloads", () => {
|
|
39
|
+
const result = NormalizedTraceSchema.safeParse(makeTrace({
|
|
40
|
+
raw_json: { events: [{ type: "session_meta" }] },
|
|
41
|
+
raw_json_format: "codex_cli.jsonl",
|
|
42
|
+
}));
|
|
43
|
+
expect(result.success).toBe(true);
|
|
44
|
+
});
|
|
37
45
|
});
|
|
38
46
|
|
|
39
47
|
describe("BatchSubmitSchema", () => {
|
package/src/validators.ts
CHANGED
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import type { JsonValue } from "./types.js";
|
|
3
|
+
|
|
4
|
+
const JsonLiteralSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]);
|
|
5
|
+
const JsonValueSchema: z.ZodType<JsonValue> = z.lazy(() =>
|
|
6
|
+
z.union([JsonLiteralSchema, z.array(JsonValueSchema), z.record(JsonValueSchema)])
|
|
7
|
+
);
|
|
2
8
|
|
|
3
9
|
export const NormalizedTraceSchema = z.object({
|
|
4
10
|
trace_id: z.string().optional(),
|
|
@@ -32,6 +38,8 @@ export const NormalizedTraceSchema = z.object({
|
|
|
32
38
|
content_fidelity: z.enum(["full", "chat_only"]).optional(),
|
|
33
39
|
env_state: z.any().nullable().optional(),
|
|
34
40
|
score: z.any().nullable().optional(),
|
|
41
|
+
raw_json: JsonValueSchema.nullable().optional(),
|
|
42
|
+
raw_json_format: z.string().nullable().optional(),
|
|
35
43
|
raw_r2_key: z.string().optional(),
|
|
36
44
|
normalized_r2_key: z.string().optional(),
|
|
37
45
|
});
|