@tangle-network/agent-interface 0.35.0 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-candidate-outcome-schema.d.ts +26 -0
- package/dist/agent-candidate-outcome-schema.js +8 -1
- package/dist/agent-candidate-promotion-schema.d.ts +159 -18
- package/dist/agent-candidate-promotion-schema.js +24 -1
- package/dist/agent-candidate-receipt-schema.d.ts +6 -0
- package/dist/agent-candidate.d.ts +35 -11
- package/dist/agent-execution-limits.d.ts +4 -0
- package/dist/agent-improvement-measurement-schema.d.ts +40 -7
- package/dist/agent-improvement-measurement-schema.js +55 -14
- package/dist/agent-profile-improvement-schema.d.ts +85 -6
- package/dist/agent-profile-improvement-schema.js +33 -1
- package/dist/agent-profile-improvement.d.ts +6 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/package.json +1 -1
|
@@ -6,6 +6,10 @@ export declare const agentCandidateFixedSpendSchema: z.ZodObject<{
|
|
|
6
6
|
reasoningTokens: z.ZodNumber;
|
|
7
7
|
modelCalls: z.ZodNumber;
|
|
8
8
|
costUsdNanos: z.ZodNumber;
|
|
9
|
+
costProvenance: z.ZodEnum<{
|
|
10
|
+
observed: "observed";
|
|
11
|
+
estimated: "estimated";
|
|
12
|
+
}>;
|
|
9
13
|
}, z.core.$strict>;
|
|
10
14
|
export declare const agentCandidateModelSettlementCallSchema: z.ZodObject<{
|
|
11
15
|
callId: z.ZodString;
|
|
@@ -23,6 +27,10 @@ export declare const agentCandidateModelSettlementCallSchema: z.ZodObject<{
|
|
|
23
27
|
cachedInputTokens: z.ZodNumber;
|
|
24
28
|
reasoningTokens: z.ZodNumber;
|
|
25
29
|
costUsdNanos: z.ZodNumber;
|
|
30
|
+
costProvenance: z.ZodEnum<{
|
|
31
|
+
observed: "observed";
|
|
32
|
+
estimated: "estimated";
|
|
33
|
+
}>;
|
|
26
34
|
}, z.core.$strict>;
|
|
27
35
|
export declare const agentCandidateModelSettlementMaterialSchema: z.ZodObject<{
|
|
28
36
|
calls: z.ZodArray<z.ZodObject<{
|
|
@@ -41,6 +49,10 @@ export declare const agentCandidateModelSettlementMaterialSchema: z.ZodObject<{
|
|
|
41
49
|
cachedInputTokens: z.ZodNumber;
|
|
42
50
|
reasoningTokens: z.ZodNumber;
|
|
43
51
|
costUsdNanos: z.ZodNumber;
|
|
52
|
+
costProvenance: z.ZodEnum<{
|
|
53
|
+
observed: "observed";
|
|
54
|
+
estimated: "estimated";
|
|
55
|
+
}>;
|
|
44
56
|
}, z.core.$strict>>;
|
|
45
57
|
kind: z.ZodLiteral<"agent-candidate-model-settlement-material">;
|
|
46
58
|
executionPlanDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
@@ -69,6 +81,10 @@ export declare const agentCandidateModelSettlementMaterialSchema: z.ZodObject<{
|
|
|
69
81
|
reasoningTokens: z.ZodNumber;
|
|
70
82
|
modelCalls: z.ZodNumber;
|
|
71
83
|
costUsdNanos: z.ZodNumber;
|
|
84
|
+
costProvenance: z.ZodEnum<{
|
|
85
|
+
observed: "observed";
|
|
86
|
+
estimated: "estimated";
|
|
87
|
+
}>;
|
|
72
88
|
}, z.core.$strict>;
|
|
73
89
|
}, z.core.$strict>;
|
|
74
90
|
export declare const agentCandidateModelSettlementEvidenceSchema: z.ZodObject<{
|
|
@@ -88,6 +104,7 @@ export declare const agentCandidateModelSettlementEvidenceSchema: z.ZodObject<{
|
|
|
88
104
|
cachedInputTokens: number;
|
|
89
105
|
reasoningTokens: number;
|
|
90
106
|
costUsdNanos: number;
|
|
107
|
+
costProvenance: "observed" | "estimated";
|
|
91
108
|
}[];
|
|
92
109
|
kind: "agent-candidate-model-settlement-material";
|
|
93
110
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -108,6 +125,7 @@ export declare const agentCandidateModelSettlementEvidenceSchema: z.ZodObject<{
|
|
|
108
125
|
reasoningTokens: number;
|
|
109
126
|
modelCalls: number;
|
|
110
127
|
costUsdNanos: number;
|
|
128
|
+
costProvenance: "observed" | "estimated";
|
|
111
129
|
};
|
|
112
130
|
}, unknown, z.core.$ZodTypeInternals<{
|
|
113
131
|
calls: {
|
|
@@ -123,6 +141,7 @@ export declare const agentCandidateModelSettlementEvidenceSchema: z.ZodObject<{
|
|
|
123
141
|
cachedInputTokens: number;
|
|
124
142
|
reasoningTokens: number;
|
|
125
143
|
costUsdNanos: number;
|
|
144
|
+
costProvenance: "observed" | "estimated";
|
|
126
145
|
}[];
|
|
127
146
|
kind: "agent-candidate-model-settlement-material";
|
|
128
147
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -143,6 +162,7 @@ export declare const agentCandidateModelSettlementEvidenceSchema: z.ZodObject<{
|
|
|
143
162
|
reasoningTokens: number;
|
|
144
163
|
modelCalls: number;
|
|
145
164
|
costUsdNanos: number;
|
|
165
|
+
costProvenance: "observed" | "estimated";
|
|
146
166
|
};
|
|
147
167
|
}, unknown>>;
|
|
148
168
|
artifact: z.ZodUnion<readonly [z.ZodObject<{
|
|
@@ -563,6 +583,10 @@ export declare const agentCandidateBenchmarkResultMaterialSchema: z.ZodObject<{
|
|
|
563
583
|
reasoningTokens: z.ZodNumber;
|
|
564
584
|
modelCalls: z.ZodNumber;
|
|
565
585
|
costUsdNanos: z.ZodNumber;
|
|
586
|
+
costProvenance: z.ZodEnum<{
|
|
587
|
+
observed: "observed";
|
|
588
|
+
estimated: "estimated";
|
|
589
|
+
}>;
|
|
566
590
|
}, z.core.$strict>;
|
|
567
591
|
timing: z.ZodObject<{
|
|
568
592
|
startedAtMs: z.ZodNumber;
|
|
@@ -625,6 +649,7 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
|
|
|
625
649
|
reasoningTokens: number;
|
|
626
650
|
modelCalls: number;
|
|
627
651
|
costUsdNanos: number;
|
|
652
|
+
costProvenance: "observed" | "estimated";
|
|
628
653
|
};
|
|
629
654
|
timing: {
|
|
630
655
|
startedAtMs: number;
|
|
@@ -683,6 +708,7 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
|
|
|
683
708
|
reasoningTokens: number;
|
|
684
709
|
modelCalls: number;
|
|
685
710
|
costUsdNanos: number;
|
|
711
|
+
costProvenance: "observed" | "estimated";
|
|
686
712
|
};
|
|
687
713
|
timing: {
|
|
688
714
|
startedAtMs: number;
|
|
@@ -21,6 +21,7 @@ export const agentCandidateFixedSpendSchema = z
|
|
|
21
21
|
reasoningTokens: safeCountSchema,
|
|
22
22
|
modelCalls: safeCountSchema,
|
|
23
23
|
costUsdNanos: safeCountSchema,
|
|
24
|
+
costProvenance: z.enum(["observed", "estimated"]),
|
|
24
25
|
})
|
|
25
26
|
.strict();
|
|
26
27
|
export const agentCandidateModelSettlementCallSchema = z
|
|
@@ -37,6 +38,7 @@ export const agentCandidateModelSettlementCallSchema = z
|
|
|
37
38
|
cachedInputTokens: safeCountSchema,
|
|
38
39
|
reasoningTokens: safeCountSchema,
|
|
39
40
|
costUsdNanos: safeCountSchema,
|
|
41
|
+
costProvenance: z.enum(["observed", "estimated"]),
|
|
40
42
|
})
|
|
41
43
|
.strict()
|
|
42
44
|
.superRefine((call, ctx) => {
|
|
@@ -82,6 +84,7 @@ function refineModelSettlementMaterial(material, ctx) {
|
|
|
82
84
|
reasoningTokens: 0,
|
|
83
85
|
modelCalls: material.calls.length,
|
|
84
86
|
costUsdNanos: 0,
|
|
87
|
+
costProvenance: "observed",
|
|
85
88
|
};
|
|
86
89
|
for (const [index, call] of material.calls.entries()) {
|
|
87
90
|
if (callIds.has(call.callId)) {
|
|
@@ -135,6 +138,9 @@ function refineModelSettlementMaterial(material, ctx) {
|
|
|
135
138
|
}
|
|
136
139
|
}
|
|
137
140
|
}
|
|
141
|
+
if (material.calls.some((call) => call.costProvenance === "estimated")) {
|
|
142
|
+
totals.costProvenance = "estimated";
|
|
143
|
+
}
|
|
138
144
|
if (!sameFixedSpend(totals, material.usage)) {
|
|
139
145
|
ctx.addIssue({
|
|
140
146
|
code: "custom",
|
|
@@ -328,7 +334,8 @@ function sameFixedSpend(left, right) {
|
|
|
328
334
|
left.cachedInputTokens === right.cachedInputTokens &&
|
|
329
335
|
left.reasoningTokens === right.reasoningTokens &&
|
|
330
336
|
left.modelCalls === right.modelCalls &&
|
|
331
|
-
left.costUsdNanos === right.costUsdNanos
|
|
337
|
+
left.costUsdNanos === right.costUsdNanos &&
|
|
338
|
+
left.costProvenance === right.costProvenance);
|
|
332
339
|
}
|
|
333
340
|
function evidenceSchema(kind, material, label) {
|
|
334
341
|
return z
|
|
@@ -2251,6 +2251,7 @@ export declare const candidateExecutionEvidenceSchema: z.ZodObject<{
|
|
|
2251
2251
|
cachedInputTokens: number;
|
|
2252
2252
|
reasoningTokens: number;
|
|
2253
2253
|
costUsdNanos: number;
|
|
2254
|
+
costProvenance: "observed" | "estimated";
|
|
2254
2255
|
}[];
|
|
2255
2256
|
kind: "agent-candidate-model-settlement-material";
|
|
2256
2257
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -2271,6 +2272,7 @@ export declare const candidateExecutionEvidenceSchema: z.ZodObject<{
|
|
|
2271
2272
|
reasoningTokens: number;
|
|
2272
2273
|
modelCalls: number;
|
|
2273
2274
|
costUsdNanos: number;
|
|
2275
|
+
costProvenance: "observed" | "estimated";
|
|
2274
2276
|
};
|
|
2275
2277
|
}, unknown, z.core.$ZodTypeInternals<{
|
|
2276
2278
|
calls: {
|
|
@@ -2286,6 +2288,7 @@ export declare const candidateExecutionEvidenceSchema: z.ZodObject<{
|
|
|
2286
2288
|
cachedInputTokens: number;
|
|
2287
2289
|
reasoningTokens: number;
|
|
2288
2290
|
costUsdNanos: number;
|
|
2291
|
+
costProvenance: "observed" | "estimated";
|
|
2289
2292
|
}[];
|
|
2290
2293
|
kind: "agent-candidate-model-settlement-material";
|
|
2291
2294
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -2306,6 +2309,7 @@ export declare const candidateExecutionEvidenceSchema: z.ZodObject<{
|
|
|
2306
2309
|
reasoningTokens: number;
|
|
2307
2310
|
modelCalls: number;
|
|
2308
2311
|
costUsdNanos: number;
|
|
2312
|
+
costProvenance: "observed" | "estimated";
|
|
2309
2313
|
};
|
|
2310
2314
|
}, unknown>>;
|
|
2311
2315
|
artifact: z.ZodUnion<readonly [z.ZodObject<{
|
|
@@ -2612,6 +2616,7 @@ export declare const candidateExecutionEvidenceSchema: z.ZodObject<{
|
|
|
2612
2616
|
reasoningTokens: number;
|
|
2613
2617
|
modelCalls: number;
|
|
2614
2618
|
costUsdNanos: number;
|
|
2619
|
+
costProvenance: "observed" | "estimated";
|
|
2615
2620
|
};
|
|
2616
2621
|
timing: {
|
|
2617
2622
|
startedAtMs: number;
|
|
@@ -2670,6 +2675,7 @@ export declare const candidateExecutionEvidenceSchema: z.ZodObject<{
|
|
|
2670
2675
|
reasoningTokens: number;
|
|
2671
2676
|
modelCalls: number;
|
|
2672
2677
|
costUsdNanos: number;
|
|
2678
|
+
costProvenance: "observed" | "estimated";
|
|
2673
2679
|
};
|
|
2674
2680
|
timing: {
|
|
2675
2681
|
startedAtMs: number;
|
|
@@ -2853,12 +2859,37 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
2853
2859
|
diff: z.ZodString;
|
|
2854
2860
|
evaluation: z.ZodObject<{
|
|
2855
2861
|
generationsExplored: z.ZodNumber;
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
+
preparation: z.ZodObject<{
|
|
2863
|
+
wallDurationMs: z.ZodNumber;
|
|
2864
|
+
cost: z.ZodObject<{
|
|
2865
|
+
usd: z.ZodNumber;
|
|
2866
|
+
provenance: z.ZodEnum<{
|
|
2867
|
+
observed: "observed";
|
|
2868
|
+
estimated: "estimated";
|
|
2869
|
+
}>;
|
|
2870
|
+
}, z.core.$strict>;
|
|
2871
|
+
}, z.core.$strict>;
|
|
2872
|
+
measurement: z.ZodObject<{
|
|
2873
|
+
wallDurationMs: z.ZodNumber;
|
|
2874
|
+
workDurationMs: z.ZodNumber;
|
|
2875
|
+
cost: z.ZodObject<{
|
|
2876
|
+
usd: z.ZodNumber;
|
|
2877
|
+
provenance: z.ZodEnum<{
|
|
2878
|
+
observed: "observed";
|
|
2879
|
+
estimated: "estimated";
|
|
2880
|
+
}>;
|
|
2881
|
+
}, z.core.$strict>;
|
|
2882
|
+
}, z.core.$strict>;
|
|
2883
|
+
total: z.ZodObject<{
|
|
2884
|
+
wallDurationMs: z.ZodNumber;
|
|
2885
|
+
cost: z.ZodObject<{
|
|
2886
|
+
usd: z.ZodNumber;
|
|
2887
|
+
provenance: z.ZodEnum<{
|
|
2888
|
+
observed: "observed";
|
|
2889
|
+
estimated: "estimated";
|
|
2890
|
+
}>;
|
|
2891
|
+
}, z.core.$strict>;
|
|
2892
|
+
}, z.core.$strict>;
|
|
2862
2893
|
}, z.core.$strict>;
|
|
2863
2894
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodCustom<AgentCandidateJsonValue, AgentCandidateJsonValue>>>;
|
|
2864
2895
|
kind: z.ZodLiteral<"agent-improvement-measured-comparison">;
|
|
@@ -5114,6 +5145,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
5114
5145
|
cachedInputTokens: number;
|
|
5115
5146
|
reasoningTokens: number;
|
|
5116
5147
|
costUsdNanos: number;
|
|
5148
|
+
costProvenance: "observed" | "estimated";
|
|
5117
5149
|
}[];
|
|
5118
5150
|
kind: "agent-candidate-model-settlement-material";
|
|
5119
5151
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -5134,6 +5166,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
5134
5166
|
reasoningTokens: number;
|
|
5135
5167
|
modelCalls: number;
|
|
5136
5168
|
costUsdNanos: number;
|
|
5169
|
+
costProvenance: "observed" | "estimated";
|
|
5137
5170
|
};
|
|
5138
5171
|
}, unknown, z.core.$ZodTypeInternals<{
|
|
5139
5172
|
calls: {
|
|
@@ -5149,6 +5182,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
5149
5182
|
cachedInputTokens: number;
|
|
5150
5183
|
reasoningTokens: number;
|
|
5151
5184
|
costUsdNanos: number;
|
|
5185
|
+
costProvenance: "observed" | "estimated";
|
|
5152
5186
|
}[];
|
|
5153
5187
|
kind: "agent-candidate-model-settlement-material";
|
|
5154
5188
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -5169,6 +5203,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
5169
5203
|
reasoningTokens: number;
|
|
5170
5204
|
modelCalls: number;
|
|
5171
5205
|
costUsdNanos: number;
|
|
5206
|
+
costProvenance: "observed" | "estimated";
|
|
5172
5207
|
};
|
|
5173
5208
|
}, unknown>>;
|
|
5174
5209
|
artifact: z.ZodUnion<readonly [z.ZodObject<{
|
|
@@ -5475,6 +5510,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
5475
5510
|
reasoningTokens: number;
|
|
5476
5511
|
modelCalls: number;
|
|
5477
5512
|
costUsdNanos: number;
|
|
5513
|
+
costProvenance: "observed" | "estimated";
|
|
5478
5514
|
};
|
|
5479
5515
|
timing: {
|
|
5480
5516
|
startedAtMs: number;
|
|
@@ -5533,6 +5569,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
5533
5569
|
reasoningTokens: number;
|
|
5534
5570
|
modelCalls: number;
|
|
5535
5571
|
costUsdNanos: number;
|
|
5572
|
+
costProvenance: "observed" | "estimated";
|
|
5536
5573
|
};
|
|
5537
5574
|
timing: {
|
|
5538
5575
|
startedAtMs: number;
|
|
@@ -6463,6 +6500,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
6463
6500
|
cachedInputTokens: number;
|
|
6464
6501
|
reasoningTokens: number;
|
|
6465
6502
|
costUsdNanos: number;
|
|
6503
|
+
costProvenance: "observed" | "estimated";
|
|
6466
6504
|
}[];
|
|
6467
6505
|
kind: "agent-candidate-model-settlement-material";
|
|
6468
6506
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -6483,6 +6521,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
6483
6521
|
reasoningTokens: number;
|
|
6484
6522
|
modelCalls: number;
|
|
6485
6523
|
costUsdNanos: number;
|
|
6524
|
+
costProvenance: "observed" | "estimated";
|
|
6486
6525
|
};
|
|
6487
6526
|
}, unknown, z.core.$ZodTypeInternals<{
|
|
6488
6527
|
calls: {
|
|
@@ -6498,6 +6537,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
6498
6537
|
cachedInputTokens: number;
|
|
6499
6538
|
reasoningTokens: number;
|
|
6500
6539
|
costUsdNanos: number;
|
|
6540
|
+
costProvenance: "observed" | "estimated";
|
|
6501
6541
|
}[];
|
|
6502
6542
|
kind: "agent-candidate-model-settlement-material";
|
|
6503
6543
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -6518,6 +6558,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
6518
6558
|
reasoningTokens: number;
|
|
6519
6559
|
modelCalls: number;
|
|
6520
6560
|
costUsdNanos: number;
|
|
6561
|
+
costProvenance: "observed" | "estimated";
|
|
6521
6562
|
};
|
|
6522
6563
|
}, unknown>>;
|
|
6523
6564
|
artifact: z.ZodUnion<readonly [z.ZodObject<{
|
|
@@ -6824,6 +6865,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
6824
6865
|
reasoningTokens: number;
|
|
6825
6866
|
modelCalls: number;
|
|
6826
6867
|
costUsdNanos: number;
|
|
6868
|
+
costProvenance: "observed" | "estimated";
|
|
6827
6869
|
};
|
|
6828
6870
|
timing: {
|
|
6829
6871
|
startedAtMs: number;
|
|
@@ -6882,6 +6924,7 @@ export declare const agentImprovementMeasuredComparisonSchema: z.ZodObject<{
|
|
|
6882
6924
|
reasoningTokens: number;
|
|
6883
6925
|
modelCalls: number;
|
|
6884
6926
|
costUsdNanos: number;
|
|
6927
|
+
costProvenance: "observed" | "estimated";
|
|
6885
6928
|
};
|
|
6886
6929
|
timing: {
|
|
6887
6930
|
startedAtMs: number;
|
|
@@ -7095,12 +7138,37 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
7095
7138
|
diff: z.ZodString;
|
|
7096
7139
|
evaluation: z.ZodObject<{
|
|
7097
7140
|
generationsExplored: z.ZodNumber;
|
|
7098
|
-
|
|
7099
|
-
|
|
7100
|
-
|
|
7101
|
-
|
|
7102
|
-
|
|
7103
|
-
|
|
7141
|
+
preparation: z.ZodObject<{
|
|
7142
|
+
wallDurationMs: z.ZodNumber;
|
|
7143
|
+
cost: z.ZodObject<{
|
|
7144
|
+
usd: z.ZodNumber;
|
|
7145
|
+
provenance: z.ZodEnum<{
|
|
7146
|
+
observed: "observed";
|
|
7147
|
+
estimated: "estimated";
|
|
7148
|
+
}>;
|
|
7149
|
+
}, z.core.$strict>;
|
|
7150
|
+
}, z.core.$strict>;
|
|
7151
|
+
measurement: z.ZodObject<{
|
|
7152
|
+
wallDurationMs: z.ZodNumber;
|
|
7153
|
+
workDurationMs: z.ZodNumber;
|
|
7154
|
+
cost: z.ZodObject<{
|
|
7155
|
+
usd: z.ZodNumber;
|
|
7156
|
+
provenance: z.ZodEnum<{
|
|
7157
|
+
observed: "observed";
|
|
7158
|
+
estimated: "estimated";
|
|
7159
|
+
}>;
|
|
7160
|
+
}, z.core.$strict>;
|
|
7161
|
+
}, z.core.$strict>;
|
|
7162
|
+
total: z.ZodObject<{
|
|
7163
|
+
wallDurationMs: z.ZodNumber;
|
|
7164
|
+
cost: z.ZodObject<{
|
|
7165
|
+
usd: z.ZodNumber;
|
|
7166
|
+
provenance: z.ZodEnum<{
|
|
7167
|
+
observed: "observed";
|
|
7168
|
+
estimated: "estimated";
|
|
7169
|
+
}>;
|
|
7170
|
+
}, z.core.$strict>;
|
|
7171
|
+
}, z.core.$strict>;
|
|
7104
7172
|
}, z.core.$strict>;
|
|
7105
7173
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodCustom<AgentCandidateJsonValue, AgentCandidateJsonValue>>>;
|
|
7106
7174
|
kind: z.ZodLiteral<"agent-improvement-measured-comparison">;
|
|
@@ -9356,6 +9424,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
9356
9424
|
cachedInputTokens: number;
|
|
9357
9425
|
reasoningTokens: number;
|
|
9358
9426
|
costUsdNanos: number;
|
|
9427
|
+
costProvenance: "observed" | "estimated";
|
|
9359
9428
|
}[];
|
|
9360
9429
|
kind: "agent-candidate-model-settlement-material";
|
|
9361
9430
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -9376,6 +9445,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
9376
9445
|
reasoningTokens: number;
|
|
9377
9446
|
modelCalls: number;
|
|
9378
9447
|
costUsdNanos: number;
|
|
9448
|
+
costProvenance: "observed" | "estimated";
|
|
9379
9449
|
};
|
|
9380
9450
|
}, unknown, z.core.$ZodTypeInternals<{
|
|
9381
9451
|
calls: {
|
|
@@ -9391,6 +9461,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
9391
9461
|
cachedInputTokens: number;
|
|
9392
9462
|
reasoningTokens: number;
|
|
9393
9463
|
costUsdNanos: number;
|
|
9464
|
+
costProvenance: "observed" | "estimated";
|
|
9394
9465
|
}[];
|
|
9395
9466
|
kind: "agent-candidate-model-settlement-material";
|
|
9396
9467
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -9411,6 +9482,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
9411
9482
|
reasoningTokens: number;
|
|
9412
9483
|
modelCalls: number;
|
|
9413
9484
|
costUsdNanos: number;
|
|
9485
|
+
costProvenance: "observed" | "estimated";
|
|
9414
9486
|
};
|
|
9415
9487
|
}, unknown>>;
|
|
9416
9488
|
artifact: z.ZodUnion<readonly [z.ZodObject<{
|
|
@@ -9717,6 +9789,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
9717
9789
|
reasoningTokens: number;
|
|
9718
9790
|
modelCalls: number;
|
|
9719
9791
|
costUsdNanos: number;
|
|
9792
|
+
costProvenance: "observed" | "estimated";
|
|
9720
9793
|
};
|
|
9721
9794
|
timing: {
|
|
9722
9795
|
startedAtMs: number;
|
|
@@ -9775,6 +9848,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
9775
9848
|
reasoningTokens: number;
|
|
9776
9849
|
modelCalls: number;
|
|
9777
9850
|
costUsdNanos: number;
|
|
9851
|
+
costProvenance: "observed" | "estimated";
|
|
9778
9852
|
};
|
|
9779
9853
|
timing: {
|
|
9780
9854
|
startedAtMs: number;
|
|
@@ -10705,6 +10779,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
10705
10779
|
cachedInputTokens: number;
|
|
10706
10780
|
reasoningTokens: number;
|
|
10707
10781
|
costUsdNanos: number;
|
|
10782
|
+
costProvenance: "observed" | "estimated";
|
|
10708
10783
|
}[];
|
|
10709
10784
|
kind: "agent-candidate-model-settlement-material";
|
|
10710
10785
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -10725,6 +10800,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
10725
10800
|
reasoningTokens: number;
|
|
10726
10801
|
modelCalls: number;
|
|
10727
10802
|
costUsdNanos: number;
|
|
10803
|
+
costProvenance: "observed" | "estimated";
|
|
10728
10804
|
};
|
|
10729
10805
|
}, unknown, z.core.$ZodTypeInternals<{
|
|
10730
10806
|
calls: {
|
|
@@ -10740,6 +10816,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
10740
10816
|
cachedInputTokens: number;
|
|
10741
10817
|
reasoningTokens: number;
|
|
10742
10818
|
costUsdNanos: number;
|
|
10819
|
+
costProvenance: "observed" | "estimated";
|
|
10743
10820
|
}[];
|
|
10744
10821
|
kind: "agent-candidate-model-settlement-material";
|
|
10745
10822
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -10760,6 +10837,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
10760
10837
|
reasoningTokens: number;
|
|
10761
10838
|
modelCalls: number;
|
|
10762
10839
|
costUsdNanos: number;
|
|
10840
|
+
costProvenance: "observed" | "estimated";
|
|
10763
10841
|
};
|
|
10764
10842
|
}, unknown>>;
|
|
10765
10843
|
artifact: z.ZodUnion<readonly [z.ZodObject<{
|
|
@@ -11066,6 +11144,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
11066
11144
|
reasoningTokens: number;
|
|
11067
11145
|
modelCalls: number;
|
|
11068
11146
|
costUsdNanos: number;
|
|
11147
|
+
costProvenance: "observed" | "estimated";
|
|
11069
11148
|
};
|
|
11070
11149
|
timing: {
|
|
11071
11150
|
startedAtMs: number;
|
|
@@ -11124,6 +11203,7 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
11124
11203
|
reasoningTokens: number;
|
|
11125
11204
|
modelCalls: number;
|
|
11126
11205
|
costUsdNanos: number;
|
|
11206
|
+
costProvenance: "observed" | "estimated";
|
|
11127
11207
|
};
|
|
11128
11208
|
timing: {
|
|
11129
11209
|
startedAtMs: number;
|
|
@@ -11308,12 +11388,37 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
11308
11388
|
diff: z.ZodString;
|
|
11309
11389
|
evaluation: z.ZodObject<{
|
|
11310
11390
|
generationsExplored: z.ZodNumber;
|
|
11311
|
-
|
|
11312
|
-
|
|
11313
|
-
|
|
11314
|
-
|
|
11315
|
-
|
|
11316
|
-
|
|
11391
|
+
preparation: z.ZodObject<{
|
|
11392
|
+
wallDurationMs: z.ZodNumber;
|
|
11393
|
+
cost: z.ZodObject<{
|
|
11394
|
+
usd: z.ZodNumber;
|
|
11395
|
+
provenance: z.ZodEnum<{
|
|
11396
|
+
observed: "observed";
|
|
11397
|
+
estimated: "estimated";
|
|
11398
|
+
}>;
|
|
11399
|
+
}, z.core.$strict>;
|
|
11400
|
+
}, z.core.$strict>;
|
|
11401
|
+
measurement: z.ZodObject<{
|
|
11402
|
+
wallDurationMs: z.ZodNumber;
|
|
11403
|
+
workDurationMs: z.ZodNumber;
|
|
11404
|
+
cost: z.ZodObject<{
|
|
11405
|
+
usd: z.ZodNumber;
|
|
11406
|
+
provenance: z.ZodEnum<{
|
|
11407
|
+
observed: "observed";
|
|
11408
|
+
estimated: "estimated";
|
|
11409
|
+
}>;
|
|
11410
|
+
}, z.core.$strict>;
|
|
11411
|
+
}, z.core.$strict>;
|
|
11412
|
+
total: z.ZodObject<{
|
|
11413
|
+
wallDurationMs: z.ZodNumber;
|
|
11414
|
+
cost: z.ZodObject<{
|
|
11415
|
+
usd: z.ZodNumber;
|
|
11416
|
+
provenance: z.ZodEnum<{
|
|
11417
|
+
observed: "observed";
|
|
11418
|
+
estimated: "estimated";
|
|
11419
|
+
}>;
|
|
11420
|
+
}, z.core.$strict>;
|
|
11421
|
+
}, z.core.$strict>;
|
|
11317
11422
|
}, z.core.$strict>;
|
|
11318
11423
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodCustom<AgentCandidateJsonValue, AgentCandidateJsonValue>>>;
|
|
11319
11424
|
kind: z.ZodLiteral<"agent-profile-improvement-measured-comparison">;
|
|
@@ -11326,6 +11431,11 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
11326
11431
|
sourceDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
11327
11432
|
sourceRevision: z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>;
|
|
11328
11433
|
}, z.core.$strict>;
|
|
11434
|
+
executionRef: z.ZodObject<{
|
|
11435
|
+
identity: z.ZodString;
|
|
11436
|
+
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
11437
|
+
kind: z.ZodLiteral<"agent-profile-improvement-execution-ref">;
|
|
11438
|
+
}, z.core.$strict>;
|
|
11329
11439
|
baseline: z.ZodObject<{
|
|
11330
11440
|
stateDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
11331
11441
|
}, z.core.$strict>;
|
|
@@ -11477,6 +11587,11 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
11477
11587
|
kind: z.ZodLiteral<"agent-profile-improvement-run">;
|
|
11478
11588
|
digestAlgorithm: z.ZodLiteral<"rfc8785-sha256">;
|
|
11479
11589
|
executionId: z.ZodString;
|
|
11590
|
+
executionRef: z.ZodObject<{
|
|
11591
|
+
identity: z.ZodString;
|
|
11592
|
+
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
11593
|
+
kind: z.ZodLiteral<"agent-profile-improvement-execution-ref">;
|
|
11594
|
+
}, z.core.$strict>;
|
|
11480
11595
|
runCell: z.ZodObject<{
|
|
11481
11596
|
kind: z.ZodLiteral<"agent-profile-improvement-run-cell">;
|
|
11482
11597
|
experimentDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
@@ -11543,6 +11658,10 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
11543
11658
|
reasoningTokens: z.ZodNumber;
|
|
11544
11659
|
modelCalls: z.ZodNumber;
|
|
11545
11660
|
costUsdNanos: z.ZodNumber;
|
|
11661
|
+
costProvenance: z.ZodEnum<{
|
|
11662
|
+
observed: "observed";
|
|
11663
|
+
estimated: "estimated";
|
|
11664
|
+
}>;
|
|
11546
11665
|
}, z.core.$strict>;
|
|
11547
11666
|
trace: z.ZodObject<{
|
|
11548
11667
|
evidence: z.ZodObject<{
|
|
@@ -11602,6 +11721,10 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
11602
11721
|
reasoningTokens: z.ZodNumber;
|
|
11603
11722
|
modelCalls: z.ZodNumber;
|
|
11604
11723
|
costUsdNanos: z.ZodNumber;
|
|
11724
|
+
costProvenance: z.ZodEnum<{
|
|
11725
|
+
observed: "observed";
|
|
11726
|
+
estimated: "estimated";
|
|
11727
|
+
}>;
|
|
11605
11728
|
}, z.core.$strict>;
|
|
11606
11729
|
score: z.ZodNumber;
|
|
11607
11730
|
passed: z.ZodBoolean;
|
|
@@ -11616,6 +11739,11 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
11616
11739
|
kind: z.ZodLiteral<"agent-profile-improvement-run">;
|
|
11617
11740
|
digestAlgorithm: z.ZodLiteral<"rfc8785-sha256">;
|
|
11618
11741
|
executionId: z.ZodString;
|
|
11742
|
+
executionRef: z.ZodObject<{
|
|
11743
|
+
identity: z.ZodString;
|
|
11744
|
+
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
11745
|
+
kind: z.ZodLiteral<"agent-profile-improvement-execution-ref">;
|
|
11746
|
+
}, z.core.$strict>;
|
|
11619
11747
|
runCell: z.ZodObject<{
|
|
11620
11748
|
kind: z.ZodLiteral<"agent-profile-improvement-run-cell">;
|
|
11621
11749
|
experimentDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
@@ -11682,6 +11810,10 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
11682
11810
|
reasoningTokens: z.ZodNumber;
|
|
11683
11811
|
modelCalls: z.ZodNumber;
|
|
11684
11812
|
costUsdNanos: z.ZodNumber;
|
|
11813
|
+
costProvenance: z.ZodEnum<{
|
|
11814
|
+
observed: "observed";
|
|
11815
|
+
estimated: "estimated";
|
|
11816
|
+
}>;
|
|
11685
11817
|
}, z.core.$strict>;
|
|
11686
11818
|
trace: z.ZodObject<{
|
|
11687
11819
|
evidence: z.ZodObject<{
|
|
@@ -11741,6 +11873,10 @@ export declare const agentImprovementProposalSchema: z.ZodObject<{
|
|
|
11741
11873
|
reasoningTokens: z.ZodNumber;
|
|
11742
11874
|
modelCalls: z.ZodNumber;
|
|
11743
11875
|
costUsdNanos: z.ZodNumber;
|
|
11876
|
+
costProvenance: z.ZodEnum<{
|
|
11877
|
+
observed: "observed";
|
|
11878
|
+
estimated: "estimated";
|
|
11879
|
+
}>;
|
|
11744
11880
|
}, z.core.$strict>;
|
|
11745
11881
|
score: z.ZodNumber;
|
|
11746
11882
|
passed: z.ZodBoolean;
|
|
@@ -11775,6 +11911,11 @@ export declare const agentImprovementActivationSchema: z.ZodObject<{
|
|
|
11775
11911
|
reviewDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
11776
11912
|
experimentDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
11777
11913
|
candidateDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
11914
|
+
executionRef: z.ZodOptional<z.ZodObject<{
|
|
11915
|
+
identity: z.ZodString;
|
|
11916
|
+
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
11917
|
+
kind: z.ZodLiteral<"agent-profile-improvement-execution-ref">;
|
|
11918
|
+
}, z.core.$strict>>;
|
|
11778
11919
|
intent: z.ZodEnum<{
|
|
11779
11920
|
"activate-candidate": "activate-candidate";
|
|
11780
11921
|
"restore-baseline": "restore-baseline";
|
|
@@ -6,7 +6,7 @@ import { canonicalCandidateDigest, isCanonicalJsonValue, sha256DigestSchema, } f
|
|
|
6
6
|
import { refineAgentExecutionWithinLimits } from "./agent-execution-limits.js";
|
|
7
7
|
import { agentCandidateMaterializationReceiptSchema, agentCandidateRunReceiptSchema, } from "./agent-candidate-receipt-schema.js";
|
|
8
8
|
import { agentCandidateEvaluationPolicySchema, canonicalJsonObjectSchema, createMeasuredComparisonIdentityRegistry, measuredComparisonCommonShape, refineMeasuredComparisonSummary, } from "./agent-improvement-measurement-schema.js";
|
|
9
|
-
import { agentProfileImprovementMeasuredComparisonSchema, changedProfileImprovementSurfaces, } from "./agent-profile-improvement-schema.js";
|
|
9
|
+
import { agentProfileImprovementExecutionRefSchema, agentProfileImprovementMeasuredComparisonSchema, changedProfileImprovementSurfaces, } from "./agent-profile-improvement-schema.js";
|
|
10
10
|
const improvementSurfaceSchema = z.enum([
|
|
11
11
|
"prompt",
|
|
12
12
|
"skills",
|
|
@@ -365,6 +365,7 @@ export const agentImprovementMeasuredComparisonSchema = z
|
|
|
365
365
|
score: (evidence) => evidence.receipt.benchmarkResult.material.score,
|
|
366
366
|
dimension: (evidence, name) => evidence.receipt.benchmarkResult.material.dimensions.find((dimension) => dimension.name === name)?.score,
|
|
367
367
|
cost: executionCostUsd,
|
|
368
|
+
costProvenance: executionCostProvenance,
|
|
368
369
|
latency: executionLatencyMs,
|
|
369
370
|
}, ctx);
|
|
370
371
|
if (!isCanonicalJsonValue(comparison)) {
|
|
@@ -442,6 +443,12 @@ function executionCostUsd(evidence) {
|
|
|
442
443
|
return (evidence.receipt.modelSettlement.material.usage.costUsdNanos +
|
|
443
444
|
evidence.receipt.benchmarkResult.material.grading.usage.costUsdNanos) / 1_000_000_000;
|
|
444
445
|
}
|
|
446
|
+
function executionCostProvenance(evidence) {
|
|
447
|
+
return evidence.receipt.modelSettlement.material.usage.costProvenance === "observed" &&
|
|
448
|
+
evidence.receipt.benchmarkResult.material.grading.usage.costProvenance === "observed"
|
|
449
|
+
? "observed"
|
|
450
|
+
: "estimated";
|
|
451
|
+
}
|
|
445
452
|
function executionLatencyMs(evidence) {
|
|
446
453
|
return (evidence.receipt.timing.durationMs +
|
|
447
454
|
evidence.receipt.benchmarkResult.material.grading.timing.durationMs);
|
|
@@ -473,6 +480,7 @@ export const agentImprovementActivationSchema = z
|
|
|
473
480
|
reviewDigest: sha256DigestSchema,
|
|
474
481
|
experimentDigest: sha256DigestSchema,
|
|
475
482
|
candidateDigest: sha256DigestSchema,
|
|
483
|
+
executionRef: agentProfileImprovementExecutionRefSchema.optional(),
|
|
476
484
|
intent: z.enum(["activate-candidate", "restore-baseline"]),
|
|
477
485
|
targets: z
|
|
478
486
|
.tuple([improvementActivationTargetSchema])
|
|
@@ -485,6 +493,21 @@ export const agentImprovementActivationSchema = z
|
|
|
485
493
|
})
|
|
486
494
|
.strict()
|
|
487
495
|
.superRefine((activation, ctx) => {
|
|
496
|
+
const targetsAgentProfile = activation.targets.some((target) => target.surface === "agent-profile");
|
|
497
|
+
if (targetsAgentProfile && !activation.executionRef) {
|
|
498
|
+
ctx.addIssue({
|
|
499
|
+
code: "custom",
|
|
500
|
+
path: ["executionRef"],
|
|
501
|
+
message: "agent-profile activation requires the measured runner reference",
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
if (!targetsAgentProfile && activation.executionRef) {
|
|
505
|
+
ctx.addIssue({
|
|
506
|
+
code: "custom",
|
|
507
|
+
path: ["executionRef"],
|
|
508
|
+
message: "executionRef is valid only for agent-profile activation",
|
|
509
|
+
});
|
|
510
|
+
}
|
|
488
511
|
const identities = activation.targets.map((target) => `${target.surface}\u0000${target.identity}`);
|
|
489
512
|
if (new Set(identities).size !== identities.length) {
|
|
490
513
|
ctx.addIssue({
|
|
@@ -1030,6 +1030,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1030
1030
|
cachedInputTokens: number;
|
|
1031
1031
|
reasoningTokens: number;
|
|
1032
1032
|
costUsdNanos: number;
|
|
1033
|
+
costProvenance: "observed" | "estimated";
|
|
1033
1034
|
}[];
|
|
1034
1035
|
kind: "agent-candidate-model-settlement-material";
|
|
1035
1036
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -1050,6 +1051,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1050
1051
|
reasoningTokens: number;
|
|
1051
1052
|
modelCalls: number;
|
|
1052
1053
|
costUsdNanos: number;
|
|
1054
|
+
costProvenance: "observed" | "estimated";
|
|
1053
1055
|
};
|
|
1054
1056
|
}, unknown, z.core.$ZodTypeInternals<{
|
|
1055
1057
|
calls: {
|
|
@@ -1065,6 +1067,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1065
1067
|
cachedInputTokens: number;
|
|
1066
1068
|
reasoningTokens: number;
|
|
1067
1069
|
costUsdNanos: number;
|
|
1070
|
+
costProvenance: "observed" | "estimated";
|
|
1068
1071
|
}[];
|
|
1069
1072
|
kind: "agent-candidate-model-settlement-material";
|
|
1070
1073
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -1085,6 +1088,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1085
1088
|
reasoningTokens: number;
|
|
1086
1089
|
modelCalls: number;
|
|
1087
1090
|
costUsdNanos: number;
|
|
1091
|
+
costProvenance: "observed" | "estimated";
|
|
1088
1092
|
};
|
|
1089
1093
|
}, unknown>>;
|
|
1090
1094
|
artifact: z.ZodUnion<readonly [z.ZodObject<{
|
|
@@ -1391,6 +1395,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1391
1395
|
reasoningTokens: number;
|
|
1392
1396
|
modelCalls: number;
|
|
1393
1397
|
costUsdNanos: number;
|
|
1398
|
+
costProvenance: "observed" | "estimated";
|
|
1394
1399
|
};
|
|
1395
1400
|
timing: {
|
|
1396
1401
|
startedAtMs: number;
|
|
@@ -1449,6 +1454,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1449
1454
|
reasoningTokens: number;
|
|
1450
1455
|
modelCalls: number;
|
|
1451
1456
|
costUsdNanos: number;
|
|
1457
|
+
costProvenance: "observed" | "estimated";
|
|
1452
1458
|
};
|
|
1453
1459
|
timing: {
|
|
1454
1460
|
startedAtMs: number;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AgentProfile, AgentProfileFileMount, AgentProfileHookCommand, AgentProfileMode, AgentProfileModelHints, AgentProfileResources, AgentSubagentProfile, ReasoningEffort } from "./agent-profile.js";
|
|
2
2
|
import type { HarnessType } from "./harness.js";
|
|
3
|
-
import type { AgentProfileImprovementMeasuredComparison } from "./agent-profile-improvement.js";
|
|
3
|
+
import type { AgentProfileImprovementExecutionRef, AgentProfileImprovementMeasuredComparison } from "./agent-profile-improvement.js";
|
|
4
4
|
/** Full SHA-256 digest with an explicit algorithm prefix. */
|
|
5
5
|
export type Sha256Digest = `sha256:${string}`;
|
|
6
6
|
/** RFC 8785 JSON Canonicalization Scheme followed by SHA-256. */
|
|
@@ -235,6 +235,33 @@ export type AgentCandidateMemoryPolicy = {
|
|
|
235
235
|
scope: "task";
|
|
236
236
|
seed?: AgentCandidateArtifactRef;
|
|
237
237
|
};
|
|
238
|
+
/** Whether a settled cost came from provider billing or a reproducible estimate. */
|
|
239
|
+
export type AgentCandidateCostProvenance = "observed" | "estimated";
|
|
240
|
+
/** One known dollar amount carried with its source. */
|
|
241
|
+
export interface AgentImprovementCost {
|
|
242
|
+
usd: number;
|
|
243
|
+
provenance: AgentCandidateCostProvenance;
|
|
244
|
+
}
|
|
245
|
+
/** Complete accounting for a measured improvement proposal. */
|
|
246
|
+
export interface AgentImprovementEvaluationAccounting {
|
|
247
|
+
generationsExplored: number;
|
|
248
|
+
/** Trace analysis and candidate search before held-out execution. */
|
|
249
|
+
preparation: {
|
|
250
|
+
wallDurationMs: number;
|
|
251
|
+
cost: AgentImprovementCost;
|
|
252
|
+
};
|
|
253
|
+
/** Held-out baseline/candidate execution. Work time sums parallel calls. */
|
|
254
|
+
measurement: {
|
|
255
|
+
wallDurationMs: number;
|
|
256
|
+
workDurationMs: number;
|
|
257
|
+
cost: AgentImprovementCost;
|
|
258
|
+
};
|
|
259
|
+
/** Wall time is preparation plus held-out measurement. */
|
|
260
|
+
total: {
|
|
261
|
+
wallDurationMs: number;
|
|
262
|
+
cost: AgentImprovementCost;
|
|
263
|
+
};
|
|
264
|
+
}
|
|
238
265
|
/** Lossless evaluator-owned usage totals for one candidate execution. */
|
|
239
266
|
export interface AgentCandidateFixedSpend {
|
|
240
267
|
inputTokens: number;
|
|
@@ -244,6 +271,7 @@ export interface AgentCandidateFixedSpend {
|
|
|
244
271
|
modelCalls: number;
|
|
245
272
|
/** Integer billionths of one US dollar. */
|
|
246
273
|
costUsdNanos: number;
|
|
274
|
+
costProvenance: AgentCandidateCostProvenance;
|
|
247
275
|
}
|
|
248
276
|
/** Evidence and ancestry that produced the immutable candidate. */
|
|
249
277
|
export interface AgentCandidateLineage {
|
|
@@ -350,13 +378,14 @@ export type AgentCandidateEffectiveMemory = {
|
|
|
350
378
|
beforeState: AgentCandidateWorkspaceSnapshotEvidence;
|
|
351
379
|
seedDigest?: Sha256Digest;
|
|
352
380
|
};
|
|
353
|
-
/** The exact evaluator-owned limits applied to every candidate arm. */
|
|
381
|
+
/** The exact evaluator-owned limits applied to every complete candidate arm. */
|
|
354
382
|
export interface AgentCandidateExecutionLimits {
|
|
355
383
|
timeoutMs: number;
|
|
356
384
|
maxSteps: number;
|
|
357
385
|
maxModelCalls: number;
|
|
358
386
|
maxInputTokens: number;
|
|
359
387
|
maxOutputTokens: number;
|
|
388
|
+
/** Total agent plus grading spend for one complete arm. */
|
|
360
389
|
maxCostUsd: number;
|
|
361
390
|
}
|
|
362
391
|
/** Counted attempt identity and the only retry class allowed by the evaluator. */
|
|
@@ -765,15 +794,7 @@ export interface AgentImprovementMeasuredComparisonBase<TExperiment, TMeasuremen
|
|
|
765
794
|
candidateContentHash: string;
|
|
766
795
|
};
|
|
767
796
|
diff: string;
|
|
768
|
-
evaluation:
|
|
769
|
-
generationsExplored: number;
|
|
770
|
-
searchDurationMs: number;
|
|
771
|
-
executionDurationMs: number;
|
|
772
|
-
durationMs: number;
|
|
773
|
-
searchCostUsd: number;
|
|
774
|
-
executionCostUsd: number;
|
|
775
|
-
totalCostUsd: number;
|
|
776
|
-
};
|
|
797
|
+
evaluation: AgentImprovementEvaluationAccounting;
|
|
777
798
|
metadata?: {
|
|
778
799
|
[key: string]: AgentCandidateJsonValue;
|
|
779
800
|
};
|
|
@@ -822,6 +843,8 @@ export interface AgentImprovementActivation {
|
|
|
822
843
|
experimentDigest: Sha256Digest;
|
|
823
844
|
/** Exact proposed state, whether it is a sealed bundle or a normal profile. */
|
|
824
845
|
candidateDigest: Sha256Digest;
|
|
846
|
+
/** Required when a target uses `agent-profile`; the schema enforces this condition. */
|
|
847
|
+
executionRef?: AgentProfileImprovementExecutionRef;
|
|
825
848
|
intent: AgentImprovementActivationIntent;
|
|
826
849
|
targets: [AgentImprovementActivationTarget, ...AgentImprovementActivationTarget[]];
|
|
827
850
|
fundingOwner: string;
|
|
@@ -895,6 +918,7 @@ export interface AgentCandidateModelSettlementCall {
|
|
|
895
918
|
cachedInputTokens: number;
|
|
896
919
|
reasoningTokens: number;
|
|
897
920
|
costUsdNanos: number;
|
|
921
|
+
costProvenance: AgentCandidateCostProvenance;
|
|
898
922
|
}
|
|
899
923
|
/** Canonical model-access ledger after the evaluator has revoked access. */
|
|
900
924
|
export interface AgentCandidateModelSettlementMaterial {
|
|
@@ -16,6 +16,10 @@ export declare const agentExecutionLimitObservationSchema: z.ZodObject<{
|
|
|
16
16
|
reasoningTokens: z.ZodNumber;
|
|
17
17
|
modelCalls: z.ZodNumber;
|
|
18
18
|
costUsdNanos: z.ZodNumber;
|
|
19
|
+
costProvenance: z.ZodEnum<{
|
|
20
|
+
observed: "observed";
|
|
21
|
+
estimated: "estimated";
|
|
22
|
+
}>;
|
|
19
23
|
}, z.core.$strict>;
|
|
20
24
|
}, z.core.$strict>;
|
|
21
25
|
export interface RefineAgentExecutionWithinLimitsOptions {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import type { AgentCandidateEvaluationPolicy, AgentCandidateJsonValue, AgentImprovementMeasuredComparisonBase } from "./agent-candidate.js";
|
|
2
|
+
import type { AgentCandidateCostProvenance, AgentCandidateEvaluationPolicy, AgentCandidateJsonValue, AgentImprovementMeasuredComparisonBase } from "./agent-candidate.js";
|
|
3
3
|
export declare const canonicalJsonSchema: z.ZodCustom<AgentCandidateJsonValue, AgentCandidateJsonValue>;
|
|
4
4
|
export declare const canonicalJsonObjectSchema: z.ZodRecord<z.ZodString, z.ZodCustom<AgentCandidateJsonValue, AgentCandidateJsonValue>>;
|
|
5
5
|
export declare const agentCandidateEvaluationPolicySchema: z.ZodObject<{
|
|
@@ -12,6 +12,13 @@ export declare const agentCandidateEvaluationPolicySchema: z.ZodObject<{
|
|
|
12
12
|
criticalDimensions: z.ZodArray<z.ZodString>;
|
|
13
13
|
regressionTolerance: z.ZodNumber;
|
|
14
14
|
}, z.core.$strict>;
|
|
15
|
+
export declare const agentImprovementCostSchema: z.ZodObject<{
|
|
16
|
+
usd: z.ZodNumber;
|
|
17
|
+
provenance: z.ZodEnum<{
|
|
18
|
+
observed: "observed";
|
|
19
|
+
estimated: "estimated";
|
|
20
|
+
}>;
|
|
21
|
+
}, z.core.$strict>;
|
|
15
22
|
export declare const measuredComparisonCommonShape: {
|
|
16
23
|
overall: z.ZodObject<{
|
|
17
24
|
direction: z.ZodLiteral<"higher-is-better">;
|
|
@@ -157,12 +164,37 @@ export declare const measuredComparisonCommonShape: {
|
|
|
157
164
|
diff: z.ZodString;
|
|
158
165
|
evaluation: z.ZodObject<{
|
|
159
166
|
generationsExplored: z.ZodNumber;
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
167
|
+
preparation: z.ZodObject<{
|
|
168
|
+
wallDurationMs: z.ZodNumber;
|
|
169
|
+
cost: z.ZodObject<{
|
|
170
|
+
usd: z.ZodNumber;
|
|
171
|
+
provenance: z.ZodEnum<{
|
|
172
|
+
observed: "observed";
|
|
173
|
+
estimated: "estimated";
|
|
174
|
+
}>;
|
|
175
|
+
}, z.core.$strict>;
|
|
176
|
+
}, z.core.$strict>;
|
|
177
|
+
measurement: z.ZodObject<{
|
|
178
|
+
wallDurationMs: z.ZodNumber;
|
|
179
|
+
workDurationMs: z.ZodNumber;
|
|
180
|
+
cost: z.ZodObject<{
|
|
181
|
+
usd: z.ZodNumber;
|
|
182
|
+
provenance: z.ZodEnum<{
|
|
183
|
+
observed: "observed";
|
|
184
|
+
estimated: "estimated";
|
|
185
|
+
}>;
|
|
186
|
+
}, z.core.$strict>;
|
|
187
|
+
}, z.core.$strict>;
|
|
188
|
+
total: z.ZodObject<{
|
|
189
|
+
wallDurationMs: z.ZodNumber;
|
|
190
|
+
cost: z.ZodObject<{
|
|
191
|
+
usd: z.ZodNumber;
|
|
192
|
+
provenance: z.ZodEnum<{
|
|
193
|
+
observed: "observed";
|
|
194
|
+
estimated: "estimated";
|
|
195
|
+
}>;
|
|
196
|
+
}, z.core.$strict>;
|
|
197
|
+
}, z.core.$strict>;
|
|
166
198
|
}, z.core.$strict>;
|
|
167
199
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodCustom<AgentCandidateJsonValue, AgentCandidateJsonValue>>>;
|
|
168
200
|
};
|
|
@@ -184,6 +216,7 @@ export declare function refineMeasuredComparisonSummary<TReceipt>(comparison: Me
|
|
|
184
216
|
score(receipt: TReceipt): number;
|
|
185
217
|
dimension(receipt: TReceipt, name: string): number | undefined;
|
|
186
218
|
cost(receipt: TReceipt): number;
|
|
219
|
+
costProvenance(receipt: TReceipt): AgentCandidateCostProvenance;
|
|
187
220
|
latency(receipt: TReceipt): number;
|
|
188
221
|
}, ctx: z.RefinementCtx): void;
|
|
189
222
|
export declare function refineEstimate(estimate: {
|
|
@@ -95,6 +95,36 @@ export const agentCandidateEvaluationPolicySchema = z
|
|
|
95
95
|
});
|
|
96
96
|
}
|
|
97
97
|
});
|
|
98
|
+
export const agentImprovementCostSchema = z
|
|
99
|
+
.object({
|
|
100
|
+
usd: z.number().finite().nonnegative(),
|
|
101
|
+
provenance: z.enum(["observed", "estimated"]),
|
|
102
|
+
})
|
|
103
|
+
.strict();
|
|
104
|
+
const agentImprovementEvaluationSchema = z
|
|
105
|
+
.object({
|
|
106
|
+
generationsExplored: z.number().int().nonnegative(),
|
|
107
|
+
preparation: z
|
|
108
|
+
.object({
|
|
109
|
+
wallDurationMs: z.number().finite().nonnegative(),
|
|
110
|
+
cost: agentImprovementCostSchema,
|
|
111
|
+
})
|
|
112
|
+
.strict(),
|
|
113
|
+
measurement: z
|
|
114
|
+
.object({
|
|
115
|
+
wallDurationMs: z.number().finite().nonnegative(),
|
|
116
|
+
workDurationMs: z.number().finite().nonnegative(),
|
|
117
|
+
cost: agentImprovementCostSchema,
|
|
118
|
+
})
|
|
119
|
+
.strict(),
|
|
120
|
+
total: z
|
|
121
|
+
.object({
|
|
122
|
+
wallDurationMs: z.number().finite().nonnegative(),
|
|
123
|
+
cost: agentImprovementCostSchema,
|
|
124
|
+
})
|
|
125
|
+
.strict(),
|
|
126
|
+
})
|
|
127
|
+
.strict();
|
|
98
128
|
export const measuredComparisonCommonShape = {
|
|
99
129
|
overall: z
|
|
100
130
|
.object({
|
|
@@ -148,17 +178,7 @@ export const measuredComparisonCommonShape = {
|
|
|
148
178
|
})
|
|
149
179
|
.strict(),
|
|
150
180
|
diff: z.string(),
|
|
151
|
-
evaluation:
|
|
152
|
-
.object({
|
|
153
|
-
generationsExplored: z.number().int().nonnegative(),
|
|
154
|
-
searchDurationMs: z.number().finite().nonnegative(),
|
|
155
|
-
executionDurationMs: z.number().finite().nonnegative(),
|
|
156
|
-
durationMs: z.number().finite().nonnegative(),
|
|
157
|
-
searchCostUsd: z.number().finite().nonnegative(),
|
|
158
|
-
executionCostUsd: z.number().finite().nonnegative(),
|
|
159
|
-
totalCostUsd: z.number().finite().nonnegative(),
|
|
160
|
-
})
|
|
161
|
-
.strict(),
|
|
181
|
+
evaluation: agentImprovementEvaluationSchema,
|
|
162
182
|
metadata: canonicalJsonObjectSchema.optional(),
|
|
163
183
|
};
|
|
164
184
|
/** Keep receipt identity reuse rules identical across measured source formats. */
|
|
@@ -181,12 +201,18 @@ export function createMeasuredComparisonIdentityRegistry(options) {
|
|
|
181
201
|
}
|
|
182
202
|
export function refineMeasuredComparisonSummary(comparison, policy, expectedN, measurements, values, ctx) {
|
|
183
203
|
refineEstimate(comparison.overall, ["overall"], ctx);
|
|
184
|
-
|
|
185
|
-
|
|
204
|
+
const totalCostProvenance = comparison.evaluation.preparation.cost.provenance === "observed" &&
|
|
205
|
+
comparison.evaluation.measurement.cost.provenance === "observed"
|
|
206
|
+
? "observed"
|
|
207
|
+
: "estimated";
|
|
208
|
+
if (!numbersApproximatelyEqual(comparison.evaluation.total.wallDurationMs, comparison.evaluation.preparation.wallDurationMs +
|
|
209
|
+
comparison.evaluation.measurement.wallDurationMs) ||
|
|
210
|
+
!numbersApproximatelyEqual(comparison.evaluation.total.cost.usd, comparison.evaluation.preparation.cost.usd + comparison.evaluation.measurement.cost.usd) ||
|
|
211
|
+
comparison.evaluation.total.cost.provenance !== totalCostProvenance) {
|
|
186
212
|
ctx.addIssue({
|
|
187
213
|
code: "custom",
|
|
188
214
|
path: ["evaluation"],
|
|
189
|
-
message: "evaluation totals must equal
|
|
215
|
+
message: "evaluation totals must equal preparation and measurement accounting",
|
|
190
216
|
});
|
|
191
217
|
}
|
|
192
218
|
if (comparison.overall.n !== expectedN) {
|
|
@@ -265,6 +291,21 @@ export function refineMeasuredComparisonSummary(comparison, policy, expectedN, m
|
|
|
265
291
|
latencyCount += 1;
|
|
266
292
|
}
|
|
267
293
|
}
|
|
294
|
+
const measurementCostUsd = measurements.reduce((sum, measurement) => sum + values.cost(measurement.baseline) + values.cost(measurement.candidate), 0);
|
|
295
|
+
const measurementWorkDurationMs = measurements.reduce((sum, measurement) => sum + values.latency(measurement.baseline) + values.latency(measurement.candidate), 0);
|
|
296
|
+
const measurementCostProvenance = measurements.every((measurement) => values.costProvenance(measurement.baseline) === "observed" &&
|
|
297
|
+
values.costProvenance(measurement.candidate) === "observed")
|
|
298
|
+
? "observed"
|
|
299
|
+
: "estimated";
|
|
300
|
+
if (!numbersApproximatelyEqual(comparison.evaluation.measurement.cost.usd, measurementCostUsd) ||
|
|
301
|
+
comparison.evaluation.measurement.cost.provenance !== measurementCostProvenance ||
|
|
302
|
+
!numbersApproximatelyEqual(comparison.evaluation.measurement.workDurationMs, measurementWorkDurationMs)) {
|
|
303
|
+
ctx.addIssue({
|
|
304
|
+
code: "custom",
|
|
305
|
+
path: ["evaluation", "measurement"],
|
|
306
|
+
message: "measurement accounting must equal the complete signed receipts",
|
|
307
|
+
});
|
|
308
|
+
}
|
|
268
309
|
if (costCount !== 1 || latencyCount !== 1) {
|
|
269
310
|
ctx.addIssue({
|
|
270
311
|
code: "custom",
|
|
@@ -178,6 +178,11 @@ export declare const agentProfileImprovementSuiteInputsSchema: z.ZodObject<{
|
|
|
178
178
|
export declare const agentProfileImprovementArmSchema: z.ZodObject<{
|
|
179
179
|
stateDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
180
180
|
}, z.core.$strict>;
|
|
181
|
+
export declare const agentProfileImprovementExecutionRefSchema: z.ZodObject<{
|
|
182
|
+
identity: z.ZodString;
|
|
183
|
+
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
184
|
+
kind: z.ZodLiteral<"agent-profile-improvement-execution-ref">;
|
|
185
|
+
}, z.core.$strict>;
|
|
181
186
|
/**
|
|
182
187
|
* The first product path changes only prompt and skills, but it uses the
|
|
183
188
|
* shared profile-diff language so execution and activation apply identical
|
|
@@ -194,6 +199,11 @@ export declare const agentProfileImprovementExperimentSchema: z.ZodObject<{
|
|
|
194
199
|
sourceDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
195
200
|
sourceRevision: z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>;
|
|
196
201
|
}, z.core.$strict>;
|
|
202
|
+
executionRef: z.ZodObject<{
|
|
203
|
+
identity: z.ZodString;
|
|
204
|
+
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
205
|
+
kind: z.ZodLiteral<"agent-profile-improvement-execution-ref">;
|
|
206
|
+
}, z.core.$strict>;
|
|
197
207
|
baseline: z.ZodObject<{
|
|
198
208
|
stateDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
199
209
|
}, z.core.$strict>;
|
|
@@ -360,6 +370,11 @@ export declare const agentProfileImprovementRunReceiptSchema: z.ZodObject<{
|
|
|
360
370
|
kind: z.ZodLiteral<"agent-profile-improvement-run">;
|
|
361
371
|
digestAlgorithm: z.ZodLiteral<"rfc8785-sha256">;
|
|
362
372
|
executionId: z.ZodString;
|
|
373
|
+
executionRef: z.ZodObject<{
|
|
374
|
+
identity: z.ZodString;
|
|
375
|
+
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
376
|
+
kind: z.ZodLiteral<"agent-profile-improvement-execution-ref">;
|
|
377
|
+
}, z.core.$strict>;
|
|
363
378
|
runCell: z.ZodObject<{
|
|
364
379
|
kind: z.ZodLiteral<"agent-profile-improvement-run-cell">;
|
|
365
380
|
experimentDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
@@ -426,6 +441,10 @@ export declare const agentProfileImprovementRunReceiptSchema: z.ZodObject<{
|
|
|
426
441
|
reasoningTokens: z.ZodNumber;
|
|
427
442
|
modelCalls: z.ZodNumber;
|
|
428
443
|
costUsdNanos: z.ZodNumber;
|
|
444
|
+
costProvenance: z.ZodEnum<{
|
|
445
|
+
observed: "observed";
|
|
446
|
+
estimated: "estimated";
|
|
447
|
+
}>;
|
|
429
448
|
}, z.core.$strict>;
|
|
430
449
|
trace: z.ZodObject<{
|
|
431
450
|
evidence: z.ZodObject<{
|
|
@@ -485,6 +504,10 @@ export declare const agentProfileImprovementRunReceiptSchema: z.ZodObject<{
|
|
|
485
504
|
reasoningTokens: z.ZodNumber;
|
|
486
505
|
modelCalls: z.ZodNumber;
|
|
487
506
|
costUsdNanos: z.ZodNumber;
|
|
507
|
+
costProvenance: z.ZodEnum<{
|
|
508
|
+
observed: "observed";
|
|
509
|
+
estimated: "estimated";
|
|
510
|
+
}>;
|
|
488
511
|
}, z.core.$strict>;
|
|
489
512
|
score: z.ZodNumber;
|
|
490
513
|
passed: z.ZodBoolean;
|
|
@@ -640,12 +663,37 @@ export declare const agentProfileImprovementMeasuredComparisonSchema: z.ZodObjec
|
|
|
640
663
|
diff: z.ZodString;
|
|
641
664
|
evaluation: z.ZodObject<{
|
|
642
665
|
generationsExplored: z.ZodNumber;
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
666
|
+
preparation: z.ZodObject<{
|
|
667
|
+
wallDurationMs: z.ZodNumber;
|
|
668
|
+
cost: z.ZodObject<{
|
|
669
|
+
usd: z.ZodNumber;
|
|
670
|
+
provenance: z.ZodEnum<{
|
|
671
|
+
observed: "observed";
|
|
672
|
+
estimated: "estimated";
|
|
673
|
+
}>;
|
|
674
|
+
}, z.core.$strict>;
|
|
675
|
+
}, z.core.$strict>;
|
|
676
|
+
measurement: z.ZodObject<{
|
|
677
|
+
wallDurationMs: z.ZodNumber;
|
|
678
|
+
workDurationMs: z.ZodNumber;
|
|
679
|
+
cost: z.ZodObject<{
|
|
680
|
+
usd: z.ZodNumber;
|
|
681
|
+
provenance: z.ZodEnum<{
|
|
682
|
+
observed: "observed";
|
|
683
|
+
estimated: "estimated";
|
|
684
|
+
}>;
|
|
685
|
+
}, z.core.$strict>;
|
|
686
|
+
}, z.core.$strict>;
|
|
687
|
+
total: z.ZodObject<{
|
|
688
|
+
wallDurationMs: z.ZodNumber;
|
|
689
|
+
cost: z.ZodObject<{
|
|
690
|
+
usd: z.ZodNumber;
|
|
691
|
+
provenance: z.ZodEnum<{
|
|
692
|
+
observed: "observed";
|
|
693
|
+
estimated: "estimated";
|
|
694
|
+
}>;
|
|
695
|
+
}, z.core.$strict>;
|
|
696
|
+
}, z.core.$strict>;
|
|
649
697
|
}, z.core.$strict>;
|
|
650
698
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodCustom<import("./agent-candidate.js").AgentCandidateJsonValue, import("./agent-candidate.js").AgentCandidateJsonValue>>>;
|
|
651
699
|
kind: z.ZodLiteral<"agent-profile-improvement-measured-comparison">;
|
|
@@ -658,6 +706,11 @@ export declare const agentProfileImprovementMeasuredComparisonSchema: z.ZodObjec
|
|
|
658
706
|
sourceDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
659
707
|
sourceRevision: z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>;
|
|
660
708
|
}, z.core.$strict>;
|
|
709
|
+
executionRef: z.ZodObject<{
|
|
710
|
+
identity: z.ZodString;
|
|
711
|
+
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
712
|
+
kind: z.ZodLiteral<"agent-profile-improvement-execution-ref">;
|
|
713
|
+
}, z.core.$strict>;
|
|
661
714
|
baseline: z.ZodObject<{
|
|
662
715
|
stateDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
663
716
|
}, z.core.$strict>;
|
|
@@ -809,6 +862,11 @@ export declare const agentProfileImprovementMeasuredComparisonSchema: z.ZodObjec
|
|
|
809
862
|
kind: z.ZodLiteral<"agent-profile-improvement-run">;
|
|
810
863
|
digestAlgorithm: z.ZodLiteral<"rfc8785-sha256">;
|
|
811
864
|
executionId: z.ZodString;
|
|
865
|
+
executionRef: z.ZodObject<{
|
|
866
|
+
identity: z.ZodString;
|
|
867
|
+
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
868
|
+
kind: z.ZodLiteral<"agent-profile-improvement-execution-ref">;
|
|
869
|
+
}, z.core.$strict>;
|
|
812
870
|
runCell: z.ZodObject<{
|
|
813
871
|
kind: z.ZodLiteral<"agent-profile-improvement-run-cell">;
|
|
814
872
|
experimentDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
@@ -875,6 +933,10 @@ export declare const agentProfileImprovementMeasuredComparisonSchema: z.ZodObjec
|
|
|
875
933
|
reasoningTokens: z.ZodNumber;
|
|
876
934
|
modelCalls: z.ZodNumber;
|
|
877
935
|
costUsdNanos: z.ZodNumber;
|
|
936
|
+
costProvenance: z.ZodEnum<{
|
|
937
|
+
observed: "observed";
|
|
938
|
+
estimated: "estimated";
|
|
939
|
+
}>;
|
|
878
940
|
}, z.core.$strict>;
|
|
879
941
|
trace: z.ZodObject<{
|
|
880
942
|
evidence: z.ZodObject<{
|
|
@@ -934,6 +996,10 @@ export declare const agentProfileImprovementMeasuredComparisonSchema: z.ZodObjec
|
|
|
934
996
|
reasoningTokens: z.ZodNumber;
|
|
935
997
|
modelCalls: z.ZodNumber;
|
|
936
998
|
costUsdNanos: z.ZodNumber;
|
|
999
|
+
costProvenance: z.ZodEnum<{
|
|
1000
|
+
observed: "observed";
|
|
1001
|
+
estimated: "estimated";
|
|
1002
|
+
}>;
|
|
937
1003
|
}, z.core.$strict>;
|
|
938
1004
|
score: z.ZodNumber;
|
|
939
1005
|
passed: z.ZodBoolean;
|
|
@@ -948,6 +1014,11 @@ export declare const agentProfileImprovementMeasuredComparisonSchema: z.ZodObjec
|
|
|
948
1014
|
kind: z.ZodLiteral<"agent-profile-improvement-run">;
|
|
949
1015
|
digestAlgorithm: z.ZodLiteral<"rfc8785-sha256">;
|
|
950
1016
|
executionId: z.ZodString;
|
|
1017
|
+
executionRef: z.ZodObject<{
|
|
1018
|
+
identity: z.ZodString;
|
|
1019
|
+
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
1020
|
+
kind: z.ZodLiteral<"agent-profile-improvement-execution-ref">;
|
|
1021
|
+
}, z.core.$strict>;
|
|
951
1022
|
runCell: z.ZodObject<{
|
|
952
1023
|
kind: z.ZodLiteral<"agent-profile-improvement-run-cell">;
|
|
953
1024
|
experimentDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
@@ -1014,6 +1085,10 @@ export declare const agentProfileImprovementMeasuredComparisonSchema: z.ZodObjec
|
|
|
1014
1085
|
reasoningTokens: z.ZodNumber;
|
|
1015
1086
|
modelCalls: z.ZodNumber;
|
|
1016
1087
|
costUsdNanos: z.ZodNumber;
|
|
1088
|
+
costProvenance: z.ZodEnum<{
|
|
1089
|
+
observed: "observed";
|
|
1090
|
+
estimated: "estimated";
|
|
1091
|
+
}>;
|
|
1017
1092
|
}, z.core.$strict>;
|
|
1018
1093
|
trace: z.ZodObject<{
|
|
1019
1094
|
evidence: z.ZodObject<{
|
|
@@ -1073,6 +1148,10 @@ export declare const agentProfileImprovementMeasuredComparisonSchema: z.ZodObjec
|
|
|
1073
1148
|
reasoningTokens: z.ZodNumber;
|
|
1074
1149
|
modelCalls: z.ZodNumber;
|
|
1075
1150
|
costUsdNanos: z.ZodNumber;
|
|
1151
|
+
costProvenance: z.ZodEnum<{
|
|
1152
|
+
observed: "observed";
|
|
1153
|
+
estimated: "estimated";
|
|
1154
|
+
}>;
|
|
1076
1155
|
}, z.core.$strict>;
|
|
1077
1156
|
score: z.ZodNumber;
|
|
1078
1157
|
passed: z.ZodBoolean;
|
|
@@ -98,6 +98,11 @@ export const agentProfileImprovementArmSchema = z
|
|
|
98
98
|
stateDigest: sha256DigestSchema,
|
|
99
99
|
})
|
|
100
100
|
.strict();
|
|
101
|
+
export const agentProfileImprovementExecutionRefSchema = profileImprovementEvidenceSchema
|
|
102
|
+
.extend({
|
|
103
|
+
kind: z.literal("agent-profile-improvement-execution-ref"),
|
|
104
|
+
})
|
|
105
|
+
.strict();
|
|
101
106
|
/**
|
|
102
107
|
* The first product path changes only prompt and skills, but it uses the
|
|
103
108
|
* shared profile-diff language so execution and activation apply identical
|
|
@@ -176,6 +181,7 @@ export const agentProfileImprovementExperimentSchema = z
|
|
|
176
181
|
kind: z.literal("agent-profile-improvement-experiment"),
|
|
177
182
|
digestAlgorithm: z.literal("rfc8785-sha256"),
|
|
178
183
|
source: agentImprovementSourceSchema,
|
|
184
|
+
executionRef: agentProfileImprovementExecutionRefSchema,
|
|
179
185
|
baseline: agentProfileImprovementArmSchema,
|
|
180
186
|
candidate: agentProfileImprovementArmSchema,
|
|
181
187
|
change: agentProfileImprovementChangeSchema,
|
|
@@ -305,6 +311,7 @@ export const agentProfileImprovementRunReceiptSchema = z
|
|
|
305
311
|
kind: z.literal("agent-profile-improvement-run"),
|
|
306
312
|
digestAlgorithm: z.literal("rfc8785-sha256"),
|
|
307
313
|
executionId: z.string().min(1).max(500),
|
|
314
|
+
executionRef: agentProfileImprovementExecutionRefSchema,
|
|
308
315
|
runCell: agentProfileImprovementRunCellSchema,
|
|
309
316
|
runRecord: profileImprovementEvidenceSchema,
|
|
310
317
|
billing: z
|
|
@@ -362,7 +369,7 @@ export const agentProfileImprovementRunReceiptSchema = z
|
|
|
362
369
|
refineAgentExecutionWithinLimits(receipt.limits, {
|
|
363
370
|
durationMs: receipt.timing.durationMs,
|
|
364
371
|
steps: receipt.steps,
|
|
365
|
-
usage: receipt.usage,
|
|
372
|
+
usage: combinedProfileUsage(receipt.usage, receipt.grading.usage),
|
|
366
373
|
}, ctx);
|
|
367
374
|
const billing = receipt.billing.map(evidenceKey);
|
|
368
375
|
if (new Set(billing).size !== billing.length) {
|
|
@@ -380,6 +387,19 @@ export const agentProfileImprovementRunReceiptSchema = z
|
|
|
380
387
|
});
|
|
381
388
|
}
|
|
382
389
|
});
|
|
390
|
+
function combinedProfileUsage(execution, grading) {
|
|
391
|
+
return {
|
|
392
|
+
inputTokens: execution.inputTokens + grading.inputTokens,
|
|
393
|
+
outputTokens: execution.outputTokens + grading.outputTokens,
|
|
394
|
+
cachedInputTokens: execution.cachedInputTokens + grading.cachedInputTokens,
|
|
395
|
+
reasoningTokens: execution.reasoningTokens + grading.reasoningTokens,
|
|
396
|
+
modelCalls: execution.modelCalls + grading.modelCalls,
|
|
397
|
+
costUsdNanos: execution.costUsdNanos + grading.costUsdNanos,
|
|
398
|
+
costProvenance: execution.costProvenance === "observed" && grading.costProvenance === "observed"
|
|
399
|
+
? "observed"
|
|
400
|
+
: "estimated",
|
|
401
|
+
};
|
|
402
|
+
}
|
|
383
403
|
const agentProfileImprovementMeasurementSchema = z
|
|
384
404
|
.object({
|
|
385
405
|
baseline: agentProfileImprovementRunReceiptSchema,
|
|
@@ -455,6 +475,11 @@ function refineProfileImprovementComparison(comparison, ctx) {
|
|
|
455
475
|
[...armPath, "runCell", "experimentDigest"],
|
|
456
476
|
"profile run receipt must bind the measured experiment",
|
|
457
477
|
],
|
|
478
|
+
[
|
|
479
|
+
JSON.stringify(receipt.executionRef) === JSON.stringify(comparison.experiment.executionRef),
|
|
480
|
+
[...armPath, "executionRef"],
|
|
481
|
+
"profile run receipt must bind the measured executor",
|
|
482
|
+
],
|
|
458
483
|
[
|
|
459
484
|
cell.arm === arm,
|
|
460
485
|
[...armPath, "runCell", "arm"],
|
|
@@ -537,6 +562,7 @@ function refineProfileImprovementComparison(comparison, ctx) {
|
|
|
537
562
|
score: (receipt) => receipt.grading.score,
|
|
538
563
|
dimension: (receipt, name) => receipt.grading.dimensions.find((dimension) => dimension.name === name)?.score,
|
|
539
564
|
cost: profileImprovementExecutionCostUsd,
|
|
565
|
+
costProvenance: profileImprovementExecutionCostProvenance,
|
|
540
566
|
latency: profileImprovementExecutionLatencyMs,
|
|
541
567
|
}, ctx);
|
|
542
568
|
if (!isCanonicalJsonValue(comparison)) {
|
|
@@ -555,6 +581,12 @@ function evidenceKey(evidence) {
|
|
|
555
581
|
function profileImprovementExecutionCostUsd(receipt) {
|
|
556
582
|
return (receipt.usage.costUsdNanos + receipt.grading.usage.costUsdNanos) / 1_000_000_000;
|
|
557
583
|
}
|
|
584
|
+
function profileImprovementExecutionCostProvenance(receipt) {
|
|
585
|
+
return receipt.usage.costProvenance === "observed" &&
|
|
586
|
+
receipt.grading.usage.costProvenance === "observed"
|
|
587
|
+
? "observed"
|
|
588
|
+
: "estimated";
|
|
589
|
+
}
|
|
558
590
|
function profileImprovementExecutionLatencyMs(receipt) {
|
|
559
591
|
return receipt.timing.durationMs + receipt.grading.timing.durationMs;
|
|
560
592
|
}
|
|
@@ -44,6 +44,10 @@ export interface AgentProfileImprovementSuiteInputs {
|
|
|
44
44
|
export interface AgentProfileImprovementArm {
|
|
45
45
|
stateDigest: Sha256Digest;
|
|
46
46
|
}
|
|
47
|
+
/** Exact non-secret identity of the runner that measured and can activate a profile change. */
|
|
48
|
+
export interface AgentProfileImprovementExecutionRef extends AgentProfileImprovementEvidence {
|
|
49
|
+
kind: "agent-profile-improvement-execution-ref";
|
|
50
|
+
}
|
|
47
51
|
/**
|
|
48
52
|
* Ordered portable profile patches retained for review and activation.
|
|
49
53
|
*
|
|
@@ -58,6 +62,7 @@ export interface AgentProfileImprovementExperimentMaterial {
|
|
|
58
62
|
kind: "agent-profile-improvement-experiment";
|
|
59
63
|
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
60
64
|
source: AgentImprovementSource;
|
|
65
|
+
executionRef: AgentProfileImprovementExecutionRef;
|
|
61
66
|
baseline: AgentProfileImprovementArm;
|
|
62
67
|
candidate: AgentProfileImprovementArm;
|
|
63
68
|
change: AgentProfileImprovementChange;
|
|
@@ -89,6 +94,7 @@ export interface AgentProfileImprovementRunReceipt {
|
|
|
89
94
|
kind: "agent-profile-improvement-run";
|
|
90
95
|
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
91
96
|
executionId: string;
|
|
97
|
+
executionRef: AgentProfileImprovementExecutionRef;
|
|
92
98
|
runCell: AgentProfileImprovementRunCell;
|
|
93
99
|
runRecord: AgentProfileImprovementEvidence;
|
|
94
100
|
billing: [AgentProfileImprovementEvidence, ...AgentProfileImprovementEvidence[]];
|
package/dist/index.d.ts
CHANGED
|
@@ -615,6 +615,7 @@ export * from "./agent-candidate.js";
|
|
|
615
615
|
export * from "./agent-candidate-schema.js";
|
|
616
616
|
export type { Sha256Digest } from "./agent-candidate.js";
|
|
617
617
|
export { canonicalCandidateBytes, canonicalCandidateDigest, canonicalCandidateJson, sha256Bytes, sha256DigestSchema, sha256Utf8, } from "./agent-candidate-schema-common.js";
|
|
618
|
+
export { numbersApproximatelyEqual } from "./number-validation.js";
|
|
618
619
|
export * from "./agent-candidate-promotion-schema.js";
|
|
619
620
|
export { agentCandidateEvaluationPolicySchema } from "./agent-improvement-measurement-schema.js";
|
|
620
621
|
export * from "./agent-improvement-source.js";
|
package/dist/index.js
CHANGED
|
@@ -127,6 +127,7 @@ export * from "./interaction.js";
|
|
|
127
127
|
export * from "./agent-candidate.js";
|
|
128
128
|
export * from "./agent-candidate-schema.js";
|
|
129
129
|
export { canonicalCandidateBytes, canonicalCandidateDigest, canonicalCandidateJson, sha256Bytes, sha256DigestSchema, sha256Utf8, } from "./agent-candidate-schema-common.js";
|
|
130
|
+
export { numbersApproximatelyEqual } from "./number-validation.js";
|
|
130
131
|
export * from "./agent-candidate-promotion-schema.js";
|
|
131
132
|
export { agentCandidateEvaluationPolicySchema } from "./agent-improvement-measurement-schema.js";
|
|
132
133
|
export * from "./agent-improvement-source.js";
|