@tangle-network/agent-interface 0.34.0 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-candidate-outcome-schema.d.ts +26 -0
- package/dist/agent-candidate-outcome-schema.js +8 -1
- package/dist/agent-candidate-promotion-schema.d.ts +166 -18
- package/dist/agent-candidate-promotion-schema.js +33 -1
- package/dist/agent-candidate-receipt-schema.d.ts +7 -0
- package/dist/agent-candidate-receipt-schema.js +1 -0
- package/dist/agent-candidate.d.ts +36 -11
- package/dist/agent-execution-limits.d.ts +32 -0
- package/dist/agent-execution-limits.js +77 -0
- package/dist/agent-improvement-measurement-schema.d.ts +40 -7
- package/dist/agent-improvement-measurement-schema.js +55 -14
- package/dist/agent-profile-improvement-schema.d.ts +88 -6
- package/dist/agent-profile-improvement-schema.js +39 -0
- package/dist/agent-profile-improvement.d.ts +7 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +3 -0
- package/package.json +1 -1
|
@@ -3,9 +3,10 @@ import { agentCandidateBundleSchema } from "./agent-candidate-schema.js";
|
|
|
3
3
|
import { agentCandidateLineageSchema } from "./agent-candidate-lineage-schema.js";
|
|
4
4
|
import { agentCandidateBenchmarkSuiteInputsSchema } from "./agent-candidate-task-schema.js";
|
|
5
5
|
import { canonicalCandidateDigest, isCanonicalJsonValue, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
|
|
6
|
+
import { refineAgentExecutionWithinLimits } from "./agent-execution-limits.js";
|
|
6
7
|
import { agentCandidateMaterializationReceiptSchema, agentCandidateRunReceiptSchema, } from "./agent-candidate-receipt-schema.js";
|
|
7
8
|
import { agentCandidateEvaluationPolicySchema, canonicalJsonObjectSchema, createMeasuredComparisonIdentityRegistry, measuredComparisonCommonShape, refineMeasuredComparisonSummary, } from "./agent-improvement-measurement-schema.js";
|
|
8
|
-
import { agentProfileImprovementMeasuredComparisonSchema, changedProfileImprovementSurfaces, } from "./agent-profile-improvement-schema.js";
|
|
9
|
+
import { agentProfileImprovementExecutionRefSchema, agentProfileImprovementMeasuredComparisonSchema, changedProfileImprovementSurfaces, } from "./agent-profile-improvement-schema.js";
|
|
9
10
|
const improvementSurfaceSchema = z.enum([
|
|
10
11
|
"prompt",
|
|
11
12
|
"skills",
|
|
@@ -112,6 +113,14 @@ export const candidateExecutionEvidenceSchema = z
|
|
|
112
113
|
if (!valid)
|
|
113
114
|
ctx.addIssue({ code: "custom", path, message });
|
|
114
115
|
}
|
|
116
|
+
refineAgentExecutionWithinLimits(plan.material.limits, {
|
|
117
|
+
durationMs: evidence.receipt.timing.durationMs,
|
|
118
|
+
steps: evidence.receipt.steps,
|
|
119
|
+
usage: evidence.receipt.modelSettlement.material.usage,
|
|
120
|
+
}, ctx, {
|
|
121
|
+
pathPrefix: ["receipt"],
|
|
122
|
+
usagePath: ["modelSettlement", "material", "usage"],
|
|
123
|
+
});
|
|
115
124
|
if (!isCanonicalJsonValue(evidence)) {
|
|
116
125
|
ctx.addIssue({
|
|
117
126
|
code: "custom",
|
|
@@ -356,6 +365,7 @@ export const agentImprovementMeasuredComparisonSchema = z
|
|
|
356
365
|
score: (evidence) => evidence.receipt.benchmarkResult.material.score,
|
|
357
366
|
dimension: (evidence, name) => evidence.receipt.benchmarkResult.material.dimensions.find((dimension) => dimension.name === name)?.score,
|
|
358
367
|
cost: executionCostUsd,
|
|
368
|
+
costProvenance: executionCostProvenance,
|
|
359
369
|
latency: executionLatencyMs,
|
|
360
370
|
}, ctx);
|
|
361
371
|
if (!isCanonicalJsonValue(comparison)) {
|
|
@@ -433,6 +443,12 @@ function executionCostUsd(evidence) {
|
|
|
433
443
|
return (evidence.receipt.modelSettlement.material.usage.costUsdNanos +
|
|
434
444
|
evidence.receipt.benchmarkResult.material.grading.usage.costUsdNanos) / 1_000_000_000;
|
|
435
445
|
}
|
|
446
|
+
function executionCostProvenance(evidence) {
|
|
447
|
+
return evidence.receipt.modelSettlement.material.usage.costProvenance === "observed" &&
|
|
448
|
+
evidence.receipt.benchmarkResult.material.grading.usage.costProvenance === "observed"
|
|
449
|
+
? "observed"
|
|
450
|
+
: "estimated";
|
|
451
|
+
}
|
|
436
452
|
function executionLatencyMs(evidence) {
|
|
437
453
|
return (evidence.receipt.timing.durationMs +
|
|
438
454
|
evidence.receipt.benchmarkResult.material.grading.timing.durationMs);
|
|
@@ -464,6 +480,7 @@ export const agentImprovementActivationSchema = z
|
|
|
464
480
|
reviewDigest: sha256DigestSchema,
|
|
465
481
|
experimentDigest: sha256DigestSchema,
|
|
466
482
|
candidateDigest: sha256DigestSchema,
|
|
483
|
+
executionRef: agentProfileImprovementExecutionRefSchema.optional(),
|
|
467
484
|
intent: z.enum(["activate-candidate", "restore-baseline"]),
|
|
468
485
|
targets: z
|
|
469
486
|
.tuple([improvementActivationTargetSchema])
|
|
@@ -476,6 +493,21 @@ export const agentImprovementActivationSchema = z
|
|
|
476
493
|
})
|
|
477
494
|
.strict()
|
|
478
495
|
.superRefine((activation, ctx) => {
|
|
496
|
+
const targetsAgentProfile = activation.targets.some((target) => target.surface === "agent-profile");
|
|
497
|
+
if (targetsAgentProfile && !activation.executionRef) {
|
|
498
|
+
ctx.addIssue({
|
|
499
|
+
code: "custom",
|
|
500
|
+
path: ["executionRef"],
|
|
501
|
+
message: "agent-profile activation requires the measured runner reference",
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
if (!targetsAgentProfile && activation.executionRef) {
|
|
505
|
+
ctx.addIssue({
|
|
506
|
+
code: "custom",
|
|
507
|
+
path: ["executionRef"],
|
|
508
|
+
message: "executionRef is valid only for agent-profile activation",
|
|
509
|
+
});
|
|
510
|
+
}
|
|
479
511
|
const identities = activation.targets.map((target) => `${target.surface}\u0000${target.identity}`);
|
|
480
512
|
if (new Set(identities).size !== identities.length) {
|
|
481
513
|
ctx.addIssue({
|
|
@@ -903,6 +903,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
903
903
|
endedAtMs: z.ZodNumber;
|
|
904
904
|
durationMs: z.ZodNumber;
|
|
905
905
|
}, z.core.$strict>;
|
|
906
|
+
steps: z.ZodNumber;
|
|
906
907
|
memory: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
907
908
|
mode: z.ZodLiteral<"disabled">;
|
|
908
909
|
}, z.core.$strict>, z.ZodObject<{
|
|
@@ -1029,6 +1030,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1029
1030
|
cachedInputTokens: number;
|
|
1030
1031
|
reasoningTokens: number;
|
|
1031
1032
|
costUsdNanos: number;
|
|
1033
|
+
costProvenance: "observed" | "estimated";
|
|
1032
1034
|
}[];
|
|
1033
1035
|
kind: "agent-candidate-model-settlement-material";
|
|
1034
1036
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -1049,6 +1051,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1049
1051
|
reasoningTokens: number;
|
|
1050
1052
|
modelCalls: number;
|
|
1051
1053
|
costUsdNanos: number;
|
|
1054
|
+
costProvenance: "observed" | "estimated";
|
|
1052
1055
|
};
|
|
1053
1056
|
}, unknown, z.core.$ZodTypeInternals<{
|
|
1054
1057
|
calls: {
|
|
@@ -1064,6 +1067,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1064
1067
|
cachedInputTokens: number;
|
|
1065
1068
|
reasoningTokens: number;
|
|
1066
1069
|
costUsdNanos: number;
|
|
1070
|
+
costProvenance: "observed" | "estimated";
|
|
1067
1071
|
}[];
|
|
1068
1072
|
kind: "agent-candidate-model-settlement-material";
|
|
1069
1073
|
executionPlanDigest: `sha256:${string}`;
|
|
@@ -1084,6 +1088,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1084
1088
|
reasoningTokens: number;
|
|
1085
1089
|
modelCalls: number;
|
|
1086
1090
|
costUsdNanos: number;
|
|
1091
|
+
costProvenance: "observed" | "estimated";
|
|
1087
1092
|
};
|
|
1088
1093
|
}, unknown>>;
|
|
1089
1094
|
artifact: z.ZodUnion<readonly [z.ZodObject<{
|
|
@@ -1390,6 +1395,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1390
1395
|
reasoningTokens: number;
|
|
1391
1396
|
modelCalls: number;
|
|
1392
1397
|
costUsdNanos: number;
|
|
1398
|
+
costProvenance: "observed" | "estimated";
|
|
1393
1399
|
};
|
|
1394
1400
|
timing: {
|
|
1395
1401
|
startedAtMs: number;
|
|
@@ -1448,6 +1454,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1448
1454
|
reasoningTokens: number;
|
|
1449
1455
|
modelCalls: number;
|
|
1450
1456
|
costUsdNanos: number;
|
|
1457
|
+
costProvenance: "observed" | "estimated";
|
|
1451
1458
|
};
|
|
1452
1459
|
timing: {
|
|
1453
1460
|
startedAtMs: number;
|
|
@@ -259,6 +259,7 @@ export const agentCandidateRunReceiptSchema = z
|
|
|
259
259
|
durationMs: z.number().int().nonnegative().safe(),
|
|
260
260
|
})
|
|
261
261
|
.strict(),
|
|
262
|
+
steps: z.number().int().nonnegative().safe(),
|
|
262
263
|
memory: agentCandidateMemoryReceiptSchema,
|
|
263
264
|
trace: agentCandidateTraceEvidenceSchema,
|
|
264
265
|
termination: agentCandidateTerminationSchema,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AgentProfile, AgentProfileFileMount, AgentProfileHookCommand, AgentProfileMode, AgentProfileModelHints, AgentProfileResources, AgentSubagentProfile, ReasoningEffort } from "./agent-profile.js";
|
|
2
2
|
import type { HarnessType } from "./harness.js";
|
|
3
|
-
import type { AgentProfileImprovementMeasuredComparison } from "./agent-profile-improvement.js";
|
|
3
|
+
import type { AgentProfileImprovementExecutionRef, AgentProfileImprovementMeasuredComparison } from "./agent-profile-improvement.js";
|
|
4
4
|
/** Full SHA-256 digest with an explicit algorithm prefix. */
|
|
5
5
|
export type Sha256Digest = `sha256:${string}`;
|
|
6
6
|
/** RFC 8785 JSON Canonicalization Scheme followed by SHA-256. */
|
|
@@ -235,6 +235,33 @@ export type AgentCandidateMemoryPolicy = {
|
|
|
235
235
|
scope: "task";
|
|
236
236
|
seed?: AgentCandidateArtifactRef;
|
|
237
237
|
};
|
|
238
|
+
/** Whether a settled cost came from provider billing or a reproducible estimate. */
|
|
239
|
+
export type AgentCandidateCostProvenance = "observed" | "estimated";
|
|
240
|
+
/** One known dollar amount carried with its source. */
|
|
241
|
+
export interface AgentImprovementCost {
|
|
242
|
+
usd: number;
|
|
243
|
+
provenance: AgentCandidateCostProvenance;
|
|
244
|
+
}
|
|
245
|
+
/** Complete accounting for a measured improvement proposal. */
|
|
246
|
+
export interface AgentImprovementEvaluationAccounting {
|
|
247
|
+
generationsExplored: number;
|
|
248
|
+
/** Trace analysis and candidate search before held-out execution. */
|
|
249
|
+
preparation: {
|
|
250
|
+
wallDurationMs: number;
|
|
251
|
+
cost: AgentImprovementCost;
|
|
252
|
+
};
|
|
253
|
+
/** Held-out baseline/candidate execution. Work time sums parallel calls. */
|
|
254
|
+
measurement: {
|
|
255
|
+
wallDurationMs: number;
|
|
256
|
+
workDurationMs: number;
|
|
257
|
+
cost: AgentImprovementCost;
|
|
258
|
+
};
|
|
259
|
+
/** Wall time is preparation plus held-out measurement. */
|
|
260
|
+
total: {
|
|
261
|
+
wallDurationMs: number;
|
|
262
|
+
cost: AgentImprovementCost;
|
|
263
|
+
};
|
|
264
|
+
}
|
|
238
265
|
/** Lossless evaluator-owned usage totals for one candidate execution. */
|
|
239
266
|
export interface AgentCandidateFixedSpend {
|
|
240
267
|
inputTokens: number;
|
|
@@ -244,6 +271,7 @@ export interface AgentCandidateFixedSpend {
|
|
|
244
271
|
modelCalls: number;
|
|
245
272
|
/** Integer billionths of one US dollar. */
|
|
246
273
|
costUsdNanos: number;
|
|
274
|
+
costProvenance: AgentCandidateCostProvenance;
|
|
247
275
|
}
|
|
248
276
|
/** Evidence and ancestry that produced the immutable candidate. */
|
|
249
277
|
export interface AgentCandidateLineage {
|
|
@@ -350,13 +378,14 @@ export type AgentCandidateEffectiveMemory = {
|
|
|
350
378
|
beforeState: AgentCandidateWorkspaceSnapshotEvidence;
|
|
351
379
|
seedDigest?: Sha256Digest;
|
|
352
380
|
};
|
|
353
|
-
/** The exact evaluator-owned limits applied to every candidate arm. */
|
|
381
|
+
/** The exact evaluator-owned limits applied to every complete candidate arm. */
|
|
354
382
|
export interface AgentCandidateExecutionLimits {
|
|
355
383
|
timeoutMs: number;
|
|
356
384
|
maxSteps: number;
|
|
357
385
|
maxModelCalls: number;
|
|
358
386
|
maxInputTokens: number;
|
|
359
387
|
maxOutputTokens: number;
|
|
388
|
+
/** Total agent plus grading spend for one complete arm. */
|
|
360
389
|
maxCostUsd: number;
|
|
361
390
|
}
|
|
362
391
|
/** Counted attempt identity and the only retry class allowed by the evaluator. */
|
|
@@ -658,6 +687,7 @@ export interface AgentCandidateRunReceipt {
|
|
|
658
687
|
endedAtMs: number;
|
|
659
688
|
durationMs: number;
|
|
660
689
|
};
|
|
690
|
+
steps: number;
|
|
661
691
|
memory: AgentCandidateMemoryReceipt;
|
|
662
692
|
trace: AgentCandidateTraceEvidence;
|
|
663
693
|
termination: AgentCandidateTermination;
|
|
@@ -764,15 +794,7 @@ export interface AgentImprovementMeasuredComparisonBase<TExperiment, TMeasuremen
|
|
|
764
794
|
candidateContentHash: string;
|
|
765
795
|
};
|
|
766
796
|
diff: string;
|
|
767
|
-
evaluation:
|
|
768
|
-
generationsExplored: number;
|
|
769
|
-
searchDurationMs: number;
|
|
770
|
-
executionDurationMs: number;
|
|
771
|
-
durationMs: number;
|
|
772
|
-
searchCostUsd: number;
|
|
773
|
-
executionCostUsd: number;
|
|
774
|
-
totalCostUsd: number;
|
|
775
|
-
};
|
|
797
|
+
evaluation: AgentImprovementEvaluationAccounting;
|
|
776
798
|
metadata?: {
|
|
777
799
|
[key: string]: AgentCandidateJsonValue;
|
|
778
800
|
};
|
|
@@ -821,6 +843,8 @@ export interface AgentImprovementActivation {
|
|
|
821
843
|
experimentDigest: Sha256Digest;
|
|
822
844
|
/** Exact proposed state, whether it is a sealed bundle or a normal profile. */
|
|
823
845
|
candidateDigest: Sha256Digest;
|
|
846
|
+
/** Required when a target uses `agent-profile`; the schema enforces this condition. */
|
|
847
|
+
executionRef?: AgentProfileImprovementExecutionRef;
|
|
824
848
|
intent: AgentImprovementActivationIntent;
|
|
825
849
|
targets: [AgentImprovementActivationTarget, ...AgentImprovementActivationTarget[]];
|
|
826
850
|
fundingOwner: string;
|
|
@@ -894,6 +918,7 @@ export interface AgentCandidateModelSettlementCall {
|
|
|
894
918
|
cachedInputTokens: number;
|
|
895
919
|
reasoningTokens: number;
|
|
896
920
|
costUsdNanos: number;
|
|
921
|
+
costProvenance: AgentCandidateCostProvenance;
|
|
897
922
|
}
|
|
898
923
|
/** Canonical model-access ledger after the evaluator has revoked access. */
|
|
899
924
|
export interface AgentCandidateModelSettlementMaterial {
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { AgentCandidateExecutionLimits, AgentCandidateFixedSpend } from "./agent-candidate.js";
|
|
3
|
+
/** The observable execution facts required to prove a frozen limit was respected. */
|
|
4
|
+
export interface AgentExecutionLimitObservation {
|
|
5
|
+
durationMs: number;
|
|
6
|
+
steps: number;
|
|
7
|
+
usage: AgentCandidateFixedSpend;
|
|
8
|
+
}
|
|
9
|
+
export declare const agentExecutionLimitObservationSchema: z.ZodObject<{
|
|
10
|
+
durationMs: z.ZodNumber;
|
|
11
|
+
steps: z.ZodNumber;
|
|
12
|
+
usage: z.ZodObject<{
|
|
13
|
+
inputTokens: z.ZodNumber;
|
|
14
|
+
outputTokens: z.ZodNumber;
|
|
15
|
+
cachedInputTokens: z.ZodNumber;
|
|
16
|
+
reasoningTokens: z.ZodNumber;
|
|
17
|
+
modelCalls: z.ZodNumber;
|
|
18
|
+
costUsdNanos: z.ZodNumber;
|
|
19
|
+
costProvenance: z.ZodEnum<{
|
|
20
|
+
observed: "observed";
|
|
21
|
+
estimated: "estimated";
|
|
22
|
+
}>;
|
|
23
|
+
}, z.core.$strict>;
|
|
24
|
+
}, z.core.$strict>;
|
|
25
|
+
export interface RefineAgentExecutionWithinLimitsOptions {
|
|
26
|
+
pathPrefix?: (string | number)[];
|
|
27
|
+
usagePath?: (string | number)[];
|
|
28
|
+
}
|
|
29
|
+
/** Reject an execution record that cannot satisfy every limit it claims to use. */
|
|
30
|
+
export declare function assertAgentExecutionWithinLimits(limits: AgentCandidateExecutionLimits, observation: AgentExecutionLimitObservation): void;
|
|
31
|
+
/** Add schema issues instead of throwing when a receipt is parsed by Zod. */
|
|
32
|
+
export declare function refineAgentExecutionWithinLimits(limits: AgentCandidateExecutionLimits, observation: AgentExecutionLimitObservation, ctx: z.RefinementCtx, options?: RefineAgentExecutionWithinLimitsOptions): void;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { agentCandidateExecutionLimitsSchema } from "./agent-candidate-execution-plan-schema.js";
|
|
3
|
+
import { agentCandidateFixedSpendSchema } from "./agent-candidate-outcome-schema.js";
|
|
4
|
+
export const agentExecutionLimitObservationSchema = z
|
|
5
|
+
.object({
|
|
6
|
+
durationMs: z.number().finite().nonnegative(),
|
|
7
|
+
steps: z.number().int().nonnegative().safe(),
|
|
8
|
+
usage: agentCandidateFixedSpendSchema,
|
|
9
|
+
})
|
|
10
|
+
.strict();
|
|
11
|
+
/** Reject an execution record that cannot satisfy every limit it claims to use. */
|
|
12
|
+
export function assertAgentExecutionWithinLimits(limits, observation) {
|
|
13
|
+
const parsedLimits = agentCandidateExecutionLimitsSchema.parse(limits);
|
|
14
|
+
const parsedObservation = agentExecutionLimitObservationSchema.parse(observation);
|
|
15
|
+
const violations = executionLimitViolations(parsedLimits, parsedObservation);
|
|
16
|
+
if (violations.length === 0)
|
|
17
|
+
return;
|
|
18
|
+
throw new Error(violations
|
|
19
|
+
.map((violation) => `execution ${violation.label} ${violation.actual} exceeds frozen limit ${violation.maximum}`)
|
|
20
|
+
.join("; "));
|
|
21
|
+
}
|
|
22
|
+
/** Add schema issues instead of throwing when a receipt is parsed by Zod. */
|
|
23
|
+
export function refineAgentExecutionWithinLimits(limits, observation, ctx, options = {}) {
|
|
24
|
+
const pathPrefix = options.pathPrefix ?? [];
|
|
25
|
+
const usagePath = options.usagePath ?? ["usage"];
|
|
26
|
+
for (const violation of executionLimitViolations(limits, observation)) {
|
|
27
|
+
const path = violation.path[0] === "usage"
|
|
28
|
+
? [...pathPrefix, ...usagePath, ...violation.path.slice(1)]
|
|
29
|
+
: [...pathPrefix, ...violation.path];
|
|
30
|
+
ctx.addIssue({
|
|
31
|
+
code: "custom",
|
|
32
|
+
path,
|
|
33
|
+
message: `execution ${violation.label} ${violation.actual} exceeds frozen limit ${violation.maximum}`,
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function executionLimitViolations(limits, observation) {
|
|
38
|
+
const checks = [
|
|
39
|
+
{
|
|
40
|
+
path: ["timing", "durationMs"],
|
|
41
|
+
label: "durationMs",
|
|
42
|
+
actual: observation.durationMs,
|
|
43
|
+
maximum: limits.timeoutMs,
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
path: ["steps"],
|
|
47
|
+
label: "steps",
|
|
48
|
+
actual: observation.steps,
|
|
49
|
+
maximum: limits.maxSteps,
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
path: ["usage", "modelCalls"],
|
|
53
|
+
label: "modelCalls",
|
|
54
|
+
actual: observation.usage.modelCalls,
|
|
55
|
+
maximum: limits.maxModelCalls,
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
path: ["usage", "inputTokens"],
|
|
59
|
+
label: "inputTokens",
|
|
60
|
+
actual: observation.usage.inputTokens,
|
|
61
|
+
maximum: limits.maxInputTokens,
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
path: ["usage", "outputTokens"],
|
|
65
|
+
label: "outputTokens",
|
|
66
|
+
actual: observation.usage.outputTokens,
|
|
67
|
+
maximum: limits.maxOutputTokens,
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
path: ["usage", "costUsdNanos"],
|
|
71
|
+
label: "costUsd",
|
|
72
|
+
actual: observation.usage.costUsdNanos / 1_000_000_000,
|
|
73
|
+
maximum: limits.maxCostUsd,
|
|
74
|
+
},
|
|
75
|
+
];
|
|
76
|
+
return checks.filter((check) => check.actual > check.maximum);
|
|
77
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import type { AgentCandidateEvaluationPolicy, AgentCandidateJsonValue, AgentImprovementMeasuredComparisonBase } from "./agent-candidate.js";
|
|
2
|
+
import type { AgentCandidateCostProvenance, AgentCandidateEvaluationPolicy, AgentCandidateJsonValue, AgentImprovementMeasuredComparisonBase } from "./agent-candidate.js";
|
|
3
3
|
export declare const canonicalJsonSchema: z.ZodCustom<AgentCandidateJsonValue, AgentCandidateJsonValue>;
|
|
4
4
|
export declare const canonicalJsonObjectSchema: z.ZodRecord<z.ZodString, z.ZodCustom<AgentCandidateJsonValue, AgentCandidateJsonValue>>;
|
|
5
5
|
export declare const agentCandidateEvaluationPolicySchema: z.ZodObject<{
|
|
@@ -12,6 +12,13 @@ export declare const agentCandidateEvaluationPolicySchema: z.ZodObject<{
|
|
|
12
12
|
criticalDimensions: z.ZodArray<z.ZodString>;
|
|
13
13
|
regressionTolerance: z.ZodNumber;
|
|
14
14
|
}, z.core.$strict>;
|
|
15
|
+
export declare const agentImprovementCostSchema: z.ZodObject<{
|
|
16
|
+
usd: z.ZodNumber;
|
|
17
|
+
provenance: z.ZodEnum<{
|
|
18
|
+
observed: "observed";
|
|
19
|
+
estimated: "estimated";
|
|
20
|
+
}>;
|
|
21
|
+
}, z.core.$strict>;
|
|
15
22
|
export declare const measuredComparisonCommonShape: {
|
|
16
23
|
overall: z.ZodObject<{
|
|
17
24
|
direction: z.ZodLiteral<"higher-is-better">;
|
|
@@ -157,12 +164,37 @@ export declare const measuredComparisonCommonShape: {
|
|
|
157
164
|
diff: z.ZodString;
|
|
158
165
|
evaluation: z.ZodObject<{
|
|
159
166
|
generationsExplored: z.ZodNumber;
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
167
|
+
preparation: z.ZodObject<{
|
|
168
|
+
wallDurationMs: z.ZodNumber;
|
|
169
|
+
cost: z.ZodObject<{
|
|
170
|
+
usd: z.ZodNumber;
|
|
171
|
+
provenance: z.ZodEnum<{
|
|
172
|
+
observed: "observed";
|
|
173
|
+
estimated: "estimated";
|
|
174
|
+
}>;
|
|
175
|
+
}, z.core.$strict>;
|
|
176
|
+
}, z.core.$strict>;
|
|
177
|
+
measurement: z.ZodObject<{
|
|
178
|
+
wallDurationMs: z.ZodNumber;
|
|
179
|
+
workDurationMs: z.ZodNumber;
|
|
180
|
+
cost: z.ZodObject<{
|
|
181
|
+
usd: z.ZodNumber;
|
|
182
|
+
provenance: z.ZodEnum<{
|
|
183
|
+
observed: "observed";
|
|
184
|
+
estimated: "estimated";
|
|
185
|
+
}>;
|
|
186
|
+
}, z.core.$strict>;
|
|
187
|
+
}, z.core.$strict>;
|
|
188
|
+
total: z.ZodObject<{
|
|
189
|
+
wallDurationMs: z.ZodNumber;
|
|
190
|
+
cost: z.ZodObject<{
|
|
191
|
+
usd: z.ZodNumber;
|
|
192
|
+
provenance: z.ZodEnum<{
|
|
193
|
+
observed: "observed";
|
|
194
|
+
estimated: "estimated";
|
|
195
|
+
}>;
|
|
196
|
+
}, z.core.$strict>;
|
|
197
|
+
}, z.core.$strict>;
|
|
166
198
|
}, z.core.$strict>;
|
|
167
199
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodCustom<AgentCandidateJsonValue, AgentCandidateJsonValue>>>;
|
|
168
200
|
};
|
|
@@ -184,6 +216,7 @@ export declare function refineMeasuredComparisonSummary<TReceipt>(comparison: Me
|
|
|
184
216
|
score(receipt: TReceipt): number;
|
|
185
217
|
dimension(receipt: TReceipt, name: string): number | undefined;
|
|
186
218
|
cost(receipt: TReceipt): number;
|
|
219
|
+
costProvenance(receipt: TReceipt): AgentCandidateCostProvenance;
|
|
187
220
|
latency(receipt: TReceipt): number;
|
|
188
221
|
}, ctx: z.RefinementCtx): void;
|
|
189
222
|
export declare function refineEstimate(estimate: {
|
|
@@ -95,6 +95,36 @@ export const agentCandidateEvaluationPolicySchema = z
|
|
|
95
95
|
});
|
|
96
96
|
}
|
|
97
97
|
});
|
|
98
|
+
export const agentImprovementCostSchema = z
|
|
99
|
+
.object({
|
|
100
|
+
usd: z.number().finite().nonnegative(),
|
|
101
|
+
provenance: z.enum(["observed", "estimated"]),
|
|
102
|
+
})
|
|
103
|
+
.strict();
|
|
104
|
+
const agentImprovementEvaluationSchema = z
|
|
105
|
+
.object({
|
|
106
|
+
generationsExplored: z.number().int().nonnegative(),
|
|
107
|
+
preparation: z
|
|
108
|
+
.object({
|
|
109
|
+
wallDurationMs: z.number().finite().nonnegative(),
|
|
110
|
+
cost: agentImprovementCostSchema,
|
|
111
|
+
})
|
|
112
|
+
.strict(),
|
|
113
|
+
measurement: z
|
|
114
|
+
.object({
|
|
115
|
+
wallDurationMs: z.number().finite().nonnegative(),
|
|
116
|
+
workDurationMs: z.number().finite().nonnegative(),
|
|
117
|
+
cost: agentImprovementCostSchema,
|
|
118
|
+
})
|
|
119
|
+
.strict(),
|
|
120
|
+
total: z
|
|
121
|
+
.object({
|
|
122
|
+
wallDurationMs: z.number().finite().nonnegative(),
|
|
123
|
+
cost: agentImprovementCostSchema,
|
|
124
|
+
})
|
|
125
|
+
.strict(),
|
|
126
|
+
})
|
|
127
|
+
.strict();
|
|
98
128
|
export const measuredComparisonCommonShape = {
|
|
99
129
|
overall: z
|
|
100
130
|
.object({
|
|
@@ -148,17 +178,7 @@ export const measuredComparisonCommonShape = {
|
|
|
148
178
|
})
|
|
149
179
|
.strict(),
|
|
150
180
|
diff: z.string(),
|
|
151
|
-
evaluation:
|
|
152
|
-
.object({
|
|
153
|
-
generationsExplored: z.number().int().nonnegative(),
|
|
154
|
-
searchDurationMs: z.number().finite().nonnegative(),
|
|
155
|
-
executionDurationMs: z.number().finite().nonnegative(),
|
|
156
|
-
durationMs: z.number().finite().nonnegative(),
|
|
157
|
-
searchCostUsd: z.number().finite().nonnegative(),
|
|
158
|
-
executionCostUsd: z.number().finite().nonnegative(),
|
|
159
|
-
totalCostUsd: z.number().finite().nonnegative(),
|
|
160
|
-
})
|
|
161
|
-
.strict(),
|
|
181
|
+
evaluation: agentImprovementEvaluationSchema,
|
|
162
182
|
metadata: canonicalJsonObjectSchema.optional(),
|
|
163
183
|
};
|
|
164
184
|
/** Keep receipt identity reuse rules identical across measured source formats. */
|
|
@@ -181,12 +201,18 @@ export function createMeasuredComparisonIdentityRegistry(options) {
|
|
|
181
201
|
}
|
|
182
202
|
export function refineMeasuredComparisonSummary(comparison, policy, expectedN, measurements, values, ctx) {
|
|
183
203
|
refineEstimate(comparison.overall, ["overall"], ctx);
|
|
184
|
-
|
|
185
|
-
|
|
204
|
+
const totalCostProvenance = comparison.evaluation.preparation.cost.provenance === "observed" &&
|
|
205
|
+
comparison.evaluation.measurement.cost.provenance === "observed"
|
|
206
|
+
? "observed"
|
|
207
|
+
: "estimated";
|
|
208
|
+
if (!numbersApproximatelyEqual(comparison.evaluation.total.wallDurationMs, comparison.evaluation.preparation.wallDurationMs +
|
|
209
|
+
comparison.evaluation.measurement.wallDurationMs) ||
|
|
210
|
+
!numbersApproximatelyEqual(comparison.evaluation.total.cost.usd, comparison.evaluation.preparation.cost.usd + comparison.evaluation.measurement.cost.usd) ||
|
|
211
|
+
comparison.evaluation.total.cost.provenance !== totalCostProvenance) {
|
|
186
212
|
ctx.addIssue({
|
|
187
213
|
code: "custom",
|
|
188
214
|
path: ["evaluation"],
|
|
189
|
-
message: "evaluation totals must equal
|
|
215
|
+
message: "evaluation totals must equal preparation and measurement accounting",
|
|
190
216
|
});
|
|
191
217
|
}
|
|
192
218
|
if (comparison.overall.n !== expectedN) {
|
|
@@ -265,6 +291,21 @@ export function refineMeasuredComparisonSummary(comparison, policy, expectedN, m
|
|
|
265
291
|
latencyCount += 1;
|
|
266
292
|
}
|
|
267
293
|
}
|
|
294
|
+
const measurementCostUsd = measurements.reduce((sum, measurement) => sum + values.cost(measurement.baseline) + values.cost(measurement.candidate), 0);
|
|
295
|
+
const measurementWorkDurationMs = measurements.reduce((sum, measurement) => sum + values.latency(measurement.baseline) + values.latency(measurement.candidate), 0);
|
|
296
|
+
const measurementCostProvenance = measurements.every((measurement) => values.costProvenance(measurement.baseline) === "observed" &&
|
|
297
|
+
values.costProvenance(measurement.candidate) === "observed")
|
|
298
|
+
? "observed"
|
|
299
|
+
: "estimated";
|
|
300
|
+
if (!numbersApproximatelyEqual(comparison.evaluation.measurement.cost.usd, measurementCostUsd) ||
|
|
301
|
+
comparison.evaluation.measurement.cost.provenance !== measurementCostProvenance ||
|
|
302
|
+
!numbersApproximatelyEqual(comparison.evaluation.measurement.workDurationMs, measurementWorkDurationMs)) {
|
|
303
|
+
ctx.addIssue({
|
|
304
|
+
code: "custom",
|
|
305
|
+
path: ["evaluation", "measurement"],
|
|
306
|
+
message: "measurement accounting must equal the complete signed receipts",
|
|
307
|
+
});
|
|
308
|
+
}
|
|
268
309
|
if (costCount !== 1 || latencyCount !== 1) {
|
|
269
310
|
ctx.addIssue({
|
|
270
311
|
code: "custom",
|