@tangle-network/agent-interface 0.28.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,17 +1,9 @@
1
1
  import { z } from "zod";
2
2
  import { agentCandidateArtifactRefSchema, agentCandidateCapturedArtifactSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js";
3
3
  import { agentCandidateContainerSchema, agentCandidateInstructionDeliverySchema, agentCandidateWorkingDirectorySchema, } from "./agent-candidate-code-schema.js";
4
+ import { reasoningEffortSchema } from "./profile-schema.js";
4
5
  import { addDuplicateIssues, agentCandidateConfigValueSchema, agentCandidateMediaTypeSchema, environmentConfigSchema, gitObjectSchema, isCanonicalJsonValue, isObviouslyPrivateHostname, isSafeExecutable, isSafeRelativePath, isWellFormedUnicode, sameGitObjectFormat, sha256Utf8, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
5
6
  import { harnessTypeSchema } from "./harness.js";
6
- const reasoningEffortSchema = z.enum([
7
- "none",
8
- "minimal",
9
- "low",
10
- "medium",
11
- "high",
12
- "xhigh",
13
- "ultracode",
14
- ]);
15
7
  function isCanonicalAbsolutePath(value) {
16
8
  return (value.startsWith("/") &&
17
9
  value !== "/" &&
@@ -58,8 +50,59 @@ export const agentCandidateResolvedModelSchema = z
58
50
  reasoningEffort: reasoningEffortSchema,
59
51
  })
60
52
  .strict();
53
+ export const agentCandidateBenchmarkGraderIdentitySchema = z
54
+ .object({
55
+ name: z.string().min(1),
56
+ version: z.string().min(1),
57
+ format: z.literal("tangle-grader"),
58
+ artifact: agentCandidateArtifactRefSchema,
59
+ })
60
+ .strict()
61
+ .superRefine((grader, ctx) => {
62
+ if (grader.artifact.byteLength === 0) {
63
+ ctx.addIssue({
64
+ code: "custom",
65
+ path: ["artifact", "byteLength"],
66
+ message: "pinned grader artifact must contain executable grader bytes",
67
+ });
68
+ }
69
+ });
70
+ export const agentCandidateTaskRepositorySchema = z
71
+ .object({
72
+ identity: z.string().min(1).max(512),
73
+ rootIdentity: z.string().min(1).max(512),
74
+ baseCommit: gitObjectSchema,
75
+ baseTree: gitObjectSchema,
76
+ })
77
+ .strict()
78
+ .superRefine((repository, ctx) => {
79
+ if (!sameGitObjectFormat(repository.baseCommit, repository.baseTree)) {
80
+ ctx.addIssue({
81
+ code: "custom",
82
+ message: "task Git object ids must use one object format",
83
+ });
84
+ }
85
+ });
86
+ export const agentCandidateResolvedTaskContainerSchema = agentCandidateContainerSchema
87
+ .extend({
88
+ source: z.literal("evaluator-task-container"),
89
+ manifestDigest: sha256DigestSchema,
90
+ platform: z
91
+ .object({
92
+ os: z.string().min(1).max(100),
93
+ architecture: z.string().min(1).max(100),
94
+ variant: z.string().min(1).max(100).optional(),
95
+ })
96
+ .strict(),
97
+ })
98
+ .strict();
99
+ export const agentCandidateRetryPolicySchema = z.enum([
100
+ "pre-model-infrastructure-only",
101
+ "none",
102
+ ]);
61
103
  export const agentCandidateProfilePlanMaterialSchema = z
62
104
  .object({
105
+ sourceProfileDigest: sha256DigestSchema,
63
106
  harness: harnessTypeSchema,
64
107
  files: z.array(z
65
108
  .object({
@@ -149,45 +192,35 @@ export const agentCandidateTaskOutcomeSpecSchema = z.discriminatedUnion("kind",
149
192
  })
150
193
  .strict(),
151
194
  ]);
195
+ export const agentCandidateBenchmarkCellRefSchema = z
196
+ .object({
197
+ suiteDigest: sha256DigestSchema,
198
+ taskIndex: z.number().int().nonnegative(),
199
+ repetition: z.number().int().nonnegative(),
200
+ })
201
+ .strict();
202
+ export const agentCandidateRunCellMaterialSchema = z
203
+ .object({
204
+ kind: z.literal("agent-candidate-run-cell"),
205
+ experimentDigest: sha256DigestSchema,
206
+ arm: z.enum(["baseline", "candidate"]),
207
+ bundleDigest: sha256DigestSchema,
208
+ suiteDigest: sha256DigestSchema,
209
+ taskDigest: sha256DigestSchema,
210
+ taskIndex: z.number().int().nonnegative(),
211
+ repetition: z.number().int().nonnegative(),
212
+ seed: z.number().int().safe(),
213
+ attempt: z.number().int().positive(),
214
+ })
215
+ .strict();
216
+ export const agentCandidateRunCellSchema = agentCandidateRunCellMaterialSchema
217
+ .extend({ digest: sha256DigestSchema })
218
+ .strict();
152
219
  export const agentCandidateExecutionPlanMaterialSchema = z
153
220
  .object({
154
221
  kind: z.literal("agent-candidate-execution-plan-material"),
155
- bundleDigest: sha256DigestSchema,
222
+ runCell: agentCandidateRunCellSchema,
156
223
  executionId: z.string().min(1),
157
- attempt: z
158
- .object({
159
- number: z.number().int().min(1),
160
- maxAttempts: z.number().int().min(1),
161
- retryPolicy: z.enum(["pre-model-infrastructure-only", "none"]),
162
- })
163
- .strict(),
164
- task: z
165
- .object({
166
- benchmark: z.string().min(1),
167
- benchmarkVersion: z.string().min(1),
168
- taskId: z.string().min(1),
169
- splitDigest: sha256DigestSchema,
170
- instruction: z
171
- .object({
172
- encoding: z.literal("utf8"),
173
- sha256: sha256DigestSchema,
174
- byteLength: z.number().int().positive(),
175
- delivery: agentCandidateInstructionDeliverySchema,
176
- })
177
- .strict(),
178
- repository: z
179
- .object({
180
- identity: z.string().min(1),
181
- rootIdentity: z.string().min(1),
182
- baseCommit: gitObjectSchema,
183
- baseTree: gitObjectSchema,
184
- })
185
- .strict()
186
- .optional(),
187
- outcome: agentCandidateTaskOutcomeSpecSchema,
188
- workspace: agentCandidateWorkspaceSnapshotEvidenceSchema,
189
- })
190
- .strict(),
191
224
  workspaces: z
192
225
  .object({
193
226
  taskRoot: z
@@ -212,6 +245,8 @@ export const agentCandidateExecutionPlanMaterialSchema = z
212
245
  .strict(),
213
246
  harness: harnessTypeSchema,
214
247
  harnessVersion: z.string().min(1),
248
+ instructionDelivery: agentCandidateInstructionDeliverySchema,
249
+ limits: agentCandidateExecutionLimitsSchema,
215
250
  container: agentCandidateContainerSchema
216
251
  .extend({
217
252
  source: z.enum(["pinned-container", "evaluator-task-container"]),
@@ -268,13 +303,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z
268
303
  .min(1),
269
304
  })
270
305
  .strict(),
271
- grader: z
272
- .object({
273
- name: z.string().min(1),
274
- version: z.string().min(1),
275
- artifact: agentCandidateArtifactRefSchema,
276
- })
277
- .strict(),
278
306
  launch: z
279
307
  .object({
280
308
  executable: z
@@ -287,42 +315,10 @@ export const agentCandidateExecutionPlanMaterialSchema = z
287
315
  .strict(),
288
316
  knowledgeManifestDigest: sha256DigestSchema.optional(),
289
317
  memory: agentCandidateEffectiveMemorySchema,
290
- limits: agentCandidateExecutionLimitsSchema,
291
318
  network: z.object({ mode: z.literal("disabled") }).strict(),
292
319
  })
293
320
  .strict()
294
321
  .superRefine((material, ctx) => {
295
- if (material.task.repository !== undefined &&
296
- !sameGitObjectFormat(material.task.repository.baseCommit, material.task.repository.baseTree)) {
297
- ctx.addIssue({
298
- code: "custom",
299
- path: ["task", "repository"],
300
- message: "task Git object ids must use one object format",
301
- });
302
- }
303
- if (material.task.outcome.kind === "workspace" &&
304
- material.task.repository === undefined) {
305
- ctx.addIssue({
306
- code: "custom",
307
- path: ["task", "repository"],
308
- message: "workspace outcomes require task repository provenance",
309
- });
310
- }
311
- if (material.attempt.number > material.attempt.maxAttempts) {
312
- ctx.addIssue({
313
- code: "custom",
314
- path: ["attempt", "number"],
315
- message: "attempt number cannot exceed the frozen maximum",
316
- });
317
- }
318
- if (material.attempt.retryPolicy === "none" &&
319
- material.attempt.maxAttempts !== 1) {
320
- ctx.addIssue({
321
- code: "custom",
322
- path: ["attempt", "maxAttempts"],
323
- message: "a no-retry plan must allow exactly one attempt",
324
- });
325
- }
326
322
  const routeIds = material.model.routes.map((route) => route.kind === "mode" || route.kind === "subagent"
327
323
  ? `${route.kind}:${route.name}`
328
324
  : route.kind);
@@ -356,20 +352,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z
356
352
  }
357
353
  }
358
354
  const modelNetwork = material.model.access.network;
359
- if (material.limits.maxModelCalls === 0 && modelNetwork.mode !== "disabled") {
360
- ctx.addIssue({
361
- code: "custom",
362
- path: ["model", "access", "network"],
363
- message: "zero-call plans cannot expose a model gateway",
364
- });
365
- }
366
- if (material.limits.maxModelCalls > 0 && modelNetwork.mode !== "gateway-only") {
367
- ctx.addIssue({
368
- code: "custom",
369
- path: ["model", "access", "network"],
370
- message: "model-calling plans require one frozen gateway allowlist",
371
- });
372
- }
373
355
  if (modelNetwork.mode === "gateway-only") {
374
356
  addDuplicateIssues(modelNetwork.domains, ["model", "access", "network", "domains"], ctx);
375
357
  for (let index = 1; index < modelNetwork.domains.length; index++) {
@@ -405,62 +387,46 @@ export const agentCandidateExecutionPlanMaterialSchema = z
405
387
  message: "task and candidate workspace roots must be disjoint",
406
388
  });
407
389
  }
408
- if (material.launch.cwd.workspace === "candidate" &&
409
- material.workspaces.candidateRoot === undefined) {
410
- ctx.addIssue({
411
- code: "custom",
412
- path: ["launch", "cwd", "workspace"],
413
- message: "candidate cwd requires a pinned candidate workspace root",
414
- });
415
- }
416
- if (material.profile.targetWorkspace === "candidate" &&
417
- material.workspaces.candidateRoot === undefined) {
418
- ctx.addIssue({
419
- code: "custom",
420
- path: ["profile", "targetWorkspace"],
421
- message: "candidate-targeted profile files require a candidate workspace root",
422
- });
423
- }
424
- const delivery = material.task.instruction.delivery;
425
390
  const taskPath = material.launch.env.TANGLE_CANDIDATE_TASK_PATH;
426
- if (delivery.kind === "utf8-file") {
427
- if (taskPath?.value !== delivery.path) {
391
+ if (taskPath !== undefined) {
392
+ if (taskPath.value !== "/tangle/input/task.txt") {
428
393
  ctx.addIssue({
429
394
  code: "custom",
430
395
  path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"],
431
- message: "file delivery requires the signed fixed task path",
396
+ message: "task file delivery must use the fixed evaluator-owned path",
432
397
  });
433
398
  }
434
- if (pathsOverlap(material.workspaces.taskRoot, delivery.path) ||
399
+ if (pathsOverlap(taskPath.value, material.workspaces.taskRoot) ||
435
400
  (material.workspaces.candidateRoot !== undefined &&
436
- pathsOverlap(material.workspaces.candidateRoot, delivery.path))) {
401
+ pathsOverlap(taskPath.value, material.workspaces.candidateRoot))) {
437
402
  ctx.addIssue({
438
403
  code: "custom",
439
- path: ["workspaces"],
440
- message: "the task instruction file must be outside both workspaces",
404
+ path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"],
405
+ message: "task file delivery must be outside both workspaces",
441
406
  });
442
407
  }
443
408
  }
444
- else if (taskPath !== undefined) {
409
+ if (material.launch.cwd.workspace === "candidate" &&
410
+ material.workspaces.candidateRoot === undefined) {
445
411
  ctx.addIssue({
446
412
  code: "custom",
447
- path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"],
448
- message: "non-file delivery cannot expose an instruction path",
413
+ path: ["launch", "cwd", "workspace"],
414
+ message: "candidate cwd requires a pinned candidate workspace root",
449
415
  });
450
416
  }
451
- if (activeCode && material.candidateWorkspace?.material.files.length === 0) {
417
+ if (material.profile.targetWorkspace === "candidate" &&
418
+ material.workspaces.candidateRoot === undefined) {
452
419
  ctx.addIssue({
453
420
  code: "custom",
454
- path: ["candidateWorkspace", "material", "files"],
455
- message: "active candidate workspaces cannot be empty",
421
+ path: ["profile", "targetWorkspace"],
422
+ message: "candidate-targeted profile files require a candidate workspace root",
456
423
  });
457
424
  }
458
- if (material.task.outcome.kind === "workspace" &&
459
- material.task.workspace.material.files.length === 0) {
425
+ if (activeCode && material.candidateWorkspace?.material.files.length === 0) {
460
426
  ctx.addIssue({
461
427
  code: "custom",
462
- path: ["task", "workspace", "material", "files"],
463
- message: "task workspace snapshots cannot be empty",
428
+ path: ["candidateWorkspace", "material", "files"],
429
+ message: "active candidate workspaces cannot be empty",
464
430
  });
465
431
  }
466
432
  if (!isCanonicalJsonValue(material)) {
@@ -120,13 +120,6 @@ export declare const agentCandidateMemoryPolicySchema: z.ZodDiscriminatedUnion<[
120
120
  byteLength: z.ZodNumber;
121
121
  }, z.core.$strict>>;
122
122
  }, z.core.$strict>], "mode">;
123
- export declare const agentCandidateSpendSchema: z.ZodObject<{
124
- costUsd: z.ZodNumber;
125
- inputTokens: z.ZodNumber;
126
- outputTokens: z.ZodNumber;
127
- cachedInputTokens: z.ZodOptional<z.ZodNumber>;
128
- modelCalls: z.ZodNumber;
129
- }, z.core.$strict>;
130
123
  export declare const agentCandidateLineageSchema: z.ZodObject<{
131
124
  source: z.ZodEnum<{
132
125
  optimizer: "optimizer";
@@ -138,25 +131,5 @@ export declare const agentCandidateLineageSchema: z.ZodObject<{
138
131
  runIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
139
132
  profileDiffIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
140
133
  modelSnapshots: z.ZodOptional<z.ZodArray<z.ZodString>>;
141
- benchmark: z.ZodOptional<z.ZodObject<{
142
- name: z.ZodString;
143
- version: z.ZodString;
144
- splitDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
145
- }, z.core.$strict>>;
146
- spend: z.ZodOptional<z.ZodObject<{
147
- proposal: z.ZodObject<{
148
- costUsd: z.ZodNumber;
149
- inputTokens: z.ZodNumber;
150
- outputTokens: z.ZodNumber;
151
- cachedInputTokens: z.ZodOptional<z.ZodNumber>;
152
- modelCalls: z.ZodNumber;
153
- }, z.core.$strict>;
154
- evaluation: z.ZodObject<{
155
- costUsd: z.ZodNumber;
156
- inputTokens: z.ZodNumber;
157
- outputTokens: z.ZodNumber;
158
- cachedInputTokens: z.ZodOptional<z.ZodNumber>;
159
- modelCalls: z.ZodNumber;
160
- }, z.core.$strict>;
161
- }, z.core.$strict>>;
134
+ developmentSplitDigest: z.ZodOptional<z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>>;
162
135
  }, z.core.$strict>;
@@ -30,15 +30,6 @@ export const agentCandidateMemoryPolicySchema = z.discriminatedUnion("mode", [
30
30
  })
31
31
  .strict(),
32
32
  ]);
33
- export const agentCandidateSpendSchema = z
34
- .object({
35
- costUsd: z.number().finite().nonnegative(),
36
- inputTokens: z.number().int().nonnegative(),
37
- outputTokens: z.number().int().nonnegative(),
38
- cachedInputTokens: z.number().int().nonnegative().optional(),
39
- modelCalls: z.number().int().nonnegative(),
40
- })
41
- .strict();
42
33
  export const agentCandidateLineageSchema = z
43
34
  .object({
44
35
  source: z.enum(["optimizer", "human", "import", "compound"]),
@@ -46,21 +37,7 @@ export const agentCandidateLineageSchema = z
46
37
  runIds: z.array(z.string().min(1)).optional(),
47
38
  profileDiffIds: z.array(z.string().min(1)).optional(),
48
39
  modelSnapshots: z.array(z.string().min(1)).optional(),
49
- benchmark: z
50
- .object({
51
- name: z.string().min(1),
52
- version: z.string().min(1),
53
- splitDigest: sha256DigestSchema,
54
- })
55
- .strict()
56
- .optional(),
57
- spend: z
58
- .object({
59
- proposal: agentCandidateSpendSchema,
60
- evaluation: agentCandidateSpendSchema,
61
- })
62
- .strict()
63
- .optional(),
40
+ developmentSplitDigest: sha256DigestSchema.optional(),
64
41
  })
65
42
  .strict()
66
43
  .superRefine((lineage, ctx) => {
@@ -91,19 +68,12 @@ export const agentCandidateLineageSchema = z
91
68
  message: "generated candidates must name their producing run",
92
69
  });
93
70
  }
94
- if (lineage.benchmark === undefined) {
71
+ if (lineage.developmentSplitDigest === undefined) {
95
72
  ctx.addIssue({
96
73
  code: "custom",
97
- path: ["benchmark"],
74
+ path: ["developmentSplitDigest"],
98
75
  message: "generated candidates must pin their development split",
99
76
  });
100
77
  }
101
- if (lineage.spend === undefined) {
102
- ctx.addIssue({
103
- code: "custom",
104
- path: ["spend"],
105
- message: "generated candidates must record proposal and evaluation spend",
106
- });
107
- }
108
78
  }
109
79
  });
@@ -522,15 +522,10 @@ export declare const agentCandidateBenchmarkResultMaterialSchema: z.ZodObject<{
522
522
  kind: z.ZodLiteral<"agent-candidate-benchmark-result-material">;
523
523
  executionPlanDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
524
524
  taskOutcomeDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
525
- benchmark: z.ZodObject<{
526
- name: z.ZodString;
527
- version: z.ZodString;
528
- taskId: z.ZodString;
529
- splitDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
530
- }, z.core.$strict>;
531
525
  grader: z.ZodObject<{
532
526
  name: z.ZodString;
533
527
  version: z.ZodString;
528
+ format: z.ZodLiteral<"tangle-grader">;
534
529
  artifact: z.ZodObject<{
535
530
  locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
536
531
  kind: z.ZodLiteral<"s3">;
@@ -560,6 +555,21 @@ export declare const agentCandidateBenchmarkResultMaterialSchema: z.ZodObject<{
560
555
  sha256: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
561
556
  byteLength: z.ZodNumber;
562
557
  }, z.core.$strict>;
558
+ grading: z.ZodObject<{
559
+ usage: z.ZodObject<{
560
+ inputTokens: z.ZodNumber;
561
+ outputTokens: z.ZodNumber;
562
+ cachedInputTokens: z.ZodNumber;
563
+ reasoningTokens: z.ZodNumber;
564
+ modelCalls: z.ZodNumber;
565
+ costUsdNanos: z.ZodNumber;
566
+ }, z.core.$strict>;
567
+ timing: z.ZodObject<{
568
+ startedAtMs: z.ZodNumber;
569
+ endedAtMs: z.ZodNumber;
570
+ durationMs: z.ZodNumber;
571
+ }, z.core.$strict>;
572
+ }, z.core.$strict>;
563
573
  score: z.ZodNumber;
564
574
  passed: z.ZodBoolean;
565
575
  dimensions: z.ZodArray<z.ZodObject<{
@@ -574,15 +584,10 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
574
584
  kind: "agent-candidate-benchmark-result-material";
575
585
  executionPlanDigest: `sha256:${string}`;
576
586
  taskOutcomeDigest: `sha256:${string}`;
577
- benchmark: {
578
- name: string;
579
- version: string;
580
- taskId: string;
581
- splitDigest: `sha256:${string}`;
582
- };
583
587
  grader: {
584
588
  name: string;
585
589
  version: string;
590
+ format: "tangle-grader";
586
591
  artifact: {
587
592
  locator: {
588
593
  kind: "s3";
@@ -612,6 +617,21 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
612
617
  sha256: `sha256:${string}`;
613
618
  byteLength: number;
614
619
  };
620
+ grading: {
621
+ usage: {
622
+ inputTokens: number;
623
+ outputTokens: number;
624
+ cachedInputTokens: number;
625
+ reasoningTokens: number;
626
+ modelCalls: number;
627
+ costUsdNanos: number;
628
+ };
629
+ timing: {
630
+ startedAtMs: number;
631
+ endedAtMs: number;
632
+ durationMs: number;
633
+ };
634
+ };
615
635
  score: number;
616
636
  passed: boolean;
617
637
  dimensions: {
@@ -622,15 +642,10 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
622
642
  kind: "agent-candidate-benchmark-result-material";
623
643
  executionPlanDigest: `sha256:${string}`;
624
644
  taskOutcomeDigest: `sha256:${string}`;
625
- benchmark: {
626
- name: string;
627
- version: string;
628
- taskId: string;
629
- splitDigest: `sha256:${string}`;
630
- };
631
645
  grader: {
632
646
  name: string;
633
647
  version: string;
648
+ format: "tangle-grader";
634
649
  artifact: {
635
650
  locator: {
636
651
  kind: "s3";
@@ -660,6 +675,21 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
660
675
  sha256: `sha256:${string}`;
661
676
  byteLength: number;
662
677
  };
678
+ grading: {
679
+ usage: {
680
+ inputTokens: number;
681
+ outputTokens: number;
682
+ cachedInputTokens: number;
683
+ reasoningTokens: number;
684
+ modelCalls: number;
685
+ costUsdNanos: number;
686
+ };
687
+ timing: {
688
+ startedAtMs: number;
689
+ endedAtMs: number;
690
+ durationMs: number;
691
+ };
692
+ };
663
693
  score: number;
664
694
  passed: boolean;
665
695
  dimensions: {
@@ -1,6 +1,6 @@
1
1
  import { z } from "zod";
2
2
  import { agentCandidateArtifactRefSchema, agentCandidateCapturedArtifactSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js";
3
- import { agentCandidateResolvedModelSchema } from "./agent-candidate-execution-plan-schema.js";
3
+ import { agentCandidateBenchmarkGraderIdentitySchema, agentCandidateResolvedModelSchema, } from "./agent-candidate-execution-plan-schema.js";
4
4
  import { agentCandidateMediaTypeSchema, gitObjectSchema, isCanonicalJsonValue, sameGitObjectFormat, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
5
5
  const safeCountSchema = z
6
6
  .number()
@@ -265,35 +265,26 @@ export const agentCandidateBenchmarkResultMaterialSchema = z
265
265
  kind: z.literal("agent-candidate-benchmark-result-material"),
266
266
  executionPlanDigest: sha256DigestSchema,
267
267
  taskOutcomeDigest: sha256DigestSchema,
268
- benchmark: z
269
- .object({
270
- name: z.string().min(1),
271
- version: z.string().min(1),
272
- taskId: z.string().min(1),
273
- splitDigest: sha256DigestSchema,
274
- })
275
- .strict(),
276
- grader: z
268
+ grader: agentCandidateBenchmarkGraderIdentitySchema,
269
+ evidence: agentCandidateArtifactRefSchema,
270
+ grading: z
277
271
  .object({
278
- name: z.string().min(1),
279
- version: z.string().min(1),
280
- artifact: agentCandidateArtifactRefSchema,
272
+ usage: agentCandidateFixedSpendSchema,
273
+ timing: z
274
+ .object({
275
+ startedAtMs: safeCountSchema,
276
+ endedAtMs: safeCountSchema,
277
+ durationMs: safeCountSchema,
278
+ })
279
+ .strict(),
281
280
  })
282
281
  .strict(),
283
- evidence: agentCandidateArtifactRefSchema,
284
282
  score: normalizedScoreSchema,
285
283
  passed: z.boolean(),
286
284
  dimensions: z.array(agentCandidateBenchmarkDimensionSchema),
287
285
  })
288
286
  .strict()
289
287
  .superRefine((material, ctx) => {
290
- if (material.grader.artifact.byteLength === 0) {
291
- ctx.addIssue({
292
- code: "custom",
293
- path: ["grader", "artifact", "byteLength"],
294
- message: "pinned grader artifact must contain executable grader bytes",
295
- });
296
- }
297
288
  if (material.evidence.byteLength === 0) {
298
289
  ctx.addIssue({
299
290
  code: "custom",
@@ -301,11 +292,15 @@ export const agentCandidateBenchmarkResultMaterialSchema = z
301
292
  message: "benchmark result must contain non-empty durable grading evidence",
302
293
  });
303
294
  }
304
- if (material.evidence.sha256 === material.grader.artifact.sha256) {
295
+ if (material.grading.timing.endedAtMs <
296
+ material.grading.timing.startedAtMs ||
297
+ material.grading.timing.durationMs !==
298
+ material.grading.timing.endedAtMs -
299
+ material.grading.timing.startedAtMs) {
305
300
  ctx.addIssue({
306
301
  code: "custom",
307
- path: ["evidence", "sha256"],
308
- message: "grading evidence must be distinct from the grader implementation",
302
+ path: ["grading", "timing"],
303
+ message: "grader timing must be ordered and internally consistent",
309
304
  });
310
305
  }
311
306
  for (let index = 1; index < material.dimensions.length; index++) {