@tangle-network/agent-interface 0.28.0 → 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/dist/agent-candidate-code-schema.d.ts +0 -8
  2. package/dist/agent-candidate-code-schema.js +0 -1
  3. package/dist/agent-candidate-execution-plan-schema.d.ts +193 -350
  4. package/dist/agent-candidate-execution-plan-schema.js +98 -131
  5. package/dist/agent-candidate-lineage-schema.d.ts +1 -28
  6. package/dist/agent-candidate-lineage-schema.js +3 -33
  7. package/dist/agent-candidate-outcome-schema.d.ts +48 -18
  8. package/dist/agent-candidate-outcome-schema.js +19 -24
  9. package/dist/agent-candidate-profile-schema.d.ts +3 -28
  10. package/dist/agent-candidate-profile-schema.js +15 -27
  11. package/dist/agent-candidate-promotion-schema.d.ts +10981 -2356
  12. package/dist/agent-candidate-promotion-schema.js +495 -105
  13. package/dist/agent-candidate-receipt-schema.d.ts +268 -303
  14. package/dist/agent-candidate-receipt-schema.js +82 -8
  15. package/dist/agent-candidate-schema-common.d.ts +8 -0
  16. package/dist/agent-candidate-schema-common.js +35 -1
  17. package/dist/agent-candidate-schema.d.ts +3 -52
  18. package/dist/agent-candidate-schema.js +3 -28
  19. package/dist/agent-candidate-task-schema.d.ts +643 -0
  20. package/dist/agent-candidate-task-schema.js +191 -0
  21. package/dist/agent-candidate.d.ts +191 -75
  22. package/dist/agent-candidate.test-fixture.d.ts +0 -26
  23. package/dist/agent-candidate.test-fixture.js +0 -26
  24. package/dist/agent-profile.d.ts +33 -9
  25. package/dist/harness-capabilities.d.ts +2 -2
  26. package/dist/harness-capabilities.js +23 -14
  27. package/dist/profile-schema.d.ts +65 -84
  28. package/dist/profile-schema.js +82 -32
  29. package/dist/profile-security.js +2 -0
  30. package/package.json +2 -2
@@ -1,17 +1,9 @@
1
1
  import { z } from "zod";
2
2
  import { agentCandidateArtifactRefSchema, agentCandidateCapturedArtifactSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js";
3
3
  import { agentCandidateContainerSchema, agentCandidateInstructionDeliverySchema, agentCandidateWorkingDirectorySchema, } from "./agent-candidate-code-schema.js";
4
+ import { reasoningEffortSchema } from "./profile-schema.js";
4
5
  import { addDuplicateIssues, agentCandidateConfigValueSchema, agentCandidateMediaTypeSchema, environmentConfigSchema, gitObjectSchema, isCanonicalJsonValue, isObviouslyPrivateHostname, isSafeExecutable, isSafeRelativePath, isWellFormedUnicode, sameGitObjectFormat, sha256Utf8, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
5
6
  import { harnessTypeSchema } from "./harness.js";
6
- const reasoningEffortSchema = z.enum([
7
- "none",
8
- "minimal",
9
- "low",
10
- "medium",
11
- "high",
12
- "xhigh",
13
- "ultracode",
14
- ]);
15
7
  function isCanonicalAbsolutePath(value) {
16
8
  return (value.startsWith("/") &&
17
9
  value !== "/" &&
@@ -58,8 +50,59 @@ export const agentCandidateResolvedModelSchema = z
58
50
  reasoningEffort: reasoningEffortSchema,
59
51
  })
60
52
  .strict();
53
+ export const agentCandidateBenchmarkGraderIdentitySchema = z
54
+ .object({
55
+ name: z.string().min(1),
56
+ version: z.string().min(1),
57
+ format: z.literal("tangle-grader"),
58
+ artifact: agentCandidateArtifactRefSchema,
59
+ })
60
+ .strict()
61
+ .superRefine((grader, ctx) => {
62
+ if (grader.artifact.byteLength === 0) {
63
+ ctx.addIssue({
64
+ code: "custom",
65
+ path: ["artifact", "byteLength"],
66
+ message: "pinned grader artifact must contain executable grader bytes",
67
+ });
68
+ }
69
+ });
70
+ export const agentCandidateTaskRepositorySchema = z
71
+ .object({
72
+ identity: z.string().min(1).max(512),
73
+ rootIdentity: z.string().min(1).max(512),
74
+ baseCommit: gitObjectSchema,
75
+ baseTree: gitObjectSchema,
76
+ })
77
+ .strict()
78
+ .superRefine((repository, ctx) => {
79
+ if (!sameGitObjectFormat(repository.baseCommit, repository.baseTree)) {
80
+ ctx.addIssue({
81
+ code: "custom",
82
+ message: "task Git object ids must use one object format",
83
+ });
84
+ }
85
+ });
86
+ export const agentCandidateResolvedTaskContainerSchema = agentCandidateContainerSchema
87
+ .extend({
88
+ source: z.literal("evaluator-task-container"),
89
+ manifestDigest: sha256DigestSchema,
90
+ platform: z
91
+ .object({
92
+ os: z.string().min(1).max(100),
93
+ architecture: z.string().min(1).max(100),
94
+ variant: z.string().min(1).max(100).optional(),
95
+ })
96
+ .strict(),
97
+ })
98
+ .strict();
99
+ export const agentCandidateRetryPolicySchema = z.enum([
100
+ "pre-model-infrastructure-only",
101
+ "none",
102
+ ]);
61
103
  export const agentCandidateProfilePlanMaterialSchema = z
62
104
  .object({
105
+ sourceProfileDigest: sha256DigestSchema,
63
106
  harness: harnessTypeSchema,
64
107
  files: z.array(z
65
108
  .object({
@@ -72,6 +115,7 @@ export const agentCandidateProfilePlanMaterialSchema = z
72
115
  .strict()),
73
116
  env: environmentConfigSchema,
74
117
  flags: z.array(agentCandidateConfigValueSchema),
118
+ systemPrompt: agentCandidateConfigValueSchema.optional(),
75
119
  unsupported: z
76
120
  .array(z
77
121
  .object({
@@ -149,45 +193,35 @@ export const agentCandidateTaskOutcomeSpecSchema = z.discriminatedUnion("kind",
149
193
  })
150
194
  .strict(),
151
195
  ]);
196
+ export const agentCandidateBenchmarkCellRefSchema = z
197
+ .object({
198
+ suiteDigest: sha256DigestSchema,
199
+ taskIndex: z.number().int().nonnegative(),
200
+ repetition: z.number().int().nonnegative(),
201
+ })
202
+ .strict();
203
+ export const agentCandidateRunCellMaterialSchema = z
204
+ .object({
205
+ kind: z.literal("agent-candidate-run-cell"),
206
+ experimentDigest: sha256DigestSchema,
207
+ arm: z.enum(["baseline", "candidate"]),
208
+ bundleDigest: sha256DigestSchema,
209
+ suiteDigest: sha256DigestSchema,
210
+ taskDigest: sha256DigestSchema,
211
+ taskIndex: z.number().int().nonnegative(),
212
+ repetition: z.number().int().nonnegative(),
213
+ seed: z.number().int().safe(),
214
+ attempt: z.number().int().positive(),
215
+ })
216
+ .strict();
217
+ export const agentCandidateRunCellSchema = agentCandidateRunCellMaterialSchema
218
+ .extend({ digest: sha256DigestSchema })
219
+ .strict();
152
220
  export const agentCandidateExecutionPlanMaterialSchema = z
153
221
  .object({
154
222
  kind: z.literal("agent-candidate-execution-plan-material"),
155
- bundleDigest: sha256DigestSchema,
223
+ runCell: agentCandidateRunCellSchema,
156
224
  executionId: z.string().min(1),
157
- attempt: z
158
- .object({
159
- number: z.number().int().min(1),
160
- maxAttempts: z.number().int().min(1),
161
- retryPolicy: z.enum(["pre-model-infrastructure-only", "none"]),
162
- })
163
- .strict(),
164
- task: z
165
- .object({
166
- benchmark: z.string().min(1),
167
- benchmarkVersion: z.string().min(1),
168
- taskId: z.string().min(1),
169
- splitDigest: sha256DigestSchema,
170
- instruction: z
171
- .object({
172
- encoding: z.literal("utf8"),
173
- sha256: sha256DigestSchema,
174
- byteLength: z.number().int().positive(),
175
- delivery: agentCandidateInstructionDeliverySchema,
176
- })
177
- .strict(),
178
- repository: z
179
- .object({
180
- identity: z.string().min(1),
181
- rootIdentity: z.string().min(1),
182
- baseCommit: gitObjectSchema,
183
- baseTree: gitObjectSchema,
184
- })
185
- .strict()
186
- .optional(),
187
- outcome: agentCandidateTaskOutcomeSpecSchema,
188
- workspace: agentCandidateWorkspaceSnapshotEvidenceSchema,
189
- })
190
- .strict(),
191
225
  workspaces: z
192
226
  .object({
193
227
  taskRoot: z
@@ -212,6 +246,8 @@ export const agentCandidateExecutionPlanMaterialSchema = z
212
246
  .strict(),
213
247
  harness: harnessTypeSchema,
214
248
  harnessVersion: z.string().min(1),
249
+ instructionDelivery: agentCandidateInstructionDeliverySchema,
250
+ limits: agentCandidateExecutionLimitsSchema,
215
251
  container: agentCandidateContainerSchema
216
252
  .extend({
217
253
  source: z.enum(["pinned-container", "evaluator-task-container"]),
@@ -268,13 +304,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z
268
304
  .min(1),
269
305
  })
270
306
  .strict(),
271
- grader: z
272
- .object({
273
- name: z.string().min(1),
274
- version: z.string().min(1),
275
- artifact: agentCandidateArtifactRefSchema,
276
- })
277
- .strict(),
278
307
  launch: z
279
308
  .object({
280
309
  executable: z
@@ -287,42 +316,10 @@ export const agentCandidateExecutionPlanMaterialSchema = z
287
316
  .strict(),
288
317
  knowledgeManifestDigest: sha256DigestSchema.optional(),
289
318
  memory: agentCandidateEffectiveMemorySchema,
290
- limits: agentCandidateExecutionLimitsSchema,
291
319
  network: z.object({ mode: z.literal("disabled") }).strict(),
292
320
  })
293
321
  .strict()
294
322
  .superRefine((material, ctx) => {
295
- if (material.task.repository !== undefined &&
296
- !sameGitObjectFormat(material.task.repository.baseCommit, material.task.repository.baseTree)) {
297
- ctx.addIssue({
298
- code: "custom",
299
- path: ["task", "repository"],
300
- message: "task Git object ids must use one object format",
301
- });
302
- }
303
- if (material.task.outcome.kind === "workspace" &&
304
- material.task.repository === undefined) {
305
- ctx.addIssue({
306
- code: "custom",
307
- path: ["task", "repository"],
308
- message: "workspace outcomes require task repository provenance",
309
- });
310
- }
311
- if (material.attempt.number > material.attempt.maxAttempts) {
312
- ctx.addIssue({
313
- code: "custom",
314
- path: ["attempt", "number"],
315
- message: "attempt number cannot exceed the frozen maximum",
316
- });
317
- }
318
- if (material.attempt.retryPolicy === "none" &&
319
- material.attempt.maxAttempts !== 1) {
320
- ctx.addIssue({
321
- code: "custom",
322
- path: ["attempt", "maxAttempts"],
323
- message: "a no-retry plan must allow exactly one attempt",
324
- });
325
- }
326
323
  const routeIds = material.model.routes.map((route) => route.kind === "mode" || route.kind === "subagent"
327
324
  ? `${route.kind}:${route.name}`
328
325
  : route.kind);
@@ -356,20 +353,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z
356
353
  }
357
354
  }
358
355
  const modelNetwork = material.model.access.network;
359
- if (material.limits.maxModelCalls === 0 && modelNetwork.mode !== "disabled") {
360
- ctx.addIssue({
361
- code: "custom",
362
- path: ["model", "access", "network"],
363
- message: "zero-call plans cannot expose a model gateway",
364
- });
365
- }
366
- if (material.limits.maxModelCalls > 0 && modelNetwork.mode !== "gateway-only") {
367
- ctx.addIssue({
368
- code: "custom",
369
- path: ["model", "access", "network"],
370
- message: "model-calling plans require one frozen gateway allowlist",
371
- });
372
- }
373
356
  if (modelNetwork.mode === "gateway-only") {
374
357
  addDuplicateIssues(modelNetwork.domains, ["model", "access", "network", "domains"], ctx);
375
358
  for (let index = 1; index < modelNetwork.domains.length; index++) {
@@ -405,62 +388,46 @@ export const agentCandidateExecutionPlanMaterialSchema = z
405
388
  message: "task and candidate workspace roots must be disjoint",
406
389
  });
407
390
  }
408
- if (material.launch.cwd.workspace === "candidate" &&
409
- material.workspaces.candidateRoot === undefined) {
410
- ctx.addIssue({
411
- code: "custom",
412
- path: ["launch", "cwd", "workspace"],
413
- message: "candidate cwd requires a pinned candidate workspace root",
414
- });
415
- }
416
- if (material.profile.targetWorkspace === "candidate" &&
417
- material.workspaces.candidateRoot === undefined) {
418
- ctx.addIssue({
419
- code: "custom",
420
- path: ["profile", "targetWorkspace"],
421
- message: "candidate-targeted profile files require a candidate workspace root",
422
- });
423
- }
424
- const delivery = material.task.instruction.delivery;
425
391
  const taskPath = material.launch.env.TANGLE_CANDIDATE_TASK_PATH;
426
- if (delivery.kind === "utf8-file") {
427
- if (taskPath?.value !== delivery.path) {
392
+ if (taskPath !== undefined) {
393
+ if (taskPath.value !== "/tangle/input/task.txt") {
428
394
  ctx.addIssue({
429
395
  code: "custom",
430
396
  path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"],
431
- message: "file delivery requires the signed fixed task path",
397
+ message: "task file delivery must use the fixed evaluator-owned path",
432
398
  });
433
399
  }
434
- if (pathsOverlap(material.workspaces.taskRoot, delivery.path) ||
400
+ if (pathsOverlap(taskPath.value, material.workspaces.taskRoot) ||
435
401
  (material.workspaces.candidateRoot !== undefined &&
436
- pathsOverlap(material.workspaces.candidateRoot, delivery.path))) {
402
+ pathsOverlap(taskPath.value, material.workspaces.candidateRoot))) {
437
403
  ctx.addIssue({
438
404
  code: "custom",
439
- path: ["workspaces"],
440
- message: "the task instruction file must be outside both workspaces",
405
+ path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"],
406
+ message: "task file delivery must be outside both workspaces",
441
407
  });
442
408
  }
443
409
  }
444
- else if (taskPath !== undefined) {
410
+ if (material.launch.cwd.workspace === "candidate" &&
411
+ material.workspaces.candidateRoot === undefined) {
445
412
  ctx.addIssue({
446
413
  code: "custom",
447
- path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"],
448
- message: "non-file delivery cannot expose an instruction path",
414
+ path: ["launch", "cwd", "workspace"],
415
+ message: "candidate cwd requires a pinned candidate workspace root",
449
416
  });
450
417
  }
451
- if (activeCode && material.candidateWorkspace?.material.files.length === 0) {
418
+ if (material.profile.targetWorkspace === "candidate" &&
419
+ material.workspaces.candidateRoot === undefined) {
452
420
  ctx.addIssue({
453
421
  code: "custom",
454
- path: ["candidateWorkspace", "material", "files"],
455
- message: "active candidate workspaces cannot be empty",
422
+ path: ["profile", "targetWorkspace"],
423
+ message: "candidate-targeted profile files require a candidate workspace root",
456
424
  });
457
425
  }
458
- if (material.task.outcome.kind === "workspace" &&
459
- material.task.workspace.material.files.length === 0) {
426
+ if (activeCode && material.candidateWorkspace?.material.files.length === 0) {
460
427
  ctx.addIssue({
461
428
  code: "custom",
462
- path: ["task", "workspace", "material", "files"],
463
- message: "task workspace snapshots cannot be empty",
429
+ path: ["candidateWorkspace", "material", "files"],
430
+ message: "active candidate workspaces cannot be empty",
464
431
  });
465
432
  }
466
433
  if (!isCanonicalJsonValue(material)) {
@@ -120,13 +120,6 @@ export declare const agentCandidateMemoryPolicySchema: z.ZodDiscriminatedUnion<[
120
120
  byteLength: z.ZodNumber;
121
121
  }, z.core.$strict>>;
122
122
  }, z.core.$strict>], "mode">;
123
- export declare const agentCandidateSpendSchema: z.ZodObject<{
124
- costUsd: z.ZodNumber;
125
- inputTokens: z.ZodNumber;
126
- outputTokens: z.ZodNumber;
127
- cachedInputTokens: z.ZodOptional<z.ZodNumber>;
128
- modelCalls: z.ZodNumber;
129
- }, z.core.$strict>;
130
123
  export declare const agentCandidateLineageSchema: z.ZodObject<{
131
124
  source: z.ZodEnum<{
132
125
  optimizer: "optimizer";
@@ -138,25 +131,5 @@ export declare const agentCandidateLineageSchema: z.ZodObject<{
138
131
  runIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
139
132
  profileDiffIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
140
133
  modelSnapshots: z.ZodOptional<z.ZodArray<z.ZodString>>;
141
- benchmark: z.ZodOptional<z.ZodObject<{
142
- name: z.ZodString;
143
- version: z.ZodString;
144
- splitDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
145
- }, z.core.$strict>>;
146
- spend: z.ZodOptional<z.ZodObject<{
147
- proposal: z.ZodObject<{
148
- costUsd: z.ZodNumber;
149
- inputTokens: z.ZodNumber;
150
- outputTokens: z.ZodNumber;
151
- cachedInputTokens: z.ZodOptional<z.ZodNumber>;
152
- modelCalls: z.ZodNumber;
153
- }, z.core.$strict>;
154
- evaluation: z.ZodObject<{
155
- costUsd: z.ZodNumber;
156
- inputTokens: z.ZodNumber;
157
- outputTokens: z.ZodNumber;
158
- cachedInputTokens: z.ZodOptional<z.ZodNumber>;
159
- modelCalls: z.ZodNumber;
160
- }, z.core.$strict>;
161
- }, z.core.$strict>>;
134
+ developmentSplitDigest: z.ZodOptional<z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>>;
162
135
  }, z.core.$strict>;
@@ -30,15 +30,6 @@ export const agentCandidateMemoryPolicySchema = z.discriminatedUnion("mode", [
30
30
  })
31
31
  .strict(),
32
32
  ]);
33
- export const agentCandidateSpendSchema = z
34
- .object({
35
- costUsd: z.number().finite().nonnegative(),
36
- inputTokens: z.number().int().nonnegative(),
37
- outputTokens: z.number().int().nonnegative(),
38
- cachedInputTokens: z.number().int().nonnegative().optional(),
39
- modelCalls: z.number().int().nonnegative(),
40
- })
41
- .strict();
42
33
  export const agentCandidateLineageSchema = z
43
34
  .object({
44
35
  source: z.enum(["optimizer", "human", "import", "compound"]),
@@ -46,21 +37,7 @@ export const agentCandidateLineageSchema = z
46
37
  runIds: z.array(z.string().min(1)).optional(),
47
38
  profileDiffIds: z.array(z.string().min(1)).optional(),
48
39
  modelSnapshots: z.array(z.string().min(1)).optional(),
49
- benchmark: z
50
- .object({
51
- name: z.string().min(1),
52
- version: z.string().min(1),
53
- splitDigest: sha256DigestSchema,
54
- })
55
- .strict()
56
- .optional(),
57
- spend: z
58
- .object({
59
- proposal: agentCandidateSpendSchema,
60
- evaluation: agentCandidateSpendSchema,
61
- })
62
- .strict()
63
- .optional(),
40
+ developmentSplitDigest: sha256DigestSchema.optional(),
64
41
  })
65
42
  .strict()
66
43
  .superRefine((lineage, ctx) => {
@@ -91,19 +68,12 @@ export const agentCandidateLineageSchema = z
91
68
  message: "generated candidates must name their producing run",
92
69
  });
93
70
  }
94
- if (lineage.benchmark === undefined) {
71
+ if (lineage.developmentSplitDigest === undefined) {
95
72
  ctx.addIssue({
96
73
  code: "custom",
97
- path: ["benchmark"],
74
+ path: ["developmentSplitDigest"],
98
75
  message: "generated candidates must pin their development split",
99
76
  });
100
77
  }
101
- if (lineage.spend === undefined) {
102
- ctx.addIssue({
103
- code: "custom",
104
- path: ["spend"],
105
- message: "generated candidates must record proposal and evaluation spend",
106
- });
107
- }
108
78
  }
109
79
  });
@@ -522,15 +522,10 @@ export declare const agentCandidateBenchmarkResultMaterialSchema: z.ZodObject<{
522
522
  kind: z.ZodLiteral<"agent-candidate-benchmark-result-material">;
523
523
  executionPlanDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
524
524
  taskOutcomeDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
525
- benchmark: z.ZodObject<{
526
- name: z.ZodString;
527
- version: z.ZodString;
528
- taskId: z.ZodString;
529
- splitDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
530
- }, z.core.$strict>;
531
525
  grader: z.ZodObject<{
532
526
  name: z.ZodString;
533
527
  version: z.ZodString;
528
+ format: z.ZodLiteral<"tangle-grader">;
534
529
  artifact: z.ZodObject<{
535
530
  locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
536
531
  kind: z.ZodLiteral<"s3">;
@@ -560,6 +555,21 @@ export declare const agentCandidateBenchmarkResultMaterialSchema: z.ZodObject<{
560
555
  sha256: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
561
556
  byteLength: z.ZodNumber;
562
557
  }, z.core.$strict>;
558
+ grading: z.ZodObject<{
559
+ usage: z.ZodObject<{
560
+ inputTokens: z.ZodNumber;
561
+ outputTokens: z.ZodNumber;
562
+ cachedInputTokens: z.ZodNumber;
563
+ reasoningTokens: z.ZodNumber;
564
+ modelCalls: z.ZodNumber;
565
+ costUsdNanos: z.ZodNumber;
566
+ }, z.core.$strict>;
567
+ timing: z.ZodObject<{
568
+ startedAtMs: z.ZodNumber;
569
+ endedAtMs: z.ZodNumber;
570
+ durationMs: z.ZodNumber;
571
+ }, z.core.$strict>;
572
+ }, z.core.$strict>;
563
573
  score: z.ZodNumber;
564
574
  passed: z.ZodBoolean;
565
575
  dimensions: z.ZodArray<z.ZodObject<{
@@ -574,15 +584,10 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
574
584
  kind: "agent-candidate-benchmark-result-material";
575
585
  executionPlanDigest: `sha256:${string}`;
576
586
  taskOutcomeDigest: `sha256:${string}`;
577
- benchmark: {
578
- name: string;
579
- version: string;
580
- taskId: string;
581
- splitDigest: `sha256:${string}`;
582
- };
583
587
  grader: {
584
588
  name: string;
585
589
  version: string;
590
+ format: "tangle-grader";
586
591
  artifact: {
587
592
  locator: {
588
593
  kind: "s3";
@@ -612,6 +617,21 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
612
617
  sha256: `sha256:${string}`;
613
618
  byteLength: number;
614
619
  };
620
+ grading: {
621
+ usage: {
622
+ inputTokens: number;
623
+ outputTokens: number;
624
+ cachedInputTokens: number;
625
+ reasoningTokens: number;
626
+ modelCalls: number;
627
+ costUsdNanos: number;
628
+ };
629
+ timing: {
630
+ startedAtMs: number;
631
+ endedAtMs: number;
632
+ durationMs: number;
633
+ };
634
+ };
615
635
  score: number;
616
636
  passed: boolean;
617
637
  dimensions: {
@@ -622,15 +642,10 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
622
642
  kind: "agent-candidate-benchmark-result-material";
623
643
  executionPlanDigest: `sha256:${string}`;
624
644
  taskOutcomeDigest: `sha256:${string}`;
625
- benchmark: {
626
- name: string;
627
- version: string;
628
- taskId: string;
629
- splitDigest: `sha256:${string}`;
630
- };
631
645
  grader: {
632
646
  name: string;
633
647
  version: string;
648
+ format: "tangle-grader";
634
649
  artifact: {
635
650
  locator: {
636
651
  kind: "s3";
@@ -660,6 +675,21 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
660
675
  sha256: `sha256:${string}`;
661
676
  byteLength: number;
662
677
  };
678
+ grading: {
679
+ usage: {
680
+ inputTokens: number;
681
+ outputTokens: number;
682
+ cachedInputTokens: number;
683
+ reasoningTokens: number;
684
+ modelCalls: number;
685
+ costUsdNanos: number;
686
+ };
687
+ timing: {
688
+ startedAtMs: number;
689
+ endedAtMs: number;
690
+ durationMs: number;
691
+ };
692
+ };
663
693
  score: number;
664
694
  passed: boolean;
665
695
  dimensions: {
@@ -1,6 +1,6 @@
1
1
  import { z } from "zod";
2
2
  import { agentCandidateArtifactRefSchema, agentCandidateCapturedArtifactSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js";
3
- import { agentCandidateResolvedModelSchema } from "./agent-candidate-execution-plan-schema.js";
3
+ import { agentCandidateBenchmarkGraderIdentitySchema, agentCandidateResolvedModelSchema, } from "./agent-candidate-execution-plan-schema.js";
4
4
  import { agentCandidateMediaTypeSchema, gitObjectSchema, isCanonicalJsonValue, sameGitObjectFormat, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
5
5
  const safeCountSchema = z
6
6
  .number()
@@ -265,35 +265,26 @@ export const agentCandidateBenchmarkResultMaterialSchema = z
265
265
  kind: z.literal("agent-candidate-benchmark-result-material"),
266
266
  executionPlanDigest: sha256DigestSchema,
267
267
  taskOutcomeDigest: sha256DigestSchema,
268
- benchmark: z
269
- .object({
270
- name: z.string().min(1),
271
- version: z.string().min(1),
272
- taskId: z.string().min(1),
273
- splitDigest: sha256DigestSchema,
274
- })
275
- .strict(),
276
- grader: z
268
+ grader: agentCandidateBenchmarkGraderIdentitySchema,
269
+ evidence: agentCandidateArtifactRefSchema,
270
+ grading: z
277
271
  .object({
278
- name: z.string().min(1),
279
- version: z.string().min(1),
280
- artifact: agentCandidateArtifactRefSchema,
272
+ usage: agentCandidateFixedSpendSchema,
273
+ timing: z
274
+ .object({
275
+ startedAtMs: safeCountSchema,
276
+ endedAtMs: safeCountSchema,
277
+ durationMs: safeCountSchema,
278
+ })
279
+ .strict(),
281
280
  })
282
281
  .strict(),
283
- evidence: agentCandidateArtifactRefSchema,
284
282
  score: normalizedScoreSchema,
285
283
  passed: z.boolean(),
286
284
  dimensions: z.array(agentCandidateBenchmarkDimensionSchema),
287
285
  })
288
286
  .strict()
289
287
  .superRefine((material, ctx) => {
290
- if (material.grader.artifact.byteLength === 0) {
291
- ctx.addIssue({
292
- code: "custom",
293
- path: ["grader", "artifact", "byteLength"],
294
- message: "pinned grader artifact must contain executable grader bytes",
295
- });
296
- }
297
288
  if (material.evidence.byteLength === 0) {
298
289
  ctx.addIssue({
299
290
  code: "custom",
@@ -301,11 +292,15 @@ export const agentCandidateBenchmarkResultMaterialSchema = z
301
292
  message: "benchmark result must contain non-empty durable grading evidence",
302
293
  });
303
294
  }
304
- if (material.evidence.sha256 === material.grader.artifact.sha256) {
295
+ if (material.grading.timing.endedAtMs <
296
+ material.grading.timing.startedAtMs ||
297
+ material.grading.timing.durationMs !==
298
+ material.grading.timing.endedAtMs -
299
+ material.grading.timing.startedAtMs) {
305
300
  ctx.addIssue({
306
301
  code: "custom",
307
- path: ["evidence", "sha256"],
308
- message: "grading evidence must be distinct from the grader implementation",
302
+ path: ["grading", "timing"],
303
+ message: "grader timing must be ordered and internally consistent",
309
304
  });
310
305
  }
311
306
  for (let index = 1; index < material.dimensions.length; index++) {
@@ -1,18 +1,6 @@
1
1
  import { z } from "zod";
2
- export declare const agentCandidateMcpServerSchema: z.ZodObject<{
3
- transport: z.ZodOptional<z.ZodLiteral<"stdio">>;
4
- command: z.ZodOptional<z.ZodString>;
5
- args: z.ZodOptional<z.ZodArray<z.ZodObject<{
6
- kind: z.ZodLiteral<"public">;
7
- value: z.ZodString;
8
- }, z.core.$strict>>>;
9
- env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
10
- kind: z.ZodLiteral<"public">;
11
- value: z.ZodString;
12
- }, z.core.$strict>>>;
13
- cwd: z.ZodOptional<z.ZodString>;
14
- enabled: z.ZodOptional<z.ZodBoolean>;
15
- }, z.core.$strict>;
2
+ import type { AgentCandidateMcpServer } from "./agent-candidate.js";
3
+ export declare const agentCandidateMcpServerSchema: z.ZodType<AgentCandidateMcpServer>;
16
4
  export declare const agentCandidateHookCommandSchema: z.ZodObject<{
17
5
  executable: z.ZodString;
18
6
  args: z.ZodOptional<z.ZodArray<z.ZodObject<{
@@ -78,20 +66,7 @@ export declare const agentCandidateProfileSchema: z.ZodObject<{
78
66
  deny: "deny";
79
67
  }>>]>>>;
80
68
  tools: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
81
- mcp: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
82
- transport: z.ZodOptional<z.ZodLiteral<"stdio">>;
83
- command: z.ZodOptional<z.ZodString>;
84
- args: z.ZodOptional<z.ZodArray<z.ZodObject<{
85
- kind: z.ZodLiteral<"public">;
86
- value: z.ZodString;
87
- }, z.core.$strict>>>;
88
- env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
89
- kind: z.ZodLiteral<"public">;
90
- value: z.ZodString;
91
- }, z.core.$strict>>>;
92
- cwd: z.ZodOptional<z.ZodString>;
93
- enabled: z.ZodOptional<z.ZodBoolean>;
94
- }, z.core.$strict>>>;
69
+ mcp: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<AgentCandidateMcpServer, unknown, z.core.$ZodTypeInternals<AgentCandidateMcpServer, unknown>>>>;
95
70
  subagents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
96
71
  prompt: z.ZodOptional<z.ZodString>;
97
72
  permissions: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodEnum<{