@tangle-network/agent-interface 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-candidate-code-schema.d.ts +0 -8
- package/dist/agent-candidate-code-schema.js +0 -1
- package/dist/agent-candidate-execution-plan-schema.d.ts +173 -350
- package/dist/agent-candidate-execution-plan-schema.js +97 -131
- package/dist/agent-candidate-lineage-schema.d.ts +1 -28
- package/dist/agent-candidate-lineage-schema.js +3 -33
- package/dist/agent-candidate-outcome-schema.d.ts +48 -18
- package/dist/agent-candidate-outcome-schema.js +19 -24
- package/dist/agent-candidate-promotion-schema.d.ts +11021 -2358
- package/dist/agent-candidate-promotion-schema.js +495 -105
- package/dist/agent-candidate-receipt-schema.d.ts +260 -303
- package/dist/agent-candidate-receipt-schema.js +82 -8
- package/dist/agent-candidate-schema-common.d.ts +8 -0
- package/dist/agent-candidate-schema-common.js +35 -1
- package/dist/agent-candidate-schema.d.ts +2 -38
- package/dist/agent-candidate-schema.js +3 -28
- package/dist/agent-candidate-task-schema.d.ts +643 -0
- package/dist/agent-candidate-task-schema.js +191 -0
- package/dist/agent-candidate.d.ts +173 -72
- package/dist/agent-candidate.test-fixture.d.ts +0 -26
- package/dist/agent-candidate.test-fixture.js +0 -26
- package/dist/profile-schema.d.ts +9 -0
- package/dist/profile-schema.js +10 -3
- package/package.json +2 -2
|
@@ -1,17 +1,9 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { agentCandidateArtifactRefSchema, agentCandidateCapturedArtifactSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js";
|
|
3
3
|
import { agentCandidateContainerSchema, agentCandidateInstructionDeliverySchema, agentCandidateWorkingDirectorySchema, } from "./agent-candidate-code-schema.js";
|
|
4
|
+
import { reasoningEffortSchema } from "./profile-schema.js";
|
|
4
5
|
import { addDuplicateIssues, agentCandidateConfigValueSchema, agentCandidateMediaTypeSchema, environmentConfigSchema, gitObjectSchema, isCanonicalJsonValue, isObviouslyPrivateHostname, isSafeExecutable, isSafeRelativePath, isWellFormedUnicode, sameGitObjectFormat, sha256Utf8, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
|
|
5
6
|
import { harnessTypeSchema } from "./harness.js";
|
|
6
|
-
const reasoningEffortSchema = z.enum([
|
|
7
|
-
"none",
|
|
8
|
-
"minimal",
|
|
9
|
-
"low",
|
|
10
|
-
"medium",
|
|
11
|
-
"high",
|
|
12
|
-
"xhigh",
|
|
13
|
-
"ultracode",
|
|
14
|
-
]);
|
|
15
7
|
function isCanonicalAbsolutePath(value) {
|
|
16
8
|
return (value.startsWith("/") &&
|
|
17
9
|
value !== "/" &&
|
|
@@ -58,8 +50,59 @@ export const agentCandidateResolvedModelSchema = z
|
|
|
58
50
|
reasoningEffort: reasoningEffortSchema,
|
|
59
51
|
})
|
|
60
52
|
.strict();
|
|
53
|
+
export const agentCandidateBenchmarkGraderIdentitySchema = z
|
|
54
|
+
.object({
|
|
55
|
+
name: z.string().min(1),
|
|
56
|
+
version: z.string().min(1),
|
|
57
|
+
format: z.literal("tangle-grader"),
|
|
58
|
+
artifact: agentCandidateArtifactRefSchema,
|
|
59
|
+
})
|
|
60
|
+
.strict()
|
|
61
|
+
.superRefine((grader, ctx) => {
|
|
62
|
+
if (grader.artifact.byteLength === 0) {
|
|
63
|
+
ctx.addIssue({
|
|
64
|
+
code: "custom",
|
|
65
|
+
path: ["artifact", "byteLength"],
|
|
66
|
+
message: "pinned grader artifact must contain executable grader bytes",
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
export const agentCandidateTaskRepositorySchema = z
|
|
71
|
+
.object({
|
|
72
|
+
identity: z.string().min(1).max(512),
|
|
73
|
+
rootIdentity: z.string().min(1).max(512),
|
|
74
|
+
baseCommit: gitObjectSchema,
|
|
75
|
+
baseTree: gitObjectSchema,
|
|
76
|
+
})
|
|
77
|
+
.strict()
|
|
78
|
+
.superRefine((repository, ctx) => {
|
|
79
|
+
if (!sameGitObjectFormat(repository.baseCommit, repository.baseTree)) {
|
|
80
|
+
ctx.addIssue({
|
|
81
|
+
code: "custom",
|
|
82
|
+
message: "task Git object ids must use one object format",
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
export const agentCandidateResolvedTaskContainerSchema = agentCandidateContainerSchema
|
|
87
|
+
.extend({
|
|
88
|
+
source: z.literal("evaluator-task-container"),
|
|
89
|
+
manifestDigest: sha256DigestSchema,
|
|
90
|
+
platform: z
|
|
91
|
+
.object({
|
|
92
|
+
os: z.string().min(1).max(100),
|
|
93
|
+
architecture: z.string().min(1).max(100),
|
|
94
|
+
variant: z.string().min(1).max(100).optional(),
|
|
95
|
+
})
|
|
96
|
+
.strict(),
|
|
97
|
+
})
|
|
98
|
+
.strict();
|
|
99
|
+
export const agentCandidateRetryPolicySchema = z.enum([
|
|
100
|
+
"pre-model-infrastructure-only",
|
|
101
|
+
"none",
|
|
102
|
+
]);
|
|
61
103
|
export const agentCandidateProfilePlanMaterialSchema = z
|
|
62
104
|
.object({
|
|
105
|
+
sourceProfileDigest: sha256DigestSchema,
|
|
63
106
|
harness: harnessTypeSchema,
|
|
64
107
|
files: z.array(z
|
|
65
108
|
.object({
|
|
@@ -149,45 +192,35 @@ export const agentCandidateTaskOutcomeSpecSchema = z.discriminatedUnion("kind",
|
|
|
149
192
|
})
|
|
150
193
|
.strict(),
|
|
151
194
|
]);
|
|
195
|
+
export const agentCandidateBenchmarkCellRefSchema = z
|
|
196
|
+
.object({
|
|
197
|
+
suiteDigest: sha256DigestSchema,
|
|
198
|
+
taskIndex: z.number().int().nonnegative(),
|
|
199
|
+
repetition: z.number().int().nonnegative(),
|
|
200
|
+
})
|
|
201
|
+
.strict();
|
|
202
|
+
export const agentCandidateRunCellMaterialSchema = z
|
|
203
|
+
.object({
|
|
204
|
+
kind: z.literal("agent-candidate-run-cell"),
|
|
205
|
+
experimentDigest: sha256DigestSchema,
|
|
206
|
+
arm: z.enum(["baseline", "candidate"]),
|
|
207
|
+
bundleDigest: sha256DigestSchema,
|
|
208
|
+
suiteDigest: sha256DigestSchema,
|
|
209
|
+
taskDigest: sha256DigestSchema,
|
|
210
|
+
taskIndex: z.number().int().nonnegative(),
|
|
211
|
+
repetition: z.number().int().nonnegative(),
|
|
212
|
+
seed: z.number().int().safe(),
|
|
213
|
+
attempt: z.number().int().positive(),
|
|
214
|
+
})
|
|
215
|
+
.strict();
|
|
216
|
+
export const agentCandidateRunCellSchema = agentCandidateRunCellMaterialSchema
|
|
217
|
+
.extend({ digest: sha256DigestSchema })
|
|
218
|
+
.strict();
|
|
152
219
|
export const agentCandidateExecutionPlanMaterialSchema = z
|
|
153
220
|
.object({
|
|
154
221
|
kind: z.literal("agent-candidate-execution-plan-material"),
|
|
155
|
-
|
|
222
|
+
runCell: agentCandidateRunCellSchema,
|
|
156
223
|
executionId: z.string().min(1),
|
|
157
|
-
attempt: z
|
|
158
|
-
.object({
|
|
159
|
-
number: z.number().int().min(1),
|
|
160
|
-
maxAttempts: z.number().int().min(1),
|
|
161
|
-
retryPolicy: z.enum(["pre-model-infrastructure-only", "none"]),
|
|
162
|
-
})
|
|
163
|
-
.strict(),
|
|
164
|
-
task: z
|
|
165
|
-
.object({
|
|
166
|
-
benchmark: z.string().min(1),
|
|
167
|
-
benchmarkVersion: z.string().min(1),
|
|
168
|
-
taskId: z.string().min(1),
|
|
169
|
-
splitDigest: sha256DigestSchema,
|
|
170
|
-
instruction: z
|
|
171
|
-
.object({
|
|
172
|
-
encoding: z.literal("utf8"),
|
|
173
|
-
sha256: sha256DigestSchema,
|
|
174
|
-
byteLength: z.number().int().positive(),
|
|
175
|
-
delivery: agentCandidateInstructionDeliverySchema,
|
|
176
|
-
})
|
|
177
|
-
.strict(),
|
|
178
|
-
repository: z
|
|
179
|
-
.object({
|
|
180
|
-
identity: z.string().min(1),
|
|
181
|
-
rootIdentity: z.string().min(1),
|
|
182
|
-
baseCommit: gitObjectSchema,
|
|
183
|
-
baseTree: gitObjectSchema,
|
|
184
|
-
})
|
|
185
|
-
.strict()
|
|
186
|
-
.optional(),
|
|
187
|
-
outcome: agentCandidateTaskOutcomeSpecSchema,
|
|
188
|
-
workspace: agentCandidateWorkspaceSnapshotEvidenceSchema,
|
|
189
|
-
})
|
|
190
|
-
.strict(),
|
|
191
224
|
workspaces: z
|
|
192
225
|
.object({
|
|
193
226
|
taskRoot: z
|
|
@@ -212,6 +245,8 @@ export const agentCandidateExecutionPlanMaterialSchema = z
|
|
|
212
245
|
.strict(),
|
|
213
246
|
harness: harnessTypeSchema,
|
|
214
247
|
harnessVersion: z.string().min(1),
|
|
248
|
+
instructionDelivery: agentCandidateInstructionDeliverySchema,
|
|
249
|
+
limits: agentCandidateExecutionLimitsSchema,
|
|
215
250
|
container: agentCandidateContainerSchema
|
|
216
251
|
.extend({
|
|
217
252
|
source: z.enum(["pinned-container", "evaluator-task-container"]),
|
|
@@ -268,13 +303,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z
|
|
|
268
303
|
.min(1),
|
|
269
304
|
})
|
|
270
305
|
.strict(),
|
|
271
|
-
grader: z
|
|
272
|
-
.object({
|
|
273
|
-
name: z.string().min(1),
|
|
274
|
-
version: z.string().min(1),
|
|
275
|
-
artifact: agentCandidateArtifactRefSchema,
|
|
276
|
-
})
|
|
277
|
-
.strict(),
|
|
278
306
|
launch: z
|
|
279
307
|
.object({
|
|
280
308
|
executable: z
|
|
@@ -287,42 +315,10 @@ export const agentCandidateExecutionPlanMaterialSchema = z
|
|
|
287
315
|
.strict(),
|
|
288
316
|
knowledgeManifestDigest: sha256DigestSchema.optional(),
|
|
289
317
|
memory: agentCandidateEffectiveMemorySchema,
|
|
290
|
-
limits: agentCandidateExecutionLimitsSchema,
|
|
291
318
|
network: z.object({ mode: z.literal("disabled") }).strict(),
|
|
292
319
|
})
|
|
293
320
|
.strict()
|
|
294
321
|
.superRefine((material, ctx) => {
|
|
295
|
-
if (material.task.repository !== undefined &&
|
|
296
|
-
!sameGitObjectFormat(material.task.repository.baseCommit, material.task.repository.baseTree)) {
|
|
297
|
-
ctx.addIssue({
|
|
298
|
-
code: "custom",
|
|
299
|
-
path: ["task", "repository"],
|
|
300
|
-
message: "task Git object ids must use one object format",
|
|
301
|
-
});
|
|
302
|
-
}
|
|
303
|
-
if (material.task.outcome.kind === "workspace" &&
|
|
304
|
-
material.task.repository === undefined) {
|
|
305
|
-
ctx.addIssue({
|
|
306
|
-
code: "custom",
|
|
307
|
-
path: ["task", "repository"],
|
|
308
|
-
message: "workspace outcomes require task repository provenance",
|
|
309
|
-
});
|
|
310
|
-
}
|
|
311
|
-
if (material.attempt.number > material.attempt.maxAttempts) {
|
|
312
|
-
ctx.addIssue({
|
|
313
|
-
code: "custom",
|
|
314
|
-
path: ["attempt", "number"],
|
|
315
|
-
message: "attempt number cannot exceed the frozen maximum",
|
|
316
|
-
});
|
|
317
|
-
}
|
|
318
|
-
if (material.attempt.retryPolicy === "none" &&
|
|
319
|
-
material.attempt.maxAttempts !== 1) {
|
|
320
|
-
ctx.addIssue({
|
|
321
|
-
code: "custom",
|
|
322
|
-
path: ["attempt", "maxAttempts"],
|
|
323
|
-
message: "a no-retry plan must allow exactly one attempt",
|
|
324
|
-
});
|
|
325
|
-
}
|
|
326
322
|
const routeIds = material.model.routes.map((route) => route.kind === "mode" || route.kind === "subagent"
|
|
327
323
|
? `${route.kind}:${route.name}`
|
|
328
324
|
: route.kind);
|
|
@@ -356,20 +352,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z
|
|
|
356
352
|
}
|
|
357
353
|
}
|
|
358
354
|
const modelNetwork = material.model.access.network;
|
|
359
|
-
if (material.limits.maxModelCalls === 0 && modelNetwork.mode !== "disabled") {
|
|
360
|
-
ctx.addIssue({
|
|
361
|
-
code: "custom",
|
|
362
|
-
path: ["model", "access", "network"],
|
|
363
|
-
message: "zero-call plans cannot expose a model gateway",
|
|
364
|
-
});
|
|
365
|
-
}
|
|
366
|
-
if (material.limits.maxModelCalls > 0 && modelNetwork.mode !== "gateway-only") {
|
|
367
|
-
ctx.addIssue({
|
|
368
|
-
code: "custom",
|
|
369
|
-
path: ["model", "access", "network"],
|
|
370
|
-
message: "model-calling plans require one frozen gateway allowlist",
|
|
371
|
-
});
|
|
372
|
-
}
|
|
373
355
|
if (modelNetwork.mode === "gateway-only") {
|
|
374
356
|
addDuplicateIssues(modelNetwork.domains, ["model", "access", "network", "domains"], ctx);
|
|
375
357
|
for (let index = 1; index < modelNetwork.domains.length; index++) {
|
|
@@ -405,62 +387,46 @@ export const agentCandidateExecutionPlanMaterialSchema = z
|
|
|
405
387
|
message: "task and candidate workspace roots must be disjoint",
|
|
406
388
|
});
|
|
407
389
|
}
|
|
408
|
-
if (material.launch.cwd.workspace === "candidate" &&
|
|
409
|
-
material.workspaces.candidateRoot === undefined) {
|
|
410
|
-
ctx.addIssue({
|
|
411
|
-
code: "custom",
|
|
412
|
-
path: ["launch", "cwd", "workspace"],
|
|
413
|
-
message: "candidate cwd requires a pinned candidate workspace root",
|
|
414
|
-
});
|
|
415
|
-
}
|
|
416
|
-
if (material.profile.targetWorkspace === "candidate" &&
|
|
417
|
-
material.workspaces.candidateRoot === undefined) {
|
|
418
|
-
ctx.addIssue({
|
|
419
|
-
code: "custom",
|
|
420
|
-
path: ["profile", "targetWorkspace"],
|
|
421
|
-
message: "candidate-targeted profile files require a candidate workspace root",
|
|
422
|
-
});
|
|
423
|
-
}
|
|
424
|
-
const delivery = material.task.instruction.delivery;
|
|
425
390
|
const taskPath = material.launch.env.TANGLE_CANDIDATE_TASK_PATH;
|
|
426
|
-
if (
|
|
427
|
-
if (taskPath
|
|
391
|
+
if (taskPath !== undefined) {
|
|
392
|
+
if (taskPath.value !== "/tangle/input/task.txt") {
|
|
428
393
|
ctx.addIssue({
|
|
429
394
|
code: "custom",
|
|
430
395
|
path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"],
|
|
431
|
-
message: "file delivery
|
|
396
|
+
message: "task file delivery must use the fixed evaluator-owned path",
|
|
432
397
|
});
|
|
433
398
|
}
|
|
434
|
-
if (pathsOverlap(material.workspaces.taskRoot
|
|
399
|
+
if (pathsOverlap(taskPath.value, material.workspaces.taskRoot) ||
|
|
435
400
|
(material.workspaces.candidateRoot !== undefined &&
|
|
436
|
-
pathsOverlap(material.workspaces.candidateRoot
|
|
401
|
+
pathsOverlap(taskPath.value, material.workspaces.candidateRoot))) {
|
|
437
402
|
ctx.addIssue({
|
|
438
403
|
code: "custom",
|
|
439
|
-
path: ["
|
|
440
|
-
message: "
|
|
404
|
+
path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"],
|
|
405
|
+
message: "task file delivery must be outside both workspaces",
|
|
441
406
|
});
|
|
442
407
|
}
|
|
443
408
|
}
|
|
444
|
-
|
|
409
|
+
if (material.launch.cwd.workspace === "candidate" &&
|
|
410
|
+
material.workspaces.candidateRoot === undefined) {
|
|
445
411
|
ctx.addIssue({
|
|
446
412
|
code: "custom",
|
|
447
|
-
path: ["launch", "
|
|
448
|
-
message: "
|
|
413
|
+
path: ["launch", "cwd", "workspace"],
|
|
414
|
+
message: "candidate cwd requires a pinned candidate workspace root",
|
|
449
415
|
});
|
|
450
416
|
}
|
|
451
|
-
if (
|
|
417
|
+
if (material.profile.targetWorkspace === "candidate" &&
|
|
418
|
+
material.workspaces.candidateRoot === undefined) {
|
|
452
419
|
ctx.addIssue({
|
|
453
420
|
code: "custom",
|
|
454
|
-
path: ["
|
|
455
|
-
message: "
|
|
421
|
+
path: ["profile", "targetWorkspace"],
|
|
422
|
+
message: "candidate-targeted profile files require a candidate workspace root",
|
|
456
423
|
});
|
|
457
424
|
}
|
|
458
|
-
if (material.
|
|
459
|
-
material.task.workspace.material.files.length === 0) {
|
|
425
|
+
if (activeCode && material.candidateWorkspace?.material.files.length === 0) {
|
|
460
426
|
ctx.addIssue({
|
|
461
427
|
code: "custom",
|
|
462
|
-
path: ["
|
|
463
|
-
message: "
|
|
428
|
+
path: ["candidateWorkspace", "material", "files"],
|
|
429
|
+
message: "active candidate workspaces cannot be empty",
|
|
464
430
|
});
|
|
465
431
|
}
|
|
466
432
|
if (!isCanonicalJsonValue(material)) {
|
|
@@ -120,13 +120,6 @@ export declare const agentCandidateMemoryPolicySchema: z.ZodDiscriminatedUnion<[
|
|
|
120
120
|
byteLength: z.ZodNumber;
|
|
121
121
|
}, z.core.$strict>>;
|
|
122
122
|
}, z.core.$strict>], "mode">;
|
|
123
|
-
export declare const agentCandidateSpendSchema: z.ZodObject<{
|
|
124
|
-
costUsd: z.ZodNumber;
|
|
125
|
-
inputTokens: z.ZodNumber;
|
|
126
|
-
outputTokens: z.ZodNumber;
|
|
127
|
-
cachedInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
128
|
-
modelCalls: z.ZodNumber;
|
|
129
|
-
}, z.core.$strict>;
|
|
130
123
|
export declare const agentCandidateLineageSchema: z.ZodObject<{
|
|
131
124
|
source: z.ZodEnum<{
|
|
132
125
|
optimizer: "optimizer";
|
|
@@ -138,25 +131,5 @@ export declare const agentCandidateLineageSchema: z.ZodObject<{
|
|
|
138
131
|
runIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
139
132
|
profileDiffIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
140
133
|
modelSnapshots: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
141
|
-
|
|
142
|
-
name: z.ZodString;
|
|
143
|
-
version: z.ZodString;
|
|
144
|
-
splitDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
145
|
-
}, z.core.$strict>>;
|
|
146
|
-
spend: z.ZodOptional<z.ZodObject<{
|
|
147
|
-
proposal: z.ZodObject<{
|
|
148
|
-
costUsd: z.ZodNumber;
|
|
149
|
-
inputTokens: z.ZodNumber;
|
|
150
|
-
outputTokens: z.ZodNumber;
|
|
151
|
-
cachedInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
152
|
-
modelCalls: z.ZodNumber;
|
|
153
|
-
}, z.core.$strict>;
|
|
154
|
-
evaluation: z.ZodObject<{
|
|
155
|
-
costUsd: z.ZodNumber;
|
|
156
|
-
inputTokens: z.ZodNumber;
|
|
157
|
-
outputTokens: z.ZodNumber;
|
|
158
|
-
cachedInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
159
|
-
modelCalls: z.ZodNumber;
|
|
160
|
-
}, z.core.$strict>;
|
|
161
|
-
}, z.core.$strict>>;
|
|
134
|
+
developmentSplitDigest: z.ZodOptional<z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>>;
|
|
162
135
|
}, z.core.$strict>;
|
|
@@ -30,15 +30,6 @@ export const agentCandidateMemoryPolicySchema = z.discriminatedUnion("mode", [
|
|
|
30
30
|
})
|
|
31
31
|
.strict(),
|
|
32
32
|
]);
|
|
33
|
-
export const agentCandidateSpendSchema = z
|
|
34
|
-
.object({
|
|
35
|
-
costUsd: z.number().finite().nonnegative(),
|
|
36
|
-
inputTokens: z.number().int().nonnegative(),
|
|
37
|
-
outputTokens: z.number().int().nonnegative(),
|
|
38
|
-
cachedInputTokens: z.number().int().nonnegative().optional(),
|
|
39
|
-
modelCalls: z.number().int().nonnegative(),
|
|
40
|
-
})
|
|
41
|
-
.strict();
|
|
42
33
|
export const agentCandidateLineageSchema = z
|
|
43
34
|
.object({
|
|
44
35
|
source: z.enum(["optimizer", "human", "import", "compound"]),
|
|
@@ -46,21 +37,7 @@ export const agentCandidateLineageSchema = z
|
|
|
46
37
|
runIds: z.array(z.string().min(1)).optional(),
|
|
47
38
|
profileDiffIds: z.array(z.string().min(1)).optional(),
|
|
48
39
|
modelSnapshots: z.array(z.string().min(1)).optional(),
|
|
49
|
-
|
|
50
|
-
.object({
|
|
51
|
-
name: z.string().min(1),
|
|
52
|
-
version: z.string().min(1),
|
|
53
|
-
splitDigest: sha256DigestSchema,
|
|
54
|
-
})
|
|
55
|
-
.strict()
|
|
56
|
-
.optional(),
|
|
57
|
-
spend: z
|
|
58
|
-
.object({
|
|
59
|
-
proposal: agentCandidateSpendSchema,
|
|
60
|
-
evaluation: agentCandidateSpendSchema,
|
|
61
|
-
})
|
|
62
|
-
.strict()
|
|
63
|
-
.optional(),
|
|
40
|
+
developmentSplitDigest: sha256DigestSchema.optional(),
|
|
64
41
|
})
|
|
65
42
|
.strict()
|
|
66
43
|
.superRefine((lineage, ctx) => {
|
|
@@ -91,19 +68,12 @@ export const agentCandidateLineageSchema = z
|
|
|
91
68
|
message: "generated candidates must name their producing run",
|
|
92
69
|
});
|
|
93
70
|
}
|
|
94
|
-
if (lineage.
|
|
71
|
+
if (lineage.developmentSplitDigest === undefined) {
|
|
95
72
|
ctx.addIssue({
|
|
96
73
|
code: "custom",
|
|
97
|
-
path: ["
|
|
74
|
+
path: ["developmentSplitDigest"],
|
|
98
75
|
message: "generated candidates must pin their development split",
|
|
99
76
|
});
|
|
100
77
|
}
|
|
101
|
-
if (lineage.spend === undefined) {
|
|
102
|
-
ctx.addIssue({
|
|
103
|
-
code: "custom",
|
|
104
|
-
path: ["spend"],
|
|
105
|
-
message: "generated candidates must record proposal and evaluation spend",
|
|
106
|
-
});
|
|
107
|
-
}
|
|
108
78
|
}
|
|
109
79
|
});
|
|
@@ -522,15 +522,10 @@ export declare const agentCandidateBenchmarkResultMaterialSchema: z.ZodObject<{
|
|
|
522
522
|
kind: z.ZodLiteral<"agent-candidate-benchmark-result-material">;
|
|
523
523
|
executionPlanDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
524
524
|
taskOutcomeDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
525
|
-
benchmark: z.ZodObject<{
|
|
526
|
-
name: z.ZodString;
|
|
527
|
-
version: z.ZodString;
|
|
528
|
-
taskId: z.ZodString;
|
|
529
|
-
splitDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
530
|
-
}, z.core.$strict>;
|
|
531
525
|
grader: z.ZodObject<{
|
|
532
526
|
name: z.ZodString;
|
|
533
527
|
version: z.ZodString;
|
|
528
|
+
format: z.ZodLiteral<"tangle-grader">;
|
|
534
529
|
artifact: z.ZodObject<{
|
|
535
530
|
locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
536
531
|
kind: z.ZodLiteral<"s3">;
|
|
@@ -560,6 +555,21 @@ export declare const agentCandidateBenchmarkResultMaterialSchema: z.ZodObject<{
|
|
|
560
555
|
sha256: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
561
556
|
byteLength: z.ZodNumber;
|
|
562
557
|
}, z.core.$strict>;
|
|
558
|
+
grading: z.ZodObject<{
|
|
559
|
+
usage: z.ZodObject<{
|
|
560
|
+
inputTokens: z.ZodNumber;
|
|
561
|
+
outputTokens: z.ZodNumber;
|
|
562
|
+
cachedInputTokens: z.ZodNumber;
|
|
563
|
+
reasoningTokens: z.ZodNumber;
|
|
564
|
+
modelCalls: z.ZodNumber;
|
|
565
|
+
costUsdNanos: z.ZodNumber;
|
|
566
|
+
}, z.core.$strict>;
|
|
567
|
+
timing: z.ZodObject<{
|
|
568
|
+
startedAtMs: z.ZodNumber;
|
|
569
|
+
endedAtMs: z.ZodNumber;
|
|
570
|
+
durationMs: z.ZodNumber;
|
|
571
|
+
}, z.core.$strict>;
|
|
572
|
+
}, z.core.$strict>;
|
|
563
573
|
score: z.ZodNumber;
|
|
564
574
|
passed: z.ZodBoolean;
|
|
565
575
|
dimensions: z.ZodArray<z.ZodObject<{
|
|
@@ -574,15 +584,10 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
|
|
|
574
584
|
kind: "agent-candidate-benchmark-result-material";
|
|
575
585
|
executionPlanDigest: `sha256:${string}`;
|
|
576
586
|
taskOutcomeDigest: `sha256:${string}`;
|
|
577
|
-
benchmark: {
|
|
578
|
-
name: string;
|
|
579
|
-
version: string;
|
|
580
|
-
taskId: string;
|
|
581
|
-
splitDigest: `sha256:${string}`;
|
|
582
|
-
};
|
|
583
587
|
grader: {
|
|
584
588
|
name: string;
|
|
585
589
|
version: string;
|
|
590
|
+
format: "tangle-grader";
|
|
586
591
|
artifact: {
|
|
587
592
|
locator: {
|
|
588
593
|
kind: "s3";
|
|
@@ -612,6 +617,21 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
|
|
|
612
617
|
sha256: `sha256:${string}`;
|
|
613
618
|
byteLength: number;
|
|
614
619
|
};
|
|
620
|
+
grading: {
|
|
621
|
+
usage: {
|
|
622
|
+
inputTokens: number;
|
|
623
|
+
outputTokens: number;
|
|
624
|
+
cachedInputTokens: number;
|
|
625
|
+
reasoningTokens: number;
|
|
626
|
+
modelCalls: number;
|
|
627
|
+
costUsdNanos: number;
|
|
628
|
+
};
|
|
629
|
+
timing: {
|
|
630
|
+
startedAtMs: number;
|
|
631
|
+
endedAtMs: number;
|
|
632
|
+
durationMs: number;
|
|
633
|
+
};
|
|
634
|
+
};
|
|
615
635
|
score: number;
|
|
616
636
|
passed: boolean;
|
|
617
637
|
dimensions: {
|
|
@@ -622,15 +642,10 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
|
|
|
622
642
|
kind: "agent-candidate-benchmark-result-material";
|
|
623
643
|
executionPlanDigest: `sha256:${string}`;
|
|
624
644
|
taskOutcomeDigest: `sha256:${string}`;
|
|
625
|
-
benchmark: {
|
|
626
|
-
name: string;
|
|
627
|
-
version: string;
|
|
628
|
-
taskId: string;
|
|
629
|
-
splitDigest: `sha256:${string}`;
|
|
630
|
-
};
|
|
631
645
|
grader: {
|
|
632
646
|
name: string;
|
|
633
647
|
version: string;
|
|
648
|
+
format: "tangle-grader";
|
|
634
649
|
artifact: {
|
|
635
650
|
locator: {
|
|
636
651
|
kind: "s3";
|
|
@@ -660,6 +675,21 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
|
|
|
660
675
|
sha256: `sha256:${string}`;
|
|
661
676
|
byteLength: number;
|
|
662
677
|
};
|
|
678
|
+
grading: {
|
|
679
|
+
usage: {
|
|
680
|
+
inputTokens: number;
|
|
681
|
+
outputTokens: number;
|
|
682
|
+
cachedInputTokens: number;
|
|
683
|
+
reasoningTokens: number;
|
|
684
|
+
modelCalls: number;
|
|
685
|
+
costUsdNanos: number;
|
|
686
|
+
};
|
|
687
|
+
timing: {
|
|
688
|
+
startedAtMs: number;
|
|
689
|
+
endedAtMs: number;
|
|
690
|
+
durationMs: number;
|
|
691
|
+
};
|
|
692
|
+
};
|
|
663
693
|
score: number;
|
|
664
694
|
passed: boolean;
|
|
665
695
|
dimensions: {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { agentCandidateArtifactRefSchema, agentCandidateCapturedArtifactSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js";
|
|
3
|
-
import { agentCandidateResolvedModelSchema } from "./agent-candidate-execution-plan-schema.js";
|
|
3
|
+
import { agentCandidateBenchmarkGraderIdentitySchema, agentCandidateResolvedModelSchema, } from "./agent-candidate-execution-plan-schema.js";
|
|
4
4
|
import { agentCandidateMediaTypeSchema, gitObjectSchema, isCanonicalJsonValue, sameGitObjectFormat, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
|
|
5
5
|
const safeCountSchema = z
|
|
6
6
|
.number()
|
|
@@ -265,35 +265,26 @@ export const agentCandidateBenchmarkResultMaterialSchema = z
|
|
|
265
265
|
kind: z.literal("agent-candidate-benchmark-result-material"),
|
|
266
266
|
executionPlanDigest: sha256DigestSchema,
|
|
267
267
|
taskOutcomeDigest: sha256DigestSchema,
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
version: z.string().min(1),
|
|
272
|
-
taskId: z.string().min(1),
|
|
273
|
-
splitDigest: sha256DigestSchema,
|
|
274
|
-
})
|
|
275
|
-
.strict(),
|
|
276
|
-
grader: z
|
|
268
|
+
grader: agentCandidateBenchmarkGraderIdentitySchema,
|
|
269
|
+
evidence: agentCandidateArtifactRefSchema,
|
|
270
|
+
grading: z
|
|
277
271
|
.object({
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
272
|
+
usage: agentCandidateFixedSpendSchema,
|
|
273
|
+
timing: z
|
|
274
|
+
.object({
|
|
275
|
+
startedAtMs: safeCountSchema,
|
|
276
|
+
endedAtMs: safeCountSchema,
|
|
277
|
+
durationMs: safeCountSchema,
|
|
278
|
+
})
|
|
279
|
+
.strict(),
|
|
281
280
|
})
|
|
282
281
|
.strict(),
|
|
283
|
-
evidence: agentCandidateArtifactRefSchema,
|
|
284
282
|
score: normalizedScoreSchema,
|
|
285
283
|
passed: z.boolean(),
|
|
286
284
|
dimensions: z.array(agentCandidateBenchmarkDimensionSchema),
|
|
287
285
|
})
|
|
288
286
|
.strict()
|
|
289
287
|
.superRefine((material, ctx) => {
|
|
290
|
-
if (material.grader.artifact.byteLength === 0) {
|
|
291
|
-
ctx.addIssue({
|
|
292
|
-
code: "custom",
|
|
293
|
-
path: ["grader", "artifact", "byteLength"],
|
|
294
|
-
message: "pinned grader artifact must contain executable grader bytes",
|
|
295
|
-
});
|
|
296
|
-
}
|
|
297
288
|
if (material.evidence.byteLength === 0) {
|
|
298
289
|
ctx.addIssue({
|
|
299
290
|
code: "custom",
|
|
@@ -301,11 +292,15 @@ export const agentCandidateBenchmarkResultMaterialSchema = z
|
|
|
301
292
|
message: "benchmark result must contain non-empty durable grading evidence",
|
|
302
293
|
});
|
|
303
294
|
}
|
|
304
|
-
if (material.
|
|
295
|
+
if (material.grading.timing.endedAtMs <
|
|
296
|
+
material.grading.timing.startedAtMs ||
|
|
297
|
+
material.grading.timing.durationMs !==
|
|
298
|
+
material.grading.timing.endedAtMs -
|
|
299
|
+
material.grading.timing.startedAtMs) {
|
|
305
300
|
ctx.addIssue({
|
|
306
301
|
code: "custom",
|
|
307
|
-
path: ["
|
|
308
|
-
message: "
|
|
302
|
+
path: ["grading", "timing"],
|
|
303
|
+
message: "grader timing must be ordered and internally consistent",
|
|
309
304
|
});
|
|
310
305
|
}
|
|
311
306
|
for (let index = 1; index < material.dimensions.length; index++) {
|