@tangle-network/agent-interface 0.28.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-candidate-code-schema.d.ts +0 -8
- package/dist/agent-candidate-code-schema.js +0 -1
- package/dist/agent-candidate-execution-plan-schema.d.ts +193 -350
- package/dist/agent-candidate-execution-plan-schema.js +98 -131
- package/dist/agent-candidate-lineage-schema.d.ts +1 -28
- package/dist/agent-candidate-lineage-schema.js +3 -33
- package/dist/agent-candidate-outcome-schema.d.ts +48 -18
- package/dist/agent-candidate-outcome-schema.js +19 -24
- package/dist/agent-candidate-profile-schema.d.ts +3 -28
- package/dist/agent-candidate-profile-schema.js +15 -27
- package/dist/agent-candidate-promotion-schema.d.ts +10981 -2356
- package/dist/agent-candidate-promotion-schema.js +495 -105
- package/dist/agent-candidate-receipt-schema.d.ts +268 -303
- package/dist/agent-candidate-receipt-schema.js +82 -8
- package/dist/agent-candidate-schema-common.d.ts +8 -0
- package/dist/agent-candidate-schema-common.js +35 -1
- package/dist/agent-candidate-schema.d.ts +3 -52
- package/dist/agent-candidate-schema.js +3 -28
- package/dist/agent-candidate-task-schema.d.ts +643 -0
- package/dist/agent-candidate-task-schema.js +191 -0
- package/dist/agent-candidate.d.ts +191 -75
- package/dist/agent-candidate.test-fixture.d.ts +0 -26
- package/dist/agent-candidate.test-fixture.js +0 -26
- package/dist/agent-profile.d.ts +33 -9
- package/dist/harness-capabilities.d.ts +2 -2
- package/dist/harness-capabilities.js +23 -14
- package/dist/profile-schema.d.ts +65 -84
- package/dist/profile-schema.js +82 -32
- package/dist/profile-security.js +2 -0
- package/package.json +2 -2
|
@@ -1,17 +1,9 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { agentCandidateArtifactRefSchema, agentCandidateCapturedArtifactSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js";
|
|
3
3
|
import { agentCandidateContainerSchema, agentCandidateInstructionDeliverySchema, agentCandidateWorkingDirectorySchema, } from "./agent-candidate-code-schema.js";
|
|
4
|
+
import { reasoningEffortSchema } from "./profile-schema.js";
|
|
4
5
|
import { addDuplicateIssues, agentCandidateConfigValueSchema, agentCandidateMediaTypeSchema, environmentConfigSchema, gitObjectSchema, isCanonicalJsonValue, isObviouslyPrivateHostname, isSafeExecutable, isSafeRelativePath, isWellFormedUnicode, sameGitObjectFormat, sha256Utf8, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
|
|
5
6
|
import { harnessTypeSchema } from "./harness.js";
|
|
6
|
-
const reasoningEffortSchema = z.enum([
|
|
7
|
-
"none",
|
|
8
|
-
"minimal",
|
|
9
|
-
"low",
|
|
10
|
-
"medium",
|
|
11
|
-
"high",
|
|
12
|
-
"xhigh",
|
|
13
|
-
"ultracode",
|
|
14
|
-
]);
|
|
15
7
|
function isCanonicalAbsolutePath(value) {
|
|
16
8
|
return (value.startsWith("/") &&
|
|
17
9
|
value !== "/" &&
|
|
@@ -58,8 +50,59 @@ export const agentCandidateResolvedModelSchema = z
|
|
|
58
50
|
reasoningEffort: reasoningEffortSchema,
|
|
59
51
|
})
|
|
60
52
|
.strict();
|
|
53
|
+
export const agentCandidateBenchmarkGraderIdentitySchema = z
|
|
54
|
+
.object({
|
|
55
|
+
name: z.string().min(1),
|
|
56
|
+
version: z.string().min(1),
|
|
57
|
+
format: z.literal("tangle-grader"),
|
|
58
|
+
artifact: agentCandidateArtifactRefSchema,
|
|
59
|
+
})
|
|
60
|
+
.strict()
|
|
61
|
+
.superRefine((grader, ctx) => {
|
|
62
|
+
if (grader.artifact.byteLength === 0) {
|
|
63
|
+
ctx.addIssue({
|
|
64
|
+
code: "custom",
|
|
65
|
+
path: ["artifact", "byteLength"],
|
|
66
|
+
message: "pinned grader artifact must contain executable grader bytes",
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
export const agentCandidateTaskRepositorySchema = z
|
|
71
|
+
.object({
|
|
72
|
+
identity: z.string().min(1).max(512),
|
|
73
|
+
rootIdentity: z.string().min(1).max(512),
|
|
74
|
+
baseCommit: gitObjectSchema,
|
|
75
|
+
baseTree: gitObjectSchema,
|
|
76
|
+
})
|
|
77
|
+
.strict()
|
|
78
|
+
.superRefine((repository, ctx) => {
|
|
79
|
+
if (!sameGitObjectFormat(repository.baseCommit, repository.baseTree)) {
|
|
80
|
+
ctx.addIssue({
|
|
81
|
+
code: "custom",
|
|
82
|
+
message: "task Git object ids must use one object format",
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
export const agentCandidateResolvedTaskContainerSchema = agentCandidateContainerSchema
|
|
87
|
+
.extend({
|
|
88
|
+
source: z.literal("evaluator-task-container"),
|
|
89
|
+
manifestDigest: sha256DigestSchema,
|
|
90
|
+
platform: z
|
|
91
|
+
.object({
|
|
92
|
+
os: z.string().min(1).max(100),
|
|
93
|
+
architecture: z.string().min(1).max(100),
|
|
94
|
+
variant: z.string().min(1).max(100).optional(),
|
|
95
|
+
})
|
|
96
|
+
.strict(),
|
|
97
|
+
})
|
|
98
|
+
.strict();
|
|
99
|
+
export const agentCandidateRetryPolicySchema = z.enum([
|
|
100
|
+
"pre-model-infrastructure-only",
|
|
101
|
+
"none",
|
|
102
|
+
]);
|
|
61
103
|
export const agentCandidateProfilePlanMaterialSchema = z
|
|
62
104
|
.object({
|
|
105
|
+
sourceProfileDigest: sha256DigestSchema,
|
|
63
106
|
harness: harnessTypeSchema,
|
|
64
107
|
files: z.array(z
|
|
65
108
|
.object({
|
|
@@ -72,6 +115,7 @@ export const agentCandidateProfilePlanMaterialSchema = z
|
|
|
72
115
|
.strict()),
|
|
73
116
|
env: environmentConfigSchema,
|
|
74
117
|
flags: z.array(agentCandidateConfigValueSchema),
|
|
118
|
+
systemPrompt: agentCandidateConfigValueSchema.optional(),
|
|
75
119
|
unsupported: z
|
|
76
120
|
.array(z
|
|
77
121
|
.object({
|
|
@@ -149,45 +193,35 @@ export const agentCandidateTaskOutcomeSpecSchema = z.discriminatedUnion("kind",
|
|
|
149
193
|
})
|
|
150
194
|
.strict(),
|
|
151
195
|
]);
|
|
196
|
+
export const agentCandidateBenchmarkCellRefSchema = z
|
|
197
|
+
.object({
|
|
198
|
+
suiteDigest: sha256DigestSchema,
|
|
199
|
+
taskIndex: z.number().int().nonnegative(),
|
|
200
|
+
repetition: z.number().int().nonnegative(),
|
|
201
|
+
})
|
|
202
|
+
.strict();
|
|
203
|
+
export const agentCandidateRunCellMaterialSchema = z
|
|
204
|
+
.object({
|
|
205
|
+
kind: z.literal("agent-candidate-run-cell"),
|
|
206
|
+
experimentDigest: sha256DigestSchema,
|
|
207
|
+
arm: z.enum(["baseline", "candidate"]),
|
|
208
|
+
bundleDigest: sha256DigestSchema,
|
|
209
|
+
suiteDigest: sha256DigestSchema,
|
|
210
|
+
taskDigest: sha256DigestSchema,
|
|
211
|
+
taskIndex: z.number().int().nonnegative(),
|
|
212
|
+
repetition: z.number().int().nonnegative(),
|
|
213
|
+
seed: z.number().int().safe(),
|
|
214
|
+
attempt: z.number().int().positive(),
|
|
215
|
+
})
|
|
216
|
+
.strict();
|
|
217
|
+
export const agentCandidateRunCellSchema = agentCandidateRunCellMaterialSchema
|
|
218
|
+
.extend({ digest: sha256DigestSchema })
|
|
219
|
+
.strict();
|
|
152
220
|
export const agentCandidateExecutionPlanMaterialSchema = z
|
|
153
221
|
.object({
|
|
154
222
|
kind: z.literal("agent-candidate-execution-plan-material"),
|
|
155
|
-
|
|
223
|
+
runCell: agentCandidateRunCellSchema,
|
|
156
224
|
executionId: z.string().min(1),
|
|
157
|
-
attempt: z
|
|
158
|
-
.object({
|
|
159
|
-
number: z.number().int().min(1),
|
|
160
|
-
maxAttempts: z.number().int().min(1),
|
|
161
|
-
retryPolicy: z.enum(["pre-model-infrastructure-only", "none"]),
|
|
162
|
-
})
|
|
163
|
-
.strict(),
|
|
164
|
-
task: z
|
|
165
|
-
.object({
|
|
166
|
-
benchmark: z.string().min(1),
|
|
167
|
-
benchmarkVersion: z.string().min(1),
|
|
168
|
-
taskId: z.string().min(1),
|
|
169
|
-
splitDigest: sha256DigestSchema,
|
|
170
|
-
instruction: z
|
|
171
|
-
.object({
|
|
172
|
-
encoding: z.literal("utf8"),
|
|
173
|
-
sha256: sha256DigestSchema,
|
|
174
|
-
byteLength: z.number().int().positive(),
|
|
175
|
-
delivery: agentCandidateInstructionDeliverySchema,
|
|
176
|
-
})
|
|
177
|
-
.strict(),
|
|
178
|
-
repository: z
|
|
179
|
-
.object({
|
|
180
|
-
identity: z.string().min(1),
|
|
181
|
-
rootIdentity: z.string().min(1),
|
|
182
|
-
baseCommit: gitObjectSchema,
|
|
183
|
-
baseTree: gitObjectSchema,
|
|
184
|
-
})
|
|
185
|
-
.strict()
|
|
186
|
-
.optional(),
|
|
187
|
-
outcome: agentCandidateTaskOutcomeSpecSchema,
|
|
188
|
-
workspace: agentCandidateWorkspaceSnapshotEvidenceSchema,
|
|
189
|
-
})
|
|
190
|
-
.strict(),
|
|
191
225
|
workspaces: z
|
|
192
226
|
.object({
|
|
193
227
|
taskRoot: z
|
|
@@ -212,6 +246,8 @@ export const agentCandidateExecutionPlanMaterialSchema = z
|
|
|
212
246
|
.strict(),
|
|
213
247
|
harness: harnessTypeSchema,
|
|
214
248
|
harnessVersion: z.string().min(1),
|
|
249
|
+
instructionDelivery: agentCandidateInstructionDeliverySchema,
|
|
250
|
+
limits: agentCandidateExecutionLimitsSchema,
|
|
215
251
|
container: agentCandidateContainerSchema
|
|
216
252
|
.extend({
|
|
217
253
|
source: z.enum(["pinned-container", "evaluator-task-container"]),
|
|
@@ -268,13 +304,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z
|
|
|
268
304
|
.min(1),
|
|
269
305
|
})
|
|
270
306
|
.strict(),
|
|
271
|
-
grader: z
|
|
272
|
-
.object({
|
|
273
|
-
name: z.string().min(1),
|
|
274
|
-
version: z.string().min(1),
|
|
275
|
-
artifact: agentCandidateArtifactRefSchema,
|
|
276
|
-
})
|
|
277
|
-
.strict(),
|
|
278
307
|
launch: z
|
|
279
308
|
.object({
|
|
280
309
|
executable: z
|
|
@@ -287,42 +316,10 @@ export const agentCandidateExecutionPlanMaterialSchema = z
|
|
|
287
316
|
.strict(),
|
|
288
317
|
knowledgeManifestDigest: sha256DigestSchema.optional(),
|
|
289
318
|
memory: agentCandidateEffectiveMemorySchema,
|
|
290
|
-
limits: agentCandidateExecutionLimitsSchema,
|
|
291
319
|
network: z.object({ mode: z.literal("disabled") }).strict(),
|
|
292
320
|
})
|
|
293
321
|
.strict()
|
|
294
322
|
.superRefine((material, ctx) => {
|
|
295
|
-
if (material.task.repository !== undefined &&
|
|
296
|
-
!sameGitObjectFormat(material.task.repository.baseCommit, material.task.repository.baseTree)) {
|
|
297
|
-
ctx.addIssue({
|
|
298
|
-
code: "custom",
|
|
299
|
-
path: ["task", "repository"],
|
|
300
|
-
message: "task Git object ids must use one object format",
|
|
301
|
-
});
|
|
302
|
-
}
|
|
303
|
-
if (material.task.outcome.kind === "workspace" &&
|
|
304
|
-
material.task.repository === undefined) {
|
|
305
|
-
ctx.addIssue({
|
|
306
|
-
code: "custom",
|
|
307
|
-
path: ["task", "repository"],
|
|
308
|
-
message: "workspace outcomes require task repository provenance",
|
|
309
|
-
});
|
|
310
|
-
}
|
|
311
|
-
if (material.attempt.number > material.attempt.maxAttempts) {
|
|
312
|
-
ctx.addIssue({
|
|
313
|
-
code: "custom",
|
|
314
|
-
path: ["attempt", "number"],
|
|
315
|
-
message: "attempt number cannot exceed the frozen maximum",
|
|
316
|
-
});
|
|
317
|
-
}
|
|
318
|
-
if (material.attempt.retryPolicy === "none" &&
|
|
319
|
-
material.attempt.maxAttempts !== 1) {
|
|
320
|
-
ctx.addIssue({
|
|
321
|
-
code: "custom",
|
|
322
|
-
path: ["attempt", "maxAttempts"],
|
|
323
|
-
message: "a no-retry plan must allow exactly one attempt",
|
|
324
|
-
});
|
|
325
|
-
}
|
|
326
323
|
const routeIds = material.model.routes.map((route) => route.kind === "mode" || route.kind === "subagent"
|
|
327
324
|
? `${route.kind}:${route.name}`
|
|
328
325
|
: route.kind);
|
|
@@ -356,20 +353,6 @@ export const agentCandidateExecutionPlanMaterialSchema = z
|
|
|
356
353
|
}
|
|
357
354
|
}
|
|
358
355
|
const modelNetwork = material.model.access.network;
|
|
359
|
-
if (material.limits.maxModelCalls === 0 && modelNetwork.mode !== "disabled") {
|
|
360
|
-
ctx.addIssue({
|
|
361
|
-
code: "custom",
|
|
362
|
-
path: ["model", "access", "network"],
|
|
363
|
-
message: "zero-call plans cannot expose a model gateway",
|
|
364
|
-
});
|
|
365
|
-
}
|
|
366
|
-
if (material.limits.maxModelCalls > 0 && modelNetwork.mode !== "gateway-only") {
|
|
367
|
-
ctx.addIssue({
|
|
368
|
-
code: "custom",
|
|
369
|
-
path: ["model", "access", "network"],
|
|
370
|
-
message: "model-calling plans require one frozen gateway allowlist",
|
|
371
|
-
});
|
|
372
|
-
}
|
|
373
356
|
if (modelNetwork.mode === "gateway-only") {
|
|
374
357
|
addDuplicateIssues(modelNetwork.domains, ["model", "access", "network", "domains"], ctx);
|
|
375
358
|
for (let index = 1; index < modelNetwork.domains.length; index++) {
|
|
@@ -405,62 +388,46 @@ export const agentCandidateExecutionPlanMaterialSchema = z
|
|
|
405
388
|
message: "task and candidate workspace roots must be disjoint",
|
|
406
389
|
});
|
|
407
390
|
}
|
|
408
|
-
if (material.launch.cwd.workspace === "candidate" &&
|
|
409
|
-
material.workspaces.candidateRoot === undefined) {
|
|
410
|
-
ctx.addIssue({
|
|
411
|
-
code: "custom",
|
|
412
|
-
path: ["launch", "cwd", "workspace"],
|
|
413
|
-
message: "candidate cwd requires a pinned candidate workspace root",
|
|
414
|
-
});
|
|
415
|
-
}
|
|
416
|
-
if (material.profile.targetWorkspace === "candidate" &&
|
|
417
|
-
material.workspaces.candidateRoot === undefined) {
|
|
418
|
-
ctx.addIssue({
|
|
419
|
-
code: "custom",
|
|
420
|
-
path: ["profile", "targetWorkspace"],
|
|
421
|
-
message: "candidate-targeted profile files require a candidate workspace root",
|
|
422
|
-
});
|
|
423
|
-
}
|
|
424
|
-
const delivery = material.task.instruction.delivery;
|
|
425
391
|
const taskPath = material.launch.env.TANGLE_CANDIDATE_TASK_PATH;
|
|
426
|
-
if (
|
|
427
|
-
if (taskPath
|
|
392
|
+
if (taskPath !== undefined) {
|
|
393
|
+
if (taskPath.value !== "/tangle/input/task.txt") {
|
|
428
394
|
ctx.addIssue({
|
|
429
395
|
code: "custom",
|
|
430
396
|
path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"],
|
|
431
|
-
message: "file delivery
|
|
397
|
+
message: "task file delivery must use the fixed evaluator-owned path",
|
|
432
398
|
});
|
|
433
399
|
}
|
|
434
|
-
if (pathsOverlap(material.workspaces.taskRoot
|
|
400
|
+
if (pathsOverlap(taskPath.value, material.workspaces.taskRoot) ||
|
|
435
401
|
(material.workspaces.candidateRoot !== undefined &&
|
|
436
|
-
pathsOverlap(material.workspaces.candidateRoot
|
|
402
|
+
pathsOverlap(taskPath.value, material.workspaces.candidateRoot))) {
|
|
437
403
|
ctx.addIssue({
|
|
438
404
|
code: "custom",
|
|
439
|
-
path: ["
|
|
440
|
-
message: "
|
|
405
|
+
path: ["launch", "env", "TANGLE_CANDIDATE_TASK_PATH"],
|
|
406
|
+
message: "task file delivery must be outside both workspaces",
|
|
441
407
|
});
|
|
442
408
|
}
|
|
443
409
|
}
|
|
444
|
-
|
|
410
|
+
if (material.launch.cwd.workspace === "candidate" &&
|
|
411
|
+
material.workspaces.candidateRoot === undefined) {
|
|
445
412
|
ctx.addIssue({
|
|
446
413
|
code: "custom",
|
|
447
|
-
path: ["launch", "
|
|
448
|
-
message: "
|
|
414
|
+
path: ["launch", "cwd", "workspace"],
|
|
415
|
+
message: "candidate cwd requires a pinned candidate workspace root",
|
|
449
416
|
});
|
|
450
417
|
}
|
|
451
|
-
if (
|
|
418
|
+
if (material.profile.targetWorkspace === "candidate" &&
|
|
419
|
+
material.workspaces.candidateRoot === undefined) {
|
|
452
420
|
ctx.addIssue({
|
|
453
421
|
code: "custom",
|
|
454
|
-
path: ["
|
|
455
|
-
message: "
|
|
422
|
+
path: ["profile", "targetWorkspace"],
|
|
423
|
+
message: "candidate-targeted profile files require a candidate workspace root",
|
|
456
424
|
});
|
|
457
425
|
}
|
|
458
|
-
if (material.
|
|
459
|
-
material.task.workspace.material.files.length === 0) {
|
|
426
|
+
if (activeCode && material.candidateWorkspace?.material.files.length === 0) {
|
|
460
427
|
ctx.addIssue({
|
|
461
428
|
code: "custom",
|
|
462
|
-
path: ["
|
|
463
|
-
message: "
|
|
429
|
+
path: ["candidateWorkspace", "material", "files"],
|
|
430
|
+
message: "active candidate workspaces cannot be empty",
|
|
464
431
|
});
|
|
465
432
|
}
|
|
466
433
|
if (!isCanonicalJsonValue(material)) {
|
|
@@ -120,13 +120,6 @@ export declare const agentCandidateMemoryPolicySchema: z.ZodDiscriminatedUnion<[
|
|
|
120
120
|
byteLength: z.ZodNumber;
|
|
121
121
|
}, z.core.$strict>>;
|
|
122
122
|
}, z.core.$strict>], "mode">;
|
|
123
|
-
export declare const agentCandidateSpendSchema: z.ZodObject<{
|
|
124
|
-
costUsd: z.ZodNumber;
|
|
125
|
-
inputTokens: z.ZodNumber;
|
|
126
|
-
outputTokens: z.ZodNumber;
|
|
127
|
-
cachedInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
128
|
-
modelCalls: z.ZodNumber;
|
|
129
|
-
}, z.core.$strict>;
|
|
130
123
|
export declare const agentCandidateLineageSchema: z.ZodObject<{
|
|
131
124
|
source: z.ZodEnum<{
|
|
132
125
|
optimizer: "optimizer";
|
|
@@ -138,25 +131,5 @@ export declare const agentCandidateLineageSchema: z.ZodObject<{
|
|
|
138
131
|
runIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
139
132
|
profileDiffIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
140
133
|
modelSnapshots: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
141
|
-
|
|
142
|
-
name: z.ZodString;
|
|
143
|
-
version: z.ZodString;
|
|
144
|
-
splitDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
145
|
-
}, z.core.$strict>>;
|
|
146
|
-
spend: z.ZodOptional<z.ZodObject<{
|
|
147
|
-
proposal: z.ZodObject<{
|
|
148
|
-
costUsd: z.ZodNumber;
|
|
149
|
-
inputTokens: z.ZodNumber;
|
|
150
|
-
outputTokens: z.ZodNumber;
|
|
151
|
-
cachedInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
152
|
-
modelCalls: z.ZodNumber;
|
|
153
|
-
}, z.core.$strict>;
|
|
154
|
-
evaluation: z.ZodObject<{
|
|
155
|
-
costUsd: z.ZodNumber;
|
|
156
|
-
inputTokens: z.ZodNumber;
|
|
157
|
-
outputTokens: z.ZodNumber;
|
|
158
|
-
cachedInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
159
|
-
modelCalls: z.ZodNumber;
|
|
160
|
-
}, z.core.$strict>;
|
|
161
|
-
}, z.core.$strict>>;
|
|
134
|
+
developmentSplitDigest: z.ZodOptional<z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>>;
|
|
162
135
|
}, z.core.$strict>;
|
|
@@ -30,15 +30,6 @@ export const agentCandidateMemoryPolicySchema = z.discriminatedUnion("mode", [
|
|
|
30
30
|
})
|
|
31
31
|
.strict(),
|
|
32
32
|
]);
|
|
33
|
-
export const agentCandidateSpendSchema = z
|
|
34
|
-
.object({
|
|
35
|
-
costUsd: z.number().finite().nonnegative(),
|
|
36
|
-
inputTokens: z.number().int().nonnegative(),
|
|
37
|
-
outputTokens: z.number().int().nonnegative(),
|
|
38
|
-
cachedInputTokens: z.number().int().nonnegative().optional(),
|
|
39
|
-
modelCalls: z.number().int().nonnegative(),
|
|
40
|
-
})
|
|
41
|
-
.strict();
|
|
42
33
|
export const agentCandidateLineageSchema = z
|
|
43
34
|
.object({
|
|
44
35
|
source: z.enum(["optimizer", "human", "import", "compound"]),
|
|
@@ -46,21 +37,7 @@ export const agentCandidateLineageSchema = z
|
|
|
46
37
|
runIds: z.array(z.string().min(1)).optional(),
|
|
47
38
|
profileDiffIds: z.array(z.string().min(1)).optional(),
|
|
48
39
|
modelSnapshots: z.array(z.string().min(1)).optional(),
|
|
49
|
-
|
|
50
|
-
.object({
|
|
51
|
-
name: z.string().min(1),
|
|
52
|
-
version: z.string().min(1),
|
|
53
|
-
splitDigest: sha256DigestSchema,
|
|
54
|
-
})
|
|
55
|
-
.strict()
|
|
56
|
-
.optional(),
|
|
57
|
-
spend: z
|
|
58
|
-
.object({
|
|
59
|
-
proposal: agentCandidateSpendSchema,
|
|
60
|
-
evaluation: agentCandidateSpendSchema,
|
|
61
|
-
})
|
|
62
|
-
.strict()
|
|
63
|
-
.optional(),
|
|
40
|
+
developmentSplitDigest: sha256DigestSchema.optional(),
|
|
64
41
|
})
|
|
65
42
|
.strict()
|
|
66
43
|
.superRefine((lineage, ctx) => {
|
|
@@ -91,19 +68,12 @@ export const agentCandidateLineageSchema = z
|
|
|
91
68
|
message: "generated candidates must name their producing run",
|
|
92
69
|
});
|
|
93
70
|
}
|
|
94
|
-
if (lineage.
|
|
71
|
+
if (lineage.developmentSplitDigest === undefined) {
|
|
95
72
|
ctx.addIssue({
|
|
96
73
|
code: "custom",
|
|
97
|
-
path: ["
|
|
74
|
+
path: ["developmentSplitDigest"],
|
|
98
75
|
message: "generated candidates must pin their development split",
|
|
99
76
|
});
|
|
100
77
|
}
|
|
101
|
-
if (lineage.spend === undefined) {
|
|
102
|
-
ctx.addIssue({
|
|
103
|
-
code: "custom",
|
|
104
|
-
path: ["spend"],
|
|
105
|
-
message: "generated candidates must record proposal and evaluation spend",
|
|
106
|
-
});
|
|
107
|
-
}
|
|
108
78
|
}
|
|
109
79
|
});
|
|
@@ -522,15 +522,10 @@ export declare const agentCandidateBenchmarkResultMaterialSchema: z.ZodObject<{
|
|
|
522
522
|
kind: z.ZodLiteral<"agent-candidate-benchmark-result-material">;
|
|
523
523
|
executionPlanDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
524
524
|
taskOutcomeDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
525
|
-
benchmark: z.ZodObject<{
|
|
526
|
-
name: z.ZodString;
|
|
527
|
-
version: z.ZodString;
|
|
528
|
-
taskId: z.ZodString;
|
|
529
|
-
splitDigest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
530
|
-
}, z.core.$strict>;
|
|
531
525
|
grader: z.ZodObject<{
|
|
532
526
|
name: z.ZodString;
|
|
533
527
|
version: z.ZodString;
|
|
528
|
+
format: z.ZodLiteral<"tangle-grader">;
|
|
534
529
|
artifact: z.ZodObject<{
|
|
535
530
|
locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
536
531
|
kind: z.ZodLiteral<"s3">;
|
|
@@ -560,6 +555,21 @@ export declare const agentCandidateBenchmarkResultMaterialSchema: z.ZodObject<{
|
|
|
560
555
|
sha256: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
561
556
|
byteLength: z.ZodNumber;
|
|
562
557
|
}, z.core.$strict>;
|
|
558
|
+
grading: z.ZodObject<{
|
|
559
|
+
usage: z.ZodObject<{
|
|
560
|
+
inputTokens: z.ZodNumber;
|
|
561
|
+
outputTokens: z.ZodNumber;
|
|
562
|
+
cachedInputTokens: z.ZodNumber;
|
|
563
|
+
reasoningTokens: z.ZodNumber;
|
|
564
|
+
modelCalls: z.ZodNumber;
|
|
565
|
+
costUsdNanos: z.ZodNumber;
|
|
566
|
+
}, z.core.$strict>;
|
|
567
|
+
timing: z.ZodObject<{
|
|
568
|
+
startedAtMs: z.ZodNumber;
|
|
569
|
+
endedAtMs: z.ZodNumber;
|
|
570
|
+
durationMs: z.ZodNumber;
|
|
571
|
+
}, z.core.$strict>;
|
|
572
|
+
}, z.core.$strict>;
|
|
563
573
|
score: z.ZodNumber;
|
|
564
574
|
passed: z.ZodBoolean;
|
|
565
575
|
dimensions: z.ZodArray<z.ZodObject<{
|
|
@@ -574,15 +584,10 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
|
|
|
574
584
|
kind: "agent-candidate-benchmark-result-material";
|
|
575
585
|
executionPlanDigest: `sha256:${string}`;
|
|
576
586
|
taskOutcomeDigest: `sha256:${string}`;
|
|
577
|
-
benchmark: {
|
|
578
|
-
name: string;
|
|
579
|
-
version: string;
|
|
580
|
-
taskId: string;
|
|
581
|
-
splitDigest: `sha256:${string}`;
|
|
582
|
-
};
|
|
583
587
|
grader: {
|
|
584
588
|
name: string;
|
|
585
589
|
version: string;
|
|
590
|
+
format: "tangle-grader";
|
|
586
591
|
artifact: {
|
|
587
592
|
locator: {
|
|
588
593
|
kind: "s3";
|
|
@@ -612,6 +617,21 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
|
|
|
612
617
|
sha256: `sha256:${string}`;
|
|
613
618
|
byteLength: number;
|
|
614
619
|
};
|
|
620
|
+
grading: {
|
|
621
|
+
usage: {
|
|
622
|
+
inputTokens: number;
|
|
623
|
+
outputTokens: number;
|
|
624
|
+
cachedInputTokens: number;
|
|
625
|
+
reasoningTokens: number;
|
|
626
|
+
modelCalls: number;
|
|
627
|
+
costUsdNanos: number;
|
|
628
|
+
};
|
|
629
|
+
timing: {
|
|
630
|
+
startedAtMs: number;
|
|
631
|
+
endedAtMs: number;
|
|
632
|
+
durationMs: number;
|
|
633
|
+
};
|
|
634
|
+
};
|
|
615
635
|
score: number;
|
|
616
636
|
passed: boolean;
|
|
617
637
|
dimensions: {
|
|
@@ -622,15 +642,10 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
|
|
|
622
642
|
kind: "agent-candidate-benchmark-result-material";
|
|
623
643
|
executionPlanDigest: `sha256:${string}`;
|
|
624
644
|
taskOutcomeDigest: `sha256:${string}`;
|
|
625
|
-
benchmark: {
|
|
626
|
-
name: string;
|
|
627
|
-
version: string;
|
|
628
|
-
taskId: string;
|
|
629
|
-
splitDigest: `sha256:${string}`;
|
|
630
|
-
};
|
|
631
645
|
grader: {
|
|
632
646
|
name: string;
|
|
633
647
|
version: string;
|
|
648
|
+
format: "tangle-grader";
|
|
634
649
|
artifact: {
|
|
635
650
|
locator: {
|
|
636
651
|
kind: "s3";
|
|
@@ -660,6 +675,21 @@ export declare const agentCandidateBenchmarkResultEvidenceSchema: z.ZodObject<{
|
|
|
660
675
|
sha256: `sha256:${string}`;
|
|
661
676
|
byteLength: number;
|
|
662
677
|
};
|
|
678
|
+
grading: {
|
|
679
|
+
usage: {
|
|
680
|
+
inputTokens: number;
|
|
681
|
+
outputTokens: number;
|
|
682
|
+
cachedInputTokens: number;
|
|
683
|
+
reasoningTokens: number;
|
|
684
|
+
modelCalls: number;
|
|
685
|
+
costUsdNanos: number;
|
|
686
|
+
};
|
|
687
|
+
timing: {
|
|
688
|
+
startedAtMs: number;
|
|
689
|
+
endedAtMs: number;
|
|
690
|
+
durationMs: number;
|
|
691
|
+
};
|
|
692
|
+
};
|
|
663
693
|
score: number;
|
|
664
694
|
passed: boolean;
|
|
665
695
|
dimensions: {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { agentCandidateArtifactRefSchema, agentCandidateCapturedArtifactSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js";
|
|
3
|
-
import { agentCandidateResolvedModelSchema } from "./agent-candidate-execution-plan-schema.js";
|
|
3
|
+
import { agentCandidateBenchmarkGraderIdentitySchema, agentCandidateResolvedModelSchema, } from "./agent-candidate-execution-plan-schema.js";
|
|
4
4
|
import { agentCandidateMediaTypeSchema, gitObjectSchema, isCanonicalJsonValue, sameGitObjectFormat, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
|
|
5
5
|
const safeCountSchema = z
|
|
6
6
|
.number()
|
|
@@ -265,35 +265,26 @@ export const agentCandidateBenchmarkResultMaterialSchema = z
|
|
|
265
265
|
kind: z.literal("agent-candidate-benchmark-result-material"),
|
|
266
266
|
executionPlanDigest: sha256DigestSchema,
|
|
267
267
|
taskOutcomeDigest: sha256DigestSchema,
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
version: z.string().min(1),
|
|
272
|
-
taskId: z.string().min(1),
|
|
273
|
-
splitDigest: sha256DigestSchema,
|
|
274
|
-
})
|
|
275
|
-
.strict(),
|
|
276
|
-
grader: z
|
|
268
|
+
grader: agentCandidateBenchmarkGraderIdentitySchema,
|
|
269
|
+
evidence: agentCandidateArtifactRefSchema,
|
|
270
|
+
grading: z
|
|
277
271
|
.object({
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
272
|
+
usage: agentCandidateFixedSpendSchema,
|
|
273
|
+
timing: z
|
|
274
|
+
.object({
|
|
275
|
+
startedAtMs: safeCountSchema,
|
|
276
|
+
endedAtMs: safeCountSchema,
|
|
277
|
+
durationMs: safeCountSchema,
|
|
278
|
+
})
|
|
279
|
+
.strict(),
|
|
281
280
|
})
|
|
282
281
|
.strict(),
|
|
283
|
-
evidence: agentCandidateArtifactRefSchema,
|
|
284
282
|
score: normalizedScoreSchema,
|
|
285
283
|
passed: z.boolean(),
|
|
286
284
|
dimensions: z.array(agentCandidateBenchmarkDimensionSchema),
|
|
287
285
|
})
|
|
288
286
|
.strict()
|
|
289
287
|
.superRefine((material, ctx) => {
|
|
290
|
-
if (material.grader.artifact.byteLength === 0) {
|
|
291
|
-
ctx.addIssue({
|
|
292
|
-
code: "custom",
|
|
293
|
-
path: ["grader", "artifact", "byteLength"],
|
|
294
|
-
message: "pinned grader artifact must contain executable grader bytes",
|
|
295
|
-
});
|
|
296
|
-
}
|
|
297
288
|
if (material.evidence.byteLength === 0) {
|
|
298
289
|
ctx.addIssue({
|
|
299
290
|
code: "custom",
|
|
@@ -301,11 +292,15 @@ export const agentCandidateBenchmarkResultMaterialSchema = z
|
|
|
301
292
|
message: "benchmark result must contain non-empty durable grading evidence",
|
|
302
293
|
});
|
|
303
294
|
}
|
|
304
|
-
if (material.
|
|
295
|
+
if (material.grading.timing.endedAtMs <
|
|
296
|
+
material.grading.timing.startedAtMs ||
|
|
297
|
+
material.grading.timing.durationMs !==
|
|
298
|
+
material.grading.timing.endedAtMs -
|
|
299
|
+
material.grading.timing.startedAtMs) {
|
|
305
300
|
ctx.addIssue({
|
|
306
301
|
code: "custom",
|
|
307
|
-
path: ["
|
|
308
|
-
message: "
|
|
302
|
+
path: ["grading", "timing"],
|
|
303
|
+
message: "grader timing must be ordered and internally consistent",
|
|
309
304
|
});
|
|
310
305
|
}
|
|
311
306
|
for (let index = 1; index < material.dimensions.length; index++) {
|
|
@@ -1,18 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
command: z.ZodOptional<z.ZodString>;
|
|
5
|
-
args: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
6
|
-
kind: z.ZodLiteral<"public">;
|
|
7
|
-
value: z.ZodString;
|
|
8
|
-
}, z.core.$strict>>>;
|
|
9
|
-
env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
10
|
-
kind: z.ZodLiteral<"public">;
|
|
11
|
-
value: z.ZodString;
|
|
12
|
-
}, z.core.$strict>>>;
|
|
13
|
-
cwd: z.ZodOptional<z.ZodString>;
|
|
14
|
-
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
15
|
-
}, z.core.$strict>;
|
|
2
|
+
import type { AgentCandidateMcpServer } from "./agent-candidate.js";
|
|
3
|
+
export declare const agentCandidateMcpServerSchema: z.ZodType<AgentCandidateMcpServer>;
|
|
16
4
|
export declare const agentCandidateHookCommandSchema: z.ZodObject<{
|
|
17
5
|
executable: z.ZodString;
|
|
18
6
|
args: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
@@ -78,20 +66,7 @@ export declare const agentCandidateProfileSchema: z.ZodObject<{
|
|
|
78
66
|
deny: "deny";
|
|
79
67
|
}>>]>>>;
|
|
80
68
|
tools: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodBoolean>>;
|
|
81
|
-
mcp: z.ZodOptional<z.ZodRecord<z.ZodString, z.
|
|
82
|
-
transport: z.ZodOptional<z.ZodLiteral<"stdio">>;
|
|
83
|
-
command: z.ZodOptional<z.ZodString>;
|
|
84
|
-
args: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
85
|
-
kind: z.ZodLiteral<"public">;
|
|
86
|
-
value: z.ZodString;
|
|
87
|
-
}, z.core.$strict>>>;
|
|
88
|
-
env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
89
|
-
kind: z.ZodLiteral<"public">;
|
|
90
|
-
value: z.ZodString;
|
|
91
|
-
}, z.core.$strict>>>;
|
|
92
|
-
cwd: z.ZodOptional<z.ZodString>;
|
|
93
|
-
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
94
|
-
}, z.core.$strict>>>;
|
|
69
|
+
mcp: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<AgentCandidateMcpServer, unknown, z.core.$ZodTypeInternals<AgentCandidateMcpServer, unknown>>>>;
|
|
95
70
|
subagents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
96
71
|
prompt: z.ZodOptional<z.ZodString>;
|
|
97
72
|
permissions: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodEnum<{
|