@tangle-network/agent-interface 0.27.2 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-candidate-artifact-schema.d.ts +0 -3
- package/dist/agent-candidate-artifact-schema.js +0 -2
- package/dist/agent-candidate-code-schema.d.ts +0 -10
- package/dist/agent-candidate-code-schema.js +0 -1
- package/dist/agent-candidate-execution-plan-schema.d.ts +173 -382
- package/dist/agent-candidate-execution-plan-schema.js +100 -138
- package/dist/agent-candidate-lineage-schema.d.ts +1 -31
- package/dist/agent-candidate-lineage-schema.js +3 -34
- package/dist/agent-candidate-outcome-schema.d.ts +48 -36
- package/dist/agent-candidate-outcome-schema.js +23 -32
- package/dist/agent-candidate-promotion-schema.d.ts +11021 -2411
- package/dist/agent-candidate-promotion-schema.js +495 -109
- package/dist/agent-candidate-receipt-schema.d.ts +260 -344
- package/dist/agent-candidate-receipt-schema.js +82 -11
- package/dist/agent-candidate-schema-common.d.ts +8 -0
- package/dist/agent-candidate-schema-common.js +35 -1
- package/dist/agent-candidate-schema.d.ts +2 -44
- package/dist/agent-candidate-schema.js +3 -29
- package/dist/agent-candidate-task-schema.d.ts +643 -0
- package/dist/agent-candidate-task-schema.js +191 -0
- package/dist/agent-candidate.d.ts +173 -94
- package/dist/agent-candidate.test-fixture.d.ts +0 -32
- package/dist/agent-candidate.test-fixture.js +0 -32
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -1
- package/dist/profile-diff.d.ts +0 -1
- package/dist/profile-schema.d.ts +9 -0
- package/dist/profile-schema.js +14 -6
- package/package.json +2 -2
- package/dist/agent-candidate-compat.d.ts +0 -1918
- package/dist/agent-candidate-compat.js +0 -340
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { agentCandidateArtifactRefSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js";
|
|
3
|
+
import { agentCandidateBenchmarkGraderIdentitySchema, agentCandidateBenchmarkCellRefSchema, agentCandidateExecutionLimitsSchema, agentCandidateResolvedModelSchema, agentCandidateResolvedTaskContainerSchema, agentCandidateRetryPolicySchema, agentCandidateTaskRepositorySchema, agentCandidateTaskOutcomeSpecSchema, } from "./agent-candidate-execution-plan-schema.js";
|
|
4
|
+
import { isCanonicalJsonValue, isWellFormedUnicode, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
|
|
5
|
+
export const agentCandidateBenchmarkTaskMaterialSchema = z
|
|
6
|
+
.object({
|
|
7
|
+
kind: z.literal("agent-candidate-benchmark-task"),
|
|
8
|
+
digestAlgorithm: z.literal("rfc8785-sha256"),
|
|
9
|
+
benchmark: z
|
|
10
|
+
.object({
|
|
11
|
+
name: z.string().min(1).max(512),
|
|
12
|
+
version: z.string().min(1).max(256),
|
|
13
|
+
splitDigest: sha256DigestSchema,
|
|
14
|
+
})
|
|
15
|
+
.strict(),
|
|
16
|
+
scenario: z
|
|
17
|
+
.object({
|
|
18
|
+
id: z.string().min(1).max(512),
|
|
19
|
+
kind: z.string().min(1).max(512),
|
|
20
|
+
scenarioDigest: sha256DigestSchema,
|
|
21
|
+
})
|
|
22
|
+
.strict(),
|
|
23
|
+
datasetSnapshot: agentCandidateArtifactRefSchema.optional(),
|
|
24
|
+
instruction: z
|
|
25
|
+
.string()
|
|
26
|
+
.min(1)
|
|
27
|
+
.max(4 * 1024 * 1024)
|
|
28
|
+
.refine(isWellFormedUnicode, "task instruction must be well-formed Unicode"),
|
|
29
|
+
repository: agentCandidateTaskRepositorySchema.optional(),
|
|
30
|
+
outcome: agentCandidateTaskOutcomeSpecSchema,
|
|
31
|
+
workspace: agentCandidateWorkspaceSnapshotEvidenceSchema,
|
|
32
|
+
grader: agentCandidateBenchmarkGraderIdentitySchema,
|
|
33
|
+
model: agentCandidateResolvedModelSchema,
|
|
34
|
+
attempt: z
|
|
35
|
+
.object({
|
|
36
|
+
maxAttempts: z.number().int().min(1),
|
|
37
|
+
retryPolicy: agentCandidateRetryPolicySchema,
|
|
38
|
+
})
|
|
39
|
+
.strict(),
|
|
40
|
+
evaluatorTaskContainer: agentCandidateResolvedTaskContainerSchema.optional(),
|
|
41
|
+
limits: agentCandidateExecutionLimitsSchema,
|
|
42
|
+
})
|
|
43
|
+
.strict()
|
|
44
|
+
.superRefine((task, ctx) => {
|
|
45
|
+
if (!isCanonicalJsonValue(task)) {
|
|
46
|
+
ctx.addIssue({
|
|
47
|
+
code: "custom",
|
|
48
|
+
message: "candidate benchmark task must contain finite, acyclic canonical JSON",
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
if (task.outcome.kind === "workspace" && task.repository === undefined) {
|
|
52
|
+
ctx.addIssue({
|
|
53
|
+
code: "custom",
|
|
54
|
+
path: ["repository"],
|
|
55
|
+
message: "workspace outcomes require an exact source repository",
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
if (task.attempt.retryPolicy === "none" && task.attempt.maxAttempts !== 1) {
|
|
59
|
+
ctx.addIssue({
|
|
60
|
+
code: "custom",
|
|
61
|
+
path: ["attempt", "maxAttempts"],
|
|
62
|
+
message: "a no-retry task must allow exactly one attempt",
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
if (task.datasetSnapshot?.byteLength === 0) {
|
|
66
|
+
ctx.addIssue({
|
|
67
|
+
code: "custom",
|
|
68
|
+
path: ["datasetSnapshot", "byteLength"],
|
|
69
|
+
message: "dataset snapshot provenance must contain bytes",
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
/** Structural parse only; Runtime recomputes the canonical digest before use. */
|
|
74
|
+
export const agentCandidateBenchmarkTaskSchema = agentCandidateBenchmarkTaskMaterialSchema
|
|
75
|
+
.extend({ digest: sha256DigestSchema })
|
|
76
|
+
.strict();
|
|
77
|
+
export const agentCandidateBenchmarkSuiteMaterialSchema = z
|
|
78
|
+
.object({
|
|
79
|
+
kind: z.literal("agent-candidate-benchmark-suite"),
|
|
80
|
+
digestAlgorithm: z.literal("rfc8785-sha256"),
|
|
81
|
+
taskDigests: z
|
|
82
|
+
.tuple([sha256DigestSchema])
|
|
83
|
+
.rest(sha256DigestSchema),
|
|
84
|
+
reps: z.number().int().positive(),
|
|
85
|
+
seeds: z
|
|
86
|
+
.tuple([
|
|
87
|
+
z
|
|
88
|
+
.number()
|
|
89
|
+
.int()
|
|
90
|
+
.min(Number.MIN_SAFE_INTEGER)
|
|
91
|
+
.max(Number.MAX_SAFE_INTEGER),
|
|
92
|
+
])
|
|
93
|
+
.rest(z
|
|
94
|
+
.number()
|
|
95
|
+
.int()
|
|
96
|
+
.min(Number.MIN_SAFE_INTEGER)
|
|
97
|
+
.max(Number.MAX_SAFE_INTEGER)),
|
|
98
|
+
})
|
|
99
|
+
.strict()
|
|
100
|
+
.superRefine((suite, ctx) => {
|
|
101
|
+
const taskDigests = new Set();
|
|
102
|
+
for (const [index, digest] of suite.taskDigests.entries()) {
|
|
103
|
+
if (taskDigests.has(digest)) {
|
|
104
|
+
ctx.addIssue({
|
|
105
|
+
code: "custom",
|
|
106
|
+
path: ["taskDigests", index],
|
|
107
|
+
message: "benchmark suite task digests must be unique",
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
taskDigests.add(digest);
|
|
111
|
+
}
|
|
112
|
+
const expectedSeeds = suite.taskDigests.length * suite.reps;
|
|
113
|
+
if (suite.seeds.length !== expectedSeeds) {
|
|
114
|
+
ctx.addIssue({
|
|
115
|
+
code: "custom",
|
|
116
|
+
path: ["seeds"],
|
|
117
|
+
message: "benchmark suite must provide one seed per task repetition",
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
if (!isCanonicalJsonValue(suite)) {
|
|
121
|
+
ctx.addIssue({
|
|
122
|
+
code: "custom",
|
|
123
|
+
message: "benchmark suite must contain finite, acyclic canonical JSON",
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
export const agentCandidateBenchmarkSuiteSchema = agentCandidateBenchmarkSuiteMaterialSchema
|
|
128
|
+
.extend({ digest: sha256DigestSchema })
|
|
129
|
+
.strict();
|
|
130
|
+
export const agentCandidateBenchmarkSuiteInputsSchema = z
|
|
131
|
+
.object({
|
|
132
|
+
suite: agentCandidateBenchmarkSuiteSchema,
|
|
133
|
+
tasks: z
|
|
134
|
+
.tuple([agentCandidateBenchmarkTaskSchema])
|
|
135
|
+
.rest(agentCandidateBenchmarkTaskSchema),
|
|
136
|
+
})
|
|
137
|
+
.strict()
|
|
138
|
+
.superRefine((input, ctx) => {
|
|
139
|
+
if (input.tasks.length !== input.suite.taskDigests.length) {
|
|
140
|
+
ctx.addIssue({
|
|
141
|
+
code: "custom",
|
|
142
|
+
path: ["tasks"],
|
|
143
|
+
message: "benchmark suite inputs must contain every signed task exactly once",
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
const taskIds = new Set();
|
|
147
|
+
const scenarioDigests = new Set();
|
|
148
|
+
const benchmark = input.tasks[0]?.benchmark;
|
|
149
|
+
for (const [index, task] of input.tasks.entries()) {
|
|
150
|
+
if (task.digest !== input.suite.taskDigests[index]) {
|
|
151
|
+
ctx.addIssue({
|
|
152
|
+
code: "custom",
|
|
153
|
+
path: ["tasks", index, "digest"],
|
|
154
|
+
message: "benchmark task order must match the signed suite",
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
if (taskIds.has(task.scenario.id)) {
|
|
158
|
+
ctx.addIssue({
|
|
159
|
+
code: "custom",
|
|
160
|
+
path: ["tasks", index, "scenario", "id"],
|
|
161
|
+
message: "benchmark suite task ids must be unique",
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
taskIds.add(task.scenario.id);
|
|
165
|
+
if (scenarioDigests.has(task.scenario.scenarioDigest)) {
|
|
166
|
+
ctx.addIssue({
|
|
167
|
+
code: "custom",
|
|
168
|
+
path: ["tasks", index, "scenario", "scenarioDigest"],
|
|
169
|
+
message: "benchmark suite scenario digests must be unique",
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
scenarioDigests.add(task.scenario.scenarioDigest);
|
|
173
|
+
if (benchmark &&
|
|
174
|
+
(task.benchmark.name !== benchmark.name ||
|
|
175
|
+
task.benchmark.version !== benchmark.version ||
|
|
176
|
+
task.benchmark.splitDigest !== benchmark.splitDigest)) {
|
|
177
|
+
ctx.addIssue({
|
|
178
|
+
code: "custom",
|
|
179
|
+
path: ["tasks", index, "benchmark"],
|
|
180
|
+
message: "benchmark suite tasks must share one benchmark identity",
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
if (!isCanonicalJsonValue(input)) {
|
|
185
|
+
ctx.addIssue({
|
|
186
|
+
code: "custom",
|
|
187
|
+
message: "benchmark suite inputs must contain finite, acyclic canonical JSON",
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
export { agentCandidateBenchmarkCellRefSchema };
|
|
@@ -106,8 +106,6 @@ export interface AgentCandidateProfile extends Omit<AgentProfile, "model" | "mcp
|
|
|
106
106
|
}
|
|
107
107
|
export interface AgentCandidateCodeDisabled {
|
|
108
108
|
kind: "disabled";
|
|
109
|
-
/** `control` marks a comparison arm; `not-applicable` disables only the code surface. */
|
|
110
|
-
reason: "control" | "not-applicable";
|
|
111
109
|
}
|
|
112
110
|
/** A code proposer ran against this exact tree and returned no change. */
|
|
113
111
|
export interface AgentCandidateCodeNoOp {
|
|
@@ -201,7 +199,6 @@ export interface AgentCandidateExecution {
|
|
|
201
199
|
}
|
|
202
200
|
/** Exact frozen knowledge candidate admitted only through an approved review. */
|
|
203
201
|
export interface AgentCandidateKnowledgeRef {
|
|
204
|
-
schemaVersion: 1;
|
|
205
202
|
kind: "knowledge-improvement-candidate";
|
|
206
203
|
runId: string;
|
|
207
204
|
candidateId: string;
|
|
@@ -225,14 +222,6 @@ export type AgentCandidateMemoryPolicy = {
|
|
|
225
222
|
scope: "task";
|
|
226
223
|
seed?: AgentCandidateArtifactRef;
|
|
227
224
|
};
|
|
228
|
-
/** Captured model spend for one phase of candidate production. */
|
|
229
|
-
export interface AgentCandidateSpend {
|
|
230
|
-
costUsd: number;
|
|
231
|
-
inputTokens: number;
|
|
232
|
-
outputTokens: number;
|
|
233
|
-
cachedInputTokens?: number;
|
|
234
|
-
modelCalls: number;
|
|
235
|
-
}
|
|
236
225
|
/** Lossless evaluator-owned usage totals for one candidate execution. */
|
|
237
226
|
export interface AgentCandidateFixedSpend {
|
|
238
227
|
inputTokens: number;
|
|
@@ -250,15 +239,8 @@ export interface AgentCandidateLineage {
|
|
|
250
239
|
runIds?: string[];
|
|
251
240
|
profileDiffIds?: string[];
|
|
252
241
|
modelSnapshots?: string[];
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
version: string;
|
|
256
|
-
splitDigest: Sha256Digest;
|
|
257
|
-
};
|
|
258
|
-
spend?: {
|
|
259
|
-
proposal: AgentCandidateSpend;
|
|
260
|
-
evaluation: AgentCandidateSpend;
|
|
261
|
-
};
|
|
242
|
+
/** Exact development split used to produce a generated candidate. */
|
|
243
|
+
developmentSplitDigest?: Sha256Digest;
|
|
262
244
|
}
|
|
263
245
|
/**
|
|
264
246
|
* Portable, immutable output of agent improvement.
|
|
@@ -269,7 +251,6 @@ export interface AgentCandidateLineage {
|
|
|
269
251
|
* artifact hashes are untrusted until recomputed.
|
|
270
252
|
*/
|
|
271
253
|
export interface AgentCandidateBundle {
|
|
272
|
-
schemaVersion: 2;
|
|
273
254
|
kind: "agent-candidate-bundle";
|
|
274
255
|
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
275
256
|
profile: AgentCandidateProfile;
|
|
@@ -277,7 +258,6 @@ export interface AgentCandidateBundle {
|
|
|
277
258
|
execution: AgentCandidateExecution;
|
|
278
259
|
knowledge?: AgentCandidateKnowledge;
|
|
279
260
|
memory: AgentCandidateMemoryPolicy;
|
|
280
|
-
lineage: AgentCandidateLineage;
|
|
281
261
|
digest: Sha256Digest;
|
|
282
262
|
}
|
|
283
263
|
export interface AgentCandidateEntrypointReceipt {
|
|
@@ -290,6 +270,14 @@ export interface AgentCandidateOciPlatform {
|
|
|
290
270
|
architecture: string;
|
|
291
271
|
variant?: string;
|
|
292
272
|
}
|
|
273
|
+
/** Exact evaluator-selected task image used when the candidate does not pin one. */
|
|
274
|
+
export interface AgentCandidateResolvedTaskContainer {
|
|
275
|
+
source: "evaluator-task-container";
|
|
276
|
+
image: string;
|
|
277
|
+
indexDigest: Sha256Digest;
|
|
278
|
+
manifestDigest: Sha256Digest;
|
|
279
|
+
platform: AgentCandidateOciPlatform;
|
|
280
|
+
}
|
|
293
281
|
export interface AgentCandidateResolvedModel {
|
|
294
282
|
requested: string;
|
|
295
283
|
provider: string;
|
|
@@ -299,7 +287,8 @@ export interface AgentCandidateResolvedModel {
|
|
|
299
287
|
}
|
|
300
288
|
/** Canonical, digest-free profile-plan identity document. */
|
|
301
289
|
export interface AgentCandidateProfilePlanMaterial {
|
|
302
|
-
|
|
290
|
+
/** Canonical digest of the complete frozen profile that produced this plan. */
|
|
291
|
+
sourceProfileDigest: Sha256Digest;
|
|
303
292
|
harness: HarnessType;
|
|
304
293
|
files: Array<{
|
|
305
294
|
relPath: string;
|
|
@@ -314,7 +303,6 @@ export interface AgentCandidateProfilePlanMaterial {
|
|
|
314
303
|
}>;
|
|
315
304
|
}
|
|
316
305
|
export interface AgentCandidateWorkspaceManifestMaterial {
|
|
317
|
-
schemaVersion: 2;
|
|
318
306
|
kind: "agent-candidate-workspace-manifest";
|
|
319
307
|
files: Array<{
|
|
320
308
|
path: string;
|
|
@@ -326,7 +314,6 @@ export interface AgentCandidateWorkspaceManifestMaterial {
|
|
|
326
314
|
}
|
|
327
315
|
/** Content-addressed manifest of every file uploaded to one workspace. */
|
|
328
316
|
export interface AgentCandidateWorkspaceSnapshotEvidence {
|
|
329
|
-
schemaVersion: 2;
|
|
330
317
|
kind: "agent-candidate-workspace-snapshot";
|
|
331
318
|
digest: Sha256Digest;
|
|
332
319
|
material: AgentCandidateWorkspaceManifestMaterial;
|
|
@@ -395,6 +382,103 @@ export type AgentCandidateTaskOutcomeSpec = {
|
|
|
395
382
|
} | ({
|
|
396
383
|
kind: "output";
|
|
397
384
|
} & AgentCandidateTaskOutputSpec);
|
|
385
|
+
/** Immutable grader identity admitted for one benchmark task. */
|
|
386
|
+
export interface AgentCandidateBenchmarkGraderIdentity {
|
|
387
|
+
name: string;
|
|
388
|
+
version: string;
|
|
389
|
+
format: "tangle-grader";
|
|
390
|
+
artifact: AgentCandidateArtifactRef;
|
|
391
|
+
}
|
|
392
|
+
/** Portable task bytes shared by evaluation, approval, and execution. */
|
|
393
|
+
export interface AgentCandidateBenchmarkTaskMaterial {
|
|
394
|
+
kind: "agent-candidate-benchmark-task";
|
|
395
|
+
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
396
|
+
benchmark: {
|
|
397
|
+
name: string;
|
|
398
|
+
version: string;
|
|
399
|
+
splitDigest: Sha256Digest;
|
|
400
|
+
};
|
|
401
|
+
scenario: {
|
|
402
|
+
id: string;
|
|
403
|
+
kind: string;
|
|
404
|
+
scenarioDigest: Sha256Digest;
|
|
405
|
+
};
|
|
406
|
+
datasetSnapshot?: AgentCandidateArtifactRef;
|
|
407
|
+
instruction: string;
|
|
408
|
+
repository?: AgentCandidateTaskRepository;
|
|
409
|
+
outcome: AgentCandidateTaskOutcomeSpec;
|
|
410
|
+
workspace: AgentCandidateWorkspaceSnapshotEvidence;
|
|
411
|
+
grader: AgentCandidateBenchmarkGraderIdentity;
|
|
412
|
+
model: AgentCandidateResolvedModel;
|
|
413
|
+
attempt: Omit<AgentCandidateAttemptPolicy, "number">;
|
|
414
|
+
evaluatorTaskContainer?: AgentCandidateResolvedTaskContainer;
|
|
415
|
+
limits: AgentCandidateExecutionLimits;
|
|
416
|
+
}
|
|
417
|
+
/** Content-addressed benchmark task approved and executed without reinterpretation. */
|
|
418
|
+
export interface AgentCandidateBenchmarkTask extends AgentCandidateBenchmarkTaskMaterial {
|
|
419
|
+
digest: Sha256Digest;
|
|
420
|
+
}
|
|
421
|
+
/** Complete measured denominator shared by evaluation and execution. */
|
|
422
|
+
export interface AgentCandidateBenchmarkSuiteMaterial {
|
|
423
|
+
kind: "agent-candidate-benchmark-suite";
|
|
424
|
+
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
425
|
+
taskDigests: [Sha256Digest, ...Sha256Digest[]];
|
|
426
|
+
reps: number;
|
|
427
|
+
/** Task-major, then repetition-major: seeds[taskIndex * reps + repetition]. */
|
|
428
|
+
seeds: [number, ...number[]];
|
|
429
|
+
}
|
|
430
|
+
export interface AgentCandidateBenchmarkSuite extends AgentCandidateBenchmarkSuiteMaterial {
|
|
431
|
+
digest: Sha256Digest;
|
|
432
|
+
}
|
|
433
|
+
/** Canonical task documents transported alongside their signed suite. */
|
|
434
|
+
export interface AgentCandidateBenchmarkSuiteInputs {
|
|
435
|
+
suite: AgentCandidateBenchmarkSuite;
|
|
436
|
+
tasks: [AgentCandidateBenchmarkTask, ...AgentCandidateBenchmarkTask[]];
|
|
437
|
+
}
|
|
438
|
+
/** One cell in a signed suite; task identity and seed are derived by position. */
|
|
439
|
+
export interface AgentCandidateBenchmarkCellRef {
|
|
440
|
+
suiteDigest: Sha256Digest;
|
|
441
|
+
taskIndex: number;
|
|
442
|
+
repetition: number;
|
|
443
|
+
}
|
|
444
|
+
/** Decision rules frozen before either experiment arm executes. */
|
|
445
|
+
export interface AgentCandidateEvaluationPolicy {
|
|
446
|
+
confidenceLevel: number;
|
|
447
|
+
resamples: number;
|
|
448
|
+
bootstrapSeed: number;
|
|
449
|
+
deltaThreshold: number;
|
|
450
|
+
minProductiveRuns: number;
|
|
451
|
+
budgetUsd?: number;
|
|
452
|
+
criticalDimensions: string[];
|
|
453
|
+
regressionTolerance: number;
|
|
454
|
+
}
|
|
455
|
+
/** Both complete agent states and the exact held-out work used to compare them. */
|
|
456
|
+
export interface AgentCandidateExperimentMaterial {
|
|
457
|
+
kind: "agent-candidate-experiment";
|
|
458
|
+
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
459
|
+
baseline: AgentCandidateBundle;
|
|
460
|
+
candidate: AgentCandidateBundle;
|
|
461
|
+
candidateLineage: AgentCandidateLineage;
|
|
462
|
+
benchmark: AgentCandidateBenchmarkSuiteInputs;
|
|
463
|
+
policy: AgentCandidateEvaluationPolicy;
|
|
464
|
+
}
|
|
465
|
+
export interface AgentCandidateExperiment extends AgentCandidateExperimentMaterial {
|
|
466
|
+
digest: Sha256Digest;
|
|
467
|
+
}
|
|
468
|
+
/** Digest-free identity of one exact attempt in a frozen experiment. */
|
|
469
|
+
export interface AgentCandidateRunCellMaterial extends AgentCandidateBenchmarkCellRef {
|
|
470
|
+
kind: "agent-candidate-run-cell";
|
|
471
|
+
experimentDigest: Sha256Digest;
|
|
472
|
+
arm: "baseline" | "candidate";
|
|
473
|
+
bundleDigest: Sha256Digest;
|
|
474
|
+
taskDigest: Sha256Digest;
|
|
475
|
+
seed: number;
|
|
476
|
+
attempt: number;
|
|
477
|
+
}
|
|
478
|
+
/** One immutable experiment attempt used by plans, receipts, traces, and results. */
|
|
479
|
+
export interface AgentCandidateRunCell extends AgentCandidateRunCellMaterial {
|
|
480
|
+
digest: Sha256Digest;
|
|
481
|
+
}
|
|
398
482
|
/**
|
|
399
483
|
* Canonical, digest-free per-task execution identity document.
|
|
400
484
|
*
|
|
@@ -402,26 +486,9 @@ export type AgentCandidateTaskOutcomeSpec = {
|
|
|
402
486
|
* its raw SHA-256 is the execution-plan digest.
|
|
403
487
|
*/
|
|
404
488
|
export interface AgentCandidateExecutionPlanMaterial {
|
|
405
|
-
schemaVersion: 2;
|
|
406
489
|
kind: "agent-candidate-execution-plan-material";
|
|
407
|
-
|
|
490
|
+
runCell: AgentCandidateRunCell;
|
|
408
491
|
executionId: string;
|
|
409
|
-
attempt: AgentCandidateAttemptPolicy;
|
|
410
|
-
task: {
|
|
411
|
-
benchmark: string;
|
|
412
|
-
benchmarkVersion: string;
|
|
413
|
-
taskId: string;
|
|
414
|
-
splitDigest: Sha256Digest;
|
|
415
|
-
instruction: {
|
|
416
|
-
encoding: "utf8";
|
|
417
|
-
sha256: Sha256Digest;
|
|
418
|
-
byteLength: number;
|
|
419
|
-
delivery: AgentCandidateInstructionDelivery;
|
|
420
|
-
};
|
|
421
|
-
repository?: AgentCandidateTaskRepository;
|
|
422
|
-
outcome: AgentCandidateTaskOutcomeSpec;
|
|
423
|
-
workspace: AgentCandidateWorkspaceSnapshotEvidence;
|
|
424
|
-
};
|
|
425
492
|
workspaces: {
|
|
426
493
|
taskRoot: string;
|
|
427
494
|
candidateRoot?: string;
|
|
@@ -431,6 +498,8 @@ export interface AgentCandidateExecutionPlanMaterial {
|
|
|
431
498
|
profile: AgentCandidateProfileApplication;
|
|
432
499
|
harness: HarnessType;
|
|
433
500
|
harnessVersion: string;
|
|
501
|
+
instructionDelivery: AgentCandidateInstructionDelivery;
|
|
502
|
+
limits: AgentCandidateExecutionLimits;
|
|
434
503
|
container: {
|
|
435
504
|
source: AgentCandidateExecutionEnvironment["kind"];
|
|
436
505
|
image: string;
|
|
@@ -462,12 +531,6 @@ export interface AgentCandidateExecutionPlanMaterial {
|
|
|
462
531
|
requested: string;
|
|
463
532
|
}>;
|
|
464
533
|
};
|
|
465
|
-
/** Exact evaluator grader implementation admitted for this plan. */
|
|
466
|
-
grader: {
|
|
467
|
-
name: string;
|
|
468
|
-
version: string;
|
|
469
|
-
artifact: AgentCandidateArtifactRef;
|
|
470
|
-
};
|
|
471
534
|
launch: {
|
|
472
535
|
executable: string;
|
|
473
536
|
args: AgentCandidateConfigValue[];
|
|
@@ -476,13 +539,11 @@ export interface AgentCandidateExecutionPlanMaterial {
|
|
|
476
539
|
};
|
|
477
540
|
knowledgeManifestDigest?: Sha256Digest;
|
|
478
541
|
memory: AgentCandidateEffectiveMemory;
|
|
479
|
-
limits: AgentCandidateExecutionLimits;
|
|
480
542
|
network: {
|
|
481
543
|
mode: "disabled";
|
|
482
544
|
};
|
|
483
545
|
}
|
|
484
546
|
export interface AgentCandidateProfilePlanEvidence {
|
|
485
|
-
schemaVersion: 1;
|
|
486
547
|
kind: "agent-profile-workspace-plan";
|
|
487
548
|
digest: Sha256Digest;
|
|
488
549
|
material: AgentCandidateProfilePlanMaterial;
|
|
@@ -490,7 +551,6 @@ export interface AgentCandidateProfilePlanEvidence {
|
|
|
490
551
|
}
|
|
491
552
|
/** Exact native profile files and the canonical plan that activated them. */
|
|
492
553
|
export interface AgentCandidateProfileActivation {
|
|
493
|
-
schemaVersion: 1;
|
|
494
554
|
kind: "agent-candidate-profile-activation";
|
|
495
555
|
profilePlan: AgentCandidateProfilePlanEvidence;
|
|
496
556
|
files: Array<{
|
|
@@ -501,14 +561,23 @@ export interface AgentCandidateProfileActivation {
|
|
|
501
561
|
digest: Sha256Digest;
|
|
502
562
|
}
|
|
503
563
|
export interface AgentCandidateExecutionPlanEvidence {
|
|
504
|
-
schemaVersion: 2;
|
|
505
564
|
kind: "agent-candidate-execution-plan";
|
|
506
565
|
digest: Sha256Digest;
|
|
507
566
|
material: AgentCandidateExecutionPlanMaterial;
|
|
508
567
|
artifact: AgentCandidateCapturedArtifact;
|
|
509
568
|
}
|
|
569
|
+
/** Exact suite and task material selected for one execution plan. */
|
|
570
|
+
export interface AgentCandidateBenchmarkInputEvidence {
|
|
571
|
+
suite: {
|
|
572
|
+
digest: Sha256Digest;
|
|
573
|
+
material: AgentCandidateCapturedArtifact;
|
|
574
|
+
};
|
|
575
|
+
task: {
|
|
576
|
+
digest: Sha256Digest;
|
|
577
|
+
material: AgentCandidateCapturedArtifact;
|
|
578
|
+
};
|
|
579
|
+
}
|
|
510
580
|
export interface AgentCandidateTraceEvidence {
|
|
511
|
-
schemaVersion: 1;
|
|
512
581
|
artifact: AgentCandidateCapturedArtifact;
|
|
513
582
|
eventCount: number;
|
|
514
583
|
modelCallCount: number;
|
|
@@ -525,11 +594,11 @@ export type AgentCandidateMemoryReceipt = {
|
|
|
525
594
|
};
|
|
526
595
|
/** Proof emitted after a runtime materializes, but before it executes, a bundle. */
|
|
527
596
|
export interface AgentCandidateMaterializationReceipt {
|
|
528
|
-
schemaVersion: 2;
|
|
529
597
|
kind: "agent-candidate-materialization";
|
|
530
598
|
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
531
599
|
bundleDigest: Sha256Digest;
|
|
532
|
-
|
|
600
|
+
benchmark: AgentCandidateBenchmarkInputEvidence;
|
|
601
|
+
profileActivation: AgentCandidateProfileActivation;
|
|
533
602
|
executionPlan: AgentCandidateExecutionPlanEvidence;
|
|
534
603
|
candidateWorkspace?: AgentCandidateWorkspaceSnapshotEvidence;
|
|
535
604
|
codeKind: AgentCandidateCode["kind"];
|
|
@@ -563,12 +632,17 @@ export type AgentCandidateTermination = {
|
|
|
563
632
|
};
|
|
564
633
|
/** Proof emitted after the exact materialized plan finishes executing. */
|
|
565
634
|
export interface AgentCandidateRunReceipt {
|
|
566
|
-
schemaVersion: 3;
|
|
567
635
|
kind: "agent-candidate-run";
|
|
568
636
|
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
569
637
|
bundleDigest: Sha256Digest;
|
|
638
|
+
runCellDigest: Sha256Digest;
|
|
570
639
|
materializationReceiptDigest: Sha256Digest;
|
|
571
640
|
executionPlanDigest: Sha256Digest;
|
|
641
|
+
timing: {
|
|
642
|
+
startedAtMs: number;
|
|
643
|
+
endedAtMs: number;
|
|
644
|
+
durationMs: number;
|
|
645
|
+
};
|
|
572
646
|
memory: AgentCandidateMemoryReceipt;
|
|
573
647
|
trace: AgentCandidateTraceEvidence;
|
|
574
648
|
termination: AgentCandidateTermination;
|
|
@@ -579,17 +653,16 @@ export interface AgentCandidateRunReceipt {
|
|
|
579
653
|
digest: Sha256Digest;
|
|
580
654
|
}
|
|
581
655
|
export type AgentImprovementSurface = "prompt" | "skills" | "tools" | "mcp" | "hooks" | "subagents" | "agent-profile" | "memory" | "code" | "knowledge";
|
|
656
|
+
/** One paired Runtime execution from the exact signed experiment. */
|
|
657
|
+
export interface AgentCandidateExperimentMeasurement {
|
|
658
|
+
baseline: CandidateExecutionEvidence;
|
|
659
|
+
candidate: CandidateExecutionEvidence;
|
|
660
|
+
}
|
|
582
661
|
/** Portable paired held-out comparison produced by an evaluation package. */
|
|
583
662
|
export interface AgentImprovementMeasuredComparison {
|
|
584
|
-
schemaVersion: 1;
|
|
585
663
|
kind: "agent-improvement-measured-comparison";
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
version: string;
|
|
589
|
-
splitDigest: Sha256Digest;
|
|
590
|
-
};
|
|
591
|
-
baselineProfileDigest: Sha256Digest;
|
|
592
|
-
candidateBundleDigest: Sha256Digest;
|
|
664
|
+
experiment: AgentCandidateExperiment;
|
|
665
|
+
measurements: AgentCandidateExperimentMeasurement[];
|
|
593
666
|
overall: {
|
|
594
667
|
name: "composite";
|
|
595
668
|
baseline: number;
|
|
@@ -678,7 +751,11 @@ export interface AgentImprovementMeasuredComparison {
|
|
|
678
751
|
diff: string;
|
|
679
752
|
evaluation: {
|
|
680
753
|
generationsExplored: number;
|
|
754
|
+
searchDurationMs: number;
|
|
755
|
+
executionDurationMs: number;
|
|
681
756
|
durationMs: number;
|
|
757
|
+
searchCostUsd: number;
|
|
758
|
+
executionCostUsd: number;
|
|
682
759
|
totalCostUsd: number;
|
|
683
760
|
};
|
|
684
761
|
metadata?: {
|
|
@@ -686,26 +763,21 @@ export interface AgentImprovementMeasuredComparison {
|
|
|
686
763
|
};
|
|
687
764
|
}
|
|
688
765
|
export interface AgentImprovementProposal {
|
|
689
|
-
schemaVersion: 1;
|
|
690
766
|
kind: "agent-improvement-proposal";
|
|
691
767
|
runId: string;
|
|
692
768
|
changedSurfaces: [AgentImprovementSurface, ...AgentImprovementSurface[]];
|
|
693
769
|
proposedAt: string;
|
|
694
|
-
baselineProfile: AgentProfile;
|
|
695
770
|
findings: {
|
|
696
771
|
[key: string]: AgentCandidateJsonValue;
|
|
697
772
|
}[];
|
|
698
773
|
evaluation: AgentImprovementMeasuredComparison;
|
|
699
|
-
candidateBundle: AgentCandidateBundle;
|
|
700
774
|
digest: Sha256Digest;
|
|
701
775
|
}
|
|
702
776
|
export type AgentImprovementReviewDecision = "approve" | "reject" | "request-changes";
|
|
703
777
|
/** Human or tenant-policy decision bound to one exact proposal. */
|
|
704
778
|
export interface AgentImprovementReview {
|
|
705
|
-
schemaVersion: 1;
|
|
706
779
|
kind: "agent-improvement-review";
|
|
707
780
|
proposalDigest: Sha256Digest;
|
|
708
|
-
candidateBundleDigest: Sha256Digest;
|
|
709
781
|
decision: AgentImprovementReviewDecision;
|
|
710
782
|
reviewedBy: string;
|
|
711
783
|
reviewedAt: string;
|
|
@@ -713,16 +785,30 @@ export interface AgentImprovementReview {
|
|
|
713
785
|
feedback?: string;
|
|
714
786
|
digest: Sha256Digest;
|
|
715
787
|
}
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
788
|
+
export interface AgentImprovementActivationTarget {
|
|
789
|
+
surface: AgentImprovementSurface;
|
|
790
|
+
/** Product-owned stable identity, such as an agent profile, repository, or knowledge base. */
|
|
791
|
+
identity: string;
|
|
792
|
+
/** Current target state that activation is allowed to replace. */
|
|
793
|
+
expectedBaseDigest: Sha256Digest;
|
|
794
|
+
}
|
|
795
|
+
/** Authority receipt permitting activation of one already-measured candidate. */
|
|
796
|
+
export interface AgentImprovementActivation {
|
|
797
|
+
kind: "agent-improvement-activation";
|
|
720
798
|
proposalDigest: Sha256Digest;
|
|
721
799
|
reviewDigest: Sha256Digest;
|
|
722
|
-
|
|
723
|
-
|
|
800
|
+
experimentDigest: Sha256Digest;
|
|
801
|
+
candidateBundleDigest: Sha256Digest;
|
|
802
|
+
targets: [AgentImprovementActivationTarget, ...AgentImprovementActivationTarget[]];
|
|
803
|
+
fundingOwner: string;
|
|
804
|
+
authorizedBy: string;
|
|
805
|
+
authorizedAt: string;
|
|
806
|
+
digest: Sha256Digest;
|
|
807
|
+
}
|
|
808
|
+
/** Complete execution of one exact experiment attempt. */
|
|
809
|
+
export interface CandidateExecutionEvidence {
|
|
810
|
+
kind: "agent-candidate-execution-evidence";
|
|
724
811
|
materializationReceipt: AgentCandidateMaterializationReceipt;
|
|
725
|
-
profileActivation: AgentCandidateProfileActivation;
|
|
726
812
|
receipt: AgentCandidateRunReceipt;
|
|
727
813
|
digest: Sha256Digest;
|
|
728
814
|
}
|
|
@@ -743,7 +829,6 @@ export interface AgentCandidateModelSettlementCall {
|
|
|
743
829
|
}
|
|
744
830
|
/** Canonical model-access ledger after the evaluator has revoked access. */
|
|
745
831
|
export interface AgentCandidateModelSettlementMaterial {
|
|
746
|
-
schemaVersion: 2;
|
|
747
832
|
kind: "agent-candidate-model-settlement-material";
|
|
748
833
|
executionPlanDigest: Sha256Digest;
|
|
749
834
|
preparationId: string;
|
|
@@ -754,7 +839,6 @@ export interface AgentCandidateModelSettlementMaterial {
|
|
|
754
839
|
usage: AgentCandidateFixedSpend;
|
|
755
840
|
}
|
|
756
841
|
export interface AgentCandidateModelSettlementEvidence {
|
|
757
|
-
schemaVersion: 2;
|
|
758
842
|
kind: "agent-candidate-model-settlement";
|
|
759
843
|
digest: Sha256Digest;
|
|
760
844
|
material: AgentCandidateModelSettlementMaterial;
|
|
@@ -769,7 +853,6 @@ export interface AgentCandidateRepositoryState {
|
|
|
769
853
|
}
|
|
770
854
|
/** Canonical result captured by the evaluator after one candidate task. */
|
|
771
855
|
export interface AgentCandidateTaskOutcomeMaterial {
|
|
772
|
-
schemaVersion: 2;
|
|
773
856
|
kind: "agent-candidate-task-outcome-material";
|
|
774
857
|
executionPlanDigest: Sha256Digest;
|
|
775
858
|
outcome: {
|
|
@@ -788,7 +871,6 @@ export interface AgentCandidateTaskOutcomeMaterial {
|
|
|
788
871
|
};
|
|
789
872
|
}
|
|
790
873
|
export interface AgentCandidateTaskOutcomeEvidence {
|
|
791
|
-
schemaVersion: 2;
|
|
792
874
|
kind: "agent-candidate-task-outcome";
|
|
793
875
|
digest: Sha256Digest;
|
|
794
876
|
material: AgentCandidateTaskOutcomeMaterial;
|
|
@@ -800,29 +882,26 @@ export interface AgentCandidateBenchmarkDimension {
|
|
|
800
882
|
}
|
|
801
883
|
/** Canonical executable-grade result for one task outcome. */
|
|
802
884
|
export interface AgentCandidateBenchmarkResultMaterial {
|
|
803
|
-
schemaVersion: 1;
|
|
804
885
|
kind: "agent-candidate-benchmark-result-material";
|
|
805
886
|
executionPlanDigest: Sha256Digest;
|
|
806
887
|
taskOutcomeDigest: Sha256Digest;
|
|
807
|
-
|
|
808
|
-
name: string;
|
|
809
|
-
version: string;
|
|
810
|
-
taskId: string;
|
|
811
|
-
splitDigest: Sha256Digest;
|
|
812
|
-
};
|
|
813
|
-
grader: {
|
|
814
|
-
name: string;
|
|
815
|
-
version: string;
|
|
816
|
-
artifact: AgentCandidateArtifactRef;
|
|
817
|
-
};
|
|
888
|
+
grader: AgentCandidateBenchmarkGraderIdentity;
|
|
818
889
|
/** Raw grader output required to independently audit the reported verdict. */
|
|
819
890
|
evidence: AgentCandidateArtifactRef;
|
|
891
|
+
/** Evaluator-owned model usage and elapsed time, separate from candidate usage. */
|
|
892
|
+
grading: {
|
|
893
|
+
usage: AgentCandidateFixedSpend;
|
|
894
|
+
timing: {
|
|
895
|
+
startedAtMs: number;
|
|
896
|
+
endedAtMs: number;
|
|
897
|
+
durationMs: number;
|
|
898
|
+
};
|
|
899
|
+
};
|
|
820
900
|
score: number;
|
|
821
901
|
passed: boolean;
|
|
822
902
|
dimensions: AgentCandidateBenchmarkDimension[];
|
|
823
903
|
}
|
|
824
904
|
export interface AgentCandidateBenchmarkResultEvidence {
|
|
825
|
-
schemaVersion: 1;
|
|
826
905
|
kind: "agent-candidate-benchmark-result";
|
|
827
906
|
digest: Sha256Digest;
|
|
828
907
|
material: AgentCandidateBenchmarkResultMaterial;
|