@tangle-network/agent-interface 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-candidate-code-schema.d.ts +0 -8
- package/dist/agent-candidate-code-schema.js +0 -1
- package/dist/agent-candidate-execution-plan-schema.d.ts +173 -350
- package/dist/agent-candidate-execution-plan-schema.js +97 -131
- package/dist/agent-candidate-lineage-schema.d.ts +1 -28
- package/dist/agent-candidate-lineage-schema.js +3 -33
- package/dist/agent-candidate-outcome-schema.d.ts +48 -18
- package/dist/agent-candidate-outcome-schema.js +19 -24
- package/dist/agent-candidate-promotion-schema.d.ts +11021 -2358
- package/dist/agent-candidate-promotion-schema.js +495 -105
- package/dist/agent-candidate-receipt-schema.d.ts +260 -303
- package/dist/agent-candidate-receipt-schema.js +82 -8
- package/dist/agent-candidate-schema-common.d.ts +8 -0
- package/dist/agent-candidate-schema-common.js +35 -1
- package/dist/agent-candidate-schema.d.ts +2 -38
- package/dist/agent-candidate-schema.js +3 -28
- package/dist/agent-candidate-task-schema.d.ts +643 -0
- package/dist/agent-candidate-task-schema.js +191 -0
- package/dist/agent-candidate.d.ts +173 -72
- package/dist/agent-candidate.test-fixture.d.ts +0 -26
- package/dist/agent-candidate.test-fixture.js +0 -26
- package/dist/profile-schema.d.ts +9 -0
- package/dist/profile-schema.js +10 -3
- package/package.json +2 -2
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { agentCandidateArtifactRefSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, } from "./agent-candidate-artifact-schema.js";
|
|
3
|
+
import { agentCandidateBenchmarkGraderIdentitySchema, agentCandidateBenchmarkCellRefSchema, agentCandidateExecutionLimitsSchema, agentCandidateResolvedModelSchema, agentCandidateResolvedTaskContainerSchema, agentCandidateRetryPolicySchema, agentCandidateTaskRepositorySchema, agentCandidateTaskOutcomeSpecSchema, } from "./agent-candidate-execution-plan-schema.js";
|
|
4
|
+
import { isCanonicalJsonValue, isWellFormedUnicode, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
|
|
5
|
+
export const agentCandidateBenchmarkTaskMaterialSchema = z
|
|
6
|
+
.object({
|
|
7
|
+
kind: z.literal("agent-candidate-benchmark-task"),
|
|
8
|
+
digestAlgorithm: z.literal("rfc8785-sha256"),
|
|
9
|
+
benchmark: z
|
|
10
|
+
.object({
|
|
11
|
+
name: z.string().min(1).max(512),
|
|
12
|
+
version: z.string().min(1).max(256),
|
|
13
|
+
splitDigest: sha256DigestSchema,
|
|
14
|
+
})
|
|
15
|
+
.strict(),
|
|
16
|
+
scenario: z
|
|
17
|
+
.object({
|
|
18
|
+
id: z.string().min(1).max(512),
|
|
19
|
+
kind: z.string().min(1).max(512),
|
|
20
|
+
scenarioDigest: sha256DigestSchema,
|
|
21
|
+
})
|
|
22
|
+
.strict(),
|
|
23
|
+
datasetSnapshot: agentCandidateArtifactRefSchema.optional(),
|
|
24
|
+
instruction: z
|
|
25
|
+
.string()
|
|
26
|
+
.min(1)
|
|
27
|
+
.max(4 * 1024 * 1024)
|
|
28
|
+
.refine(isWellFormedUnicode, "task instruction must be well-formed Unicode"),
|
|
29
|
+
repository: agentCandidateTaskRepositorySchema.optional(),
|
|
30
|
+
outcome: agentCandidateTaskOutcomeSpecSchema,
|
|
31
|
+
workspace: agentCandidateWorkspaceSnapshotEvidenceSchema,
|
|
32
|
+
grader: agentCandidateBenchmarkGraderIdentitySchema,
|
|
33
|
+
model: agentCandidateResolvedModelSchema,
|
|
34
|
+
attempt: z
|
|
35
|
+
.object({
|
|
36
|
+
maxAttempts: z.number().int().min(1),
|
|
37
|
+
retryPolicy: agentCandidateRetryPolicySchema,
|
|
38
|
+
})
|
|
39
|
+
.strict(),
|
|
40
|
+
evaluatorTaskContainer: agentCandidateResolvedTaskContainerSchema.optional(),
|
|
41
|
+
limits: agentCandidateExecutionLimitsSchema,
|
|
42
|
+
})
|
|
43
|
+
.strict()
|
|
44
|
+
.superRefine((task, ctx) => {
|
|
45
|
+
if (!isCanonicalJsonValue(task)) {
|
|
46
|
+
ctx.addIssue({
|
|
47
|
+
code: "custom",
|
|
48
|
+
message: "candidate benchmark task must contain finite, acyclic canonical JSON",
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
if (task.outcome.kind === "workspace" && task.repository === undefined) {
|
|
52
|
+
ctx.addIssue({
|
|
53
|
+
code: "custom",
|
|
54
|
+
path: ["repository"],
|
|
55
|
+
message: "workspace outcomes require an exact source repository",
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
if (task.attempt.retryPolicy === "none" && task.attempt.maxAttempts !== 1) {
|
|
59
|
+
ctx.addIssue({
|
|
60
|
+
code: "custom",
|
|
61
|
+
path: ["attempt", "maxAttempts"],
|
|
62
|
+
message: "a no-retry task must allow exactly one attempt",
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
if (task.datasetSnapshot?.byteLength === 0) {
|
|
66
|
+
ctx.addIssue({
|
|
67
|
+
code: "custom",
|
|
68
|
+
path: ["datasetSnapshot", "byteLength"],
|
|
69
|
+
message: "dataset snapshot provenance must contain bytes",
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
/** Structural parse only; Runtime recomputes the canonical digest before use. */
|
|
74
|
+
export const agentCandidateBenchmarkTaskSchema = agentCandidateBenchmarkTaskMaterialSchema
|
|
75
|
+
.extend({ digest: sha256DigestSchema })
|
|
76
|
+
.strict();
|
|
77
|
+
export const agentCandidateBenchmarkSuiteMaterialSchema = z
|
|
78
|
+
.object({
|
|
79
|
+
kind: z.literal("agent-candidate-benchmark-suite"),
|
|
80
|
+
digestAlgorithm: z.literal("rfc8785-sha256"),
|
|
81
|
+
taskDigests: z
|
|
82
|
+
.tuple([sha256DigestSchema])
|
|
83
|
+
.rest(sha256DigestSchema),
|
|
84
|
+
reps: z.number().int().positive(),
|
|
85
|
+
seeds: z
|
|
86
|
+
.tuple([
|
|
87
|
+
z
|
|
88
|
+
.number()
|
|
89
|
+
.int()
|
|
90
|
+
.min(Number.MIN_SAFE_INTEGER)
|
|
91
|
+
.max(Number.MAX_SAFE_INTEGER),
|
|
92
|
+
])
|
|
93
|
+
.rest(z
|
|
94
|
+
.number()
|
|
95
|
+
.int()
|
|
96
|
+
.min(Number.MIN_SAFE_INTEGER)
|
|
97
|
+
.max(Number.MAX_SAFE_INTEGER)),
|
|
98
|
+
})
|
|
99
|
+
.strict()
|
|
100
|
+
.superRefine((suite, ctx) => {
|
|
101
|
+
const taskDigests = new Set();
|
|
102
|
+
for (const [index, digest] of suite.taskDigests.entries()) {
|
|
103
|
+
if (taskDigests.has(digest)) {
|
|
104
|
+
ctx.addIssue({
|
|
105
|
+
code: "custom",
|
|
106
|
+
path: ["taskDigests", index],
|
|
107
|
+
message: "benchmark suite task digests must be unique",
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
taskDigests.add(digest);
|
|
111
|
+
}
|
|
112
|
+
const expectedSeeds = suite.taskDigests.length * suite.reps;
|
|
113
|
+
if (suite.seeds.length !== expectedSeeds) {
|
|
114
|
+
ctx.addIssue({
|
|
115
|
+
code: "custom",
|
|
116
|
+
path: ["seeds"],
|
|
117
|
+
message: "benchmark suite must provide one seed per task repetition",
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
if (!isCanonicalJsonValue(suite)) {
|
|
121
|
+
ctx.addIssue({
|
|
122
|
+
code: "custom",
|
|
123
|
+
message: "benchmark suite must contain finite, acyclic canonical JSON",
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
export const agentCandidateBenchmarkSuiteSchema = agentCandidateBenchmarkSuiteMaterialSchema
|
|
128
|
+
.extend({ digest: sha256DigestSchema })
|
|
129
|
+
.strict();
|
|
130
|
+
export const agentCandidateBenchmarkSuiteInputsSchema = z
|
|
131
|
+
.object({
|
|
132
|
+
suite: agentCandidateBenchmarkSuiteSchema,
|
|
133
|
+
tasks: z
|
|
134
|
+
.tuple([agentCandidateBenchmarkTaskSchema])
|
|
135
|
+
.rest(agentCandidateBenchmarkTaskSchema),
|
|
136
|
+
})
|
|
137
|
+
.strict()
|
|
138
|
+
.superRefine((input, ctx) => {
|
|
139
|
+
if (input.tasks.length !== input.suite.taskDigests.length) {
|
|
140
|
+
ctx.addIssue({
|
|
141
|
+
code: "custom",
|
|
142
|
+
path: ["tasks"],
|
|
143
|
+
message: "benchmark suite inputs must contain every signed task exactly once",
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
const taskIds = new Set();
|
|
147
|
+
const scenarioDigests = new Set();
|
|
148
|
+
const benchmark = input.tasks[0]?.benchmark;
|
|
149
|
+
for (const [index, task] of input.tasks.entries()) {
|
|
150
|
+
if (task.digest !== input.suite.taskDigests[index]) {
|
|
151
|
+
ctx.addIssue({
|
|
152
|
+
code: "custom",
|
|
153
|
+
path: ["tasks", index, "digest"],
|
|
154
|
+
message: "benchmark task order must match the signed suite",
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
if (taskIds.has(task.scenario.id)) {
|
|
158
|
+
ctx.addIssue({
|
|
159
|
+
code: "custom",
|
|
160
|
+
path: ["tasks", index, "scenario", "id"],
|
|
161
|
+
message: "benchmark suite task ids must be unique",
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
taskIds.add(task.scenario.id);
|
|
165
|
+
if (scenarioDigests.has(task.scenario.scenarioDigest)) {
|
|
166
|
+
ctx.addIssue({
|
|
167
|
+
code: "custom",
|
|
168
|
+
path: ["tasks", index, "scenario", "scenarioDigest"],
|
|
169
|
+
message: "benchmark suite scenario digests must be unique",
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
scenarioDigests.add(task.scenario.scenarioDigest);
|
|
173
|
+
if (benchmark &&
|
|
174
|
+
(task.benchmark.name !== benchmark.name ||
|
|
175
|
+
task.benchmark.version !== benchmark.version ||
|
|
176
|
+
task.benchmark.splitDigest !== benchmark.splitDigest)) {
|
|
177
|
+
ctx.addIssue({
|
|
178
|
+
code: "custom",
|
|
179
|
+
path: ["tasks", index, "benchmark"],
|
|
180
|
+
message: "benchmark suite tasks must share one benchmark identity",
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
if (!isCanonicalJsonValue(input)) {
|
|
185
|
+
ctx.addIssue({
|
|
186
|
+
code: "custom",
|
|
187
|
+
message: "benchmark suite inputs must contain finite, acyclic canonical JSON",
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
export { agentCandidateBenchmarkCellRefSchema };
|
|
@@ -106,8 +106,6 @@ export interface AgentCandidateProfile extends Omit<AgentProfile, "model" | "mcp
|
|
|
106
106
|
}
|
|
107
107
|
export interface AgentCandidateCodeDisabled {
|
|
108
108
|
kind: "disabled";
|
|
109
|
-
/** `control` marks a comparison arm; `not-applicable` disables only the code surface. */
|
|
110
|
-
reason: "control" | "not-applicable";
|
|
111
109
|
}
|
|
112
110
|
/** A code proposer ran against this exact tree and returned no change. */
|
|
113
111
|
export interface AgentCandidateCodeNoOp {
|
|
@@ -224,14 +222,6 @@ export type AgentCandidateMemoryPolicy = {
|
|
|
224
222
|
scope: "task";
|
|
225
223
|
seed?: AgentCandidateArtifactRef;
|
|
226
224
|
};
|
|
227
|
-
/** Captured model spend for one phase of candidate production. */
|
|
228
|
-
export interface AgentCandidateSpend {
|
|
229
|
-
costUsd: number;
|
|
230
|
-
inputTokens: number;
|
|
231
|
-
outputTokens: number;
|
|
232
|
-
cachedInputTokens?: number;
|
|
233
|
-
modelCalls: number;
|
|
234
|
-
}
|
|
235
225
|
/** Lossless evaluator-owned usage totals for one candidate execution. */
|
|
236
226
|
export interface AgentCandidateFixedSpend {
|
|
237
227
|
inputTokens: number;
|
|
@@ -249,15 +239,8 @@ export interface AgentCandidateLineage {
|
|
|
249
239
|
runIds?: string[];
|
|
250
240
|
profileDiffIds?: string[];
|
|
251
241
|
modelSnapshots?: string[];
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
version: string;
|
|
255
|
-
splitDigest: Sha256Digest;
|
|
256
|
-
};
|
|
257
|
-
spend?: {
|
|
258
|
-
proposal: AgentCandidateSpend;
|
|
259
|
-
evaluation: AgentCandidateSpend;
|
|
260
|
-
};
|
|
242
|
+
/** Exact development split used to produce a generated candidate. */
|
|
243
|
+
developmentSplitDigest?: Sha256Digest;
|
|
261
244
|
}
|
|
262
245
|
/**
|
|
263
246
|
* Portable, immutable output of agent improvement.
|
|
@@ -275,7 +258,6 @@ export interface AgentCandidateBundle {
|
|
|
275
258
|
execution: AgentCandidateExecution;
|
|
276
259
|
knowledge?: AgentCandidateKnowledge;
|
|
277
260
|
memory: AgentCandidateMemoryPolicy;
|
|
278
|
-
lineage: AgentCandidateLineage;
|
|
279
261
|
digest: Sha256Digest;
|
|
280
262
|
}
|
|
281
263
|
export interface AgentCandidateEntrypointReceipt {
|
|
@@ -288,6 +270,14 @@ export interface AgentCandidateOciPlatform {
|
|
|
288
270
|
architecture: string;
|
|
289
271
|
variant?: string;
|
|
290
272
|
}
|
|
273
|
+
/** Exact evaluator-selected task image used when the candidate does not pin one. */
|
|
274
|
+
export interface AgentCandidateResolvedTaskContainer {
|
|
275
|
+
source: "evaluator-task-container";
|
|
276
|
+
image: string;
|
|
277
|
+
indexDigest: Sha256Digest;
|
|
278
|
+
manifestDigest: Sha256Digest;
|
|
279
|
+
platform: AgentCandidateOciPlatform;
|
|
280
|
+
}
|
|
291
281
|
export interface AgentCandidateResolvedModel {
|
|
292
282
|
requested: string;
|
|
293
283
|
provider: string;
|
|
@@ -297,6 +287,8 @@ export interface AgentCandidateResolvedModel {
|
|
|
297
287
|
}
|
|
298
288
|
/** Canonical, digest-free profile-plan identity document. */
|
|
299
289
|
export interface AgentCandidateProfilePlanMaterial {
|
|
290
|
+
/** Canonical digest of the complete frozen profile that produced this plan. */
|
|
291
|
+
sourceProfileDigest: Sha256Digest;
|
|
300
292
|
harness: HarnessType;
|
|
301
293
|
files: Array<{
|
|
302
294
|
relPath: string;
|
|
@@ -390,6 +382,103 @@ export type AgentCandidateTaskOutcomeSpec = {
|
|
|
390
382
|
} | ({
|
|
391
383
|
kind: "output";
|
|
392
384
|
} & AgentCandidateTaskOutputSpec);
|
|
385
|
+
/** Immutable grader identity admitted for one benchmark task. */
|
|
386
|
+
export interface AgentCandidateBenchmarkGraderIdentity {
|
|
387
|
+
name: string;
|
|
388
|
+
version: string;
|
|
389
|
+
format: "tangle-grader";
|
|
390
|
+
artifact: AgentCandidateArtifactRef;
|
|
391
|
+
}
|
|
392
|
+
/** Portable task bytes shared by evaluation, approval, and execution. */
|
|
393
|
+
export interface AgentCandidateBenchmarkTaskMaterial {
|
|
394
|
+
kind: "agent-candidate-benchmark-task";
|
|
395
|
+
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
396
|
+
benchmark: {
|
|
397
|
+
name: string;
|
|
398
|
+
version: string;
|
|
399
|
+
splitDigest: Sha256Digest;
|
|
400
|
+
};
|
|
401
|
+
scenario: {
|
|
402
|
+
id: string;
|
|
403
|
+
kind: string;
|
|
404
|
+
scenarioDigest: Sha256Digest;
|
|
405
|
+
};
|
|
406
|
+
datasetSnapshot?: AgentCandidateArtifactRef;
|
|
407
|
+
instruction: string;
|
|
408
|
+
repository?: AgentCandidateTaskRepository;
|
|
409
|
+
outcome: AgentCandidateTaskOutcomeSpec;
|
|
410
|
+
workspace: AgentCandidateWorkspaceSnapshotEvidence;
|
|
411
|
+
grader: AgentCandidateBenchmarkGraderIdentity;
|
|
412
|
+
model: AgentCandidateResolvedModel;
|
|
413
|
+
attempt: Omit<AgentCandidateAttemptPolicy, "number">;
|
|
414
|
+
evaluatorTaskContainer?: AgentCandidateResolvedTaskContainer;
|
|
415
|
+
limits: AgentCandidateExecutionLimits;
|
|
416
|
+
}
|
|
417
|
+
/** Content-addressed benchmark task approved and executed without reinterpretation. */
|
|
418
|
+
export interface AgentCandidateBenchmarkTask extends AgentCandidateBenchmarkTaskMaterial {
|
|
419
|
+
digest: Sha256Digest;
|
|
420
|
+
}
|
|
421
|
+
/** Complete measured denominator shared by evaluation and execution. */
|
|
422
|
+
export interface AgentCandidateBenchmarkSuiteMaterial {
|
|
423
|
+
kind: "agent-candidate-benchmark-suite";
|
|
424
|
+
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
425
|
+
taskDigests: [Sha256Digest, ...Sha256Digest[]];
|
|
426
|
+
reps: number;
|
|
427
|
+
/** Task-major, then repetition-major: seeds[taskIndex * reps + repetition]. */
|
|
428
|
+
seeds: [number, ...number[]];
|
|
429
|
+
}
|
|
430
|
+
export interface AgentCandidateBenchmarkSuite extends AgentCandidateBenchmarkSuiteMaterial {
|
|
431
|
+
digest: Sha256Digest;
|
|
432
|
+
}
|
|
433
|
+
/** Canonical task documents transported alongside their signed suite. */
|
|
434
|
+
export interface AgentCandidateBenchmarkSuiteInputs {
|
|
435
|
+
suite: AgentCandidateBenchmarkSuite;
|
|
436
|
+
tasks: [AgentCandidateBenchmarkTask, ...AgentCandidateBenchmarkTask[]];
|
|
437
|
+
}
|
|
438
|
+
/** One cell in a signed suite; task identity and seed are derived by position. */
|
|
439
|
+
export interface AgentCandidateBenchmarkCellRef {
|
|
440
|
+
suiteDigest: Sha256Digest;
|
|
441
|
+
taskIndex: number;
|
|
442
|
+
repetition: number;
|
|
443
|
+
}
|
|
444
|
+
/** Decision rules frozen before either experiment arm executes. */
|
|
445
|
+
export interface AgentCandidateEvaluationPolicy {
|
|
446
|
+
confidenceLevel: number;
|
|
447
|
+
resamples: number;
|
|
448
|
+
bootstrapSeed: number;
|
|
449
|
+
deltaThreshold: number;
|
|
450
|
+
minProductiveRuns: number;
|
|
451
|
+
budgetUsd?: number;
|
|
452
|
+
criticalDimensions: string[];
|
|
453
|
+
regressionTolerance: number;
|
|
454
|
+
}
|
|
455
|
+
/** Both complete agent states and the exact held-out work used to compare them. */
|
|
456
|
+
export interface AgentCandidateExperimentMaterial {
|
|
457
|
+
kind: "agent-candidate-experiment";
|
|
458
|
+
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
459
|
+
baseline: AgentCandidateBundle;
|
|
460
|
+
candidate: AgentCandidateBundle;
|
|
461
|
+
candidateLineage: AgentCandidateLineage;
|
|
462
|
+
benchmark: AgentCandidateBenchmarkSuiteInputs;
|
|
463
|
+
policy: AgentCandidateEvaluationPolicy;
|
|
464
|
+
}
|
|
465
|
+
export interface AgentCandidateExperiment extends AgentCandidateExperimentMaterial {
|
|
466
|
+
digest: Sha256Digest;
|
|
467
|
+
}
|
|
468
|
+
/** Digest-free identity of one exact attempt in a frozen experiment. */
|
|
469
|
+
export interface AgentCandidateRunCellMaterial extends AgentCandidateBenchmarkCellRef {
|
|
470
|
+
kind: "agent-candidate-run-cell";
|
|
471
|
+
experimentDigest: Sha256Digest;
|
|
472
|
+
arm: "baseline" | "candidate";
|
|
473
|
+
bundleDigest: Sha256Digest;
|
|
474
|
+
taskDigest: Sha256Digest;
|
|
475
|
+
seed: number;
|
|
476
|
+
attempt: number;
|
|
477
|
+
}
|
|
478
|
+
/** One immutable experiment attempt used by plans, receipts, traces, and results. */
|
|
479
|
+
export interface AgentCandidateRunCell extends AgentCandidateRunCellMaterial {
|
|
480
|
+
digest: Sha256Digest;
|
|
481
|
+
}
|
|
393
482
|
/**
|
|
394
483
|
* Canonical, digest-free per-task execution identity document.
|
|
395
484
|
*
|
|
@@ -398,24 +487,8 @@ export type AgentCandidateTaskOutcomeSpec = {
|
|
|
398
487
|
*/
|
|
399
488
|
export interface AgentCandidateExecutionPlanMaterial {
|
|
400
489
|
kind: "agent-candidate-execution-plan-material";
|
|
401
|
-
|
|
490
|
+
runCell: AgentCandidateRunCell;
|
|
402
491
|
executionId: string;
|
|
403
|
-
attempt: AgentCandidateAttemptPolicy;
|
|
404
|
-
task: {
|
|
405
|
-
benchmark: string;
|
|
406
|
-
benchmarkVersion: string;
|
|
407
|
-
taskId: string;
|
|
408
|
-
splitDigest: Sha256Digest;
|
|
409
|
-
instruction: {
|
|
410
|
-
encoding: "utf8";
|
|
411
|
-
sha256: Sha256Digest;
|
|
412
|
-
byteLength: number;
|
|
413
|
-
delivery: AgentCandidateInstructionDelivery;
|
|
414
|
-
};
|
|
415
|
-
repository?: AgentCandidateTaskRepository;
|
|
416
|
-
outcome: AgentCandidateTaskOutcomeSpec;
|
|
417
|
-
workspace: AgentCandidateWorkspaceSnapshotEvidence;
|
|
418
|
-
};
|
|
419
492
|
workspaces: {
|
|
420
493
|
taskRoot: string;
|
|
421
494
|
candidateRoot?: string;
|
|
@@ -425,6 +498,8 @@ export interface AgentCandidateExecutionPlanMaterial {
|
|
|
425
498
|
profile: AgentCandidateProfileApplication;
|
|
426
499
|
harness: HarnessType;
|
|
427
500
|
harnessVersion: string;
|
|
501
|
+
instructionDelivery: AgentCandidateInstructionDelivery;
|
|
502
|
+
limits: AgentCandidateExecutionLimits;
|
|
428
503
|
container: {
|
|
429
504
|
source: AgentCandidateExecutionEnvironment["kind"];
|
|
430
505
|
image: string;
|
|
@@ -456,12 +531,6 @@ export interface AgentCandidateExecutionPlanMaterial {
|
|
|
456
531
|
requested: string;
|
|
457
532
|
}>;
|
|
458
533
|
};
|
|
459
|
-
/** Exact evaluator grader implementation admitted for this plan. */
|
|
460
|
-
grader: {
|
|
461
|
-
name: string;
|
|
462
|
-
version: string;
|
|
463
|
-
artifact: AgentCandidateArtifactRef;
|
|
464
|
-
};
|
|
465
534
|
launch: {
|
|
466
535
|
executable: string;
|
|
467
536
|
args: AgentCandidateConfigValue[];
|
|
@@ -470,7 +539,6 @@ export interface AgentCandidateExecutionPlanMaterial {
|
|
|
470
539
|
};
|
|
471
540
|
knowledgeManifestDigest?: Sha256Digest;
|
|
472
541
|
memory: AgentCandidateEffectiveMemory;
|
|
473
|
-
limits: AgentCandidateExecutionLimits;
|
|
474
542
|
network: {
|
|
475
543
|
mode: "disabled";
|
|
476
544
|
};
|
|
@@ -498,6 +566,17 @@ export interface AgentCandidateExecutionPlanEvidence {
|
|
|
498
566
|
material: AgentCandidateExecutionPlanMaterial;
|
|
499
567
|
artifact: AgentCandidateCapturedArtifact;
|
|
500
568
|
}
|
|
569
|
+
/** Exact suite and task material selected for one execution plan. */
|
|
570
|
+
export interface AgentCandidateBenchmarkInputEvidence {
|
|
571
|
+
suite: {
|
|
572
|
+
digest: Sha256Digest;
|
|
573
|
+
material: AgentCandidateCapturedArtifact;
|
|
574
|
+
};
|
|
575
|
+
task: {
|
|
576
|
+
digest: Sha256Digest;
|
|
577
|
+
material: AgentCandidateCapturedArtifact;
|
|
578
|
+
};
|
|
579
|
+
}
|
|
501
580
|
export interface AgentCandidateTraceEvidence {
|
|
502
581
|
artifact: AgentCandidateCapturedArtifact;
|
|
503
582
|
eventCount: number;
|
|
@@ -518,7 +597,8 @@ export interface AgentCandidateMaterializationReceipt {
|
|
|
518
597
|
kind: "agent-candidate-materialization";
|
|
519
598
|
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
520
599
|
bundleDigest: Sha256Digest;
|
|
521
|
-
|
|
600
|
+
benchmark: AgentCandidateBenchmarkInputEvidence;
|
|
601
|
+
profileActivation: AgentCandidateProfileActivation;
|
|
522
602
|
executionPlan: AgentCandidateExecutionPlanEvidence;
|
|
523
603
|
candidateWorkspace?: AgentCandidateWorkspaceSnapshotEvidence;
|
|
524
604
|
codeKind: AgentCandidateCode["kind"];
|
|
@@ -555,8 +635,14 @@ export interface AgentCandidateRunReceipt {
|
|
|
555
635
|
kind: "agent-candidate-run";
|
|
556
636
|
digestAlgorithm: AgentCandidateDigestAlgorithm;
|
|
557
637
|
bundleDigest: Sha256Digest;
|
|
638
|
+
runCellDigest: Sha256Digest;
|
|
558
639
|
materializationReceiptDigest: Sha256Digest;
|
|
559
640
|
executionPlanDigest: Sha256Digest;
|
|
641
|
+
timing: {
|
|
642
|
+
startedAtMs: number;
|
|
643
|
+
endedAtMs: number;
|
|
644
|
+
durationMs: number;
|
|
645
|
+
};
|
|
560
646
|
memory: AgentCandidateMemoryReceipt;
|
|
561
647
|
trace: AgentCandidateTraceEvidence;
|
|
562
648
|
termination: AgentCandidateTermination;
|
|
@@ -567,16 +653,16 @@ export interface AgentCandidateRunReceipt {
|
|
|
567
653
|
digest: Sha256Digest;
|
|
568
654
|
}
|
|
569
655
|
export type AgentImprovementSurface = "prompt" | "skills" | "tools" | "mcp" | "hooks" | "subagents" | "agent-profile" | "memory" | "code" | "knowledge";
|
|
656
|
+
/** One paired Runtime execution from the exact signed experiment. */
|
|
657
|
+
export interface AgentCandidateExperimentMeasurement {
|
|
658
|
+
baseline: CandidateExecutionEvidence;
|
|
659
|
+
candidate: CandidateExecutionEvidence;
|
|
660
|
+
}
|
|
570
661
|
/** Portable paired held-out comparison produced by an evaluation package. */
|
|
571
662
|
export interface AgentImprovementMeasuredComparison {
|
|
572
663
|
kind: "agent-improvement-measured-comparison";
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
version: string;
|
|
576
|
-
splitDigest: Sha256Digest;
|
|
577
|
-
};
|
|
578
|
-
baselineProfileDigest: Sha256Digest;
|
|
579
|
-
candidateBundleDigest: Sha256Digest;
|
|
664
|
+
experiment: AgentCandidateExperiment;
|
|
665
|
+
measurements: AgentCandidateExperimentMeasurement[];
|
|
580
666
|
overall: {
|
|
581
667
|
name: "composite";
|
|
582
668
|
baseline: number;
|
|
@@ -665,7 +751,11 @@ export interface AgentImprovementMeasuredComparison {
|
|
|
665
751
|
diff: string;
|
|
666
752
|
evaluation: {
|
|
667
753
|
generationsExplored: number;
|
|
754
|
+
searchDurationMs: number;
|
|
755
|
+
executionDurationMs: number;
|
|
668
756
|
durationMs: number;
|
|
757
|
+
searchCostUsd: number;
|
|
758
|
+
executionCostUsd: number;
|
|
669
759
|
totalCostUsd: number;
|
|
670
760
|
};
|
|
671
761
|
metadata?: {
|
|
@@ -677,12 +767,10 @@ export interface AgentImprovementProposal {
|
|
|
677
767
|
runId: string;
|
|
678
768
|
changedSurfaces: [AgentImprovementSurface, ...AgentImprovementSurface[]];
|
|
679
769
|
proposedAt: string;
|
|
680
|
-
baselineProfile: AgentProfile;
|
|
681
770
|
findings: {
|
|
682
771
|
[key: string]: AgentCandidateJsonValue;
|
|
683
772
|
}[];
|
|
684
773
|
evaluation: AgentImprovementMeasuredComparison;
|
|
685
|
-
candidateBundle: AgentCandidateBundle;
|
|
686
774
|
digest: Sha256Digest;
|
|
687
775
|
}
|
|
688
776
|
export type AgentImprovementReviewDecision = "approve" | "reject" | "request-changes";
|
|
@@ -690,7 +778,6 @@ export type AgentImprovementReviewDecision = "approve" | "reject" | "request-cha
|
|
|
690
778
|
export interface AgentImprovementReview {
|
|
691
779
|
kind: "agent-improvement-review";
|
|
692
780
|
proposalDigest: Sha256Digest;
|
|
693
|
-
candidateBundleDigest: Sha256Digest;
|
|
694
781
|
decision: AgentImprovementReviewDecision;
|
|
695
782
|
reviewedBy: string;
|
|
696
783
|
reviewedAt: string;
|
|
@@ -698,15 +785,30 @@ export interface AgentImprovementReview {
|
|
|
698
785
|
feedback?: string;
|
|
699
786
|
digest: Sha256Digest;
|
|
700
787
|
}
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
788
|
+
export interface AgentImprovementActivationTarget {
|
|
789
|
+
surface: AgentImprovementSurface;
|
|
790
|
+
/** Product-owned stable identity, such as an agent profile, repository, or knowledge base. */
|
|
791
|
+
identity: string;
|
|
792
|
+
/** Current target state that activation is allowed to replace. */
|
|
793
|
+
expectedBaseDigest: Sha256Digest;
|
|
794
|
+
}
|
|
795
|
+
/** Authority receipt permitting activation of one already-measured candidate. */
|
|
796
|
+
export interface AgentImprovementActivation {
|
|
797
|
+
kind: "agent-improvement-activation";
|
|
704
798
|
proposalDigest: Sha256Digest;
|
|
705
799
|
reviewDigest: Sha256Digest;
|
|
706
|
-
|
|
707
|
-
|
|
800
|
+
experimentDigest: Sha256Digest;
|
|
801
|
+
candidateBundleDigest: Sha256Digest;
|
|
802
|
+
targets: [AgentImprovementActivationTarget, ...AgentImprovementActivationTarget[]];
|
|
803
|
+
fundingOwner: string;
|
|
804
|
+
authorizedBy: string;
|
|
805
|
+
authorizedAt: string;
|
|
806
|
+
digest: Sha256Digest;
|
|
807
|
+
}
|
|
808
|
+
/** Complete execution of one exact experiment attempt. */
|
|
809
|
+
export interface CandidateExecutionEvidence {
|
|
810
|
+
kind: "agent-candidate-execution-evidence";
|
|
708
811
|
materializationReceipt: AgentCandidateMaterializationReceipt;
|
|
709
|
-
profileActivation: AgentCandidateProfileActivation;
|
|
710
812
|
receipt: AgentCandidateRunReceipt;
|
|
711
813
|
digest: Sha256Digest;
|
|
712
814
|
}
|
|
@@ -783,19 +885,18 @@ export interface AgentCandidateBenchmarkResultMaterial {
|
|
|
783
885
|
kind: "agent-candidate-benchmark-result-material";
|
|
784
886
|
executionPlanDigest: Sha256Digest;
|
|
785
887
|
taskOutcomeDigest: Sha256Digest;
|
|
786
|
-
|
|
787
|
-
name: string;
|
|
788
|
-
version: string;
|
|
789
|
-
taskId: string;
|
|
790
|
-
splitDigest: Sha256Digest;
|
|
791
|
-
};
|
|
792
|
-
grader: {
|
|
793
|
-
name: string;
|
|
794
|
-
version: string;
|
|
795
|
-
artifact: AgentCandidateArtifactRef;
|
|
796
|
-
};
|
|
888
|
+
grader: AgentCandidateBenchmarkGraderIdentity;
|
|
797
889
|
/** Raw grader output required to independently audit the reported verdict. */
|
|
798
890
|
evidence: AgentCandidateArtifactRef;
|
|
891
|
+
/** Evaluator-owned model usage and elapsed time, separate from candidate usage. */
|
|
892
|
+
grading: {
|
|
893
|
+
usage: AgentCandidateFixedSpend;
|
|
894
|
+
timing: {
|
|
895
|
+
startedAtMs: number;
|
|
896
|
+
endedAtMs: number;
|
|
897
|
+
durationMs: number;
|
|
898
|
+
};
|
|
899
|
+
};
|
|
799
900
|
score: number;
|
|
800
901
|
passed: boolean;
|
|
801
902
|
dimensions: AgentCandidateBenchmarkDimension[];
|
|
@@ -158,31 +158,5 @@ export declare function candidateFixture(): {
|
|
|
158
158
|
mode: "isolated";
|
|
159
159
|
scope: "task";
|
|
160
160
|
};
|
|
161
|
-
lineage: {
|
|
162
|
-
source: "optimizer";
|
|
163
|
-
parentDigests: `sha256:${string}`[];
|
|
164
|
-
runIds: string[];
|
|
165
|
-
profileDiffIds: string[];
|
|
166
|
-
modelSnapshots: string[];
|
|
167
|
-
benchmark: {
|
|
168
|
-
name: string;
|
|
169
|
-
version: string;
|
|
170
|
-
splitDigest: `sha256:${string}`;
|
|
171
|
-
};
|
|
172
|
-
spend: {
|
|
173
|
-
proposal: {
|
|
174
|
-
costUsd: number;
|
|
175
|
-
inputTokens: number;
|
|
176
|
-
outputTokens: number;
|
|
177
|
-
modelCalls: number;
|
|
178
|
-
};
|
|
179
|
-
evaluation: {
|
|
180
|
-
costUsd: number;
|
|
181
|
-
inputTokens: number;
|
|
182
|
-
outputTokens: number;
|
|
183
|
-
modelCalls: number;
|
|
184
|
-
};
|
|
185
|
-
};
|
|
186
|
-
};
|
|
187
161
|
digest: `sha256:${string}`;
|
|
188
162
|
};
|
|
@@ -148,32 +148,6 @@ export function candidateFixture() {
|
|
|
148
148
|
mode: "isolated",
|
|
149
149
|
scope: "task",
|
|
150
150
|
},
|
|
151
|
-
lineage: {
|
|
152
|
-
source: "optimizer",
|
|
153
|
-
parentDigests: [candidateSha("8")],
|
|
154
|
-
runIds: ["r360-search"],
|
|
155
|
-
profileDiffIds: ["profile-diff-3"],
|
|
156
|
-
modelSnapshots: ["openai/gpt-5.4-2026-06-15"],
|
|
157
|
-
benchmark: {
|
|
158
|
-
name: "pier",
|
|
159
|
-
version: "0.3",
|
|
160
|
-
splitDigest: candidateSha("9"),
|
|
161
|
-
},
|
|
162
|
-
spend: {
|
|
163
|
-
proposal: {
|
|
164
|
-
costUsd: 3.5,
|
|
165
|
-
inputTokens: 100,
|
|
166
|
-
outputTokens: 20,
|
|
167
|
-
modelCalls: 2,
|
|
168
|
-
},
|
|
169
|
-
evaluation: {
|
|
170
|
-
costUsd: 9,
|
|
171
|
-
inputTokens: 900,
|
|
172
|
-
outputTokens: 200,
|
|
173
|
-
modelCalls: 8,
|
|
174
|
-
},
|
|
175
|
-
},
|
|
176
|
-
},
|
|
177
151
|
digest: candidateSha("a"),
|
|
178
152
|
});
|
|
179
153
|
}
|
package/dist/profile-schema.d.ts
CHANGED
|
@@ -115,6 +115,15 @@ export declare const agentProfileResourcesSchema: z.ZodObject<{
|
|
|
115
115
|
}, z.core.$strip>]>]>>;
|
|
116
116
|
failOnError: z.ZodOptional<z.ZodBoolean>;
|
|
117
117
|
}, z.core.$strip>;
|
|
118
|
+
export declare const reasoningEffortSchema: z.ZodEnum<{
|
|
119
|
+
none: "none";
|
|
120
|
+
minimal: "minimal";
|
|
121
|
+
low: "low";
|
|
122
|
+
medium: "medium";
|
|
123
|
+
high: "high";
|
|
124
|
+
xhigh: "xhigh";
|
|
125
|
+
ultracode: "ultracode";
|
|
126
|
+
}>;
|
|
118
127
|
export declare const agentProfileModelHintsSchema: z.ZodObject<{
|
|
119
128
|
default: z.ZodOptional<z.ZodString>;
|
|
120
129
|
small: z.ZodOptional<z.ZodString>;
|
package/dist/profile-schema.js
CHANGED
|
@@ -39,13 +39,20 @@ export const agentProfileResourcesSchema = z.object({
|
|
|
39
39
|
instructions: z.union([z.string(), agentProfileResourceRefSchema]).optional(),
|
|
40
40
|
failOnError: z.boolean().optional(),
|
|
41
41
|
});
|
|
42
|
+
export const reasoningEffortSchema = z.enum([
|
|
43
|
+
"none",
|
|
44
|
+
"minimal",
|
|
45
|
+
"low",
|
|
46
|
+
"medium",
|
|
47
|
+
"high",
|
|
48
|
+
"xhigh",
|
|
49
|
+
"ultracode",
|
|
50
|
+
]);
|
|
42
51
|
export const agentProfileModelHintsSchema = z.object({
|
|
43
52
|
default: z.string().optional(),
|
|
44
53
|
small: z.string().optional(),
|
|
45
54
|
provider: z.string().optional(),
|
|
46
|
-
reasoningEffort:
|
|
47
|
-
.enum(["none", "minimal", "low", "medium", "high", "xhigh", "ultracode"])
|
|
48
|
-
.optional(),
|
|
55
|
+
reasoningEffort: reasoningEffortSchema.optional(),
|
|
49
56
|
metadata: z.record(z.string(), z.unknown()).optional(),
|
|
50
57
|
});
|
|
51
58
|
export const agentProfilePromptSchema = z.object({
|