@tangle-network/agent-interface 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-candidate-code-schema.d.ts +0 -3
- package/dist/agent-candidate-execution-plan-schema.d.ts +11 -17
- package/dist/agent-candidate-lineage-schema.d.ts +2 -2
- package/dist/agent-candidate-outcome-schema.d.ts +4 -4
- package/dist/agent-candidate-profile-schema.d.ts +0 -3
- package/dist/agent-candidate-promotion-schema.d.ts +1418 -885
- package/dist/agent-candidate-promotion-schema.js +55 -346
- package/dist/agent-candidate-receipt-schema.d.ts +9 -11
- package/dist/agent-candidate-receipt-schema.js +1 -0
- package/dist/agent-candidate-schema.d.ts +0 -6
- package/dist/agent-candidate-schema.js +1 -3
- package/dist/agent-candidate.d.ts +15 -7
- package/dist/agent-execution-limits.d.ts +28 -0
- package/dist/agent-execution-limits.js +77 -0
- package/dist/agent-improvement-measurement-schema.d.ts +198 -0
- package/dist/agent-improvement-measurement-schema.js +349 -0
- package/dist/agent-improvement-source.d.ts +23 -0
- package/dist/agent-improvement-source.js +38 -0
- package/dist/agent-profile-improvement-schema.d.ts +1088 -0
- package/dist/agent-profile-improvement-schema.js +560 -0
- package/dist/agent-profile-improvement.d.ts +140 -0
- package/dist/agent-profile-improvement.js +1 -0
- package/dist/agent-profile.d.ts +2 -2
- package/dist/agent-profile.js +2 -2
- package/dist/harness-capabilities.d.ts +1 -1
- package/dist/harness-capabilities.js +9 -12
- package/dist/harness.d.ts +1 -10
- package/dist/harness.js +0 -13
- package/dist/index.d.ts +8 -1
- package/dist/index.js +6 -0
- package/dist/interaction.d.ts +0 -17
- package/dist/interaction.js +0 -23
- package/dist/number-validation.d.ts +1 -0
- package/dist/number-validation.js +4 -0
- package/dist/profile-schema.d.ts +0 -3
- package/package.json +1 -1
|
@@ -3,79 +3,10 @@ import { agentCandidateBundleSchema } from "./agent-candidate-schema.js";
|
|
|
3
3
|
import { agentCandidateLineageSchema } from "./agent-candidate-lineage-schema.js";
|
|
4
4
|
import { agentCandidateBenchmarkSuiteInputsSchema } from "./agent-candidate-task-schema.js";
|
|
5
5
|
import { canonicalCandidateDigest, isCanonicalJsonValue, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
|
|
6
|
+
import { refineAgentExecutionWithinLimits } from "./agent-execution-limits.js";
|
|
6
7
|
import { agentCandidateMaterializationReceiptSchema, agentCandidateRunReceiptSchema, } from "./agent-candidate-receipt-schema.js";
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
.record(z.string(), canonicalJsonSchema)
|
|
10
|
-
.refine(isCanonicalJsonValue, "value must be finite, acyclic RFC 8785 JSON");
|
|
11
|
-
const confidenceIntervalSchema = z
|
|
12
|
-
.object({
|
|
13
|
-
level: z.number().finite().gt(0).lt(1),
|
|
14
|
-
lower: z.number().finite(),
|
|
15
|
-
upper: z.number().finite(),
|
|
16
|
-
method: z.literal("paired-bootstrap"),
|
|
17
|
-
statistic: z.literal("mean"),
|
|
18
|
-
resamples: z.number().int().positive(),
|
|
19
|
-
})
|
|
20
|
-
.strict();
|
|
21
|
-
const measuredEstimateFields = {
|
|
22
|
-
baseline: z.number().finite(),
|
|
23
|
-
candidate: z.number().finite(),
|
|
24
|
-
delta: z.number().finite(),
|
|
25
|
-
confidenceInterval: confidenceIntervalSchema,
|
|
26
|
-
n: z.number().int().positive(),
|
|
27
|
-
};
|
|
28
|
-
const qualityObjectiveFields = {
|
|
29
|
-
kind: z.literal("objective"),
|
|
30
|
-
name: z.string().min(1),
|
|
31
|
-
direction: z.literal("higher-is-better"),
|
|
32
|
-
unit: z.literal("score"),
|
|
33
|
-
};
|
|
34
|
-
const qualityDimensionFields = {
|
|
35
|
-
kind: z.literal("dimension"),
|
|
36
|
-
objective: z.string().min(1),
|
|
37
|
-
name: z.string().min(1),
|
|
38
|
-
direction: z.literal("higher-is-better"),
|
|
39
|
-
unit: z.literal("score"),
|
|
40
|
-
};
|
|
41
|
-
const costObjectiveFields = {
|
|
42
|
-
kind: z.literal("cost"),
|
|
43
|
-
name: z.literal("cost"),
|
|
44
|
-
direction: z.literal("lower-is-better"),
|
|
45
|
-
unit: z.literal("usd"),
|
|
46
|
-
};
|
|
47
|
-
const latencyObjectiveFields = {
|
|
48
|
-
kind: z.literal("latency"),
|
|
49
|
-
name: z.literal("latency"),
|
|
50
|
-
direction: z.literal("lower-is-better"),
|
|
51
|
-
unit: z.literal("milliseconds"),
|
|
52
|
-
};
|
|
53
|
-
function measuredObjectiveVariant(fields) {
|
|
54
|
-
return z
|
|
55
|
-
.object({
|
|
56
|
-
...fields,
|
|
57
|
-
availability: z.literal("measured"),
|
|
58
|
-
...measuredEstimateFields,
|
|
59
|
-
})
|
|
60
|
-
.strict();
|
|
61
|
-
}
|
|
62
|
-
function unavailableObjectiveVariant(fields) {
|
|
63
|
-
return z
|
|
64
|
-
.object({
|
|
65
|
-
...fields,
|
|
66
|
-
availability: z.literal("unavailable"),
|
|
67
|
-
reason: z.string().min(1),
|
|
68
|
-
})
|
|
69
|
-
.strict();
|
|
70
|
-
}
|
|
71
|
-
const measuredObjectiveSchema = z.union([
|
|
72
|
-
measuredObjectiveVariant(qualityObjectiveFields),
|
|
73
|
-
unavailableObjectiveVariant(qualityObjectiveFields),
|
|
74
|
-
measuredObjectiveVariant(qualityDimensionFields),
|
|
75
|
-
unavailableObjectiveVariant(qualityDimensionFields),
|
|
76
|
-
measuredObjectiveVariant(costObjectiveFields),
|
|
77
|
-
measuredObjectiveVariant(latencyObjectiveFields),
|
|
78
|
-
]);
|
|
8
|
+
import { agentCandidateEvaluationPolicySchema, canonicalJsonObjectSchema, createMeasuredComparisonIdentityRegistry, measuredComparisonCommonShape, refineMeasuredComparisonSummary, } from "./agent-improvement-measurement-schema.js";
|
|
9
|
+
import { agentProfileImprovementMeasuredComparisonSchema, changedProfileImprovementSurfaces, } from "./agent-profile-improvement-schema.js";
|
|
79
10
|
const improvementSurfaceSchema = z.enum([
|
|
80
11
|
"prompt",
|
|
81
12
|
"skills",
|
|
@@ -88,28 +19,6 @@ const improvementSurfaceSchema = z.enum([
|
|
|
88
19
|
"code",
|
|
89
20
|
"knowledge",
|
|
90
21
|
]);
|
|
91
|
-
export const agentCandidateEvaluationPolicySchema = z
|
|
92
|
-
.object({
|
|
93
|
-
confidenceLevel: z.number().finite().gt(0).lt(1),
|
|
94
|
-
resamples: z.number().int().min(100),
|
|
95
|
-
bootstrapSeed: z.number().int().safe(),
|
|
96
|
-
deltaThreshold: z.number().finite().nonnegative(),
|
|
97
|
-
minProductiveRuns: z.number().int().min(3),
|
|
98
|
-
budgetUsd: z.number().finite().nonnegative().optional(),
|
|
99
|
-
criticalDimensions: z.array(z.string().min(1)),
|
|
100
|
-
regressionTolerance: z.number().finite().nonnegative(),
|
|
101
|
-
})
|
|
102
|
-
.strict()
|
|
103
|
-
.superRefine((policy, ctx) => {
|
|
104
|
-
if (new Set(policy.criticalDimensions).size !== policy.criticalDimensions.length ||
|
|
105
|
-
policy.criticalDimensions.some((name, index) => index > 0 && policy.criticalDimensions[index - 1] >= name)) {
|
|
106
|
-
ctx.addIssue({
|
|
107
|
-
code: "custom",
|
|
108
|
-
path: ["criticalDimensions"],
|
|
109
|
-
message: "critical dimensions must be sorted and unique",
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
});
|
|
113
22
|
export const agentCandidateExperimentSchema = z
|
|
114
23
|
.object({
|
|
115
24
|
kind: z.literal("agent-candidate-experiment"),
|
|
@@ -204,6 +113,14 @@ export const candidateExecutionEvidenceSchema = z
|
|
|
204
113
|
if (!valid)
|
|
205
114
|
ctx.addIssue({ code: "custom", path, message });
|
|
206
115
|
}
|
|
116
|
+
refineAgentExecutionWithinLimits(plan.material.limits, {
|
|
117
|
+
durationMs: evidence.receipt.timing.durationMs,
|
|
118
|
+
steps: evidence.receipt.steps,
|
|
119
|
+
usage: evidence.receipt.modelSettlement.material.usage,
|
|
120
|
+
}, ctx, {
|
|
121
|
+
pathPrefix: ["receipt"],
|
|
122
|
+
usagePath: ["modelSettlement", "material", "usage"],
|
|
123
|
+
});
|
|
207
124
|
if (!isCanonicalJsonValue(evidence)) {
|
|
208
125
|
ctx.addIssue({
|
|
209
126
|
code: "custom",
|
|
@@ -221,82 +138,10 @@ export const agentImprovementMeasuredComparisonSchema = z
|
|
|
221
138
|
candidate: candidateExecutionEvidenceSchema,
|
|
222
139
|
})
|
|
223
140
|
.strict()),
|
|
224
|
-
|
|
225
|
-
.object({
|
|
226
|
-
name: z.literal("composite"),
|
|
227
|
-
...measuredEstimateFields,
|
|
228
|
-
direction: z.literal("higher-is-better"),
|
|
229
|
-
unit: z.literal("score"),
|
|
230
|
-
})
|
|
231
|
-
.strict(),
|
|
232
|
-
objectives: z.array(measuredObjectiveSchema),
|
|
233
|
-
candidate: z
|
|
234
|
-
.object({
|
|
235
|
-
label: z.string().min(1).optional(),
|
|
236
|
-
rationale: z.string().min(1).optional(),
|
|
237
|
-
})
|
|
238
|
-
.strict()
|
|
239
|
-
.refine((candidate) => candidate.label !== undefined || candidate.rationale !== undefined, "candidate metadata requires a label or rationale")
|
|
240
|
-
.optional(),
|
|
241
|
-
decision: z
|
|
242
|
-
.object({
|
|
243
|
-
outcome: z.enum([
|
|
244
|
-
"ship",
|
|
245
|
-
"hold",
|
|
246
|
-
"need_more_work",
|
|
247
|
-
"model_ceiling",
|
|
248
|
-
"arch_ceiling",
|
|
249
|
-
]),
|
|
250
|
-
reasons: z.array(z.string().min(1)).min(1),
|
|
251
|
-
contributingChecks: z.array(z.object({ name: z.string().min(1), passed: z.boolean() }).strict()),
|
|
252
|
-
})
|
|
253
|
-
.strict(),
|
|
254
|
-
power: z
|
|
255
|
-
.object({
|
|
256
|
-
sufficient: z.boolean(),
|
|
257
|
-
n: z.number().int().positive(),
|
|
258
|
-
minimumDetectableDelta: z.number().finite().nonnegative(),
|
|
259
|
-
confidenceLevel: z.number().finite().gt(0).lt(1),
|
|
260
|
-
scaleAssumed: z.boolean(),
|
|
261
|
-
sharedScorerChannel: z.boolean(),
|
|
262
|
-
reason: z.string().min(1),
|
|
263
|
-
})
|
|
264
|
-
.strict(),
|
|
265
|
-
provenance: z
|
|
266
|
-
.object({
|
|
267
|
-
kind: z.literal("agent-eval-loop"),
|
|
268
|
-
schema: z.string().min(1),
|
|
269
|
-
runId: z.string().min(1),
|
|
270
|
-
recordDigest: sha256DigestSchema,
|
|
271
|
-
baselineContentHash: z.string().regex(/^(?:sha256:)?[a-f0-9]{64}$/),
|
|
272
|
-
candidateContentHash: z.string().regex(/^(?:sha256:)?[a-f0-9]{64}$/),
|
|
273
|
-
})
|
|
274
|
-
.strict(),
|
|
275
|
-
diff: z.string(),
|
|
276
|
-
evaluation: z
|
|
277
|
-
.object({
|
|
278
|
-
generationsExplored: z.number().int().nonnegative(),
|
|
279
|
-
searchDurationMs: z.number().finite().nonnegative(),
|
|
280
|
-
executionDurationMs: z.number().finite().nonnegative(),
|
|
281
|
-
durationMs: z.number().finite().nonnegative(),
|
|
282
|
-
searchCostUsd: z.number().finite().nonnegative(),
|
|
283
|
-
executionCostUsd: z.number().finite().nonnegative(),
|
|
284
|
-
totalCostUsd: z.number().finite().nonnegative(),
|
|
285
|
-
})
|
|
286
|
-
.strict(),
|
|
287
|
-
metadata: canonicalJsonObjectSchema.optional(),
|
|
141
|
+
...measuredComparisonCommonShape,
|
|
288
142
|
})
|
|
289
143
|
.strict()
|
|
290
144
|
.superRefine((comparison, ctx) => {
|
|
291
|
-
refineEstimate(comparison.overall, ["overall"], ctx);
|
|
292
|
-
if (!approximatelyEqual(comparison.evaluation.durationMs, comparison.evaluation.searchDurationMs + comparison.evaluation.executionDurationMs) ||
|
|
293
|
-
!approximatelyEqual(comparison.evaluation.totalCostUsd, comparison.evaluation.searchCostUsd + comparison.evaluation.executionCostUsd)) {
|
|
294
|
-
ctx.addIssue({
|
|
295
|
-
code: "custom",
|
|
296
|
-
path: ["evaluation"],
|
|
297
|
-
message: "evaluation totals must equal their search and execution components",
|
|
298
|
-
});
|
|
299
|
-
}
|
|
300
145
|
const { suite, tasks } = comparison.experiment.benchmark;
|
|
301
146
|
const expectedN = suite.taskDigests.length * suite.reps;
|
|
302
147
|
if (comparison.measurements.length !== expectedN) {
|
|
@@ -306,13 +151,10 @@ export const agentImprovementMeasuredComparisonSchema = z
|
|
|
306
151
|
message: "measured comparison must contain every signed benchmark cell",
|
|
307
152
|
});
|
|
308
153
|
}
|
|
309
|
-
const
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
receipt: new Set(),
|
|
314
|
-
evidence: new Set(),
|
|
315
|
-
};
|
|
154
|
+
const recordExecutionIdentities = createMeasuredComparisonIdentityRegistry({
|
|
155
|
+
ctx,
|
|
156
|
+
identityLabel: "measured executions",
|
|
157
|
+
});
|
|
316
158
|
for (let taskIndex = 0; taskIndex < suite.taskDigests.length; taskIndex += 1) {
|
|
317
159
|
const task = tasks[taskIndex];
|
|
318
160
|
if (!task)
|
|
@@ -509,138 +351,22 @@ export const agentImprovementMeasuredComparisonSchema = z
|
|
|
509
351
|
if (!valid)
|
|
510
352
|
ctx.addIssue({ code: "custom", path, message });
|
|
511
353
|
}
|
|
512
|
-
|
|
513
|
-
execution: plan.material.executionId,
|
|
514
|
-
runCell: runCell.digest,
|
|
515
|
-
materialization: materialization.digest,
|
|
516
|
-
receipt: evidence.receipt.digest,
|
|
517
|
-
evidence: evidence.digest,
|
|
518
|
-
|
|
519
|
-
for (const [kind, identity] of Object.entries(identitiesForRun)) {
|
|
520
|
-
if (executionIdentities[kind].has(identity)) {
|
|
521
|
-
ctx.addIssue({
|
|
522
|
-
code: "custom",
|
|
523
|
-
path: armPath,
|
|
524
|
-
message: `measured executions must not reuse ${kind} identity`,
|
|
525
|
-
});
|
|
526
|
-
}
|
|
527
|
-
executionIdentities[kind].add(identity);
|
|
528
|
-
}
|
|
354
|
+
recordExecutionIdentities([
|
|
355
|
+
{ kind: "execution", value: plan.material.executionId },
|
|
356
|
+
{ kind: "runCell", value: runCell.digest },
|
|
357
|
+
{ kind: "materialization", value: materialization.digest },
|
|
358
|
+
{ kind: "receipt", value: evidence.receipt.digest },
|
|
359
|
+
{ kind: "evidence", value: evidence.digest },
|
|
360
|
+
], armPath);
|
|
529
361
|
}
|
|
530
362
|
}
|
|
531
363
|
}
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
}
|
|
539
|
-
if (comparison.measurements.length > 0) {
|
|
540
|
-
refineMeasuredMean(comparison.overall.baseline, comparison.measurements.map((row) => row.baseline.receipt.benchmarkResult.material.score), ["overall", "baseline"], ctx);
|
|
541
|
-
refineMeasuredMean(comparison.overall.candidate, comparison.measurements.map((row) => row.candidate.receipt.benchmarkResult.material.score), ["overall", "candidate"], ctx);
|
|
542
|
-
}
|
|
543
|
-
const identities = new Set();
|
|
544
|
-
const qualityObjectives = new Set();
|
|
545
|
-
const dimensionParents = [];
|
|
546
|
-
let costCount = 0;
|
|
547
|
-
let latencyCount = 0;
|
|
548
|
-
for (const [index, objective] of comparison.objectives.entries()) {
|
|
549
|
-
if (objective.availability === "measured") {
|
|
550
|
-
refineEstimate(objective, ["objectives", index], ctx);
|
|
551
|
-
if (objective.n !== expectedN) {
|
|
552
|
-
ctx.addIssue({
|
|
553
|
-
code: "custom",
|
|
554
|
-
path: ["objectives", index, "n"],
|
|
555
|
-
message: "measured objective count must equal the complete benchmark suite",
|
|
556
|
-
});
|
|
557
|
-
}
|
|
558
|
-
if (comparison.measurements.length > 0 && objective.kind === "cost") {
|
|
559
|
-
refineMeasuredMean(objective.baseline, comparison.measurements.map((row) => executionCostUsd(row.baseline)), ["objectives", index, "baseline"], ctx);
|
|
560
|
-
refineMeasuredMean(objective.candidate, comparison.measurements.map((row) => executionCostUsd(row.candidate)), ["objectives", index, "candidate"], ctx);
|
|
561
|
-
}
|
|
562
|
-
if (comparison.measurements.length > 0 && objective.kind === "latency") {
|
|
563
|
-
refineMeasuredMean(objective.baseline, comparison.measurements.map((row) => executionLatencyMs(row.baseline)), ["objectives", index, "baseline"], ctx);
|
|
564
|
-
refineMeasuredMean(objective.candidate, comparison.measurements.map((row) => executionLatencyMs(row.candidate)), ["objectives", index, "candidate"], ctx);
|
|
565
|
-
}
|
|
566
|
-
}
|
|
567
|
-
const identity = objective.kind === "dimension"
|
|
568
|
-
? `${objective.kind}:${objective.objective}:${objective.name}`
|
|
569
|
-
: `${objective.kind}:${objective.name}`;
|
|
570
|
-
if (identities.has(identity)) {
|
|
571
|
-
ctx.addIssue({
|
|
572
|
-
code: "custom",
|
|
573
|
-
path: ["objectives", index, "name"],
|
|
574
|
-
message: "measured objective identities must be unique",
|
|
575
|
-
});
|
|
576
|
-
}
|
|
577
|
-
identities.add(identity);
|
|
578
|
-
if (objective.kind === "objective") {
|
|
579
|
-
qualityObjectives.add(objective.name);
|
|
580
|
-
}
|
|
581
|
-
else if (objective.kind === "dimension") {
|
|
582
|
-
dimensionParents.push({ index, objective: objective.objective });
|
|
583
|
-
}
|
|
584
|
-
else if (objective.kind === "cost") {
|
|
585
|
-
costCount += 1;
|
|
586
|
-
}
|
|
587
|
-
else if (objective.kind === "latency") {
|
|
588
|
-
latencyCount += 1;
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
if (costCount !== 1 || latencyCount !== 1) {
|
|
592
|
-
ctx.addIssue({
|
|
593
|
-
code: "custom",
|
|
594
|
-
path: ["objectives"],
|
|
595
|
-
message: "measured comparison must contain exactly one cost and latency objective",
|
|
596
|
-
});
|
|
597
|
-
}
|
|
598
|
-
if (qualityObjectives.size === 0) {
|
|
599
|
-
ctx.addIssue({
|
|
600
|
-
code: "custom",
|
|
601
|
-
path: ["objectives"],
|
|
602
|
-
message: "measured comparison must contain at least one quality objective",
|
|
603
|
-
});
|
|
604
|
-
}
|
|
605
|
-
for (const parent of dimensionParents) {
|
|
606
|
-
if (!qualityObjectives.has(parent.objective)) {
|
|
607
|
-
ctx.addIssue({
|
|
608
|
-
code: "custom",
|
|
609
|
-
path: ["objectives", parent.index, "objective"],
|
|
610
|
-
message: "measured dimension must name a present quality objective",
|
|
611
|
-
});
|
|
612
|
-
}
|
|
613
|
-
}
|
|
614
|
-
if (comparison.power.n !== comparison.overall.n) {
|
|
615
|
-
ctx.addIssue({
|
|
616
|
-
code: "custom",
|
|
617
|
-
path: ["power", "n"],
|
|
618
|
-
message: "power analysis must use the paired held-out sample",
|
|
619
|
-
});
|
|
620
|
-
}
|
|
621
|
-
if (comparison.overall.confidenceInterval.level !==
|
|
622
|
-
comparison.experiment.policy.confidenceLevel ||
|
|
623
|
-
comparison.overall.confidenceInterval.resamples !==
|
|
624
|
-
comparison.experiment.policy.resamples ||
|
|
625
|
-
comparison.power.confidenceLevel !== comparison.experiment.policy.confidenceLevel) {
|
|
626
|
-
ctx.addIssue({
|
|
627
|
-
code: "custom",
|
|
628
|
-
path: ["experiment", "policy"],
|
|
629
|
-
message: "reported uncertainty must use the frozen evaluation policy",
|
|
630
|
-
});
|
|
631
|
-
}
|
|
632
|
-
for (const [index, objective] of comparison.objectives.entries()) {
|
|
633
|
-
if (objective.availability === "measured" &&
|
|
634
|
-
(objective.confidenceInterval.level !==
|
|
635
|
-
comparison.experiment.policy.confidenceLevel ||
|
|
636
|
-
objective.confidenceInterval.resamples !== comparison.experiment.policy.resamples)) {
|
|
637
|
-
ctx.addIssue({
|
|
638
|
-
code: "custom",
|
|
639
|
-
path: ["objectives", index, "confidenceInterval"],
|
|
640
|
-
message: "objective uncertainty must use the frozen evaluation policy",
|
|
641
|
-
});
|
|
642
|
-
}
|
|
643
|
-
}
|
|
364
|
+
refineMeasuredComparisonSummary(comparison, comparison.experiment.policy, expectedN, comparison.measurements, {
|
|
365
|
+
score: (evidence) => evidence.receipt.benchmarkResult.material.score,
|
|
366
|
+
dimension: (evidence, name) => evidence.receipt.benchmarkResult.material.dimensions.find((dimension) => dimension.name === name)?.score,
|
|
367
|
+
cost: executionCostUsd,
|
|
368
|
+
latency: executionLatencyMs,
|
|
369
|
+
}, ctx);
|
|
644
370
|
if (!isCanonicalJsonValue(comparison)) {
|
|
645
371
|
ctx.addIssue({
|
|
646
372
|
code: "custom",
|
|
@@ -658,7 +384,10 @@ export const agentImprovementProposalSchema = z
|
|
|
658
384
|
.refine((surfaces) => new Set(surfaces).size === surfaces.length, "changed surfaces must be unique"),
|
|
659
385
|
proposedAt: z.iso.datetime(),
|
|
660
386
|
findings: z.array(canonicalJsonObjectSchema),
|
|
661
|
-
evaluation:
|
|
387
|
+
evaluation: z.discriminatedUnion("kind", [
|
|
388
|
+
agentImprovementMeasuredComparisonSchema,
|
|
389
|
+
agentProfileImprovementMeasuredComparisonSchema,
|
|
390
|
+
]),
|
|
662
391
|
digest: sha256DigestSchema,
|
|
663
392
|
})
|
|
664
393
|
.strict()
|
|
@@ -679,13 +408,25 @@ export const agentImprovementProposalSchema = z
|
|
|
679
408
|
message: "an improvement proposal requires sufficient pre-registered power",
|
|
680
409
|
});
|
|
681
410
|
}
|
|
682
|
-
if (proposal.evaluation.
|
|
683
|
-
proposal.evaluation.experiment.
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
411
|
+
if (proposal.evaluation.kind === "agent-improvement-measured-comparison") {
|
|
412
|
+
if (proposal.evaluation.experiment.baseline.digest ===
|
|
413
|
+
proposal.evaluation.experiment.candidate.digest) {
|
|
414
|
+
ctx.addIssue({
|
|
415
|
+
code: "custom",
|
|
416
|
+
path: ["evaluation", "experiment", "candidate", "digest"],
|
|
417
|
+
message: "an improvement proposal requires a changed candidate bundle",
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
else {
|
|
422
|
+
const changed = changedProfileImprovementSurfaces(proposal.evaluation.experiment.change);
|
|
423
|
+
if (!sameSurfaces(proposal.changedSurfaces, changed)) {
|
|
424
|
+
ctx.addIssue({
|
|
425
|
+
code: "custom",
|
|
426
|
+
path: ["changedSurfaces"],
|
|
427
|
+
message: "proposal changed surfaces must equal the measured profile changes",
|
|
428
|
+
});
|
|
429
|
+
}
|
|
689
430
|
}
|
|
690
431
|
if (!isCanonicalJsonValue(proposal)) {
|
|
691
432
|
ctx.addIssue({
|
|
@@ -694,40 +435,8 @@ export const agentImprovementProposalSchema = z
|
|
|
694
435
|
});
|
|
695
436
|
}
|
|
696
437
|
});
|
|
697
|
-
function
|
|
698
|
-
|
|
699
|
-
const tolerance = Number.EPSILON * Math.max(1, Math.abs(expectedDelta)) * 8;
|
|
700
|
-
if (Math.abs(estimate.delta - expectedDelta) > tolerance) {
|
|
701
|
-
ctx.addIssue({
|
|
702
|
-
code: "custom",
|
|
703
|
-
path: [...path, "delta"],
|
|
704
|
-
message: "measured delta must equal candidate minus baseline",
|
|
705
|
-
});
|
|
706
|
-
}
|
|
707
|
-
if (estimate.confidenceInterval.lower > estimate.confidenceInterval.upper ||
|
|
708
|
-
estimate.delta < estimate.confidenceInterval.lower ||
|
|
709
|
-
estimate.delta > estimate.confidenceInterval.upper) {
|
|
710
|
-
ctx.addIssue({
|
|
711
|
-
code: "custom",
|
|
712
|
-
path: [...path, "confidenceInterval"],
|
|
713
|
-
message: "confidence interval must be ordered and contain the measured delta",
|
|
714
|
-
});
|
|
715
|
-
}
|
|
716
|
-
}
|
|
717
|
-
function refineMeasuredMean(reported, values, path, ctx) {
|
|
718
|
-
const measured = values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
719
|
-
const tolerance = Number.EPSILON * Math.max(1, Math.abs(measured)) * values.length * 8;
|
|
720
|
-
if (Math.abs(reported - measured) > tolerance) {
|
|
721
|
-
ctx.addIssue({
|
|
722
|
-
code: "custom",
|
|
723
|
-
path,
|
|
724
|
-
message: "reported mean must equal the signed per-cell results",
|
|
725
|
-
});
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
function approximatelyEqual(left, right) {
|
|
729
|
-
const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 16;
|
|
730
|
-
return Math.abs(left - right) <= tolerance;
|
|
438
|
+
function sameSurfaces(left, right) {
|
|
439
|
+
return left.length === right.length && left.every((surface) => right.includes(surface));
|
|
731
440
|
}
|
|
732
441
|
function executionCostUsd(evidence) {
|
|
733
442
|
return (evidence.receipt.modelSettlement.material.usage.costUsdNanos +
|
|
@@ -763,7 +472,7 @@ export const agentImprovementActivationSchema = z
|
|
|
763
472
|
proposalDigest: sha256DigestSchema,
|
|
764
473
|
reviewDigest: sha256DigestSchema,
|
|
765
474
|
experimentDigest: sha256DigestSchema,
|
|
766
|
-
|
|
475
|
+
candidateDigest: sha256DigestSchema,
|
|
767
476
|
intent: z.enum(["activate-candidate", "restore-baseline"]),
|
|
768
477
|
targets: z
|
|
769
478
|
.tuple([improvementActivationTargetSchema])
|
|
@@ -197,7 +197,7 @@ export declare const agentCandidateMaterializationReceiptSchema: z.ZodObject<{
|
|
|
197
197
|
digest: z.ZodType<`sha256:${string}`, unknown, z.core.$ZodTypeInternals<`sha256:${string}`, unknown>>;
|
|
198
198
|
material: z.ZodType<{
|
|
199
199
|
sourceProfileDigest: `sha256:${string}`;
|
|
200
|
-
harness: "claude-code" | "
|
|
200
|
+
harness: "claude-code" | "nanoclaw" | "codex" | "opencode" | "kimi-code" | "pi" | "gemini" | "hermes" | "openclaw" | "amp" | "factory-droids" | "acp" | "cli-base";
|
|
201
201
|
files: {
|
|
202
202
|
relPath: string;
|
|
203
203
|
mode: number;
|
|
@@ -221,7 +221,7 @@ export declare const agentCandidateMaterializationReceiptSchema: z.ZodObject<{
|
|
|
221
221
|
} | undefined;
|
|
222
222
|
}, unknown, z.core.$ZodTypeInternals<{
|
|
223
223
|
sourceProfileDigest: `sha256:${string}`;
|
|
224
|
-
harness: "claude-code" | "
|
|
224
|
+
harness: "claude-code" | "nanoclaw" | "codex" | "opencode" | "kimi-code" | "pi" | "gemini" | "hermes" | "openclaw" | "amp" | "factory-droids" | "acp" | "cli-base";
|
|
225
225
|
files: {
|
|
226
226
|
relPath: string;
|
|
227
227
|
mode: number;
|
|
@@ -279,7 +279,7 @@ export declare const agentCandidateMaterializationReceiptSchema: z.ZodObject<{
|
|
|
279
279
|
runCell: {
|
|
280
280
|
kind: "agent-candidate-run-cell";
|
|
281
281
|
experimentDigest: `sha256:${string}`;
|
|
282
|
-
arm: "
|
|
282
|
+
arm: "baseline" | "candidate";
|
|
283
283
|
bundleDigest: `sha256:${string}`;
|
|
284
284
|
suiteDigest: `sha256:${string}`;
|
|
285
285
|
taskDigest: `sha256:${string}`;
|
|
@@ -300,7 +300,7 @@ export declare const agentCandidateMaterializationReceiptSchema: z.ZodObject<{
|
|
|
300
300
|
targetWorkspace: "candidate" | "task";
|
|
301
301
|
mountPaths: string[];
|
|
302
302
|
};
|
|
303
|
-
harness: "claude-code" | "
|
|
303
|
+
harness: "claude-code" | "nanoclaw" | "codex" | "opencode" | "kimi-code" | "pi" | "gemini" | "hermes" | "openclaw" | "amp" | "factory-droids" | "acp" | "cli-base";
|
|
304
304
|
harnessVersion: string;
|
|
305
305
|
instructionDelivery: {
|
|
306
306
|
kind: "argv-append";
|
|
@@ -522,7 +522,7 @@ export declare const agentCandidateMaterializationReceiptSchema: z.ZodObject<{
|
|
|
522
522
|
runCell: {
|
|
523
523
|
kind: "agent-candidate-run-cell";
|
|
524
524
|
experimentDigest: `sha256:${string}`;
|
|
525
|
-
arm: "
|
|
525
|
+
arm: "baseline" | "candidate";
|
|
526
526
|
bundleDigest: `sha256:${string}`;
|
|
527
527
|
suiteDigest: `sha256:${string}`;
|
|
528
528
|
taskDigest: `sha256:${string}`;
|
|
@@ -543,7 +543,7 @@ export declare const agentCandidateMaterializationReceiptSchema: z.ZodObject<{
|
|
|
543
543
|
targetWorkspace: "candidate" | "task";
|
|
544
544
|
mountPaths: string[];
|
|
545
545
|
};
|
|
546
|
-
harness: "claude-code" | "
|
|
546
|
+
harness: "claude-code" | "nanoclaw" | "codex" | "opencode" | "kimi-code" | "pi" | "gemini" | "hermes" | "openclaw" | "amp" | "factory-droids" | "acp" | "cli-base";
|
|
547
547
|
harnessVersion: string;
|
|
548
548
|
instructionDelivery: {
|
|
549
549
|
kind: "argv-append";
|
|
@@ -840,13 +840,10 @@ export declare const agentCandidateMaterializationReceiptSchema: z.ZodObject<{
|
|
|
840
840
|
materializedTree: z.ZodOptional<z.ZodString>;
|
|
841
841
|
harness: z.ZodEnum<{
|
|
842
842
|
"claude-code": "claude-code";
|
|
843
|
-
claude: "claude";
|
|
844
|
-
claudish: "claudish";
|
|
845
843
|
nanoclaw: "nanoclaw";
|
|
846
844
|
codex: "codex";
|
|
847
845
|
opencode: "opencode";
|
|
848
846
|
"kimi-code": "kimi-code";
|
|
849
|
-
kimi: "kimi";
|
|
850
847
|
pi: "pi";
|
|
851
848
|
gemini: "gemini";
|
|
852
849
|
hermes: "hermes";
|
|
@@ -906,6 +903,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
906
903
|
endedAtMs: z.ZodNumber;
|
|
907
904
|
durationMs: z.ZodNumber;
|
|
908
905
|
}, z.core.$strict>;
|
|
906
|
+
steps: z.ZodNumber;
|
|
909
907
|
memory: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
910
908
|
mode: z.ZodLiteral<"disabled">;
|
|
911
909
|
}, z.core.$strict>, z.ZodObject<{
|
|
@@ -1023,7 +1021,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1023
1021
|
callId: string;
|
|
1024
1022
|
generationId: string;
|
|
1025
1023
|
traceSpanId: string;
|
|
1026
|
-
status: "
|
|
1024
|
+
status: "succeeded" | "failed";
|
|
1027
1025
|
model: string;
|
|
1028
1026
|
startedAtMs: number;
|
|
1029
1027
|
endedAtMs: number;
|
|
@@ -1058,7 +1056,7 @@ export declare const agentCandidateRunReceiptSchema: z.ZodObject<{
|
|
|
1058
1056
|
callId: string;
|
|
1059
1057
|
generationId: string;
|
|
1060
1058
|
traceSpanId: string;
|
|
1061
|
-
status: "
|
|
1059
|
+
status: "succeeded" | "failed";
|
|
1062
1060
|
model: string;
|
|
1063
1061
|
startedAtMs: number;
|
|
1064
1062
|
endedAtMs: number;
|
|
@@ -259,6 +259,7 @@ export const agentCandidateRunReceiptSchema = z
|
|
|
259
259
|
durationMs: z.number().int().nonnegative().safe(),
|
|
260
260
|
})
|
|
261
261
|
.strict(),
|
|
262
|
+
steps: z.number().int().nonnegative().safe(),
|
|
262
263
|
memory: agentCandidateMemoryReceiptSchema,
|
|
263
264
|
trace: agentCandidateTraceEvidenceSchema,
|
|
264
265
|
termination: agentCandidateTerminationSchema,
|
|
@@ -35,13 +35,10 @@ export declare const agentCandidateBundleSchema: z.ZodObject<{
|
|
|
35
35
|
}, z.core.$strict>>;
|
|
36
36
|
harness: z.ZodOptional<z.ZodEnum<{
|
|
37
37
|
"claude-code": "claude-code";
|
|
38
|
-
claude: "claude";
|
|
39
|
-
claudish: "claudish";
|
|
40
38
|
nanoclaw: "nanoclaw";
|
|
41
39
|
codex: "codex";
|
|
42
40
|
opencode: "opencode";
|
|
43
41
|
"kimi-code": "kimi-code";
|
|
44
|
-
kimi: "kimi";
|
|
45
42
|
pi: "pi";
|
|
46
43
|
gemini: "gemini";
|
|
47
44
|
hermes: "hermes";
|
|
@@ -270,13 +267,10 @@ export declare const agentCandidateBundleSchema: z.ZodObject<{
|
|
|
270
267
|
execution: z.ZodObject<{
|
|
271
268
|
harness: z.ZodEnum<{
|
|
272
269
|
"claude-code": "claude-code";
|
|
273
|
-
claude: "claude";
|
|
274
|
-
claudish: "claudish";
|
|
275
270
|
nanoclaw: "nanoclaw";
|
|
276
271
|
codex: "codex";
|
|
277
272
|
opencode: "opencode";
|
|
278
273
|
"kimi-code": "kimi-code";
|
|
279
|
-
kimi: "kimi";
|
|
280
274
|
pi: "pi";
|
|
281
275
|
gemini: "gemini";
|
|
282
276
|
hermes: "hermes";
|
|
@@ -3,7 +3,6 @@ import { agentCandidateCodeSchema, agentCandidateExecutionSchema } from "./agent
|
|
|
3
3
|
import { agentCandidateProfileSchema } from "./agent-candidate-profile-schema.js";
|
|
4
4
|
import { agentCandidateKnowledgeSchema, agentCandidateMemoryPolicySchema, } from "./agent-candidate-lineage-schema.js";
|
|
5
5
|
import { isCanonicalJsonValue, sha256DigestSchema, } from "./agent-candidate-schema-common.js";
|
|
6
|
-
import { canonicalizeHarness } from "./harness.js";
|
|
7
6
|
/**
|
|
8
7
|
* Structural parser for a frozen candidate.
|
|
9
8
|
*
|
|
@@ -32,8 +31,7 @@ export const agentCandidateBundleSchema = z
|
|
|
32
31
|
});
|
|
33
32
|
}
|
|
34
33
|
if (bundle.profile.harness !== undefined &&
|
|
35
|
-
|
|
36
|
-
canonicalizeHarness(bundle.execution.harness)) {
|
|
34
|
+
bundle.profile.harness !== bundle.execution.harness) {
|
|
37
35
|
ctx.addIssue({
|
|
38
36
|
code: "custom",
|
|
39
37
|
path: ["execution", "harness"],
|