@openpond/evals 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +13 -2
- package/RELEASING.md +6 -5
- package/dist/compatibility.js +45 -0
- package/dist/conformance.js +6 -9
- package/dist/harness-improvements.js +329 -0
- package/dist/harness-workspaces.js +368 -0
- package/dist/harness.js +4 -6
- package/dist/index.js +3 -0
- package/dist/runs.js +33 -5
- package/dist/tasksets.js +2 -3
- package/dist/types/compatibility.d.ts +11 -0
- package/dist/types/compatibility.d.ts.map +1 -0
- package/dist/types/conformance.d.ts +8 -48
- package/dist/types/conformance.d.ts.map +1 -1
- package/dist/types/evidence/conformance.d.ts +12 -12
- package/dist/types/evidence/contracts.d.ts +10 -10
- package/dist/types/harness-improvements.d.ts +523 -0
- package/dist/types/harness-improvements.d.ts.map +1 -0
- package/dist/types/harness-workspaces.d.ts +802 -0
- package/dist/types/harness-workspaces.d.ts.map +1 -0
- package/dist/types/harness.d.ts +6 -68
- package/dist/types/harness.d.ts.map +1 -1
- package/dist/types/index.d.ts +3 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/runs.d.ts +46 -1
- package/dist/types/runs.d.ts.map +1 -1
- package/dist/types/tasksets.d.ts +2 -10
- package/dist/types/tasksets.d.ts.map +1 -1
- package/package.json +9 -1
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { contentHash, ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, } from "./common.js";
|
|
3
|
+
const MetadataSchema = z.record(z.string(), z.unknown()).default({});
|
|
4
|
+
const RevisionSchema = z.number().int().nonnegative();
|
|
5
|
+
const BoundedTextSchema = z.string().trim().min(1).max(100_000);
|
|
6
|
+
export const HarnessOwnerScopeSchema = z
|
|
7
|
+
.object({
|
|
8
|
+
kind: z.enum(["personal", "team"]),
|
|
9
|
+
id: ReleaseIdSchema,
|
|
10
|
+
})
|
|
11
|
+
.strict();
|
|
12
|
+
export const HarnessCurrentChannelSchema = z
|
|
13
|
+
.object({
|
|
14
|
+
name: ReleaseIdSchema,
|
|
15
|
+
release: ImmutableReleaseRefSchema.nullable(),
|
|
16
|
+
revision: RevisionSchema,
|
|
17
|
+
})
|
|
18
|
+
.strict();
|
|
19
|
+
/**
|
|
20
|
+
* Mutable, host-owned authoring state. Portable runs never consume this object;
|
|
21
|
+
* they consume the immutable release selected from currentChannel.
|
|
22
|
+
*/
|
|
23
|
+
export const HarnessWorkspaceSchema = z
|
|
24
|
+
.object({
|
|
25
|
+
schemaVersion: z.literal("openpond.harnessWorkspace.v1"),
|
|
26
|
+
id: ReleaseIdSchema,
|
|
27
|
+
ownerScope: HarnessOwnerScopeSchema,
|
|
28
|
+
name: z.string().trim().min(1).max(240),
|
|
29
|
+
location: z.enum(["local", "hosted"]),
|
|
30
|
+
sourceRevision: z.string().trim().min(1).max(500),
|
|
31
|
+
revision: RevisionSchema,
|
|
32
|
+
dirty: z.boolean(),
|
|
33
|
+
currentChannel: HarnessCurrentChannelSchema,
|
|
34
|
+
createdAt: ReleaseTimestampSchema,
|
|
35
|
+
updatedAt: ReleaseTimestampSchema,
|
|
36
|
+
metadata: MetadataSchema,
|
|
37
|
+
})
|
|
38
|
+
.strict();
|
|
39
|
+
export const HarnessImprovementRouteSchema = z.enum([
|
|
40
|
+
"runtime",
|
|
41
|
+
"memory",
|
|
42
|
+
"prompt",
|
|
43
|
+
"skill",
|
|
44
|
+
"agent",
|
|
45
|
+
"product",
|
|
46
|
+
"taskset",
|
|
47
|
+
"training",
|
|
48
|
+
]);
|
|
49
|
+
export const HarnessChangeEffectSchema = z.enum([
|
|
50
|
+
"text_instruction",
|
|
51
|
+
"memory",
|
|
52
|
+
"dependency_selection",
|
|
53
|
+
"executable_code",
|
|
54
|
+
"business_logic",
|
|
55
|
+
"financial_logic",
|
|
56
|
+
"permission",
|
|
57
|
+
"connected_app",
|
|
58
|
+
"publication",
|
|
59
|
+
"deployment",
|
|
60
|
+
"training",
|
|
61
|
+
"model_binding",
|
|
62
|
+
"team_or_global",
|
|
63
|
+
]);
|
|
64
|
+
export const HarnessOverlayEditSchema = z
|
|
65
|
+
.object({
|
|
66
|
+
id: ReleaseIdSchema,
|
|
67
|
+
route: HarnessImprovementRouteSchema,
|
|
68
|
+
operation: z.enum(["create", "update", "delete"]),
|
|
69
|
+
target: z.string().trim().min(1).max(2_000),
|
|
70
|
+
summary: z.string().trim().min(1).max(2_000),
|
|
71
|
+
content: z.string().max(1_000_000).nullable(),
|
|
72
|
+
contentHash: ReleaseHashSchema.nullable(),
|
|
73
|
+
effects: z.array(HarnessChangeEffectSchema).min(1).max(20),
|
|
74
|
+
})
|
|
75
|
+
.strict()
|
|
76
|
+
.superRefine((edit, context) => {
|
|
77
|
+
if (edit.operation === "delete") {
|
|
78
|
+
if (edit.content !== null || edit.contentHash !== null) {
|
|
79
|
+
context.addIssue({
|
|
80
|
+
code: "custom",
|
|
81
|
+
message: "delete edits cannot include replacement content",
|
|
82
|
+
path: ["content"],
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
if (edit.content === null || edit.contentHash === null) {
|
|
88
|
+
context.addIssue({
|
|
89
|
+
code: "custom",
|
|
90
|
+
message: "create and update edits require content and contentHash",
|
|
91
|
+
path: ["content"],
|
|
92
|
+
});
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
const expected = contentHash(edit.content);
|
|
96
|
+
if (edit.contentHash !== expected) {
|
|
97
|
+
context.addIssue({
|
|
98
|
+
code: "custom",
|
|
99
|
+
message: `edit contentHash is ${edit.contentHash}; expected ${expected}`,
|
|
100
|
+
path: ["contentHash"],
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
export const HarnessWorkspaceRevisionRefSchema = z
|
|
105
|
+
.object({
|
|
106
|
+
workspaceId: ReleaseIdSchema,
|
|
107
|
+
revision: RevisionSchema,
|
|
108
|
+
sourceRevision: z.string().trim().min(1).max(500),
|
|
109
|
+
channelRevision: RevisionSchema,
|
|
110
|
+
})
|
|
111
|
+
.strict();
|
|
112
|
+
export const HarnessOverlaySnapshotRefSchema = z
|
|
113
|
+
.object({
|
|
114
|
+
id: ReleaseIdSchema,
|
|
115
|
+
revision: RevisionSchema,
|
|
116
|
+
contentHash: ReleaseHashSchema,
|
|
117
|
+
})
|
|
118
|
+
.strict();
|
|
119
|
+
export const HarnessTurnSnapshotSchema = z
|
|
120
|
+
.object({
|
|
121
|
+
schemaVersion: z.literal("openpond.harnessTurnSnapshot.v1"),
|
|
122
|
+
workspaceId: ReleaseIdSchema,
|
|
123
|
+
workspaceRevision: RevisionSchema,
|
|
124
|
+
sourceRevision: z.string().trim().min(1).max(500),
|
|
125
|
+
channelName: ReleaseIdSchema,
|
|
126
|
+
channelRevision: RevisionSchema,
|
|
127
|
+
harnessRelease: ImmutableReleaseRefSchema,
|
|
128
|
+
overlay: HarnessOverlaySnapshotRefSchema.nullable().optional().default(null),
|
|
129
|
+
})
|
|
130
|
+
.strict();
|
|
131
|
+
export const HarnessRunOverlayContentSchema = z
|
|
132
|
+
.object({
|
|
133
|
+
schemaVersion: z.literal("openpond.harnessRunOverlay.v1"),
|
|
134
|
+
id: ReleaseIdSchema,
|
|
135
|
+
runId: ReleaseIdSchema,
|
|
136
|
+
baseHarnessRelease: ImmutableReleaseRefSchema,
|
|
137
|
+
workspace: HarnessWorkspaceRevisionRefSchema,
|
|
138
|
+
revision: RevisionSchema,
|
|
139
|
+
status: z.enum(["active", "frozen", "abandoned"]),
|
|
140
|
+
edits: z.array(HarnessOverlayEditSchema).max(1_000),
|
|
141
|
+
createdAt: ReleaseTimestampSchema,
|
|
142
|
+
updatedAt: ReleaseTimestampSchema,
|
|
143
|
+
metadata: MetadataSchema,
|
|
144
|
+
})
|
|
145
|
+
.strict();
|
|
146
|
+
export const HarnessRunOverlaySchema = HarnessRunOverlayContentSchema.extend({
|
|
147
|
+
contentHash: ReleaseHashSchema,
|
|
148
|
+
}).strict();
|
|
149
|
+
export const HarnessProposalEvidenceRefSchema = z
|
|
150
|
+
.object({
|
|
151
|
+
kind: z.enum([
|
|
152
|
+
"tool_event",
|
|
153
|
+
"recovery",
|
|
154
|
+
"validation",
|
|
155
|
+
"user_correction",
|
|
156
|
+
"work_output",
|
|
157
|
+
"work_receipt",
|
|
158
|
+
]),
|
|
159
|
+
id: ReleaseIdSchema,
|
|
160
|
+
contentHash: ReleaseHashSchema.nullable(),
|
|
161
|
+
})
|
|
162
|
+
.strict();
|
|
163
|
+
export const HarnessValidationPlanItemSchema = z
|
|
164
|
+
.object({
|
|
165
|
+
id: ReleaseIdSchema,
|
|
166
|
+
kind: z.enum([
|
|
167
|
+
"dependency",
|
|
168
|
+
"schema",
|
|
169
|
+
"package",
|
|
170
|
+
"file_render",
|
|
171
|
+
"skill",
|
|
172
|
+
"prompt",
|
|
173
|
+
"business_formula",
|
|
174
|
+
"targeted_evaluation",
|
|
175
|
+
"observed_recovery",
|
|
176
|
+
]),
|
|
177
|
+
description: BoundedTextSchema,
|
|
178
|
+
required: z.boolean(),
|
|
179
|
+
})
|
|
180
|
+
.strict();
|
|
181
|
+
export const HarnessImprovementProposalContentSchema = z
|
|
182
|
+
.object({
|
|
183
|
+
schemaVersion: z.literal("openpond.harnessImprovementProposal.v1"),
|
|
184
|
+
id: ReleaseIdSchema,
|
|
185
|
+
overlay: HarnessOverlaySnapshotRefSchema,
|
|
186
|
+
baseHarnessRelease: ImmutableReleaseRefSchema,
|
|
187
|
+
expectedWorkspace: HarnessWorkspaceRevisionRefSchema,
|
|
188
|
+
requestedScope: z.enum(["personal", "team", "global"]),
|
|
189
|
+
route: HarnessImprovementRouteSchema,
|
|
190
|
+
risk: z.enum(["low", "review", "restricted"]),
|
|
191
|
+
effects: z.array(HarnessChangeEffectSchema).min(1).max(20),
|
|
192
|
+
evidence: z.array(HarnessProposalEvidenceRefSchema).min(1).max(1_000),
|
|
193
|
+
edits: z.array(HarnessOverlayEditSchema).min(1).max(1_000),
|
|
194
|
+
validationPlan: z.array(HarnessValidationPlanItemSchema).min(1).max(100),
|
|
195
|
+
expectedOutcome: BoundedTextSchema,
|
|
196
|
+
createdAt: ReleaseTimestampSchema,
|
|
197
|
+
metadata: MetadataSchema,
|
|
198
|
+
})
|
|
199
|
+
.strict()
|
|
200
|
+
.superRefine((proposal, context) => {
|
|
201
|
+
const declaredEffects = new Set(proposal.effects);
|
|
202
|
+
for (const [index, edit] of proposal.edits.entries()) {
|
|
203
|
+
if (edit.route !== proposal.route) {
|
|
204
|
+
context.addIssue({
|
|
205
|
+
code: "custom",
|
|
206
|
+
message: "every edit route must match the proposal route",
|
|
207
|
+
path: ["edits", index, "route"],
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
for (const effect of edit.effects) {
|
|
211
|
+
if (!declaredEffects.has(effect)) {
|
|
212
|
+
context.addIssue({
|
|
213
|
+
code: "custom",
|
|
214
|
+
message: `edit effect ${effect} is missing from proposal effects`,
|
|
215
|
+
path: ["edits", index, "effects"],
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
if (new Set(proposal.validationPlan.map((item) => item.id)).size !== proposal.validationPlan.length) {
|
|
221
|
+
context.addIssue({
|
|
222
|
+
code: "custom",
|
|
223
|
+
message: "validation plan ids must be unique",
|
|
224
|
+
path: ["validationPlan"],
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
});
|
|
228
|
+
export const HarnessImprovementProposalSchema = HarnessImprovementProposalContentSchema.extend({
|
|
229
|
+
contentHash: ReleaseHashSchema,
|
|
230
|
+
}).strict();
|
|
231
|
+
export const HarnessTargetedValidationReceiptContentSchema = z
|
|
232
|
+
.object({
|
|
233
|
+
schemaVersion: z.literal("openpond.harnessTargetedValidationReceipt.v1"),
|
|
234
|
+
id: ReleaseIdSchema,
|
|
235
|
+
proposal: ImmutableReleaseRefSchema,
|
|
236
|
+
validationId: ReleaseIdSchema,
|
|
237
|
+
kind: HarnessValidationPlanItemSchema.shape.kind,
|
|
238
|
+
status: z.enum(["passed", "failed", "blocked", "skipped"]),
|
|
239
|
+
summary: BoundedTextSchema,
|
|
240
|
+
evidenceRefs: z.array(HarnessProposalEvidenceRefSchema).max(1_000),
|
|
241
|
+
createdAt: ReleaseTimestampSchema,
|
|
242
|
+
metadata: MetadataSchema,
|
|
243
|
+
})
|
|
244
|
+
.strict();
|
|
245
|
+
export const HarnessTargetedValidationReceiptSchema = HarnessTargetedValidationReceiptContentSchema.extend({
|
|
246
|
+
contentHash: ReleaseHashSchema,
|
|
247
|
+
}).strict();
|
|
248
|
+
export const HarnessAdvanceDecisionSchema = z.enum([
|
|
249
|
+
"advanced",
|
|
250
|
+
"retained",
|
|
251
|
+
"conflict",
|
|
252
|
+
"rolled_back",
|
|
253
|
+
]);
|
|
254
|
+
export const HarnessAdvanceReceiptContentSchema = z
|
|
255
|
+
.object({
|
|
256
|
+
schemaVersion: z.literal("openpond.harnessAdvanceReceipt.v1"),
|
|
257
|
+
id: ReleaseIdSchema,
|
|
258
|
+
workspaceId: ReleaseIdSchema,
|
|
259
|
+
ownerScope: HarnessOwnerScopeSchema,
|
|
260
|
+
proposal: ImmutableReleaseRefSchema.nullable(),
|
|
261
|
+
expectedWorkspaceRevision: RevisionSchema,
|
|
262
|
+
observedWorkspaceRevision: RevisionSchema,
|
|
263
|
+
previousChannelRevision: RevisionSchema,
|
|
264
|
+
nextChannelRevision: RevisionSchema,
|
|
265
|
+
previousRelease: ImmutableReleaseRefSchema.nullable(),
|
|
266
|
+
nextRelease: ImmutableReleaseRefSchema.nullable(),
|
|
267
|
+
validationReceipts: z.array(ImmutableReleaseRefSchema).max(100),
|
|
268
|
+
decision: HarnessAdvanceDecisionSchema,
|
|
269
|
+
reason: BoundedTextSchema,
|
|
270
|
+
rollbackOf: ImmutableReleaseRefSchema.nullable(),
|
|
271
|
+
createdAt: ReleaseTimestampSchema,
|
|
272
|
+
metadata: MetadataSchema,
|
|
273
|
+
})
|
|
274
|
+
.strict()
|
|
275
|
+
.superRefine((receipt, context) => {
|
|
276
|
+
const changed = receipt.nextChannelRevision === receipt.previousChannelRevision + 1;
|
|
277
|
+
if (["advanced", "rolled_back"].includes(receipt.decision) !== changed) {
|
|
278
|
+
context.addIssue({
|
|
279
|
+
code: "custom",
|
|
280
|
+
message: "only successful advancement or rollback increments the channel revision",
|
|
281
|
+
path: ["nextChannelRevision"],
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
if (receipt.decision === "rolled_back" && receipt.rollbackOf === null) {
|
|
285
|
+
context.addIssue({
|
|
286
|
+
code: "custom",
|
|
287
|
+
message: "rollback receipts require rollbackOf",
|
|
288
|
+
path: ["rollbackOf"],
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
if (receipt.decision !== "rolled_back" && receipt.rollbackOf !== null) {
|
|
292
|
+
context.addIssue({
|
|
293
|
+
code: "custom",
|
|
294
|
+
message: "rollbackOf is only valid for rollback receipts",
|
|
295
|
+
path: ["rollbackOf"],
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
export const HarnessAdvanceReceiptSchema = HarnessAdvanceReceiptContentSchema.extend({
|
|
300
|
+
contentHash: ReleaseHashSchema,
|
|
301
|
+
}).strict();
|
|
302
|
+
export const HarnessOverlayMergeReceiptContentSchema = z
|
|
303
|
+
.object({
|
|
304
|
+
schemaVersion: z.literal("openpond.harnessOverlayMergeReceipt.v1"),
|
|
305
|
+
id: ReleaseIdSchema,
|
|
306
|
+
workspaceId: ReleaseIdSchema,
|
|
307
|
+
baseHarnessRelease: ImmutableReleaseRefSchema,
|
|
308
|
+
leftOverlay: HarnessOverlaySnapshotRefSchema,
|
|
309
|
+
rightOverlay: HarnessOverlaySnapshotRefSchema,
|
|
310
|
+
decision: z.enum(["merged", "conflict"]),
|
|
311
|
+
mergedOverlay: HarnessOverlaySnapshotRefSchema.nullable(),
|
|
312
|
+
conflictTargets: z.array(z.string().trim().min(1).max(2_000)).max(1_000),
|
|
313
|
+
requiresRevalidation: z.literal(true),
|
|
314
|
+
createdAt: ReleaseTimestampSchema,
|
|
315
|
+
metadata: MetadataSchema,
|
|
316
|
+
})
|
|
317
|
+
.strict()
|
|
318
|
+
.superRefine((receipt, context) => {
|
|
319
|
+
if ((receipt.decision === "merged") !== (receipt.mergedOverlay !== null)) {
|
|
320
|
+
context.addIssue({
|
|
321
|
+
code: "custom",
|
|
322
|
+
message: "merged decisions require a merged overlay; conflicts cannot provide one",
|
|
323
|
+
path: ["mergedOverlay"],
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
if ((receipt.decision === "conflict") !== (receipt.conflictTargets.length > 0)) {
|
|
327
|
+
context.addIssue({
|
|
328
|
+
code: "custom",
|
|
329
|
+
message: "conflict decisions require conflict targets and merged decisions cannot include them",
|
|
330
|
+
path: ["conflictTargets"],
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
});
|
|
334
|
+
export const HarnessOverlayMergeReceiptSchema = HarnessOverlayMergeReceiptContentSchema.extend({
|
|
335
|
+
contentHash: ReleaseHashSchema,
|
|
336
|
+
}).strict();
|
|
337
|
+
export function createHarnessRunOverlay(content) {
|
|
338
|
+
const parsed = HarnessRunOverlayContentSchema.parse(content);
|
|
339
|
+
return HarnessRunOverlaySchema.parse({ ...parsed, contentHash: contentHash(parsed) });
|
|
340
|
+
}
|
|
341
|
+
export function createHarnessImprovementProposal(content) {
|
|
342
|
+
const parsed = HarnessImprovementProposalContentSchema.parse(content);
|
|
343
|
+
return HarnessImprovementProposalSchema.parse({
|
|
344
|
+
...parsed,
|
|
345
|
+
contentHash: contentHash(parsed),
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
export function createHarnessTargetedValidationReceipt(content) {
|
|
349
|
+
const parsed = HarnessTargetedValidationReceiptContentSchema.parse(content);
|
|
350
|
+
return HarnessTargetedValidationReceiptSchema.parse({
|
|
351
|
+
...parsed,
|
|
352
|
+
contentHash: contentHash(parsed),
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
export function createHarnessAdvanceReceipt(content) {
|
|
356
|
+
const parsed = HarnessAdvanceReceiptContentSchema.parse(content);
|
|
357
|
+
return HarnessAdvanceReceiptSchema.parse({
|
|
358
|
+
...parsed,
|
|
359
|
+
contentHash: contentHash(parsed),
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
export function createHarnessOverlayMergeReceipt(content) {
|
|
363
|
+
const parsed = HarnessOverlayMergeReceiptContentSchema.parse(content);
|
|
364
|
+
return HarnessOverlayMergeReceiptSchema.parse({
|
|
365
|
+
...parsed,
|
|
366
|
+
contentHash: contentHash(parsed),
|
|
367
|
+
});
|
|
368
|
+
}
|
package/dist/harness.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { FailureClassSchema, ImmutableArtifactRefSchema, ImmutableAssetRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash, } from "./common.js";
|
|
3
|
-
import {
|
|
3
|
+
import { ToolDeclarationSchema } from "./tasksets.js";
|
|
4
4
|
import { createAttemptReceipt } from "./runs.js";
|
|
5
5
|
export const PortabilityReportSchema = z.object({
|
|
6
6
|
portable: z.boolean(),
|
|
@@ -9,9 +9,9 @@ export const PortabilityReportSchema = z.object({
|
|
|
9
9
|
hostPrivateAssetRefs: z.array(ReleaseIdSchema).max(10_000),
|
|
10
10
|
}).strict();
|
|
11
11
|
export const AgentSnapshotContentSchema = z.object({
|
|
12
|
-
schemaVersion: z.literal("openpond.agentSnapshot.
|
|
12
|
+
schemaVersion: z.literal("openpond.agentSnapshot.v2"),
|
|
13
13
|
id: ReleaseIdSchema,
|
|
14
|
-
|
|
14
|
+
sourceRelease: ImmutableReleaseRefSchema.nullable(),
|
|
15
15
|
instructions: z.array(ImmutableAssetRefSchema).max(10_000),
|
|
16
16
|
skills: z.array(ImmutableAssetRefSchema).max(10_000),
|
|
17
17
|
agents: z.array(ImmutableAssetRefSchema).max(10_000),
|
|
@@ -40,15 +40,13 @@ export const GraderInterfaceContractSchema = z.object({
|
|
|
40
40
|
privateVerifierIsolation: z.boolean(),
|
|
41
41
|
}).strict();
|
|
42
42
|
export const HarnessReleaseContentSchema = z.object({
|
|
43
|
-
schemaVersion: z.literal("openpond.harnessRelease.
|
|
43
|
+
schemaVersion: z.literal("openpond.harnessRelease.v2"),
|
|
44
44
|
id: ReleaseIdSchema,
|
|
45
45
|
agentSnapshot: ImmutableReleaseRefSchema,
|
|
46
46
|
program: ImmutableAssetRefSchema,
|
|
47
|
-
environment: EnvironmentContractSchema,
|
|
48
47
|
tools: z.array(ToolDeclarationSchema).max(200),
|
|
49
48
|
lifecycle: LifecycleContractSchema,
|
|
50
49
|
graderInterface: GraderInterfaceContractSchema,
|
|
51
|
-
policy: PolicyBoundarySchema,
|
|
52
50
|
files: z.array(ImmutableAssetRefSchema).max(100_000),
|
|
53
51
|
metadata: MetadataSchema,
|
|
54
52
|
}).strict();
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
export * from "./common.js";
|
|
2
|
+
export * from "./compatibility.js";
|
|
2
3
|
export * from "./evidence/index.js";
|
|
3
4
|
export * from "./graders.js";
|
|
4
5
|
export * from "./harness.js";
|
|
6
|
+
export * from "./harness-improvements.js";
|
|
7
|
+
export * from "./harness-workspaces.js";
|
|
5
8
|
export * from "./runs.js";
|
|
6
9
|
export * from "./tasksets.js";
|
package/dist/runs.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import { FailureClassSchema, ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "./common.js";
|
|
2
|
+
import { FailureClassSchema, ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "./common.js";
|
|
3
3
|
export const ModelRefSchema = z.object({
|
|
4
4
|
provider: ReleaseIdSchema,
|
|
5
5
|
model: ReleaseIdSchema,
|
|
@@ -69,6 +69,21 @@ export const EvaluationResultContentSchema = z.object({
|
|
|
69
69
|
metadata: MetadataSchema,
|
|
70
70
|
}).strict();
|
|
71
71
|
export const EvaluationResultSchema = EvaluationResultContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
|
|
72
|
+
export const HarnessCompatibilityReceiptContentSchema = z.object({
|
|
73
|
+
schemaVersion: z.literal("openpond.harnessCompatibility.v1"),
|
|
74
|
+
id: ReleaseIdSchema,
|
|
75
|
+
baseHarnessRelease: ImmutableReleaseRefSchema,
|
|
76
|
+
candidateHarnessRelease: ImmutableReleaseRefSchema,
|
|
77
|
+
tasksetRelease: ImmutableReleaseRefSchema,
|
|
78
|
+
environmentHash: ReleaseHashSchema,
|
|
79
|
+
toolContractHash: ReleaseHashSchema,
|
|
80
|
+
policyHash: ReleaseHashSchema,
|
|
81
|
+
graderInterfaceHash: ReleaseHashSchema,
|
|
82
|
+
metadata: MetadataSchema,
|
|
83
|
+
}).strict();
|
|
84
|
+
export const HarnessCompatibilityReceiptSchema = HarnessCompatibilityReceiptContentSchema
|
|
85
|
+
.extend({ contentHash: ReleaseHashSchema })
|
|
86
|
+
.strict();
|
|
72
87
|
export function createRunManifest(input) {
|
|
73
88
|
const content = RunManifestContentSchema.parse(input);
|
|
74
89
|
return RunManifestSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
@@ -115,13 +130,26 @@ export function aggregateEvaluationReceipts(input) {
|
|
|
115
130
|
});
|
|
116
131
|
return EvaluationResultSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
117
132
|
}
|
|
118
|
-
export function
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
133
|
+
export function createHarnessCompatibilityReceipt(input) {
|
|
134
|
+
const content = HarnessCompatibilityReceiptContentSchema.parse(input);
|
|
135
|
+
return HarnessCompatibilityReceiptSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
136
|
+
}
|
|
137
|
+
export function assertComparableRunManifests(base, candidate, compatibility) {
|
|
122
138
|
if (base.tasksetRelease.contentHash !== candidate.tasksetRelease.contentHash) {
|
|
123
139
|
throw new Error("Evaluation runs use different Taskset Releases.");
|
|
124
140
|
}
|
|
141
|
+
if (base.harnessRelease.contentHash === candidate.harnessRelease.contentHash)
|
|
142
|
+
return;
|
|
143
|
+
if (!compatibility) {
|
|
144
|
+
throw new Error("Evaluation runs use different Harness Releases without a compatibility receipt.");
|
|
145
|
+
}
|
|
146
|
+
const receipt = HarnessCompatibilityReceiptSchema.parse(compatibility);
|
|
147
|
+
assertContentHash(receipt, "Harness compatibility receipt");
|
|
148
|
+
if (receipt.baseHarnessRelease.contentHash !== base.harnessRelease.contentHash
|
|
149
|
+
|| receipt.candidateHarnessRelease.contentHash !== candidate.harnessRelease.contentHash
|
|
150
|
+
|| receipt.tasksetRelease.contentHash !== base.tasksetRelease.contentHash) {
|
|
151
|
+
throw new Error("Harness compatibility receipt does not match the compared runs.");
|
|
152
|
+
}
|
|
125
153
|
}
|
|
126
154
|
export function rewardEligibleReceipts(receipts) {
|
|
127
155
|
return receipts.filter((receipt) => receipt.terminal
|
package/dist/tasksets.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import { ImmutableAssetRefSchema,
|
|
2
|
+
import { ImmutableAssetRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, assertContentHash, contentHash, } from "./common.js";
|
|
3
3
|
export const TaskSplitSchema = z.enum(["train", "validation", "test", "frozen_eval"]);
|
|
4
4
|
export const PolicyBoundarySchema = z.object({
|
|
5
5
|
policyVisibleFields: z.array(ReleaseIdSchema).max(1_000).default([]),
|
|
@@ -77,10 +77,9 @@ export const TaskRecordSchema = z.object({
|
|
|
77
77
|
tags: z.array(ReleaseIdSchema).max(100).default([]),
|
|
78
78
|
}).strict();
|
|
79
79
|
export const TasksetReleaseContentSchema = z.object({
|
|
80
|
-
schemaVersion: z.literal("openpond.tasksetRelease.
|
|
80
|
+
schemaVersion: z.literal("openpond.tasksetRelease.v2"),
|
|
81
81
|
id: ReleaseIdSchema,
|
|
82
82
|
revision: z.number().int().positive(),
|
|
83
|
-
harnessRelease: ImmutableReleaseRefSchema,
|
|
84
83
|
policy: PolicyBoundarySchema,
|
|
85
84
|
environment: EnvironmentContractSchema,
|
|
86
85
|
tools: z.array(ToolDeclarationSchema).max(200),
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type HarnessRelease } from "./harness.js";
|
|
2
|
+
import { type HarnessCompatibilityReceipt } from "./runs.js";
|
|
3
|
+
import { type TasksetRelease } from "./tasksets.js";
|
|
4
|
+
export declare function createVerifiedHarnessCompatibilityReceipt(input: {
|
|
5
|
+
id: string;
|
|
6
|
+
baseHarnessRelease: HarnessRelease;
|
|
7
|
+
candidateHarnessRelease: HarnessRelease;
|
|
8
|
+
tasksetRelease: TasksetRelease;
|
|
9
|
+
metadata?: Record<string, unknown>;
|
|
10
|
+
}): HarnessCompatibilityReceipt;
|
|
11
|
+
//# sourceMappingURL=compatibility.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compatibility.d.ts","sourceRoot":"","sources":["../../../../src/compatibility.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,WAAW,CAAC;AACnB,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,eAAe,CAAC;AAEvB,wBAAgB,yCAAyC,CAAC,KAAK,EAAE;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,kBAAkB,EAAE,cAAc,CAAC;IACnC,uBAAuB,EAAE,cAAc,CAAC;IACxC,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,2BAA2B,CA4C9B"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
export declare const genericToolConformance: {
|
|
2
2
|
snapshot: {
|
|
3
|
-
schemaVersion: "openpond.agentSnapshot.
|
|
3
|
+
schemaVersion: "openpond.agentSnapshot.v2";
|
|
4
4
|
id: string;
|
|
5
|
-
|
|
5
|
+
sourceRelease: {
|
|
6
6
|
id: string;
|
|
7
7
|
contentHash: string;
|
|
8
8
|
} | null;
|
|
@@ -61,7 +61,7 @@ export declare const genericToolConformance: {
|
|
|
61
61
|
contentHash: string;
|
|
62
62
|
};
|
|
63
63
|
harness: {
|
|
64
|
-
schemaVersion: "openpond.harnessRelease.
|
|
64
|
+
schemaVersion: "openpond.harnessRelease.v2";
|
|
65
65
|
id: string;
|
|
66
66
|
agentSnapshot: {
|
|
67
67
|
id: string;
|
|
@@ -75,16 +75,6 @@ export declare const genericToolConformance: {
|
|
|
75
75
|
mediaType: string;
|
|
76
76
|
visibility: "policy" | "verifier" | "host_private";
|
|
77
77
|
};
|
|
78
|
-
environment: {
|
|
79
|
-
protocolVersion: "openpond.environment.v1";
|
|
80
|
-
kind: "text" | "agent" | "work" | "custom_program";
|
|
81
|
-
entrypoint: string;
|
|
82
|
-
stateful: boolean;
|
|
83
|
-
deterministicSeeds: boolean;
|
|
84
|
-
lifecycle: ("create" | "reset" | "step" | "collect" | "destroy")[];
|
|
85
|
-
networkPolicy: "none" | "declared_read_only" | "declared_scoped";
|
|
86
|
-
defaultTimeoutMs: number;
|
|
87
|
-
};
|
|
88
78
|
tools: {
|
|
89
79
|
name: string;
|
|
90
80
|
description: string;
|
|
@@ -106,12 +96,6 @@ export declare const genericToolConformance: {
|
|
|
106
96
|
privilegedEvidence: string[];
|
|
107
97
|
privateVerifierIsolation: boolean;
|
|
108
98
|
};
|
|
109
|
-
policy: {
|
|
110
|
-
policyVisibleFields: string[];
|
|
111
|
-
privilegedFields: string[];
|
|
112
|
-
hiddenGraderRefs: string[];
|
|
113
|
-
connectedAppScopes: string[];
|
|
114
|
-
};
|
|
115
99
|
files: {
|
|
116
100
|
id: string;
|
|
117
101
|
path: string;
|
|
@@ -124,13 +108,9 @@ export declare const genericToolConformance: {
|
|
|
124
108
|
contentHash: string;
|
|
125
109
|
};
|
|
126
110
|
taskset: {
|
|
127
|
-
schemaVersion: "openpond.tasksetRelease.
|
|
111
|
+
schemaVersion: "openpond.tasksetRelease.v2";
|
|
128
112
|
id: string;
|
|
129
113
|
revision: number;
|
|
130
|
-
harnessRelease: {
|
|
131
|
-
id: string;
|
|
132
|
-
contentHash: string;
|
|
133
|
-
};
|
|
134
114
|
policy: {
|
|
135
115
|
policyVisibleFields: string[];
|
|
136
116
|
privilegedFields: string[];
|
|
@@ -286,9 +266,9 @@ export declare const genericToolConformance: {
|
|
|
286
266
|
};
|
|
287
267
|
export declare const marketingPortfolioConformance: {
|
|
288
268
|
snapshot: {
|
|
289
|
-
schemaVersion: "openpond.agentSnapshot.
|
|
269
|
+
schemaVersion: "openpond.agentSnapshot.v2";
|
|
290
270
|
id: string;
|
|
291
|
-
|
|
271
|
+
sourceRelease: {
|
|
292
272
|
id: string;
|
|
293
273
|
contentHash: string;
|
|
294
274
|
} | null;
|
|
@@ -347,7 +327,7 @@ export declare const marketingPortfolioConformance: {
|
|
|
347
327
|
contentHash: string;
|
|
348
328
|
};
|
|
349
329
|
harness: {
|
|
350
|
-
schemaVersion: "openpond.harnessRelease.
|
|
330
|
+
schemaVersion: "openpond.harnessRelease.v2";
|
|
351
331
|
id: string;
|
|
352
332
|
agentSnapshot: {
|
|
353
333
|
id: string;
|
|
@@ -361,16 +341,6 @@ export declare const marketingPortfolioConformance: {
|
|
|
361
341
|
mediaType: string;
|
|
362
342
|
visibility: "policy" | "verifier" | "host_private";
|
|
363
343
|
};
|
|
364
|
-
environment: {
|
|
365
|
-
protocolVersion: "openpond.environment.v1";
|
|
366
|
-
kind: "text" | "agent" | "work" | "custom_program";
|
|
367
|
-
entrypoint: string;
|
|
368
|
-
stateful: boolean;
|
|
369
|
-
deterministicSeeds: boolean;
|
|
370
|
-
lifecycle: ("create" | "reset" | "step" | "collect" | "destroy")[];
|
|
371
|
-
networkPolicy: "none" | "declared_read_only" | "declared_scoped";
|
|
372
|
-
defaultTimeoutMs: number;
|
|
373
|
-
};
|
|
374
344
|
tools: {
|
|
375
345
|
name: string;
|
|
376
346
|
description: string;
|
|
@@ -392,12 +362,6 @@ export declare const marketingPortfolioConformance: {
|
|
|
392
362
|
privilegedEvidence: string[];
|
|
393
363
|
privateVerifierIsolation: boolean;
|
|
394
364
|
};
|
|
395
|
-
policy: {
|
|
396
|
-
policyVisibleFields: string[];
|
|
397
|
-
privilegedFields: string[];
|
|
398
|
-
hiddenGraderRefs: string[];
|
|
399
|
-
connectedAppScopes: string[];
|
|
400
|
-
};
|
|
401
365
|
files: {
|
|
402
366
|
id: string;
|
|
403
367
|
path: string;
|
|
@@ -410,13 +374,9 @@ export declare const marketingPortfolioConformance: {
|
|
|
410
374
|
contentHash: string;
|
|
411
375
|
};
|
|
412
376
|
taskset: {
|
|
413
|
-
schemaVersion: "openpond.tasksetRelease.
|
|
377
|
+
schemaVersion: "openpond.tasksetRelease.v2";
|
|
414
378
|
id: string;
|
|
415
379
|
revision: number;
|
|
416
|
-
harnessRelease: {
|
|
417
|
-
id: string;
|
|
418
|
-
contentHash: string;
|
|
419
|
-
};
|
|
420
380
|
policy: {
|
|
421
381
|
policyVisibleFields: string[];
|
|
422
382
|
privilegedFields: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AASA,eAAO,MAAM,sBAAsB
|
|
1
|
+
{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AASA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}
|