@openpond/evals 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +24 -5
- package/README.md +27 -8
- package/RELEASING.md +13 -45
- package/dist/compatibility.js +45 -0
- package/dist/conformance.js +7 -11
- package/dist/evidence/authoring.js +1 -1
- package/dist/evidence/conformance.js +1 -1
- package/dist/evidence/contracts.js +1 -2
- package/dist/evidence/eligibility.js +1 -1
- package/dist/graders.js +1 -1
- package/dist/harness.js +33 -103
- package/dist/index.js +3 -1
- package/dist/model-improvement-qualification.js +125 -0
- package/dist/review-conformance.js +209 -0
- package/dist/runs.js +33 -13
- package/dist/tasksets.js +3 -17
- package/dist/types/compatibility.d.ts +11 -0
- package/dist/types/compatibility.d.ts.map +1 -0
- package/dist/types/conformance.d.ts +10 -48
- package/dist/types/conformance.d.ts.map +1 -1
- package/dist/types/evidence/conformance.d.ts +8 -8
- package/dist/types/evidence/contracts.d.ts +8 -8
- package/dist/types/harness.d.ts +3 -471
- package/dist/types/harness.d.ts.map +1 -1
- package/dist/types/index.d.ts +3 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/model-improvement-qualification.d.ts +222 -0
- package/dist/types/model-improvement-qualification.d.ts.map +1 -0
- package/dist/types/review-conformance.d.ts +607 -0
- package/dist/types/review-conformance.d.ts.map +1 -0
- package/dist/types/runs.d.ts +47 -10
- package/dist/types/runs.d.ts.map +1 -1
- package/dist/types/tasksets.d.ts +4 -35
- package/dist/types/tasksets.d.ts.map +1 -1
- package/package.json +14 -2
- package/dist/common.js +0 -67
- package/dist/sha256.js +0 -91
- package/dist/types/common.d.ts +0 -51
- package/dist/types/common.d.ts.map +0 -1
- package/dist/types/sha256.d.ts +0 -2
- package/dist/types/sha256.d.ts.map +0 -1
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const ModelImprovementDecisionSchema: z.ZodEnum<{
|
|
3
|
+
no_training: "no_training";
|
|
4
|
+
sft: "sft";
|
|
5
|
+
preference: "preference";
|
|
6
|
+
rl: "rl";
|
|
7
|
+
}>;
|
|
8
|
+
export declare const ModelImprovementSignalSchema: z.ZodObject<{
|
|
9
|
+
kind: z.ZodEnum<{
|
|
10
|
+
none: "none";
|
|
11
|
+
demonstrations: "demonstrations";
|
|
12
|
+
chosen_rejected: "chosen_rejected";
|
|
13
|
+
scalar_reward: "scalar_reward";
|
|
14
|
+
}>;
|
|
15
|
+
strength: z.ZodEnum<{
|
|
16
|
+
absent: "absent";
|
|
17
|
+
weak: "weak";
|
|
18
|
+
usable: "usable";
|
|
19
|
+
}>;
|
|
20
|
+
calibrated: z.ZodBoolean;
|
|
21
|
+
confounded: z.ZodBoolean;
|
|
22
|
+
variance: z.ZodNullable<z.ZodNumber>;
|
|
23
|
+
evidenceRefs: z.ZodArray<z.ZodObject<{
|
|
24
|
+
id: z.ZodString;
|
|
25
|
+
contentHash: z.ZodString;
|
|
26
|
+
}, z.core.$strict>>;
|
|
27
|
+
}, z.core.$strict>;
|
|
28
|
+
export declare const ModelImprovementQualificationReceiptContentSchema: z.ZodObject<{
|
|
29
|
+
schemaVersion: z.ZodLiteral<"openpond.modelImprovementQualificationReceipt.v1">;
|
|
30
|
+
id: z.ZodString;
|
|
31
|
+
review: z.ZodObject<{
|
|
32
|
+
id: z.ZodString;
|
|
33
|
+
contentHash: z.ZodString;
|
|
34
|
+
}, z.core.$strict>;
|
|
35
|
+
harnessRelease: z.ZodObject<{
|
|
36
|
+
id: z.ZodString;
|
|
37
|
+
contentHash: z.ZodString;
|
|
38
|
+
}, z.core.$strict>;
|
|
39
|
+
tasksetRelease: z.ZodNullable<z.ZodObject<{
|
|
40
|
+
id: z.ZodString;
|
|
41
|
+
contentHash: z.ZodString;
|
|
42
|
+
}, z.core.$strict>>;
|
|
43
|
+
baselineEvaluation: z.ZodNullable<z.ZodObject<{
|
|
44
|
+
id: z.ZodString;
|
|
45
|
+
contentHash: z.ZodString;
|
|
46
|
+
}, z.core.$strict>>;
|
|
47
|
+
model: z.ZodObject<{
|
|
48
|
+
provider: z.ZodString;
|
|
49
|
+
model: z.ZodString;
|
|
50
|
+
revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
51
|
+
artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
52
|
+
tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
53
|
+
chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
54
|
+
}, z.core.$strict>;
|
|
55
|
+
environmentHash: z.ZodNullable<z.ZodString>;
|
|
56
|
+
toolContractHash: z.ZodNullable<z.ZodString>;
|
|
57
|
+
permissionContractHash: z.ZodNullable<z.ZodString>;
|
|
58
|
+
policyHash: z.ZodNullable<z.ZodString>;
|
|
59
|
+
verifierRef: z.ZodNullable<z.ZodObject<{
|
|
60
|
+
id: z.ZodString;
|
|
61
|
+
contentHash: z.ZodString;
|
|
62
|
+
}, z.core.$strict>>;
|
|
63
|
+
sourcePolicies: z.ZodArray<z.ZodObject<{
|
|
64
|
+
policy: z.ZodObject<{
|
|
65
|
+
id: z.ZodString;
|
|
66
|
+
contentHash: z.ZodString;
|
|
67
|
+
}, z.core.$strict>;
|
|
68
|
+
state: z.ZodEnum<{
|
|
69
|
+
authorized: "authorized";
|
|
70
|
+
revoked: "revoked";
|
|
71
|
+
deleted: "deleted";
|
|
72
|
+
expired: "expired";
|
|
73
|
+
}>;
|
|
74
|
+
checkedAt: z.ZodString;
|
|
75
|
+
}, z.core.$strict>>;
|
|
76
|
+
trainingEvidenceRefs: z.ZodArray<z.ZodObject<{
|
|
77
|
+
id: z.ZodString;
|
|
78
|
+
contentHash: z.ZodString;
|
|
79
|
+
}, z.core.$strict>>;
|
|
80
|
+
frozenEvaluationEvidenceRefs: z.ZodArray<z.ZodObject<{
|
|
81
|
+
id: z.ZodString;
|
|
82
|
+
contentHash: z.ZodString;
|
|
83
|
+
}, z.core.$strict>>;
|
|
84
|
+
privacyApproval: z.ZodNullable<z.ZodObject<{
|
|
85
|
+
id: z.ZodString;
|
|
86
|
+
contentHash: z.ZodString;
|
|
87
|
+
}, z.core.$strict>>;
|
|
88
|
+
budgetApproval: z.ZodNullable<z.ZodObject<{
|
|
89
|
+
id: z.ZodString;
|
|
90
|
+
contentHash: z.ZodString;
|
|
91
|
+
}, z.core.$strict>>;
|
|
92
|
+
maximumCostUsd: z.ZodNumber;
|
|
93
|
+
signal: z.ZodObject<{
|
|
94
|
+
kind: z.ZodEnum<{
|
|
95
|
+
none: "none";
|
|
96
|
+
demonstrations: "demonstrations";
|
|
97
|
+
chosen_rejected: "chosen_rejected";
|
|
98
|
+
scalar_reward: "scalar_reward";
|
|
99
|
+
}>;
|
|
100
|
+
strength: z.ZodEnum<{
|
|
101
|
+
absent: "absent";
|
|
102
|
+
weak: "weak";
|
|
103
|
+
usable: "usable";
|
|
104
|
+
}>;
|
|
105
|
+
calibrated: z.ZodBoolean;
|
|
106
|
+
confounded: z.ZodBoolean;
|
|
107
|
+
variance: z.ZodNullable<z.ZodNumber>;
|
|
108
|
+
evidenceRefs: z.ZodArray<z.ZodObject<{
|
|
109
|
+
id: z.ZodString;
|
|
110
|
+
contentHash: z.ZodString;
|
|
111
|
+
}, z.core.$strict>>;
|
|
112
|
+
}, z.core.$strict>;
|
|
113
|
+
decision: z.ZodEnum<{
|
|
114
|
+
no_training: "no_training";
|
|
115
|
+
sft: "sft";
|
|
116
|
+
preference: "preference";
|
|
117
|
+
rl: "rl";
|
|
118
|
+
}>;
|
|
119
|
+
reasons: z.ZodArray<z.ZodString>;
|
|
120
|
+
createdAt: z.ZodString;
|
|
121
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
122
|
+
}, z.core.$strict>;
|
|
123
|
+
export declare const ModelImprovementQualificationReceiptSchema: z.ZodObject<{
|
|
124
|
+
schemaVersion: z.ZodLiteral<"openpond.modelImprovementQualificationReceipt.v1">;
|
|
125
|
+
id: z.ZodString;
|
|
126
|
+
review: z.ZodObject<{
|
|
127
|
+
id: z.ZodString;
|
|
128
|
+
contentHash: z.ZodString;
|
|
129
|
+
}, z.core.$strict>;
|
|
130
|
+
harnessRelease: z.ZodObject<{
|
|
131
|
+
id: z.ZodString;
|
|
132
|
+
contentHash: z.ZodString;
|
|
133
|
+
}, z.core.$strict>;
|
|
134
|
+
tasksetRelease: z.ZodNullable<z.ZodObject<{
|
|
135
|
+
id: z.ZodString;
|
|
136
|
+
contentHash: z.ZodString;
|
|
137
|
+
}, z.core.$strict>>;
|
|
138
|
+
baselineEvaluation: z.ZodNullable<z.ZodObject<{
|
|
139
|
+
id: z.ZodString;
|
|
140
|
+
contentHash: z.ZodString;
|
|
141
|
+
}, z.core.$strict>>;
|
|
142
|
+
model: z.ZodObject<{
|
|
143
|
+
provider: z.ZodString;
|
|
144
|
+
model: z.ZodString;
|
|
145
|
+
revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
146
|
+
artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
147
|
+
tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
148
|
+
chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
149
|
+
}, z.core.$strict>;
|
|
150
|
+
environmentHash: z.ZodNullable<z.ZodString>;
|
|
151
|
+
toolContractHash: z.ZodNullable<z.ZodString>;
|
|
152
|
+
permissionContractHash: z.ZodNullable<z.ZodString>;
|
|
153
|
+
policyHash: z.ZodNullable<z.ZodString>;
|
|
154
|
+
verifierRef: z.ZodNullable<z.ZodObject<{
|
|
155
|
+
id: z.ZodString;
|
|
156
|
+
contentHash: z.ZodString;
|
|
157
|
+
}, z.core.$strict>>;
|
|
158
|
+
sourcePolicies: z.ZodArray<z.ZodObject<{
|
|
159
|
+
policy: z.ZodObject<{
|
|
160
|
+
id: z.ZodString;
|
|
161
|
+
contentHash: z.ZodString;
|
|
162
|
+
}, z.core.$strict>;
|
|
163
|
+
state: z.ZodEnum<{
|
|
164
|
+
authorized: "authorized";
|
|
165
|
+
revoked: "revoked";
|
|
166
|
+
deleted: "deleted";
|
|
167
|
+
expired: "expired";
|
|
168
|
+
}>;
|
|
169
|
+
checkedAt: z.ZodString;
|
|
170
|
+
}, z.core.$strict>>;
|
|
171
|
+
trainingEvidenceRefs: z.ZodArray<z.ZodObject<{
|
|
172
|
+
id: z.ZodString;
|
|
173
|
+
contentHash: z.ZodString;
|
|
174
|
+
}, z.core.$strict>>;
|
|
175
|
+
frozenEvaluationEvidenceRefs: z.ZodArray<z.ZodObject<{
|
|
176
|
+
id: z.ZodString;
|
|
177
|
+
contentHash: z.ZodString;
|
|
178
|
+
}, z.core.$strict>>;
|
|
179
|
+
privacyApproval: z.ZodNullable<z.ZodObject<{
|
|
180
|
+
id: z.ZodString;
|
|
181
|
+
contentHash: z.ZodString;
|
|
182
|
+
}, z.core.$strict>>;
|
|
183
|
+
budgetApproval: z.ZodNullable<z.ZodObject<{
|
|
184
|
+
id: z.ZodString;
|
|
185
|
+
contentHash: z.ZodString;
|
|
186
|
+
}, z.core.$strict>>;
|
|
187
|
+
maximumCostUsd: z.ZodNumber;
|
|
188
|
+
signal: z.ZodObject<{
|
|
189
|
+
kind: z.ZodEnum<{
|
|
190
|
+
none: "none";
|
|
191
|
+
demonstrations: "demonstrations";
|
|
192
|
+
chosen_rejected: "chosen_rejected";
|
|
193
|
+
scalar_reward: "scalar_reward";
|
|
194
|
+
}>;
|
|
195
|
+
strength: z.ZodEnum<{
|
|
196
|
+
absent: "absent";
|
|
197
|
+
weak: "weak";
|
|
198
|
+
usable: "usable";
|
|
199
|
+
}>;
|
|
200
|
+
calibrated: z.ZodBoolean;
|
|
201
|
+
confounded: z.ZodBoolean;
|
|
202
|
+
variance: z.ZodNullable<z.ZodNumber>;
|
|
203
|
+
evidenceRefs: z.ZodArray<z.ZodObject<{
|
|
204
|
+
id: z.ZodString;
|
|
205
|
+
contentHash: z.ZodString;
|
|
206
|
+
}, z.core.$strict>>;
|
|
207
|
+
}, z.core.$strict>;
|
|
208
|
+
decision: z.ZodEnum<{
|
|
209
|
+
no_training: "no_training";
|
|
210
|
+
sft: "sft";
|
|
211
|
+
preference: "preference";
|
|
212
|
+
rl: "rl";
|
|
213
|
+
}>;
|
|
214
|
+
reasons: z.ZodArray<z.ZodString>;
|
|
215
|
+
createdAt: z.ZodString;
|
|
216
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
217
|
+
contentHash: z.ZodString;
|
|
218
|
+
}, z.core.$strict>;
|
|
219
|
+
export declare function createModelImprovementQualificationReceipt(input: z.input<typeof ModelImprovementQualificationReceiptContentSchema>): ModelImprovementQualificationReceipt;
|
|
220
|
+
export declare function verifyModelImprovementQualificationReceipt(value: unknown): value is ModelImprovementQualificationReceipt;
|
|
221
|
+
export type ModelImprovementQualificationReceipt = z.infer<typeof ModelImprovementQualificationReceiptSchema>;
|
|
222
|
+
//# sourceMappingURL=model-improvement-qualification.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-improvement-qualification.d.ts","sourceRoot":"","sources":["../../../../src/model-improvement-qualification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,eAAO,MAAM,8BAA8B;;;;;EAKzC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;kBAS9B,CAAC;AAEZ,eAAO,MAAM,iDAAiD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkG1D,CAAC;AAEL,eAAO,MAAM,0CAA0C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAG1C,CAAC;AAEd,wBAAgB,0CAA0C,CACxD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,iDAAiD,CAAC,GACvE,oCAAoC,CAMtC;AAED,wBAAgB,0CAA0C,CACxD,KAAK,EAAE,OAAO,GACb,KAAK,IAAI,oCAAoC,CAS/C;AAMD,MAAM,MAAM,oCAAoC,GAAG,CAAC,CAAC,KAAK,CACxD,OAAO,0CAA0C,CAClD,CAAC"}
|