@openpond/evals 0.11.12 → 0.11.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +4 -0
- package/dist/experiments.js +1 -0
- package/dist/profile-evaluation-execution.js +10 -0
- package/dist/types/experiments.d.ts +16 -0
- package/dist/types/experiments.d.ts.map +1 -1
- package/dist/types/profile-evaluation-execution.d.ts +8 -0
- package/dist/types/profile-evaluation-execution.d.ts.map +1 -1
- package/package.json +1 -1
package/CONTRACT.md
CHANGED
|
@@ -182,3 +182,7 @@ numeric or Boolean feedback contribute. Failed, cancelled, absent or ungraded
|
|
|
182
182
|
cases never become zero-valued measurements. Categorical feedback remains
|
|
183
183
|
inspectable without a numeric delta. This operation reads retained evidence and
|
|
184
184
|
has no execution, grading or billing authority.
|
|
185
|
+
|
|
186
|
+
## Retained Profile case evidence
|
|
187
|
+
|
|
188
|
+
Profile execution may return an installation-owned `retainedEvidenceRef` with its exact session and turn identities. The runner validates it and binds it into immutable receipt metadata. A reference grants no access: the installation resolves it through its owner-scoped store and checks the source, member, output and trace hashes before inspection. Full outputs and bounded trace pages stay separate from private grading inputs. Missing historic references remain unavailable rather than reconstructed from a newer source.
|
package/dist/experiments.js
CHANGED
|
@@ -12,6 +12,7 @@ export const ExperimentTargetSchema = z.discriminatedUnion("kind", [
|
|
|
12
12
|
z.object({
|
|
13
13
|
kind: z.literal("harness"),
|
|
14
14
|
source: ProfileEvaluationRunSourceSchema,
|
|
15
|
+
model: z.object({ modelId: ReleaseIdSchema, configurationHash: ReleaseHashSchema }).strict(),
|
|
15
16
|
}).strict(),
|
|
16
17
|
z.object({ kind: z.literal("fixture"), configurationHash: ReleaseHashSchema }).strict(),
|
|
17
18
|
]);
|
|
@@ -4,6 +4,13 @@ import { aggregateTasksetRunReceipts, assertTasksetMetricSource } from "./metric
|
|
|
4
4
|
import { createAttemptReceipt, verifyAttemptReceipt } from "./runs.js";
|
|
5
5
|
import { assertProfileEvaluationRunAdmission } from "./taskset-run-contract.js";
|
|
6
6
|
import { policyTaskView } from "./tasksets.js";
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
/** An installation-owned policy trace. Access remains with that installation;
|
|
9
|
+
* the immutable receipt binds the reference without copying private graders. */
|
|
10
|
+
export const ProfileEvaluationEvidenceRefSchema = z.object({
|
|
11
|
+
sessionId: z.string().trim().min(1).max(500),
|
|
12
|
+
turnId: z.string().trim().min(1).max(500),
|
|
13
|
+
}).strict();
|
|
7
14
|
/** The installation owns execution, tools, approvals and storage. This runner
|
|
8
15
|
* owns frozen population dispatch and private grading for both Desktop and web. */
|
|
9
16
|
export async function executeProfileEvaluationRun(input) {
|
|
@@ -78,6 +85,9 @@ export async function executeProfileEvaluationRun(input) {
|
|
|
78
85
|
sourceRevision: source.sourceRevision,
|
|
79
86
|
definitionHash: source.definitionHash,
|
|
80
87
|
environmentHash: source.environmentHash,
|
|
88
|
+
...(execution.retainedEvidenceRef ? {
|
|
89
|
+
retainedEvidenceRef: ProfileEvaluationEvidenceRefSchema.parse(execution.retainedEvidenceRef),
|
|
90
|
+
} : {}),
|
|
81
91
|
},
|
|
82
92
|
});
|
|
83
93
|
await input.saveReceipt(receipt);
|
|
@@ -29,6 +29,10 @@ export declare const ExperimentTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObjec
|
|
|
29
29
|
}, z.core.$strict>], "kind">;
|
|
30
30
|
environmentHash: z.ZodString;
|
|
31
31
|
}, z.core.$strict>;
|
|
32
|
+
model: z.ZodObject<{
|
|
33
|
+
modelId: z.ZodString;
|
|
34
|
+
configurationHash: z.ZodString;
|
|
35
|
+
}, z.core.$strict>;
|
|
32
36
|
}, z.core.$strict>, z.ZodObject<{
|
|
33
37
|
kind: z.ZodLiteral<"fixture">;
|
|
34
38
|
configurationHash: z.ZodString;
|
|
@@ -97,6 +101,10 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
|
|
|
97
101
|
}, z.core.$strict>], "kind">;
|
|
98
102
|
environmentHash: z.ZodString;
|
|
99
103
|
}, z.core.$strict>;
|
|
104
|
+
model: z.ZodObject<{
|
|
105
|
+
modelId: z.ZodString;
|
|
106
|
+
configurationHash: z.ZodString;
|
|
107
|
+
}, z.core.$strict>;
|
|
100
108
|
}, z.core.$strict>, z.ZodObject<{
|
|
101
109
|
kind: z.ZodLiteral<"fixture">;
|
|
102
110
|
configurationHash: z.ZodString;
|
|
@@ -182,6 +190,10 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
|
|
|
182
190
|
}, z.core.$strict>], "kind">;
|
|
183
191
|
environmentHash: z.ZodString;
|
|
184
192
|
}, z.core.$strict>;
|
|
193
|
+
model: z.ZodObject<{
|
|
194
|
+
modelId: z.ZodString;
|
|
195
|
+
configurationHash: z.ZodString;
|
|
196
|
+
}, z.core.$strict>;
|
|
185
197
|
}, z.core.$strict>, z.ZodObject<{
|
|
186
198
|
kind: z.ZodLiteral<"fixture">;
|
|
187
199
|
configurationHash: z.ZodString;
|
|
@@ -534,6 +546,10 @@ export declare function verifyExperimentEvidence(input: {
|
|
|
534
546
|
};
|
|
535
547
|
environmentHash: string;
|
|
536
548
|
};
|
|
549
|
+
model: {
|
|
550
|
+
modelId: string;
|
|
551
|
+
configurationHash: string;
|
|
552
|
+
};
|
|
537
553
|
} | {
|
|
538
554
|
kind: "fixture";
|
|
539
555
|
configurationHash: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,eAAO,MAAM,sBAAsB
|
|
1
|
+
{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAYjC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;kBAepC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;kBAI9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAG1E;AAED,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA0B1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAC5H,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;kBAQ1B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;kBAUhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU5B,CAAC;AAEZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AACxH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,EAAE,QAAQ,EAAE,kBAAkB,GAAG,gBAAgB,CAgC3I;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAC7H,QAAQ,EAAE,4BAA4B,EAAE,CAAA;KAAE,CAAC,CAAC;IAC9C,OAAO,EAAE,0BAA0B,EAAE,CAAC;CACvC,CAAC;AACF,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,4BAA4B,GAAG;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,0BAA0B,GAAG,cAAc,GAAG,oBAAoB,GAAG,sBAAsB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IACpI,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;IAC/B,eAAe,EAAE,OAAO,GAAG,IAAI,CAAC;CACjC,CAAC;AACF,MAAM,MAAM,0BAA0B,GAAG;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAOrF;AAED,iFAAiF;AACjF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,EACpE,SAAS,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,GACpE,oBAAoB,CAsEtB"}
|
|
@@ -5,6 +5,13 @@ import { type AttemptReceipt } from "./runs.js";
|
|
|
5
5
|
import { type TasksetRunManifest } from "./taskset-run-contract.js";
|
|
6
6
|
import { policyTaskView, type TasksetRelease } from "./tasksets.js";
|
|
7
7
|
import type { ProfileEvaluationCatalog, ProfileEvaluationRunSource } from "./profile-evaluations.js";
|
|
8
|
+
import { z } from "zod";
|
|
9
|
+
/** An installation-owned policy trace. Access remains with that installation;
|
|
10
|
+
* the immutable receipt binds the reference without copying private graders. */
|
|
11
|
+
export declare const ProfileEvaluationEvidenceRefSchema: z.ZodObject<{
|
|
12
|
+
sessionId: z.ZodString;
|
|
13
|
+
turnId: z.ZodString;
|
|
14
|
+
}, z.core.$strict>;
|
|
8
15
|
type PolicyTask = ReturnType<typeof policyTaskView>;
|
|
9
16
|
/** The installation owns execution, tools, approvals and storage. This runner
|
|
10
17
|
* owns frozen population dispatch and private grading for both Desktop and web. */
|
|
@@ -20,6 +27,7 @@ export declare function executeProfileEvaluationRun(input: {
|
|
|
20
27
|
}) => Promise<{
|
|
21
28
|
evidence: AttemptEvidence;
|
|
22
29
|
traceHash: string;
|
|
30
|
+
retainedEvidenceRef?: z.infer<typeof ProfileEvaluationEvidenceRefSchema>;
|
|
23
31
|
artifactRefs: ImmutableArtifactRef[];
|
|
24
32
|
startedAt: string;
|
|
25
33
|
completedAt: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"profile-evaluation-execution.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEvG,OAAO,EAAsC,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAC1J,OAAO,EAA0D,KAAK,qBAAqB,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAC5I,OAAO,EAA8C,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAC5F,OAAO,EAAuC,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACzG,OAAO,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,KAAK,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"profile-evaluation-execution.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEvG,OAAO,EAAsC,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAC1J,OAAO,EAA0D,KAAK,qBAAqB,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAC5I,OAAO,EAA8C,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAC5F,OAAO,EAAuC,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACzG,OAAO,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,KAAK,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AACrG,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;gFACgF;AAChF,eAAO,MAAM,kCAAkC;;;kBAGpC,CAAC;AAEZ,KAAK,UAAU,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;AAEpD;mFACmF;AACnF,wBAAsB,2BAA2B,CAAC,KAAK,EAAE;IACvD,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,CAAC,MAAM,EAAE;QAChB,IAAI,EAAE,UAAU,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,0BAA0B,CAAC;QACnC,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,KAAK,OAAO,CAAC;QACZ,QAAQ,EAAE,eAAe,CAAC;QAC1B,SAAS,EAAE,MAAM,CAAC;QAClB,mBAAmB,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,kCAAkC,CAAC,CAAC;QACzE,YAAY,EAAE,oBAAoB,EAAE,CAAC;QACrC,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,QAAQ,EAAE,OAAO,CAAC;QAClB,YAAY,CAAC,EAAE,cAAc,CAAC,cAAc,CAAC,CAAC;KAC/C,CAAC,CAAC;IACH,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/D,WAAW,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD;mFAC+E;IAC/E,mBAAmB,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC;QAC7F,OAAO,EAAE,cAAc,CAAC;QACxB,KAAK,EAAE,SAAS,CAAC;KAClB,GAAG,IAAI,CAAC,CAAC;IACV,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC;IACV,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,MAAM,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC,CA2FD"}
|
package/package.json
CHANGED