@openpond/evals 0.11.5 → 0.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js
CHANGED
|
@@ -14,6 +14,7 @@ export * from "./model-improvement-qualification.js";
|
|
|
14
14
|
export * from "./preferences.js";
|
|
15
15
|
export * from "./profile-evaluations.js";
|
|
16
16
|
export * from "./profile-evaluation-comparisons.js";
|
|
17
|
+
export * from "./profile-evaluation-reports.js";
|
|
17
18
|
export * from "./profile-evaluation-suites.js";
|
|
18
19
|
export * from "./profile-evaluation-execution.js";
|
|
19
20
|
export * from "./review-conformance.js";
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "@openpond/harness";
|
|
3
|
+
const TestedSourceSchema = z.object({
|
|
4
|
+
profileId: ReleaseIdSchema,
|
|
5
|
+
sourceRevision: z.string().trim().min(1).max(240),
|
|
6
|
+
harnessRelease: ImmutableReleaseRefSchema,
|
|
7
|
+
}).strict();
|
|
8
|
+
const EvidenceSchema = z.object({
|
|
9
|
+
kind: z.enum(["run", "suite", "comparison"]),
|
|
10
|
+
id: ReleaseIdSchema,
|
|
11
|
+
contentHash: ReleaseHashSchema,
|
|
12
|
+
}).strict();
|
|
13
|
+
/** Portable summary only. Private cases, outputs, traces, and credentials stay
|
|
14
|
+
* with the installation's retained evidence, outside Profile source. */
|
|
15
|
+
export const ProfileEvaluationReportContentSchema = z.object({
|
|
16
|
+
schemaVersion: z.literal("openpond.profileEvaluationReport.v1"),
|
|
17
|
+
id: ReleaseIdSchema,
|
|
18
|
+
createdAt: ReleaseTimestampSchema,
|
|
19
|
+
testedSources: z.array(TestedSourceSchema).min(1).max(100),
|
|
20
|
+
tasksetReleases: z.array(ImmutableReleaseRefSchema).min(1).max(100),
|
|
21
|
+
metricPolicyHashes: z.array(ReleaseHashSchema).min(1).max(100),
|
|
22
|
+
modelConfigurationHashes: z.array(ReleaseHashSchema).min(1).max(100),
|
|
23
|
+
evidence: z.array(EvidenceSchema).min(1).max(100),
|
|
24
|
+
summary: z.object({
|
|
25
|
+
totalChecks: z.number().int().nonnegative(),
|
|
26
|
+
passedChecks: z.number().int().nonnegative(),
|
|
27
|
+
failedChecks: z.number().int().nonnegative(),
|
|
28
|
+
unscoredChecks: z.number().int().nonnegative(),
|
|
29
|
+
meanScore: z.number().min(0).max(1).nullable(),
|
|
30
|
+
}).strict(),
|
|
31
|
+
}).strict().superRefine((report, context) => {
|
|
32
|
+
const { summary } = report;
|
|
33
|
+
if (summary.passedChecks + summary.failedChecks + summary.unscoredChecks !== summary.totalChecks) {
|
|
34
|
+
context.addIssue({ code: "custom", path: ["summary"], message: "Report check counts must add up." });
|
|
35
|
+
}
|
|
36
|
+
if (new Set(report.evidence.map(({ kind, id }) => `${kind}:${id}`)).size !== report.evidence.length) {
|
|
37
|
+
context.addIssue({ code: "custom", path: ["evidence"], message: "Report evidence must be unique." });
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
export const ProfileEvaluationReportSchema = ProfileEvaluationReportContentSchema.safeExtend({
|
|
41
|
+
contentHash: ReleaseHashSchema,
|
|
42
|
+
});
|
|
43
|
+
export function createProfileEvaluationReport(input) {
|
|
44
|
+
const content = ProfileEvaluationReportContentSchema.parse(input);
|
|
45
|
+
return ProfileEvaluationReportSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
46
|
+
}
|
|
47
|
+
export function assertProfileEvaluationReport(value) {
|
|
48
|
+
const report = ProfileEvaluationReportSchema.parse(value);
|
|
49
|
+
const { contentHash: expected, ...content } = report;
|
|
50
|
+
if (contentHash(content) !== expected)
|
|
51
|
+
throw new Error("Profile evaluation report content hash differs from its source bytes.");
|
|
52
|
+
return report;
|
|
53
|
+
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export * from "./model-improvement-qualification.js";
|
|
|
14
14
|
export * from "./preferences.js";
|
|
15
15
|
export * from "./profile-evaluations.js";
|
|
16
16
|
export * from "./profile-evaluation-comparisons.js";
|
|
17
|
+
export * from "./profile-evaluation-reports.js";
|
|
17
18
|
export * from "./profile-evaluation-suites.js";
|
|
18
19
|
export * from "./profile-evaluation-execution.js";
|
|
19
20
|
export * from "./review-conformance.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,mCAAmC,CAAC;AAClD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,mCAAmC,CAAC;AAClD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/** Portable summary only. Private cases, outputs, traces, and credentials stay
|
|
3
|
+
* with the installation's retained evidence, outside Profile source. */
|
|
4
|
+
export declare const ProfileEvaluationReportContentSchema: z.ZodObject<{
|
|
5
|
+
schemaVersion: z.ZodLiteral<"openpond.profileEvaluationReport.v1">;
|
|
6
|
+
id: z.ZodString;
|
|
7
|
+
createdAt: z.ZodString;
|
|
8
|
+
testedSources: z.ZodArray<z.ZodObject<{
|
|
9
|
+
profileId: z.ZodString;
|
|
10
|
+
sourceRevision: z.ZodString;
|
|
11
|
+
harnessRelease: z.ZodObject<{
|
|
12
|
+
id: z.ZodString;
|
|
13
|
+
contentHash: z.ZodString;
|
|
14
|
+
}, z.core.$strict>;
|
|
15
|
+
}, z.core.$strict>>;
|
|
16
|
+
tasksetReleases: z.ZodArray<z.ZodObject<{
|
|
17
|
+
id: z.ZodString;
|
|
18
|
+
contentHash: z.ZodString;
|
|
19
|
+
}, z.core.$strict>>;
|
|
20
|
+
metricPolicyHashes: z.ZodArray<z.ZodString>;
|
|
21
|
+
modelConfigurationHashes: z.ZodArray<z.ZodString>;
|
|
22
|
+
evidence: z.ZodArray<z.ZodObject<{
|
|
23
|
+
kind: z.ZodEnum<{
|
|
24
|
+
run: "run";
|
|
25
|
+
suite: "suite";
|
|
26
|
+
comparison: "comparison";
|
|
27
|
+
}>;
|
|
28
|
+
id: z.ZodString;
|
|
29
|
+
contentHash: z.ZodString;
|
|
30
|
+
}, z.core.$strict>>;
|
|
31
|
+
summary: z.ZodObject<{
|
|
32
|
+
totalChecks: z.ZodNumber;
|
|
33
|
+
passedChecks: z.ZodNumber;
|
|
34
|
+
failedChecks: z.ZodNumber;
|
|
35
|
+
unscoredChecks: z.ZodNumber;
|
|
36
|
+
meanScore: z.ZodNullable<z.ZodNumber>;
|
|
37
|
+
}, z.core.$strict>;
|
|
38
|
+
}, z.core.$strict>;
|
|
39
|
+
export declare const ProfileEvaluationReportSchema: z.ZodObject<{
|
|
40
|
+
schemaVersion: z.ZodLiteral<"openpond.profileEvaluationReport.v1">;
|
|
41
|
+
id: z.ZodString;
|
|
42
|
+
createdAt: z.ZodString;
|
|
43
|
+
testedSources: z.ZodArray<z.ZodObject<{
|
|
44
|
+
profileId: z.ZodString;
|
|
45
|
+
sourceRevision: z.ZodString;
|
|
46
|
+
harnessRelease: z.ZodObject<{
|
|
47
|
+
id: z.ZodString;
|
|
48
|
+
contentHash: z.ZodString;
|
|
49
|
+
}, z.core.$strict>;
|
|
50
|
+
}, z.core.$strict>>;
|
|
51
|
+
tasksetReleases: z.ZodArray<z.ZodObject<{
|
|
52
|
+
id: z.ZodString;
|
|
53
|
+
contentHash: z.ZodString;
|
|
54
|
+
}, z.core.$strict>>;
|
|
55
|
+
metricPolicyHashes: z.ZodArray<z.ZodString>;
|
|
56
|
+
modelConfigurationHashes: z.ZodArray<z.ZodString>;
|
|
57
|
+
evidence: z.ZodArray<z.ZodObject<{
|
|
58
|
+
kind: z.ZodEnum<{
|
|
59
|
+
run: "run";
|
|
60
|
+
suite: "suite";
|
|
61
|
+
comparison: "comparison";
|
|
62
|
+
}>;
|
|
63
|
+
id: z.ZodString;
|
|
64
|
+
contentHash: z.ZodString;
|
|
65
|
+
}, z.core.$strict>>;
|
|
66
|
+
summary: z.ZodObject<{
|
|
67
|
+
totalChecks: z.ZodNumber;
|
|
68
|
+
passedChecks: z.ZodNumber;
|
|
69
|
+
failedChecks: z.ZodNumber;
|
|
70
|
+
unscoredChecks: z.ZodNumber;
|
|
71
|
+
meanScore: z.ZodNullable<z.ZodNumber>;
|
|
72
|
+
}, z.core.$strict>;
|
|
73
|
+
contentHash: z.ZodString;
|
|
74
|
+
}, z.core.$strict>;
|
|
75
|
+
export type ProfileEvaluationReport = z.infer<typeof ProfileEvaluationReportSchema>;
|
|
76
|
+
export declare function createProfileEvaluationReport(input: z.input<typeof ProfileEvaluationReportContentSchema>): ProfileEvaluationReport;
|
|
77
|
+
export declare function assertProfileEvaluationReport(value: unknown): ProfileEvaluationReport;
|
|
78
|
+
//# sourceMappingURL=profile-evaluation-reports.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"profile-evaluation-reports.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-reports.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAmBxB;wEACwE;AACxE,eAAO,MAAM,oCAAoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAwB/C,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAExC,CAAC;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AAEpF,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,oCAAoC,CAAC,GAAG,uBAAuB,CAGlI;AAED,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,OAAO,GAAG,uBAAuB,CAKrF"}
|
package/package.json
CHANGED