@openpond/evals 0.11.4 → 0.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +2 -0
- package/dist/profile-evaluation-reports.js +53 -0
- package/dist/profile-evaluation-suites.js +96 -0
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/profile-evaluation-reports.d.ts +78 -0
- package/dist/types/profile-evaluation-reports.d.ts.map +1 -0
- package/dist/types/profile-evaluation-suites.d.ts +75 -0
- package/dist/types/profile-evaluation-suites.d.ts.map +1 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -14,6 +14,8 @@ export * from "./model-improvement-qualification.js";
|
|
|
14
14
|
export * from "./preferences.js";
|
|
15
15
|
export * from "./profile-evaluations.js";
|
|
16
16
|
export * from "./profile-evaluation-comparisons.js";
|
|
17
|
+
export * from "./profile-evaluation-reports.js";
|
|
18
|
+
export * from "./profile-evaluation-suites.js";
|
|
17
19
|
export * from "./profile-evaluation-execution.js";
|
|
18
20
|
export * from "./review-conformance.js";
|
|
19
21
|
export * from "./rollouts.js";
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "@openpond/harness";
|
|
3
|
+
const TestedSourceSchema = z.object({
|
|
4
|
+
profileId: ReleaseIdSchema,
|
|
5
|
+
sourceRevision: z.string().trim().min(1).max(240),
|
|
6
|
+
harnessRelease: ImmutableReleaseRefSchema,
|
|
7
|
+
}).strict();
|
|
8
|
+
const EvidenceSchema = z.object({
|
|
9
|
+
kind: z.enum(["run", "suite", "comparison"]),
|
|
10
|
+
id: ReleaseIdSchema,
|
|
11
|
+
contentHash: ReleaseHashSchema,
|
|
12
|
+
}).strict();
|
|
13
|
+
/** Portable summary only. Private cases, outputs, traces, and credentials stay
|
|
14
|
+
* with the installation's retained evidence, outside Profile source. */
|
|
15
|
+
export const ProfileEvaluationReportContentSchema = z.object({
|
|
16
|
+
schemaVersion: z.literal("openpond.profileEvaluationReport.v1"),
|
|
17
|
+
id: ReleaseIdSchema,
|
|
18
|
+
createdAt: ReleaseTimestampSchema,
|
|
19
|
+
testedSources: z.array(TestedSourceSchema).min(1).max(100),
|
|
20
|
+
tasksetReleases: z.array(ImmutableReleaseRefSchema).min(1).max(100),
|
|
21
|
+
metricPolicyHashes: z.array(ReleaseHashSchema).min(1).max(100),
|
|
22
|
+
modelConfigurationHashes: z.array(ReleaseHashSchema).min(1).max(100),
|
|
23
|
+
evidence: z.array(EvidenceSchema).min(1).max(100),
|
|
24
|
+
summary: z.object({
|
|
25
|
+
totalChecks: z.number().int().nonnegative(),
|
|
26
|
+
passedChecks: z.number().int().nonnegative(),
|
|
27
|
+
failedChecks: z.number().int().nonnegative(),
|
|
28
|
+
unscoredChecks: z.number().int().nonnegative(),
|
|
29
|
+
meanScore: z.number().min(0).max(1).nullable(),
|
|
30
|
+
}).strict(),
|
|
31
|
+
}).strict().superRefine((report, context) => {
|
|
32
|
+
const { summary } = report;
|
|
33
|
+
if (summary.passedChecks + summary.failedChecks + summary.unscoredChecks !== summary.totalChecks) {
|
|
34
|
+
context.addIssue({ code: "custom", path: ["summary"], message: "Report check counts must add up." });
|
|
35
|
+
}
|
|
36
|
+
if (new Set(report.evidence.map(({ kind, id }) => `${kind}:${id}`)).size !== report.evidence.length) {
|
|
37
|
+
context.addIssue({ code: "custom", path: ["evidence"], message: "Report evidence must be unique." });
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
export const ProfileEvaluationReportSchema = ProfileEvaluationReportContentSchema.safeExtend({
|
|
41
|
+
contentHash: ReleaseHashSchema,
|
|
42
|
+
});
|
|
43
|
+
export function createProfileEvaluationReport(input) {
|
|
44
|
+
const content = ProfileEvaluationReportContentSchema.parse(input);
|
|
45
|
+
return ProfileEvaluationReportSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
46
|
+
}
|
|
47
|
+
export function assertProfileEvaluationReport(value) {
|
|
48
|
+
const report = ProfileEvaluationReportSchema.parse(value);
|
|
49
|
+
const { contentHash: expected, ...content } = report;
|
|
50
|
+
if (contentHash(content) !== expected)
|
|
51
|
+
throw new Error("Profile evaluation report content hash differs from its source bytes.");
|
|
52
|
+
return report;
|
|
53
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "@openpond/harness";
|
|
3
|
+
import { ProfileEvaluationCatalogSchema } from "./profile-evaluations.js";
|
|
4
|
+
import { TasksetRunManifestSchema } from "./taskset-run-contract.js";
|
|
5
|
+
export const ProfileEvaluationSuiteRunContentSchema = z.object({
|
|
6
|
+
schemaVersion: z.literal("openpond.profileEvaluationSuiteRun.v1"),
|
|
7
|
+
id: ReleaseIdSchema,
|
|
8
|
+
suiteId: ReleaseIdSchema,
|
|
9
|
+
suiteHash: ReleaseHashSchema,
|
|
10
|
+
catalogHash: ReleaseHashSchema,
|
|
11
|
+
profileId: ReleaseIdSchema,
|
|
12
|
+
sourceRevision: z.string().trim().min(1).max(240),
|
|
13
|
+
harnessRelease: ImmutableReleaseRefSchema,
|
|
14
|
+
members: z.array(z.object({
|
|
15
|
+
definitionId: ReleaseIdSchema,
|
|
16
|
+
runManifest: ImmutableReleaseRefSchema,
|
|
17
|
+
runHash: ReleaseHashSchema,
|
|
18
|
+
passed: z.boolean(),
|
|
19
|
+
score: z.number().min(0).max(1).nullable(),
|
|
20
|
+
}).strict()).min(1).max(1_000),
|
|
21
|
+
passed: z.boolean(),
|
|
22
|
+
createdAt: ReleaseTimestampSchema,
|
|
23
|
+
completedAt: ReleaseTimestampSchema,
|
|
24
|
+
}).strict();
|
|
25
|
+
export const ProfileEvaluationSuiteRunSchema = ProfileEvaluationSuiteRunContentSchema.extend({
|
|
26
|
+
contentHash: ReleaseHashSchema,
|
|
27
|
+
}).strict();
|
|
28
|
+
/** Suite membership preserves each definition's own Taskset and metric. A
|
|
29
|
+
* suite passes only when every declared check passes; unlike scores are never
|
|
30
|
+
* averaged. */
|
|
31
|
+
export function createProfileEvaluationSuiteRun(input) {
|
|
32
|
+
const catalog = ProfileEvaluationCatalogSchema.parse(input.catalog);
|
|
33
|
+
const suite = catalog.suites.find((candidate) => candidate.id === input.suiteId);
|
|
34
|
+
if (!suite)
|
|
35
|
+
throw new Error(`Profile evaluation suite ${input.suiteId} is absent from its catalog.`);
|
|
36
|
+
if (input.members.length !== suite.definitionIds.length) {
|
|
37
|
+
throw new Error("Profile evaluation suite has an incomplete run population.");
|
|
38
|
+
}
|
|
39
|
+
const parsed = input.members.map((member, index) => {
|
|
40
|
+
const definitionId = suite.definitionIds[index];
|
|
41
|
+
const definition = catalog.definitions.find((candidate) => candidate.id === definitionId);
|
|
42
|
+
if (!definition || member.definitionId !== definitionId) {
|
|
43
|
+
throw new Error("Profile evaluation suite members differ from declared definition order.");
|
|
44
|
+
}
|
|
45
|
+
const manifest = TasksetRunManifestSchema.parse(member.manifest);
|
|
46
|
+
assertContentHash(manifest, "Profile evaluation suite member manifest");
|
|
47
|
+
const source = manifest.profileEvaluation;
|
|
48
|
+
if (!source || manifest.execution.kind !== "harness"
|
|
49
|
+
|| source.definitionId !== definitionId
|
|
50
|
+
|| source.definitionHash !== contentHash(definition)
|
|
51
|
+
|| contentHash(source.target) !== contentHash(definition.target)
|
|
52
|
+
|| source.catalogHash !== contentHash(catalog)
|
|
53
|
+
|| manifest.tasksetRelease.id !== definition.tasksetRelease.id
|
|
54
|
+
|| manifest.tasksetRelease.contentHash !== definition.tasksetRelease.contentHash) {
|
|
55
|
+
throw new Error("Profile evaluation suite member differs from its released definition.");
|
|
56
|
+
}
|
|
57
|
+
return { ...member, manifest, source };
|
|
58
|
+
});
|
|
59
|
+
const first = parsed[0].source;
|
|
60
|
+
const firstPolicy = parsed[0].manifest.policy;
|
|
61
|
+
if (firstPolicy.kind !== "model")
|
|
62
|
+
throw new Error("Profile evaluation suite requires model runs.");
|
|
63
|
+
for (const member of parsed) {
|
|
64
|
+
if (member.source.profileId !== first.profileId
|
|
65
|
+
|| member.source.sourceRevision !== first.sourceRevision
|
|
66
|
+
|| member.source.harnessRelease.id !== first.harnessRelease.id
|
|
67
|
+
|| member.source.harnessRelease.contentHash !== first.harnessRelease.contentHash) {
|
|
68
|
+
throw new Error("Profile evaluation suite members use different Profile releases.");
|
|
69
|
+
}
|
|
70
|
+
const policy = member.manifest.policy;
|
|
71
|
+
if (policy.kind !== "model" || contentHash(policy) !== contentHash(firstPolicy)) {
|
|
72
|
+
throw new Error("Profile evaluation suite members use different model configurations.");
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const content = ProfileEvaluationSuiteRunContentSchema.parse({
|
|
76
|
+
schemaVersion: "openpond.profileEvaluationSuiteRun.v1",
|
|
77
|
+
id: input.id,
|
|
78
|
+
suiteId: suite.id,
|
|
79
|
+
suiteHash: contentHash(suite),
|
|
80
|
+
catalogHash: contentHash(catalog),
|
|
81
|
+
profileId: first.profileId,
|
|
82
|
+
sourceRevision: first.sourceRevision,
|
|
83
|
+
harnessRelease: first.harnessRelease,
|
|
84
|
+
members: parsed.map((member) => ({
|
|
85
|
+
definitionId: member.definitionId,
|
|
86
|
+
runManifest: { id: member.manifest.id, contentHash: member.manifest.contentHash },
|
|
87
|
+
runHash: member.runHash,
|
|
88
|
+
passed: member.passed,
|
|
89
|
+
score: member.score,
|
|
90
|
+
})),
|
|
91
|
+
passed: parsed.every((member) => member.passed),
|
|
92
|
+
createdAt: input.createdAt,
|
|
93
|
+
completedAt: input.completedAt,
|
|
94
|
+
});
|
|
95
|
+
return ProfileEvaluationSuiteRunSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
96
|
+
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -14,6 +14,8 @@ export * from "./model-improvement-qualification.js";
|
|
|
14
14
|
export * from "./preferences.js";
|
|
15
15
|
export * from "./profile-evaluations.js";
|
|
16
16
|
export * from "./profile-evaluation-comparisons.js";
|
|
17
|
+
export * from "./profile-evaluation-reports.js";
|
|
18
|
+
export * from "./profile-evaluation-suites.js";
|
|
17
19
|
export * from "./profile-evaluation-execution.js";
|
|
18
20
|
export * from "./review-conformance.js";
|
|
19
21
|
export * from "./rollouts.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,mCAAmC,CAAC;AAClD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,mCAAmC,CAAC;AAClD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/** Portable summary only. Private cases, outputs, traces, and credentials stay
|
|
3
|
+
* with the installation's retained evidence, outside Profile source. */
|
|
4
|
+
export declare const ProfileEvaluationReportContentSchema: z.ZodObject<{
|
|
5
|
+
schemaVersion: z.ZodLiteral<"openpond.profileEvaluationReport.v1">;
|
|
6
|
+
id: z.ZodString;
|
|
7
|
+
createdAt: z.ZodString;
|
|
8
|
+
testedSources: z.ZodArray<z.ZodObject<{
|
|
9
|
+
profileId: z.ZodString;
|
|
10
|
+
sourceRevision: z.ZodString;
|
|
11
|
+
harnessRelease: z.ZodObject<{
|
|
12
|
+
id: z.ZodString;
|
|
13
|
+
contentHash: z.ZodString;
|
|
14
|
+
}, z.core.$strict>;
|
|
15
|
+
}, z.core.$strict>>;
|
|
16
|
+
tasksetReleases: z.ZodArray<z.ZodObject<{
|
|
17
|
+
id: z.ZodString;
|
|
18
|
+
contentHash: z.ZodString;
|
|
19
|
+
}, z.core.$strict>>;
|
|
20
|
+
metricPolicyHashes: z.ZodArray<z.ZodString>;
|
|
21
|
+
modelConfigurationHashes: z.ZodArray<z.ZodString>;
|
|
22
|
+
evidence: z.ZodArray<z.ZodObject<{
|
|
23
|
+
kind: z.ZodEnum<{
|
|
24
|
+
run: "run";
|
|
25
|
+
suite: "suite";
|
|
26
|
+
comparison: "comparison";
|
|
27
|
+
}>;
|
|
28
|
+
id: z.ZodString;
|
|
29
|
+
contentHash: z.ZodString;
|
|
30
|
+
}, z.core.$strict>>;
|
|
31
|
+
summary: z.ZodObject<{
|
|
32
|
+
totalChecks: z.ZodNumber;
|
|
33
|
+
passedChecks: z.ZodNumber;
|
|
34
|
+
failedChecks: z.ZodNumber;
|
|
35
|
+
unscoredChecks: z.ZodNumber;
|
|
36
|
+
meanScore: z.ZodNullable<z.ZodNumber>;
|
|
37
|
+
}, z.core.$strict>;
|
|
38
|
+
}, z.core.$strict>;
|
|
39
|
+
export declare const ProfileEvaluationReportSchema: z.ZodObject<{
|
|
40
|
+
schemaVersion: z.ZodLiteral<"openpond.profileEvaluationReport.v1">;
|
|
41
|
+
id: z.ZodString;
|
|
42
|
+
createdAt: z.ZodString;
|
|
43
|
+
testedSources: z.ZodArray<z.ZodObject<{
|
|
44
|
+
profileId: z.ZodString;
|
|
45
|
+
sourceRevision: z.ZodString;
|
|
46
|
+
harnessRelease: z.ZodObject<{
|
|
47
|
+
id: z.ZodString;
|
|
48
|
+
contentHash: z.ZodString;
|
|
49
|
+
}, z.core.$strict>;
|
|
50
|
+
}, z.core.$strict>>;
|
|
51
|
+
tasksetReleases: z.ZodArray<z.ZodObject<{
|
|
52
|
+
id: z.ZodString;
|
|
53
|
+
contentHash: z.ZodString;
|
|
54
|
+
}, z.core.$strict>>;
|
|
55
|
+
metricPolicyHashes: z.ZodArray<z.ZodString>;
|
|
56
|
+
modelConfigurationHashes: z.ZodArray<z.ZodString>;
|
|
57
|
+
evidence: z.ZodArray<z.ZodObject<{
|
|
58
|
+
kind: z.ZodEnum<{
|
|
59
|
+
run: "run";
|
|
60
|
+
suite: "suite";
|
|
61
|
+
comparison: "comparison";
|
|
62
|
+
}>;
|
|
63
|
+
id: z.ZodString;
|
|
64
|
+
contentHash: z.ZodString;
|
|
65
|
+
}, z.core.$strict>>;
|
|
66
|
+
summary: z.ZodObject<{
|
|
67
|
+
totalChecks: z.ZodNumber;
|
|
68
|
+
passedChecks: z.ZodNumber;
|
|
69
|
+
failedChecks: z.ZodNumber;
|
|
70
|
+
unscoredChecks: z.ZodNumber;
|
|
71
|
+
meanScore: z.ZodNullable<z.ZodNumber>;
|
|
72
|
+
}, z.core.$strict>;
|
|
73
|
+
contentHash: z.ZodString;
|
|
74
|
+
}, z.core.$strict>;
|
|
75
|
+
export type ProfileEvaluationReport = z.infer<typeof ProfileEvaluationReportSchema>;
|
|
76
|
+
export declare function createProfileEvaluationReport(input: z.input<typeof ProfileEvaluationReportContentSchema>): ProfileEvaluationReport;
|
|
77
|
+
export declare function assertProfileEvaluationReport(value: unknown): ProfileEvaluationReport;
|
|
78
|
+
//# sourceMappingURL=profile-evaluation-reports.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"profile-evaluation-reports.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-reports.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAmBxB;wEACwE;AACxE,eAAO,MAAM,oCAAoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAwB/C,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAExC,CAAC;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AAEpF,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,oCAAoC,CAAC,GAAG,uBAAuB,CAGlI;AAED,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,OAAO,GAAG,uBAAuB,CAKrF"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { type ProfileEvaluationCatalog } from "./profile-evaluations.js";
|
|
3
|
+
import { type TasksetRunManifest } from "./taskset-run-contract.js";
|
|
4
|
+
export declare const ProfileEvaluationSuiteRunContentSchema: z.ZodObject<{
|
|
5
|
+
schemaVersion: z.ZodLiteral<"openpond.profileEvaluationSuiteRun.v1">;
|
|
6
|
+
id: z.ZodString;
|
|
7
|
+
suiteId: z.ZodString;
|
|
8
|
+
suiteHash: z.ZodString;
|
|
9
|
+
catalogHash: z.ZodString;
|
|
10
|
+
profileId: z.ZodString;
|
|
11
|
+
sourceRevision: z.ZodString;
|
|
12
|
+
harnessRelease: z.ZodObject<{
|
|
13
|
+
id: z.ZodString;
|
|
14
|
+
contentHash: z.ZodString;
|
|
15
|
+
}, z.core.$strict>;
|
|
16
|
+
members: z.ZodArray<z.ZodObject<{
|
|
17
|
+
definitionId: z.ZodString;
|
|
18
|
+
runManifest: z.ZodObject<{
|
|
19
|
+
id: z.ZodString;
|
|
20
|
+
contentHash: z.ZodString;
|
|
21
|
+
}, z.core.$strict>;
|
|
22
|
+
runHash: z.ZodString;
|
|
23
|
+
passed: z.ZodBoolean;
|
|
24
|
+
score: z.ZodNullable<z.ZodNumber>;
|
|
25
|
+
}, z.core.$strict>>;
|
|
26
|
+
passed: z.ZodBoolean;
|
|
27
|
+
createdAt: z.ZodString;
|
|
28
|
+
completedAt: z.ZodString;
|
|
29
|
+
}, z.core.$strict>;
|
|
30
|
+
export declare const ProfileEvaluationSuiteRunSchema: z.ZodObject<{
|
|
31
|
+
schemaVersion: z.ZodLiteral<"openpond.profileEvaluationSuiteRun.v1">;
|
|
32
|
+
id: z.ZodString;
|
|
33
|
+
suiteId: z.ZodString;
|
|
34
|
+
suiteHash: z.ZodString;
|
|
35
|
+
catalogHash: z.ZodString;
|
|
36
|
+
profileId: z.ZodString;
|
|
37
|
+
sourceRevision: z.ZodString;
|
|
38
|
+
harnessRelease: z.ZodObject<{
|
|
39
|
+
id: z.ZodString;
|
|
40
|
+
contentHash: z.ZodString;
|
|
41
|
+
}, z.core.$strict>;
|
|
42
|
+
members: z.ZodArray<z.ZodObject<{
|
|
43
|
+
definitionId: z.ZodString;
|
|
44
|
+
runManifest: z.ZodObject<{
|
|
45
|
+
id: z.ZodString;
|
|
46
|
+
contentHash: z.ZodString;
|
|
47
|
+
}, z.core.$strict>;
|
|
48
|
+
runHash: z.ZodString;
|
|
49
|
+
passed: z.ZodBoolean;
|
|
50
|
+
score: z.ZodNullable<z.ZodNumber>;
|
|
51
|
+
}, z.core.$strict>>;
|
|
52
|
+
passed: z.ZodBoolean;
|
|
53
|
+
createdAt: z.ZodString;
|
|
54
|
+
completedAt: z.ZodString;
|
|
55
|
+
contentHash: z.ZodString;
|
|
56
|
+
}, z.core.$strict>;
|
|
57
|
+
export type ProfileEvaluationSuiteRun = z.infer<typeof ProfileEvaluationSuiteRunSchema>;
|
|
58
|
+
/** Suite membership preserves each definition's own Taskset and metric. A
|
|
59
|
+
* suite passes only when every declared check passes; unlike scores are never
|
|
60
|
+
* averaged. */
|
|
61
|
+
export declare function createProfileEvaluationSuiteRun(input: {
|
|
62
|
+
id: string;
|
|
63
|
+
suiteId: string;
|
|
64
|
+
catalog: ProfileEvaluationCatalog;
|
|
65
|
+
members: Array<{
|
|
66
|
+
definitionId: string;
|
|
67
|
+
manifest: TasksetRunManifest;
|
|
68
|
+
runHash: string;
|
|
69
|
+
passed: boolean;
|
|
70
|
+
score: number | null;
|
|
71
|
+
}>;
|
|
72
|
+
createdAt: string;
|
|
73
|
+
completedAt: string;
|
|
74
|
+
}): ProfileEvaluationSuiteRun;
|
|
75
|
+
//# sourceMappingURL=profile-evaluation-suites.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"profile-evaluation-suites.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-suites.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,OAAO,EAAkC,KAAK,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzG,OAAO,EAA4B,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE9F,eAAO,MAAM,sCAAsC;;;;;;;;;;;;;;;;;;;;;;;;;kBAmBxC,CAAC;AACZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;kBAEjC,CAAC;AACZ,MAAM,MAAM,yBAAyB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,CAAC;AAExF;;eAEe;AACf,wBAAgB,+BAA+B,CAAC,KAAK,EAAE;IACrD,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,KAAK,CAAC;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,kBAAkB,CAAC;QAC7B,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,OAAO,CAAC;QAChB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;KACtB,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB,GAAG,yBAAyB,CA+D5B"}
|
package/package.json
CHANGED