@openpond/evals 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CONTRACT.md +24 -5
  2. package/README.md +27 -8
  3. package/RELEASING.md +13 -45
  4. package/dist/compatibility.js +45 -0
  5. package/dist/conformance.js +7 -11
  6. package/dist/evidence/authoring.js +1 -1
  7. package/dist/evidence/conformance.js +1 -1
  8. package/dist/evidence/contracts.js +1 -2
  9. package/dist/evidence/eligibility.js +1 -1
  10. package/dist/graders.js +1 -1
  11. package/dist/harness.js +33 -103
  12. package/dist/index.js +3 -1
  13. package/dist/model-improvement-qualification.js +125 -0
  14. package/dist/review-conformance.js +209 -0
  15. package/dist/runs.js +33 -13
  16. package/dist/tasksets.js +3 -17
  17. package/dist/types/compatibility.d.ts +11 -0
  18. package/dist/types/compatibility.d.ts.map +1 -0
  19. package/dist/types/conformance.d.ts +10 -48
  20. package/dist/types/conformance.d.ts.map +1 -1
  21. package/dist/types/evidence/conformance.d.ts +8 -8
  22. package/dist/types/evidence/contracts.d.ts +8 -8
  23. package/dist/types/harness.d.ts +3 -471
  24. package/dist/types/harness.d.ts.map +1 -1
  25. package/dist/types/index.d.ts +3 -1
  26. package/dist/types/index.d.ts.map +1 -1
  27. package/dist/types/model-improvement-qualification.d.ts +222 -0
  28. package/dist/types/model-improvement-qualification.d.ts.map +1 -0
  29. package/dist/types/review-conformance.d.ts +607 -0
  30. package/dist/types/review-conformance.d.ts.map +1 -0
  31. package/dist/types/runs.d.ts +47 -10
  32. package/dist/types/runs.d.ts.map +1 -1
  33. package/dist/types/tasksets.d.ts +4 -35
  34. package/dist/types/tasksets.d.ts.map +1 -1
  35. package/package.json +14 -2
  36. package/dist/common.js +0 -67
  37. package/dist/sha256.js +0 -91
  38. package/dist/types/common.d.ts +0 -51
  39. package/dist/types/common.d.ts.map +0 -1
  40. package/dist/types/sha256.d.ts +0 -2
  41. package/dist/types/sha256.d.ts.map +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../../../../src/runs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,eAAO,MAAM,cAAc;;;;;;;kBAOhB,CAAC;AAEZ,eAAO,MAAM,0BAA0B;;;;;;;;;;kBAK5B,CAAC;AAEZ,eAAO,MAAM,eAAe;;;;;kBAKjB,CAAC;AAEZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAW1B,CAAC;AACZ,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA+E,CAAC;AAE9G,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkB7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAc/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AAExH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,GAAG,WAAW,CAG9F;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,GAAG,cAAc,CAGvG;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAKrE;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,gBAAgB,CAgCnB;AAED,wBAAgB,4BAA4B,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,GAAG,IAAI,CAO5F;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CASnF;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC"}
1
+ {"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../../../../src/runs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,eAAO,MAAM,0BAA0B;;;;;;;;;;kBAK5B,CAAC;AAEZ,eAAO,MAAM,eAAe;;;;;kBAKjB,CAAC;AAEZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAW1B,CAAC;AACZ,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA+E,CAAC;AAE9G,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkB7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAc/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AAExH,eAAO,MAAM,wCAAwC;;;;;;;;;;;;;;;;;;;;kBAW1C,CAAC;AACZ,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;kBAEnC,CAAC;AAEZ,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,GAAG,WAAW,CAG9F;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,GAAG,cAAc,CAGvG;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAKrE;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,gBAAgB,CAgCnB;AAED,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wCAAwC,CAAC,GAC9D,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAGnD;AAED,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,WAAW,EACtB,aAAa,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,GAChE,IAAI,CAiBN;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CASnF;AAED,YAAY,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAAC"}
@@ -1,5 +1,5 @@
1
1
  import { z } from "zod";
2
- import { ImmutableAssetRefSchema } from "./common.js";
2
+ import { ImmutableAssetRefSchema } from "@openpond/harness";
3
3
  export declare const TaskSplitSchema: z.ZodEnum<{
4
4
  train: "train";
5
5
  validation: "validation";
@@ -12,28 +12,6 @@ export declare const PolicyBoundarySchema: z.ZodObject<{
12
12
  hiddenGraderRefs: z.ZodDefault<z.ZodArray<z.ZodString>>;
13
13
  connectedAppScopes: z.ZodDefault<z.ZodArray<z.ZodString>>;
14
14
  }, z.core.$strict>;
15
- export declare const ToolDeclarationSchema: z.ZodObject<{
16
- name: z.ZodString;
17
- description: z.ZodString;
18
- inputSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
19
- inputSchemaHash: z.ZodString;
20
- sideEffect: z.ZodEnum<{
21
- read: "read";
22
- write: "write";
23
- }>;
24
- timeoutMs: z.ZodNumber;
25
- }, z.core.$strict>;
26
- export declare const CapabilityRequirementSchema: z.ZodObject<{
27
- id: z.ZodString;
28
- required: z.ZodBoolean;
29
- scopes: z.ZodDefault<z.ZodArray<z.ZodString>>;
30
- portability: z.ZodEnum<{
31
- portable: "portable";
32
- host_adapter: "host_adapter";
33
- local_only: "local_only";
34
- hosted_only: "hosted_only";
35
- }>;
36
- }, z.core.$strict>;
37
15
  export declare const EnvironmentContractSchema: z.ZodObject<{
38
16
  protocolVersion: z.ZodLiteral<"openpond.environment.v1">;
39
17
  kind: z.ZodEnum<{
@@ -258,13 +236,9 @@ export declare const TaskRecordSchema: z.ZodObject<{
258
236
  tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
259
237
  }, z.core.$strict>;
260
238
  export declare const TasksetReleaseContentSchema: z.ZodObject<{
261
- schemaVersion: z.ZodLiteral<"openpond.tasksetRelease.v1">;
239
+ schemaVersion: z.ZodLiteral<"openpond.tasksetRelease.v2">;
262
240
  id: z.ZodString;
263
241
  revision: z.ZodNumber;
264
- harnessRelease: z.ZodObject<{
265
- id: z.ZodString;
266
- contentHash: z.ZodString;
267
- }, z.core.$strict>;
268
242
  policy: z.ZodObject<{
269
243
  policyVisibleFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
270
244
  privilegedFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
@@ -432,13 +406,9 @@ export declare const TasksetReleaseContentSchema: z.ZodObject<{
432
406
  metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
433
407
  }, z.core.$strict>;
434
408
  export declare const TasksetReleaseSchema: z.ZodObject<{
435
- schemaVersion: z.ZodLiteral<"openpond.tasksetRelease.v1">;
409
+ schemaVersion: z.ZodLiteral<"openpond.tasksetRelease.v2">;
436
410
  id: z.ZodString;
437
411
  revision: z.ZodNumber;
438
- harnessRelease: z.ZodObject<{
439
- id: z.ZodString;
440
- contentHash: z.ZodString;
441
- }, z.core.$strict>;
442
412
  policy: z.ZodObject<{
443
413
  policyVisibleFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
444
414
  privilegedFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
@@ -627,11 +597,10 @@ export declare function policyTaskView(task: TaskRecord): {
627
597
  tags: string[];
628
598
  };
629
599
  export declare function trainingPolicyTaskViews(taskset: TasksetRelease): ReturnType<typeof policyTaskView>[];
630
- export type ToolDeclaration = z.infer<typeof ToolDeclarationSchema>;
631
- export type CapabilityRequirement = z.infer<typeof CapabilityRequirementSchema>;
632
600
  export type EnvironmentContract = z.infer<typeof EnvironmentContractSchema>;
633
601
  export type GraderSpec = z.infer<typeof GraderSpecSchema>;
634
602
  export type DeterministicGraderSpec = z.infer<typeof DeterministicGraderSpecSchema>;
635
603
  export type TaskRecord = z.infer<typeof TaskRecordSchema>;
636
604
  export type TasksetRelease = z.infer<typeof TasksetReleaseSchema>;
605
+ export { CapabilityRequirementSchema, ToolDeclarationSchema, type CapabilityRequirement, type ToolDeclaration, } from "@openpond/harness";
637
606
  //# sourceMappingURL=tasksets.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"tasksets.d.ts","sourceRoot":"","sources":["../../../../src/tasksets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EACL,uBAAuB,EAOxB,MAAM,aAAa,CAAC;AAErB,eAAO,MAAM,eAAe;;;;;EAAyD,CAAC;AACtF,eAAO,MAAM,oBAAoB;;;;;kBAKtB,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;;;;;kBAOvB,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;;;;kBAK7B,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAWZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;kBAG/B,CAAC;AACZ,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;kBAI5B,CAAC;AACZ,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;kBAKhC,CAAC;AACZ,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;kBAIvB,CAAC;AACZ,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAK3B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUlB,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAY7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG;IACtD,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,cAAc,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC,CA8CA;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,CAKlE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9C,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC,CAAC;IAC7D,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,CAQA;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,cAAc,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,EAAE,CAIpG;AAED,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAChF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AACpF,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}
1
+ {"version":3,"file":"tasksets.d.ts","sourceRoot":"","sources":["../../../../src/tasksets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAEL,uBAAuB,EAOxB,MAAM,mBAAmB,CAAC;AAE3B,eAAO,MAAM,eAAe;;;;;EAAyD,CAAC;AACtF,eAAO,MAAM,oBAAoB;;;;;kBAKtB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAWZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;kBAG/B,CAAC;AACZ,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;kBAI5B,CAAC;AACZ,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;kBAKhC,CAAC;AACZ,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;kBAIvB,CAAC;AACZ,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAK3B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUlB,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG;IACtD,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,cAAc,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC,CA8CA;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,CAKlE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9C,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC,CAAC;IAC7D,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,CAQA;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,cAAc,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,EAAE,CAIpG;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AACpF,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,OAAO,EACL,2BAA2B,EAC3B,qBAAqB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,eAAe,GACrB,MAAM,mBAAmB,CAAC"}
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@openpond/evals",
3
- "version": "0.2.0",
4
- "description": "Portable evaluation, Work evidence, Harness, Taskset, grader, and receipt contracts for OpenPond",
3
+ "version": "0.4.0",
4
+ "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
5
5
  "license": "MIT",
6
6
  "type": "module",
7
7
  "main": "./dist/index.js",
@@ -34,6 +34,14 @@
34
34
  "types": "./dist/types/runs.d.ts",
35
35
  "import": "./dist/runs.js"
36
36
  },
37
+ "./review": {
38
+ "types": "./dist/types/review-conformance.d.ts",
39
+ "import": "./dist/review-conformance.js"
40
+ },
41
+ "./model-improvement-qualification": {
42
+ "types": "./dist/types/model-improvement-qualification.d.ts",
43
+ "import": "./dist/model-improvement-qualification.js"
44
+ },
37
45
  "./conformance": {
38
46
  "types": "./dist/types/conformance.d.ts",
39
47
  "import": "./dist/conformance.js"
@@ -71,7 +79,11 @@
71
79
  "dependencies": {
72
80
  "zod": "^4.1.13"
73
81
  },
82
+ "peerDependencies": {
83
+ "@openpond/harness": "^0.2.0"
84
+ },
74
85
  "devDependencies": {
86
+ "@openpond/harness": "workspace:*",
75
87
  "@types/node": "24.13.3",
76
88
  "tsx": "4.23.1",
77
89
  "typescript": "^5.9.3",
package/dist/common.js DELETED
@@ -1,67 +0,0 @@
1
- import { z } from "zod";
2
- import { sha256Hex } from "./sha256.js";
3
- export const MAX_PORTABLE_PATH_BYTES = 2_000;
4
- export const MAX_PORTABLE_ASSET_BYTES = 250_000_000;
5
- export const ReleaseIdSchema = z.string().trim().min(1).max(240);
6
- export const ReleaseHashSchema = z.string().regex(/^[a-f0-9]{64}$/);
7
- export const ReleaseTimestampSchema = z.string().datetime({ offset: true });
8
- export const MetadataSchema = z.record(z.string(), z.unknown()).default({});
9
- export const ImmutableReleaseRefSchema = z.object({
10
- id: ReleaseIdSchema,
11
- contentHash: ReleaseHashSchema,
12
- }).strict();
13
- export const ImmutableAssetRefSchema = z.object({
14
- id: ReleaseIdSchema,
15
- path: z.string().trim().min(1).max(MAX_PORTABLE_PATH_BYTES).refine(safeRelativePath),
16
- contentHash: ReleaseHashSchema,
17
- sizeBytes: z.number().int().nonnegative().max(MAX_PORTABLE_ASSET_BYTES),
18
- mediaType: z.string().trim().min(1).max(200),
19
- visibility: z.enum(["policy", "verifier", "host_private"]),
20
- }).strict();
21
- export const ImmutableArtifactRefSchema = z.object({
22
- id: ReleaseIdSchema,
23
- contentHash: ReleaseHashSchema,
24
- mediaType: z.string().trim().min(1).max(200).nullable().default(null),
25
- sizeBytes: z.number().int().nonnegative().max(MAX_PORTABLE_ASSET_BYTES).nullable().default(null),
26
- }).strict();
27
- export const FailureClassSchema = z.enum([
28
- "policy_failure",
29
- "grader_failure",
30
- "environment_failure",
31
- "infrastructure_failure",
32
- "timeout",
33
- "cancelled",
34
- ]);
35
- export function canonicalJson(value) {
36
- return `${JSON.stringify(sortValue(value), null, 2)}\n`;
37
- }
38
- export function sha256(value) {
39
- return sha256Hex(value);
40
- }
41
- export function contentHash(value) {
42
- return sha256(canonicalJson(value));
43
- }
44
- export function withContentHash(value) {
45
- return { ...value, contentHash: contentHash(value) };
46
- }
47
- export function assertContentHash(value, label) {
48
- const { contentHash: actual, ...hashable } = value;
49
- const expected = contentHash(hashable);
50
- if (actual !== expected)
51
- throw new Error(`${label} contentHash is ${actual}; expected ${expected}.`);
52
- }
53
- function safeRelativePath(value) {
54
- const normalized = value.replaceAll("\\", "/");
55
- if (!normalized || normalized.startsWith("/") || normalized.includes("\0"))
56
- return false;
57
- return !normalized.split("/").some((part) => !part || part === "." || part === "..");
58
- }
59
- function sortValue(value) {
60
- if (Array.isArray(value))
61
- return value.map(sortValue);
62
- if (!value || typeof value !== "object")
63
- return value;
64
- return Object.fromEntries(Object.entries(value)
65
- .sort(([left], [right]) => left.localeCompare(right))
66
- .map(([key, child]) => [key, sortValue(child)]));
67
- }
package/dist/sha256.js DELETED
@@ -1,91 +0,0 @@
1
- // FIPS 180-4 SHA-256 implemented over platform-neutral typed arrays so the
2
- // portable contracts keep the same synchronous content-hash API in any host.
3
- const INITIAL_STATE = new Uint32Array([
4
- 0x6a09e667,
5
- 0xbb67ae85,
6
- 0x3c6ef372,
7
- 0xa54ff53a,
8
- 0x510e527f,
9
- 0x9b05688c,
10
- 0x1f83d9ab,
11
- 0x5be0cd19,
12
- ]);
13
- const ROUND_CONSTANTS = new Uint32Array([
14
- 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
15
- 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
16
- 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
17
- 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
18
- 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
19
- 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
20
- 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
21
- 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
22
- 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
23
- 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
24
- 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
25
- 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
26
- 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
27
- 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
28
- 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
29
- 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
30
- ]);
31
- export function sha256Hex(value) {
32
- const bytes = typeof value === "string" ? new TextEncoder().encode(value) : value;
33
- const paddedLength = Math.ceil((bytes.byteLength + 9) / 64) * 64;
34
- const padded = new Uint8Array(paddedLength);
35
- padded.set(bytes);
36
- padded[bytes.byteLength] = 0x80;
37
- const bitLength = bytes.byteLength * 8;
38
- const view = new DataView(padded.buffer);
39
- view.setUint32(paddedLength - 8, Math.floor(bitLength / 0x1_0000_0000), false);
40
- view.setUint32(paddedLength - 4, bitLength >>> 0, false);
41
- const state = new Uint32Array(INITIAL_STATE);
42
- const words = new Uint32Array(64);
43
- for (let offset = 0; offset < paddedLength; offset += 64) {
44
- for (let index = 0; index < 16; index += 1) {
45
- words[index] = view.getUint32(offset + index * 4, false);
46
- }
47
- for (let index = 16; index < 64; index += 1) {
48
- const previous = words[index - 15];
49
- const recent = words[index - 2];
50
- const sigma0 = rotateRight(previous, 7) ^ rotateRight(previous, 18) ^ (previous >>> 3);
51
- const sigma1 = rotateRight(recent, 17) ^ rotateRight(recent, 19) ^ (recent >>> 10);
52
- words[index] = (words[index - 16] + sigma0 + words[index - 7] + sigma1) >>> 0;
53
- }
54
- let a = state[0];
55
- let b = state[1];
56
- let c = state[2];
57
- let d = state[3];
58
- let e = state[4];
59
- let f = state[5];
60
- let g = state[6];
61
- let h = state[7];
62
- for (let index = 0; index < 64; index += 1) {
63
- const choice = (e & f) ^ (~e & g);
64
- const majority = (a & b) ^ (a & c) ^ (b & c);
65
- const sum0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
66
- const sum1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
67
- const first = (h + sum1 + choice + ROUND_CONSTANTS[index] + words[index]) >>> 0;
68
- const second = (sum0 + majority) >>> 0;
69
- h = g;
70
- g = f;
71
- f = e;
72
- e = (d + first) >>> 0;
73
- d = c;
74
- c = b;
75
- b = a;
76
- a = (first + second) >>> 0;
77
- }
78
- state[0] = (state[0] + a) >>> 0;
79
- state[1] = (state[1] + b) >>> 0;
80
- state[2] = (state[2] + c) >>> 0;
81
- state[3] = (state[3] + d) >>> 0;
82
- state[4] = (state[4] + e) >>> 0;
83
- state[5] = (state[5] + f) >>> 0;
84
- state[6] = (state[6] + g) >>> 0;
85
- state[7] = (state[7] + h) >>> 0;
86
- }
87
- return Array.from(state, (word) => word.toString(16).padStart(8, "0")).join("");
88
- }
89
- function rotateRight(value, bits) {
90
- return (value >>> bits) | (value << (32 - bits));
91
- }
@@ -1,51 +0,0 @@
1
- import { z } from "zod";
2
- export declare const MAX_PORTABLE_PATH_BYTES = 2000;
3
- export declare const MAX_PORTABLE_ASSET_BYTES = 250000000;
4
- export declare const ReleaseIdSchema: z.ZodString;
5
- export declare const ReleaseHashSchema: z.ZodString;
6
- export declare const ReleaseTimestampSchema: z.ZodString;
7
- export declare const MetadataSchema: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
8
- export declare const ImmutableReleaseRefSchema: z.ZodObject<{
9
- id: z.ZodString;
10
- contentHash: z.ZodString;
11
- }, z.core.$strict>;
12
- export declare const ImmutableAssetRefSchema: z.ZodObject<{
13
- id: z.ZodString;
14
- path: z.ZodString;
15
- contentHash: z.ZodString;
16
- sizeBytes: z.ZodNumber;
17
- mediaType: z.ZodString;
18
- visibility: z.ZodEnum<{
19
- policy: "policy";
20
- verifier: "verifier";
21
- host_private: "host_private";
22
- }>;
23
- }, z.core.$strict>;
24
- export declare const ImmutableArtifactRefSchema: z.ZodObject<{
25
- id: z.ZodString;
26
- contentHash: z.ZodString;
27
- mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
28
- sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
29
- }, z.core.$strict>;
30
- export declare const FailureClassSchema: z.ZodEnum<{
31
- policy_failure: "policy_failure";
32
- grader_failure: "grader_failure";
33
- environment_failure: "environment_failure";
34
- infrastructure_failure: "infrastructure_failure";
35
- timeout: "timeout";
36
- cancelled: "cancelled";
37
- }>;
38
- export declare function canonicalJson(value: unknown): string;
39
- export declare function sha256(value: string | Uint8Array): string;
40
- export declare function contentHash(value: unknown): string;
41
- export declare function withContentHash<T extends Record<string, unknown>>(value: T): T & {
42
- contentHash: string;
43
- };
44
- export declare function assertContentHash(value: {
45
- contentHash: string;
46
- } & Record<string, unknown>, label: string): void;
47
- export type ImmutableReleaseRef = z.infer<typeof ImmutableReleaseRefSchema>;
48
- export type ImmutableAssetRef = z.infer<typeof ImmutableAssetRefSchema>;
49
- export type ImmutableArtifactRef = z.infer<typeof ImmutableArtifactRefSchema>;
50
- export type FailureClass = z.infer<typeof FailureClassSchema>;
51
- //# sourceMappingURL=common.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../../../src/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,uBAAuB,OAAQ,CAAC;AAC7C,eAAO,MAAM,wBAAwB,YAAc,CAAC;AACpD,eAAO,MAAM,eAAe,aAAoC,CAAC;AACjE,eAAO,MAAM,iBAAiB,aAAqC,CAAC;AACpE,eAAO,MAAM,sBAAsB,aAAwC,CAAC;AAC5E,eAAO,MAAM,cAAc,sDAAgD,CAAC;AAE5E,eAAO,MAAM,yBAAyB;;;kBAG3B,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;kBAOzB,CAAC;AAEZ,eAAO,MAAM,0BAA0B;;;;;kBAK5B,CAAC;AAEZ,eAAO,MAAM,kBAAkB;;;;;;;EAO7B,CAAC;AAEH,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEpD;AAED,wBAAgB,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAEzD;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAElD;AAED,wBAAgB,eAAe,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,CAExG;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAI/G;AAkBD,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC"}
@@ -1,2 +0,0 @@
1
- export declare function sha256Hex(value: string | Uint8Array): string;
2
- //# sourceMappingURL=sha256.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"sha256.d.ts","sourceRoot":"","sources":["../../../../src/sha256.ts"],"names":[],"mappings":"AAgCA,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAgE5D"}