@openpond/evals 0.11.16 → 0.11.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/dist/connected-evidence/contracts.js +72 -0
  2. package/dist/connected-evidence/imports.js +71 -0
  3. package/dist/connected-evidence/index.js +5 -0
  4. package/dist/connected-evidence/normalize.js +97 -0
  5. package/dist/connected-evidence/python-correctness.js +73 -0
  6. package/dist/connected-evidence/sources/claude-code.js +71 -0
  7. package/dist/connected-evidence/sources/codex.js +52 -0
  8. package/dist/connected-evidence/sources/history.js +86 -0
  9. package/dist/connected-evidence/sources/usage.js +15 -0
  10. package/dist/connected-evidence/usage-summary.js +52 -0
  11. package/dist/experiments.js +16 -4
  12. package/dist/graders.js +9 -6
  13. package/dist/human-review/comparison.js +161 -0
  14. package/dist/human-review/contracts.js +110 -0
  15. package/dist/human-review/execution.js +61 -0
  16. package/dist/human-review/handoff.js +18 -0
  17. package/dist/human-review/index.js +9 -0
  18. package/dist/human-review/local-publication.js +18 -0
  19. package/dist/human-review/repository.js +1 -0
  20. package/dist/human-review/results.js +18 -0
  21. package/dist/human-review/service.js +268 -0
  22. package/dist/human-review/task-proposals.js +39 -0
  23. package/dist/index.js +1 -0
  24. package/dist/javascript-verifier-contract.js +2 -2
  25. package/dist/learning/authoring.js +2 -1
  26. package/dist/learning/contracts.js +353 -81
  27. package/dist/learning/index.js +1 -0
  28. package/dist/learning/reward-authoring.js +5 -2
  29. package/dist/learning/training-handoff.js +32 -0
  30. package/dist/metrics-node.js +2 -0
  31. package/dist/profile-evaluation-target.js +7 -0
  32. package/dist/profile-evaluations.js +7 -10
  33. package/dist/profile-external-evaluation.js +35 -0
  34. package/dist/profile-source-candidate.js +5 -0
  35. package/dist/rewards.js +5 -3
  36. package/dist/taskset-run-contract.js +10 -0
  37. package/dist/tasksets.js +2 -0
  38. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +75 -1
  39. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  40. package/dist/types/conformance.d.ts +150 -2
  41. package/dist/types/conformance.d.ts.map +1 -1
  42. package/dist/types/connected-evidence/contracts.d.ts +317 -0
  43. package/dist/types/connected-evidence/contracts.d.ts.map +1 -0
  44. package/dist/types/connected-evidence/imports.d.ts +19 -0
  45. package/dist/types/connected-evidence/imports.d.ts.map +1 -0
  46. package/dist/types/connected-evidence/index.d.ts +6 -0
  47. package/dist/types/connected-evidence/index.d.ts.map +1 -0
  48. package/dist/types/connected-evidence/normalize.d.ts +25 -0
  49. package/dist/types/connected-evidence/normalize.d.ts.map +1 -0
  50. package/dist/types/connected-evidence/python-correctness.d.ts +196 -0
  51. package/dist/types/connected-evidence/python-correctness.d.ts.map +1 -0
  52. package/dist/types/connected-evidence/sources/claude-code.d.ts +74 -0
  53. package/dist/types/connected-evidence/sources/claude-code.d.ts.map +1 -0
  54. package/dist/types/connected-evidence/sources/codex.d.ts +74 -0
  55. package/dist/types/connected-evidence/sources/codex.d.ts.map +1 -0
  56. package/dist/types/connected-evidence/sources/history.d.ts +146 -0
  57. package/dist/types/connected-evidence/sources/history.d.ts.map +1 -0
  58. package/dist/types/connected-evidence/sources/usage.d.ts +3 -0
  59. package/dist/types/connected-evidence/sources/usage.d.ts.map +1 -0
  60. package/dist/types/connected-evidence/usage-summary.d.ts +162 -0
  61. package/dist/types/connected-evidence/usage-summary.d.ts.map +1 -0
  62. package/dist/types/evidence/conformance.d.ts +28 -28
  63. package/dist/types/evidence/contracts.d.ts +27 -27
  64. package/dist/types/execution-contracts.d.ts +159 -5
  65. package/dist/types/execution-contracts.d.ts.map +1 -1
  66. package/dist/types/experiments.d.ts +476 -7
  67. package/dist/types/experiments.d.ts.map +1 -1
  68. package/dist/types/graders.d.ts +21 -1
  69. package/dist/types/graders.d.ts.map +1 -1
  70. package/dist/types/human-review/comparison.d.ts +1959 -0
  71. package/dist/types/human-review/comparison.d.ts.map +1 -0
  72. package/dist/types/human-review/contracts.d.ts +2719 -0
  73. package/dist/types/human-review/contracts.d.ts.map +1 -0
  74. package/dist/types/human-review/execution.d.ts +298 -0
  75. package/dist/types/human-review/execution.d.ts.map +1 -0
  76. package/dist/types/human-review/handoff.d.ts +124 -0
  77. package/dist/types/human-review/handoff.d.ts.map +1 -0
  78. package/dist/types/human-review/index.d.ts +10 -0
  79. package/dist/types/human-review/index.d.ts.map +1 -0
  80. package/dist/types/human-review/local-publication.d.ts +165 -0
  81. package/dist/types/human-review/local-publication.d.ts.map +1 -0
  82. package/dist/types/human-review/repository.d.ts +43 -0
  83. package/dist/types/human-review/repository.d.ts.map +1 -0
  84. package/dist/types/human-review/results.d.ts +118 -0
  85. package/dist/types/human-review/results.d.ts.map +1 -0
  86. package/dist/types/human-review/service.d.ts +878 -0
  87. package/dist/types/human-review/service.d.ts.map +1 -0
  88. package/dist/types/human-review/task-proposals.d.ts +70 -0
  89. package/dist/types/human-review/task-proposals.d.ts.map +1 -0
  90. package/dist/types/index.d.ts +1 -0
  91. package/dist/types/index.d.ts.map +1 -1
  92. package/dist/types/javascript-environment-contract.d.ts +1 -1
  93. package/dist/types/javascript-verifier-contract.d.ts +1 -1
  94. package/dist/types/javascript-verifier-contract.d.ts.map +1 -1
  95. package/dist/types/learned-preference.d.ts +2 -2
  96. package/dist/types/learning/acceptance-group.d.ts +13 -13
  97. package/dist/types/learning/acceptance-plan.d.ts +1 -1
  98. package/dist/types/learning/admission.d.ts +78 -1
  99. package/dist/types/learning/admission.d.ts.map +1 -1
  100. package/dist/types/learning/authoring-service.d.ts +2 -0
  101. package/dist/types/learning/authoring-service.d.ts.map +1 -1
  102. package/dist/types/learning/authoring.d.ts +20 -0
  103. package/dist/types/learning/authoring.d.ts.map +1 -1
  104. package/dist/types/learning/batch-service.d.ts +76 -2
  105. package/dist/types/learning/batch-service.d.ts.map +1 -1
  106. package/dist/types/learning/contracts.d.ts +759 -24
  107. package/dist/types/learning/contracts.d.ts.map +1 -1
  108. package/dist/types/learning/index.d.ts +1 -0
  109. package/dist/types/learning/index.d.ts.map +1 -1
  110. package/dist/types/learning/iteration-candidate-decision.d.ts +2 -2
  111. package/dist/types/learning/iteration-dispatch-contracts.d.ts +2 -2
  112. package/dist/types/learning/operations.d.ts +1118 -8
  113. package/dist/types/learning/operations.d.ts.map +1 -1
  114. package/dist/types/learning/repository.d.ts +402 -15
  115. package/dist/types/learning/repository.d.ts.map +1 -1
  116. package/dist/types/learning/reward-authoring.d.ts.map +1 -1
  117. package/dist/types/learning/reward-calibration.d.ts +74 -0
  118. package/dist/types/learning/reward-calibration.d.ts.map +1 -1
  119. package/dist/types/learning/reward-checks.d.ts +14 -2
  120. package/dist/types/learning/reward-checks.d.ts.map +1 -1
  121. package/dist/types/learning/training-handoff.d.ts +201 -0
  122. package/dist/types/learning/training-handoff.d.ts.map +1 -0
  123. package/dist/types/learning/transport.d.ts +963 -21
  124. package/dist/types/learning/transport.d.ts.map +1 -1
  125. package/dist/types/metrics-node.d.ts +3 -1
  126. package/dist/types/metrics-node.d.ts.map +1 -1
  127. package/dist/types/model-improvement-qualification.d.ts +3 -3
  128. package/dist/types/native-harness-experiment-attempt.d.ts +2 -2
  129. package/dist/types/profile-evaluation-comparisons.d.ts +230 -0
  130. package/dist/types/profile-evaluation-comparisons.d.ts.map +1 -1
  131. package/dist/types/profile-evaluation-target.d.ts +14 -0
  132. package/dist/types/profile-evaluation-target.d.ts.map +1 -0
  133. package/dist/types/profile-evaluations.d.ts +117 -14
  134. package/dist/types/profile-evaluations.d.ts.map +1 -1
  135. package/dist/types/profile-external-evaluation.d.ts +404 -0
  136. package/dist/types/profile-external-evaluation.d.ts.map +1 -0
  137. package/dist/types/profile-source-candidate.d.ts +15 -0
  138. package/dist/types/profile-source-candidate.d.ts.map +1 -0
  139. package/dist/types/review-conformance.d.ts +7 -7
  140. package/dist/types/rewards.d.ts +253 -0
  141. package/dist/types/rewards.d.ts.map +1 -1
  142. package/dist/types/rollouts.d.ts +3 -3
  143. package/dist/types/taskset-run-contract.d.ts +230 -0
  144. package/dist/types/taskset-run-contract.d.ts.map +1 -1
  145. package/dist/types/tasksets.d.ts +311 -3
  146. package/dist/types/tasksets.d.ts.map +1 -1
  147. package/dist/types/telemetry-analysis.d.ts +2 -2
  148. package/dist/types/telemetry-bundle.d.ts +6 -6
  149. package/dist/types/telemetry.d.ts +3 -3
  150. package/package.json +13 -1
  151. package/schemas/learning/v1/command-request.schema.json +1745 -409
  152. package/schemas/learning/v1/decision.schema.json +11 -0
  153. package/schemas/learning/v1/draft.schema.json +11 -0
  154. package/schemas/learning/v1/grade.schema.json +11 -0
  155. package/schemas/learning/v1/package.schema.json +532 -12
  156. package/schemas/learning/v1/policy.schema.json +836 -28
  157. package/schemas/learning/v1/reward.schema.json +520 -0
  158. package/schemas/learning/v1/reward_check.schema.json +9 -0
@@ -0,0 +1,118 @@
1
+ import { z } from "zod";
2
+ import { type HumanReview } from "./contracts.js";
3
+ export declare const HumanResultViewSchema: z.ZodObject<{
4
+ schemaVersion: z.ZodLiteral<"openpond.humanResultView.v1">;
5
+ executionId: z.ZodString;
6
+ executionHash: z.ZodString;
7
+ selection: z.ZodArray<z.ZodObject<{
8
+ id: z.ZodString;
9
+ revision: z.ZodNumber;
10
+ contentHash: z.ZodString;
11
+ }, z.core.$strict>>;
12
+ coverage: z.ZodObject<{
13
+ pending: z.ZodNumber;
14
+ submitted: z.ZodNumber;
15
+ accepted: z.ZodNumber;
16
+ rejected: z.ZodNumber;
17
+ cancelled: z.ZodNumber;
18
+ }, z.core.$strict>;
19
+ results: z.ZodArray<z.ZodObject<{
20
+ review: z.ZodObject<{
21
+ id: z.ZodString;
22
+ revision: z.ZodNumber;
23
+ contentHash: z.ZodString;
24
+ }, z.core.$strict>;
25
+ decision: z.ZodNullable<z.ZodObject<{
26
+ id: z.ZodString;
27
+ revision: z.ZodNumber;
28
+ contentHash: z.ZodString;
29
+ }, z.core.$strict>>;
30
+ attemptIds: z.ZodArray<z.ZodString>;
31
+ mode: z.ZodEnum<{
32
+ individual: "individual";
33
+ pairwise: "pairwise";
34
+ }>;
35
+ origin: z.ZodEnum<{
36
+ owner_attested_local: "owner_attested_local";
37
+ native_local: "native_local";
38
+ recorded_evidence: "recorded_evidence";
39
+ hosted_retained: "hosted_retained";
40
+ }>;
41
+ status: z.ZodEnum<{
42
+ submitted: "submitted";
43
+ accepted: "accepted";
44
+ rejected: "rejected";
45
+ cancelled: "cancelled";
46
+ pending: "pending";
47
+ }>;
48
+ grader: z.ZodObject<{
49
+ id: z.ZodString;
50
+ revision: z.ZodNumber;
51
+ contentHash: z.ZodString;
52
+ }, z.core.$strict>;
53
+ answers: z.ZodArray<z.ZodObject<{
54
+ criterionId: z.ZodString;
55
+ value: z.ZodUnion<readonly [z.ZodNumber, z.ZodBoolean, z.ZodString, z.ZodNull]>;
56
+ abstain: z.ZodBoolean;
57
+ note: z.ZodString;
58
+ strength: z.ZodOptional<z.ZodEnum<{
59
+ slight: "slight";
60
+ moderate: "moderate";
61
+ strong: "strong";
62
+ }>>;
63
+ }, z.core.$strict>>;
64
+ }, z.core.$strict>>;
65
+ contentHash: z.ZodString;
66
+ }, z.core.$strict>;
67
+ /** A content-addressed view retains selected review revisions; automatic execution receipts remain untouched. */
68
+ export declare function createHumanResultView(input: {
69
+ executionId: string;
70
+ executionHash: string;
71
+ reviews: HumanReview[];
72
+ }): {
73
+ schemaVersion: "openpond.humanResultView.v1";
74
+ executionId: string;
75
+ executionHash: string;
76
+ selection: {
77
+ id: string;
78
+ revision: number;
79
+ contentHash: string;
80
+ }[];
81
+ coverage: {
82
+ pending: number;
83
+ submitted: number;
84
+ accepted: number;
85
+ rejected: number;
86
+ cancelled: number;
87
+ };
88
+ results: {
89
+ review: {
90
+ id: string;
91
+ revision: number;
92
+ contentHash: string;
93
+ };
94
+ decision: {
95
+ id: string;
96
+ revision: number;
97
+ contentHash: string;
98
+ } | null;
99
+ attemptIds: string[];
100
+ mode: "individual" | "pairwise";
101
+ origin: "owner_attested_local" | "native_local" | "recorded_evidence" | "hosted_retained";
102
+ status: "submitted" | "accepted" | "rejected" | "cancelled" | "pending";
103
+ grader: {
104
+ id: string;
105
+ revision: number;
106
+ contentHash: string;
107
+ };
108
+ answers: {
109
+ criterionId: string;
110
+ value: string | number | boolean | null;
111
+ abstain: boolean;
112
+ note: string;
113
+ strength?: "slight" | "moderate" | "strong" | undefined;
114
+ }[];
115
+ }[];
116
+ contentHash: string;
117
+ };
118
+ //# sourceMappingURL=results.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../../../../src/human-review/results.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAA8E,KAAK,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAG9H,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA0E,CAAC;AAC7G,iHAAiH;AACjH,wBAAgB,qBAAqB,CAAC,KAAK,EAAC;IAAC,WAAW,EAAC,MAAM,CAAC;IAAA,aAAa,EAAC,MAAM,CAAC;IAAA,OAAO,EAAC,WAAW,EAAE,CAAA;CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAM1G"}