openpond-sdk 0.12.24 → 0.12.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/LEARNING.md +5 -7
  2. package/OPENTELEMETRY.md +1 -1
  3. package/PRODUCT_INTEGRATION.md +129 -0
  4. package/README.md +13 -3
  5. package/SERVERLESS_SERVING.md +37 -0
  6. package/TASKSETS.md +1 -2
  7. package/TASK_INGESTION.md +87 -86
  8. package/TRAINING_PROTOCOL.md +7 -3
  9. package/TRAINING_WORKFLOWS.md +11 -6
  10. package/dist/advanced-refiner-evaluations.js +10 -9
  11. package/dist/ai-sdk.js +225 -0
  12. package/dist/ai-sdk.js.map +1 -0
  13. package/dist/candidate-evaluations.js +7 -6
  14. package/dist/{chunk-ZQFROKKS.js → chunk-2U6BQ5C4.js} +3 -3
  15. package/dist/{chunk-BNBPKCIR.js → chunk-3BILL33A.js} +4 -4
  16. package/dist/{chunk-3B5KSH3V.js → chunk-456F3BXD.js} +5 -2
  17. package/dist/chunk-456F3BXD.js.map +1 -0
  18. package/dist/{chunk-GCUWJUUM.js → chunk-4A4FNG4R.js} +2 -2
  19. package/dist/{chunk-EZME7DV4.js → chunk-4CDLPMJK.js} +2 -2
  20. package/dist/{chunk-SP7DQEQM.js → chunk-4CRLQMJ2.js} +5 -5
  21. package/dist/{chunk-QOX57Y27.js → chunk-4PEK3YOW.js} +6 -3
  22. package/dist/chunk-4PEK3YOW.js.map +1 -0
  23. package/dist/{chunk-GS5HEE4M.js → chunk-4REK5TTE.js} +2 -2
  24. package/dist/{chunk-AG6UJ3DI.js → chunk-57ROW3XF.js} +3 -3
  25. package/dist/{chunk-3BYFS4RO.js → chunk-6LVLWRBJ.js} +11 -8
  26. package/dist/chunk-6LVLWRBJ.js.map +1 -0
  27. package/dist/{chunk-3VUF5H3B.js → chunk-AMZ2SWHR.js} +2 -2
  28. package/dist/{chunk-ZQ6QTYRE.js → chunk-AQV7RQZH.js} +11 -11
  29. package/dist/{chunk-ZQ6QTYRE.js.map → chunk-AQV7RQZH.js.map} +1 -1
  30. package/dist/{chunk-BJMENXR4.js → chunk-C7I4ZG6Q.js} +3 -3
  31. package/dist/{chunk-JMFYNIYZ.js → chunk-CAOMKM55.js} +6 -3
  32. package/dist/chunk-CAOMKM55.js.map +1 -0
  33. package/dist/{chunk-QNOCM5JX.js → chunk-D26RJQU7.js} +2 -2
  34. package/dist/{chunk-GBDBKGXW.js → chunk-EL6KZO43.js} +3 -3
  35. package/dist/{chunk-MVAEE5EI.js → chunk-H5KHLBC5.js} +5 -2
  36. package/dist/chunk-H5KHLBC5.js.map +1 -0
  37. package/dist/{chunk-7G47YXQT.js → chunk-HCUZYGUK.js} +8 -5
  38. package/dist/chunk-HCUZYGUK.js.map +1 -0
  39. package/dist/{chunk-Q4RGCEOI.js → chunk-JINWKUKR.js} +3 -3
  40. package/dist/{chunk-626JLH4D.js → chunk-JWEIAEQ6.js} +2 -2
  41. package/dist/{chunk-DH6O4JFB.js → chunk-KJTX4A27.js} +5 -2
  42. package/dist/chunk-KJTX4A27.js.map +1 -0
  43. package/dist/{chunk-OBI2MQYU.js → chunk-MN4FE52A.js} +17 -14
  44. package/dist/chunk-MN4FE52A.js.map +1 -0
  45. package/dist/{chunk-NZRGRUYG.js → chunk-MUG4OQKL.js} +21 -7
  46. package/dist/chunk-MUG4OQKL.js.map +1 -0
  47. package/dist/{chunk-C3TS766N.js → chunk-N6LGT7NT.js} +5 -2
  48. package/dist/chunk-N6LGT7NT.js.map +1 -0
  49. package/dist/{chunk-QIGMNB6M.js → chunk-P56JWQYV.js} +3 -3
  50. package/dist/{chunk-GX6KMHW2.js → chunk-QKLZZOZD.js} +5 -2
  51. package/dist/chunk-QKLZZOZD.js.map +1 -0
  52. package/dist/chunk-QYWPNNP3.js +9 -0
  53. package/dist/chunk-QYWPNNP3.js.map +1 -0
  54. package/dist/{chunk-QCRNQMN4.js → chunk-UZAVG2WK.js} +5 -5
  55. package/dist/chunk-UZAVG2WK.js.map +1 -0
  56. package/dist/{chunk-Z252BNAZ.js → chunk-VE5JD5FN.js} +6 -2
  57. package/dist/chunk-VE5JD5FN.js.map +1 -0
  58. package/dist/{chunk-7F43U2NC.js → chunk-WZKKQNYY.js} +3 -3
  59. package/dist/{chunk-6EZAGTZG.js → chunk-X5AT253K.js} +5 -5
  60. package/dist/{chunk-ZRTNR4U7.js → chunk-XWUTYFOU.js} +3 -3
  61. package/dist/{chunk-KEG5QS7W.js → chunk-XYK3MAIC.js} +193 -193
  62. package/dist/chunk-XYK3MAIC.js.map +1 -0
  63. package/dist/chunk-ZG4HVCJB.js +248 -0
  64. package/dist/chunk-ZG4HVCJB.js.map +1 -0
  65. package/dist/connected-evidence.js +11 -10
  66. package/dist/dataset-commerce.js +4 -1
  67. package/dist/dataset-commerce.js.map +1 -1
  68. package/dist/dataset-marketplace.js +4 -1
  69. package/dist/dataset-marketplace.js.map +1 -1
  70. package/dist/dataset-workspaces.js +157 -93
  71. package/dist/dataset-workspaces.js.map +1 -1
  72. package/dist/errors.js +2 -1
  73. package/dist/experiment-evaluation-schedules.js +20 -19
  74. package/dist/experiment-improvements.js +8 -7
  75. package/dist/experiments.js +21 -20
  76. package/dist/human-review.js +2 -1
  77. package/dist/index.js +87 -56
  78. package/dist/index.js.map +1 -1
  79. package/dist/learning.js +6 -3
  80. package/dist/learning.js.map +1 -1
  81. package/dist/model-projects.js +2 -1
  82. package/dist/model-starter-attempts.js +3 -2
  83. package/dist/model-starter-catalog.js +7 -4
  84. package/dist/model-starter-catalog.js.map +1 -1
  85. package/dist/model-starters.js +4 -3
  86. package/dist/model-taskset-runs.js +7 -4
  87. package/dist/model-taskset-runs.js.map +1 -1
  88. package/dist/post-training.js +7 -6
  89. package/dist/taskset-catalog.js +2 -1
  90. package/dist/taskset-drafts.js +13 -10
  91. package/dist/taskset-drafts.js.map +1 -1
  92. package/dist/taskset-packages.js +16 -15
  93. package/dist/tool-learning.js +4 -1
  94. package/dist/tool-learning.js.map +1 -1
  95. package/dist/training-bundle.js +19 -18
  96. package/dist/training-bundle.js.map +1 -1
  97. package/dist/training-hardware.js +3 -2
  98. package/dist/training-projects.js +4 -1
  99. package/dist/training-projects.js.map +1 -1
  100. package/dist/training.js +2761 -2626
  101. package/dist/training.js.map +1 -1
  102. package/dist/types/packages/sdk/src/ai-sdk.d.ts +6 -0
  103. package/dist/types/packages/sdk/src/ai-sdk.d.ts.map +1 -0
  104. package/dist/types/packages/sdk/src/candidate-evaluation-client.d.ts +21 -21
  105. package/dist/types/packages/sdk/src/connected-evidence-client.d.ts +1 -1
  106. package/dist/types/packages/sdk/src/connected-evidence-client.d.ts.map +1 -1
  107. package/dist/types/packages/sdk/src/connected-sync.d.ts +1 -1
  108. package/dist/types/packages/sdk/src/connected-sync.d.ts.map +1 -1
  109. package/dist/types/packages/sdk/src/dataset-commerce.d.ts +1 -1
  110. package/dist/types/packages/sdk/src/dataset-commerce.d.ts.map +1 -1
  111. package/dist/types/packages/sdk/src/dataset-marketplace.d.ts +1 -1
  112. package/dist/types/packages/sdk/src/dataset-marketplace.d.ts.map +1 -1
  113. package/dist/types/packages/sdk/src/dataset-population.d.ts +1 -1
  114. package/dist/types/packages/sdk/src/dataset-population.d.ts.map +1 -1
  115. package/dist/types/packages/sdk/src/dataset-preparation-client.d.ts +1 -1
  116. package/dist/types/packages/sdk/src/dataset-preparation-client.d.ts.map +1 -1
  117. package/dist/types/packages/sdk/src/dataset-task-ingestion.d.ts +9 -2
  118. package/dist/types/packages/sdk/src/dataset-task-ingestion.d.ts.map +1 -1
  119. package/dist/types/packages/sdk/src/dataset-workspace-client.d.ts +141 -141
  120. package/dist/types/packages/sdk/src/dataset-workspace-client.d.ts.map +1 -1
  121. package/dist/types/packages/sdk/src/dataset-workspace-error.d.ts +28 -0
  122. package/dist/types/packages/sdk/src/dataset-workspace-error.d.ts.map +1 -0
  123. package/dist/types/packages/sdk/src/dataset-workspaces.d.ts +1 -0
  124. package/dist/types/packages/sdk/src/dataset-workspaces.d.ts.map +1 -1
  125. package/dist/types/packages/sdk/src/endpoints.d.ts +4 -0
  126. package/dist/types/packages/sdk/src/endpoints.d.ts.map +1 -0
  127. package/dist/types/packages/sdk/src/evaluation-coordination-http.d.ts +1 -1
  128. package/dist/types/packages/sdk/src/evaluation-coordination-http.d.ts.map +1 -1
  129. package/dist/types/packages/sdk/src/experiment-client.d.ts +637 -637
  130. package/dist/types/packages/sdk/src/experiment-client.d.ts.map +1 -1
  131. package/dist/types/packages/sdk/src/experiment-draft-storage.d.ts +28 -28
  132. package/dist/types/packages/sdk/src/experiment-evaluation-schedules.d.ts +28 -28
  133. package/dist/types/packages/sdk/src/experiment-improvement-client.d.ts +1623 -1623
  134. package/dist/types/packages/sdk/src/experiment-improvement-client.d.ts.map +1 -1
  135. package/dist/types/packages/sdk/src/experiment-inspection.d.ts +1 -1
  136. package/dist/types/packages/sdk/src/experiment-inspection.d.ts.map +1 -1
  137. package/dist/types/packages/sdk/src/experiment-run-details.d.ts +7 -7
  138. package/dist/types/packages/sdk/src/grader-inspection.d.ts +1 -1
  139. package/dist/types/packages/sdk/src/grader-inspection.d.ts.map +1 -1
  140. package/dist/types/packages/sdk/src/human-review-client.d.ts +1 -1
  141. package/dist/types/packages/sdk/src/human-review-client.d.ts.map +1 -1
  142. package/dist/types/packages/sdk/src/index.d.ts +13 -4
  143. package/dist/types/packages/sdk/src/index.d.ts.map +1 -1
  144. package/dist/types/packages/sdk/src/learning-client.d.ts +1 -1
  145. package/dist/types/packages/sdk/src/learning-client.d.ts.map +1 -1
  146. package/dist/types/packages/sdk/src/learning-correction-metrics.d.ts +1 -1
  147. package/dist/types/packages/sdk/src/learning-correction-metrics.d.ts.map +1 -1
  148. package/dist/types/packages/sdk/src/managed-decision-executions.d.ts +347 -347
  149. package/dist/types/packages/sdk/src/managed-training-preparation.d.ts +25 -25
  150. package/dist/types/packages/sdk/src/model-projects.d.ts +1 -1
  151. package/dist/types/packages/sdk/src/model-projects.d.ts.map +1 -1
  152. package/dist/types/packages/sdk/src/model-starter-attempts.d.ts +1 -1
  153. package/dist/types/packages/sdk/src/model-starter-attempts.d.ts.map +1 -1
  154. package/dist/types/packages/sdk/src/model-starter-catalog.d.ts +1 -1
  155. package/dist/types/packages/sdk/src/model-starter-catalog.d.ts.map +1 -1
  156. package/dist/types/packages/sdk/src/model-taskset-runs.d.ts +29 -29
  157. package/dist/types/packages/sdk/src/model-taskset-runs.d.ts.map +1 -1
  158. package/dist/types/packages/sdk/src/post-training.d.ts +7 -7
  159. package/dist/types/packages/sdk/src/product-model-response.d.ts +4 -0
  160. package/dist/types/packages/sdk/src/product-model-response.d.ts.map +1 -0
  161. package/dist/types/packages/sdk/src/product-models.d.ts +76 -0
  162. package/dist/types/packages/sdk/src/product-models.d.ts.map +1 -0
  163. package/dist/types/packages/sdk/src/profile-evaluation-discovery.d.ts +1 -1
  164. package/dist/types/packages/sdk/src/profile-evaluation-discovery.d.ts.map +1 -1
  165. package/dist/types/packages/sdk/src/serverless-serving.d.ts +157 -0
  166. package/dist/types/packages/sdk/src/serverless-serving.d.ts.map +1 -0
  167. package/dist/types/packages/sdk/src/steering-model-serving.d.ts +276 -0
  168. package/dist/types/packages/sdk/src/steering-model-serving.d.ts.map +1 -1
  169. package/dist/types/packages/sdk/src/taskset-catalog.d.ts +1 -1
  170. package/dist/types/packages/sdk/src/taskset-catalog.d.ts.map +1 -1
  171. package/dist/types/packages/sdk/src/taskset-draft-client.d.ts +121 -121
  172. package/dist/types/packages/sdk/src/taskset-draft-client.d.ts.map +1 -1
  173. package/dist/types/packages/sdk/src/taskset-draft-source.d.ts +20 -20
  174. package/dist/types/packages/sdk/src/taskset-package-client.d.ts +1 -1
  175. package/dist/types/packages/sdk/src/taskset-package-client.d.ts.map +1 -1
  176. package/dist/types/packages/sdk/src/taskset-package-projection.d.ts +20 -20
  177. package/dist/types/packages/sdk/src/tool-learning-client.d.ts +1 -1
  178. package/dist/types/packages/sdk/src/tool-learning-client.d.ts.map +1 -1
  179. package/dist/types/packages/sdk/src/training-evidence.d.ts +54 -0
  180. package/dist/types/packages/sdk/src/training-evidence.d.ts.map +1 -0
  181. package/dist/types/packages/sdk/src/training-harness-sources.d.ts +2 -2
  182. package/dist/types/packages/sdk/src/training-model-serving-control.d.ts +232 -0
  183. package/dist/types/packages/sdk/src/training-model-serving-control.d.ts.map +1 -1
  184. package/dist/types/packages/sdk/src/training-model-serving.d.ts +552 -0
  185. package/dist/types/packages/sdk/src/training-model-serving.d.ts.map +1 -1
  186. package/dist/types/packages/sdk/src/training-project-client.d.ts +1 -1
  187. package/dist/types/packages/sdk/src/training-project-client.d.ts.map +1 -1
  188. package/dist/types/packages/sdk/src/training-transport.d.ts +1 -1
  189. package/dist/types/packages/sdk/src/training-transport.d.ts.map +1 -1
  190. package/dist/types/packages/sdk/src/training.d.ts +2489 -2148
  191. package/dist/types/packages/sdk/src/training.d.ts.map +1 -1
  192. package/dist/types/packages/sdk/src/types.d.ts +14 -1
  193. package/dist/types/packages/sdk/src/types.d.ts.map +1 -1
  194. package/examples/learning/submit.py +1 -1
  195. package/examples/learning/submit.ts +2 -2
  196. package/examples/training/continual.ts +4 -4
  197. package/examples/training/manual.ts +1 -3
  198. package/fixtures/personalized-inference-request.schema.json +316 -0
  199. package/fixtures/python-foundation/schemas.json +4873 -4484
  200. package/package.json +19 -3
  201. package/dist/chunk-3B5KSH3V.js.map +0 -1
  202. package/dist/chunk-3BYFS4RO.js.map +0 -1
  203. package/dist/chunk-7G47YXQT.js.map +0 -1
  204. package/dist/chunk-C3TS766N.js.map +0 -1
  205. package/dist/chunk-DH6O4JFB.js.map +0 -1
  206. package/dist/chunk-GX6KMHW2.js.map +0 -1
  207. package/dist/chunk-JMFYNIYZ.js.map +0 -1
  208. package/dist/chunk-KEG5QS7W.js.map +0 -1
  209. package/dist/chunk-MVAEE5EI.js.map +0 -1
  210. package/dist/chunk-NZRGRUYG.js.map +0 -1
  211. package/dist/chunk-OBI2MQYU.js.map +0 -1
  212. package/dist/chunk-QCRNQMN4.js.map +0 -1
  213. package/dist/chunk-QOX57Y27.js.map +0 -1
  214. package/dist/chunk-Z252BNAZ.js.map +0 -1
  215. /package/dist/{chunk-ZQFROKKS.js.map → chunk-2U6BQ5C4.js.map} +0 -0
  216. /package/dist/{chunk-BNBPKCIR.js.map → chunk-3BILL33A.js.map} +0 -0
  217. /package/dist/{chunk-GCUWJUUM.js.map → chunk-4A4FNG4R.js.map} +0 -0
  218. /package/dist/{chunk-EZME7DV4.js.map → chunk-4CDLPMJK.js.map} +0 -0
  219. /package/dist/{chunk-SP7DQEQM.js.map → chunk-4CRLQMJ2.js.map} +0 -0
  220. /package/dist/{chunk-GS5HEE4M.js.map → chunk-4REK5TTE.js.map} +0 -0
  221. /package/dist/{chunk-AG6UJ3DI.js.map → chunk-57ROW3XF.js.map} +0 -0
  222. /package/dist/{chunk-3VUF5H3B.js.map → chunk-AMZ2SWHR.js.map} +0 -0
  223. /package/dist/{chunk-BJMENXR4.js.map → chunk-C7I4ZG6Q.js.map} +0 -0
  224. /package/dist/{chunk-QNOCM5JX.js.map → chunk-D26RJQU7.js.map} +0 -0
  225. /package/dist/{chunk-GBDBKGXW.js.map → chunk-EL6KZO43.js.map} +0 -0
  226. /package/dist/{chunk-Q4RGCEOI.js.map → chunk-JINWKUKR.js.map} +0 -0
  227. /package/dist/{chunk-626JLH4D.js.map → chunk-JWEIAEQ6.js.map} +0 -0
  228. /package/dist/{chunk-QIGMNB6M.js.map → chunk-P56JWQYV.js.map} +0 -0
  229. /package/dist/{chunk-7F43U2NC.js.map → chunk-WZKKQNYY.js.map} +0 -0
  230. /package/dist/{chunk-6EZAGTZG.js.map → chunk-X5AT253K.js.map} +0 -0
  231. /package/dist/{chunk-ZRTNR4U7.js.map → chunk-XWUTYFOU.js.map} +0 -0
@@ -4,7 +4,7 @@ export declare class ExperimentImprovementClient {
4
4
  private readonly options;
5
5
  private readonly origin;
6
6
  constructor(options: {
7
- baseUrl: string;
7
+ baseUrl?: string;
8
8
  apiKey: string;
9
9
  teamId: string;
10
10
  actorId: string;
@@ -164,149 +164,27 @@ export declare class ExperimentImprovementClient {
164
164
  contentHash: string;
165
165
  }>;
166
166
  command(raw: ExperimentImprovementCommand, signal?: AbortSignal): Promise<{
167
+ schemaVersion: "openpond.experimentImprovement.v1";
168
+ id: string;
169
+ operationId: string;
170
+ requestHash: string;
171
+ actorId: string;
167
172
  teamId: string;
168
- evidence: {
173
+ projectId: string | null;
174
+ ownerWorkspaceId: string;
175
+ ownerRevision: number;
176
+ ownerSourceRevision: string;
177
+ baseRelease: {
169
178
  id: string;
170
179
  contentHash: string;
171
180
  };
172
- scoring?: {
173
- sourcePass: {
174
- id: string;
175
- contentHash: string;
176
- };
177
- graders: {
178
- id: string;
179
- contentHash: string;
180
- revision: number;
181
- }[];
182
- mappings?: {
183
- graderId: string;
184
- fields: {
185
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
186
- path: string[];
187
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
188
- sourcePath: string[];
189
- }[];
190
- }[] | undefined;
191
- maximumCostUsd: number;
192
- } | undefined;
193
- profiles: {
194
- id: string;
195
- name: string;
196
- profileRef: {
197
- source: "github" | "local" | "openpond_git";
198
- repositoryId: string;
199
- profileId: string;
200
- };
201
- ownerWorkspaceId: string;
202
- ownerRevision: number;
203
- baseRelease: {
204
- id: string;
205
- contentHash: string;
206
- };
207
- profileSourceRevision: string;
208
- components: {
209
- kind: "agent" | "instruction" | "skill" | "workflow";
210
- path: string;
211
- agentId?: string | undefined;
212
- workflowId?: string | undefined;
213
- label: string;
214
- binding: {
215
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
216
- dataset: {
217
- id: string;
218
- contentHash: string;
219
- revision: number;
220
- };
221
- packageHash: string;
222
- profileId: string;
223
- target: {
224
- kind: "profile";
225
- } | {
226
- kind: "workflow";
227
- workflowId: string;
228
- } | {
229
- kind: "skill";
230
- skillPath: string;
231
- } | {
232
- kind: "agent_action";
233
- actionId: string;
234
- };
235
- declaredProfileCatalogHash: string | null;
236
- protectedProfileClosureHash: string;
237
- recordedOrigin?: {
238
- execution: {
239
- id: string;
240
- contentHash: string;
241
- };
242
- originalEvidence: {
243
- id: string;
244
- contentHash: string;
245
- };
246
- members: {
247
- caseId: string;
248
- seed: string;
249
- fixtureId: string;
250
- sourceId: string;
251
- snapshotHash: string;
252
- boundaryRevisionHash: string;
253
- }[];
254
- } | undefined;
255
- gradingSource?: {
256
- pass: {
257
- id: string;
258
- contentHash: string;
259
- };
260
- evidence: {
261
- id: string;
262
- contentHash: string;
263
- };
264
- } | undefined;
265
- split: "frozen_eval" | "test" | "train" | "validation";
266
- population: {
267
- taskId: string;
268
- seed: string;
269
- fixtureId: string | null;
270
- }[];
271
- evaluators: {
272
- release: {
273
- id: string;
274
- contentHash: string;
275
- revision: string | number;
276
- };
277
- feedbackKey: string;
278
- configurationHash: string;
279
- output: "boolean" | "category" | "score";
280
- categories: string[];
281
- }[];
282
- fieldMappingsHash: string;
283
- fieldMappings: {
284
- graderId: string;
285
- fields: {
286
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
287
- path: string[];
288
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
289
- sourcePath: string[];
290
- }[];
291
- }[];
292
- environmentHash: string;
293
- privateDatasetClosureHash: string;
294
- criterion: {
295
- minimumPassRate: number;
296
- requireComplete: boolean;
297
- };
298
- contentHash: string;
299
- };
300
- }[];
301
- }[];
302
- } | {
303
- candidateId: string;
304
- actorId: string;
305
- teamId: string;
306
- projectId: string | null;
307
- candidateRevision: number;
308
- recipeHash: string;
309
- baseline: {
181
+ profileRef: {
182
+ source: "github" | "local" | "openpond_git";
183
+ repositoryId: string;
184
+ profileId: string;
185
+ };
186
+ baseProfileSourceRevision: string;
187
+ evidence: {
310
188
  manifest: {
311
189
  schemaVersion: "openpond.experimentManifest.v1";
312
190
  id: string;
@@ -575,57 +453,198 @@ export declare class ExperimentImprovementClient {
575
453
  contentHash: string;
576
454
  };
577
455
  };
578
- candidate: {
579
- manifest: {
580
- schemaVersion: "openpond.experimentManifest.v1";
456
+ component: {
457
+ kind: "agent" | "instruction" | "skill" | "workflow";
458
+ path: string;
459
+ workflowId?: string | undefined;
460
+ };
461
+ mode: "llm_assisted" | "manual";
462
+ status: "accepted" | "adopting" | "authoring" | "authoring_queued" | "cancelled" | "declined" | "draft" | "failed" | "frozen" | "ready_for_review" | "rolled_back" | "testing";
463
+ revision: number;
464
+ limits: {
465
+ maximumCostUsd: number;
466
+ maximumAuthoringSteps: number;
467
+ maximumDurationMs: number;
468
+ };
469
+ authoringSteps: number;
470
+ work: {
471
+ sessionId: string;
472
+ turnId: string;
473
+ revision: number;
474
+ startedAt: string;
475
+ } | null;
476
+ frozen: {
477
+ release: {
581
478
  id: string;
582
- name: string;
583
- teamId: string;
584
- operationId: string;
585
- maximumCostUsd: number | null;
586
- dataset: {
587
- id: string;
588
- contentHash: string;
589
- revision: number;
590
- };
591
- target: {
592
- kind: "model";
593
- modelId: string;
594
- configurationHash: string;
595
- } | {
596
- kind: "harness";
597
- source: {
598
- profileId: string;
599
- sourceRevision: string;
600
- harnessRelease: {
479
+ contentHash: string;
480
+ };
481
+ sourceRevision: string;
482
+ profileSourceRevision: string;
483
+ authoringHash: string;
484
+ partitionHash: string;
485
+ executableSourceHash: string;
486
+ protectedClosureHash: string;
487
+ candidateRevision: number;
488
+ diff: {
489
+ path: string;
490
+ before: string | null;
491
+ after: string | null;
492
+ }[];
493
+ } | null;
494
+ test: {
495
+ recipe: {
496
+ externalDatasetBinding: {
497
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
498
+ dataset: {
499
+ id: string;
500
+ contentHash: string;
501
+ revision: number;
502
+ };
503
+ packageHash: string;
504
+ profileId: string;
505
+ target: {
506
+ kind: "profile";
507
+ } | {
508
+ kind: "workflow";
509
+ workflowId: string;
510
+ } | {
511
+ kind: "skill";
512
+ skillPath: string;
513
+ } | {
514
+ kind: "agent_action";
515
+ actionId: string;
516
+ };
517
+ declaredProfileCatalogHash: string | null;
518
+ protectedProfileClosureHash: string;
519
+ recordedOrigin?: {
520
+ execution: {
601
521
  id: string;
602
522
  contentHash: string;
603
523
  };
604
- catalogHash: string;
605
- definitionId: string;
606
- definitionHash: string;
607
- target: {
608
- kind: "profile";
609
- } | {
610
- kind: "workflow";
611
- workflowId: string;
612
- } | {
613
- kind: "skill";
614
- skillPath: string;
615
- } | {
616
- kind: "agent_action";
617
- actionId: string;
524
+ originalEvidence: {
525
+ id: string;
526
+ contentHash: string;
618
527
  };
619
- environmentHash: string;
620
- externalDatasetBinding?: {
621
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
622
- dataset: {
528
+ members: {
529
+ caseId: string;
530
+ seed: string;
531
+ fixtureId: string;
532
+ sourceId: string;
533
+ snapshotHash: string;
534
+ boundaryRevisionHash: string;
535
+ }[];
536
+ } | undefined;
537
+ gradingSource?: {
538
+ pass: {
539
+ id: string;
540
+ contentHash: string;
541
+ };
542
+ evidence: {
543
+ id: string;
544
+ contentHash: string;
545
+ };
546
+ } | undefined;
547
+ split: "frozen_eval" | "test" | "train" | "validation";
548
+ population: {
549
+ taskId: string;
550
+ seed: string;
551
+ fixtureId: string | null;
552
+ }[];
553
+ evaluators: {
554
+ release: {
555
+ id: string;
556
+ contentHash: string;
557
+ revision: string | number;
558
+ };
559
+ feedbackKey: string;
560
+ configurationHash: string;
561
+ output: "boolean" | "category" | "score";
562
+ categories: string[];
563
+ }[];
564
+ fieldMappingsHash: string;
565
+ fieldMappings: {
566
+ graderId: string;
567
+ fields: {
568
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
569
+ path: string[];
570
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
571
+ sourcePath: string[];
572
+ }[];
573
+ }[];
574
+ environmentHash: string;
575
+ privateDatasetClosureHash: string;
576
+ criterion: {
577
+ minimumPassRate: number;
578
+ requireComplete: boolean;
579
+ };
580
+ contentHash: string;
581
+ };
582
+ scoring?: {
583
+ sourcePass: {
584
+ id: string;
585
+ contentHash: string;
586
+ };
587
+ graders: {
588
+ id: string;
589
+ contentHash: string;
590
+ revision: number;
591
+ }[];
592
+ mappings?: {
593
+ graderId: string;
594
+ fields: {
595
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
596
+ path: string[];
597
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
598
+ sourcePath: string[];
599
+ }[];
600
+ }[] | undefined;
601
+ maximumCostUsd: number;
602
+ } | undefined;
603
+ modelId: string;
604
+ maximumCostUsd: number;
605
+ };
606
+ recipeHash: string;
607
+ baselineOperationId: string;
608
+ candidateOperationId: string;
609
+ cancelRequestedAt: string | null;
610
+ dispatching: {
611
+ kind: "baseline" | "candidate";
612
+ token: string;
613
+ pid: number;
614
+ processStart: string | null;
615
+ startedAt: string;
616
+ } | null;
617
+ } | null;
618
+ comparison: {
619
+ baseline: {
620
+ manifest: {
621
+ schemaVersion: "openpond.experimentManifest.v1";
622
+ id: string;
623
+ name: string;
624
+ teamId: string;
625
+ operationId: string;
626
+ maximumCostUsd: number | null;
627
+ dataset: {
628
+ id: string;
629
+ contentHash: string;
630
+ revision: number;
631
+ };
632
+ target: {
633
+ kind: "model";
634
+ modelId: string;
635
+ configurationHash: string;
636
+ } | {
637
+ kind: "harness";
638
+ source: {
639
+ profileId: string;
640
+ sourceRevision: string;
641
+ harnessRelease: {
623
642
  id: string;
624
643
  contentHash: string;
625
- revision: number;
626
644
  };
627
- packageHash: string;
628
- profileId: string;
645
+ catalogHash: string;
646
+ definitionId: string;
647
+ definitionHash: string;
629
648
  target: {
630
649
  kind: "profile";
631
650
  } | {
@@ -638,292 +657,263 @@ export declare class ExperimentImprovementClient {
638
657
  kind: "agent_action";
639
658
  actionId: string;
640
659
  };
641
- declaredProfileCatalogHash: string | null;
642
- protectedProfileClosureHash: string;
643
- recordedOrigin?: {
644
- execution: {
660
+ environmentHash: string;
661
+ externalDatasetBinding?: {
662
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
663
+ dataset: {
645
664
  id: string;
646
665
  contentHash: string;
666
+ revision: number;
647
667
  };
648
- originalEvidence: {
649
- id: string;
650
- contentHash: string;
668
+ packageHash: string;
669
+ profileId: string;
670
+ target: {
671
+ kind: "profile";
672
+ } | {
673
+ kind: "workflow";
674
+ workflowId: string;
675
+ } | {
676
+ kind: "skill";
677
+ skillPath: string;
678
+ } | {
679
+ kind: "agent_action";
680
+ actionId: string;
651
681
  };
652
- members: {
653
- caseId: string;
682
+ declaredProfileCatalogHash: string | null;
683
+ protectedProfileClosureHash: string;
684
+ recordedOrigin?: {
685
+ execution: {
686
+ id: string;
687
+ contentHash: string;
688
+ };
689
+ originalEvidence: {
690
+ id: string;
691
+ contentHash: string;
692
+ };
693
+ members: {
694
+ caseId: string;
695
+ seed: string;
696
+ fixtureId: string;
697
+ sourceId: string;
698
+ snapshotHash: string;
699
+ boundaryRevisionHash: string;
700
+ }[];
701
+ } | undefined;
702
+ gradingSource?: {
703
+ pass: {
704
+ id: string;
705
+ contentHash: string;
706
+ };
707
+ evidence: {
708
+ id: string;
709
+ contentHash: string;
710
+ };
711
+ } | undefined;
712
+ split: "frozen_eval" | "test" | "train" | "validation";
713
+ population: {
714
+ taskId: string;
654
715
  seed: string;
655
- fixtureId: string;
656
- sourceId: string;
657
- snapshotHash: string;
658
- boundaryRevisionHash: string;
716
+ fixtureId: string | null;
659
717
  }[];
660
- } | undefined;
661
- gradingSource?: {
662
- pass: {
663
- id: string;
664
- contentHash: string;
665
- };
666
- evidence: {
667
- id: string;
668
- contentHash: string;
718
+ evaluators: {
719
+ release: {
720
+ id: string;
721
+ contentHash: string;
722
+ revision: string | number;
723
+ };
724
+ feedbackKey: string;
725
+ configurationHash: string;
726
+ output: "boolean" | "category" | "score";
727
+ categories: string[];
728
+ }[];
729
+ fieldMappingsHash: string;
730
+ fieldMappings: {
731
+ graderId: string;
732
+ fields: {
733
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
734
+ path: string[];
735
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
736
+ sourcePath: string[];
737
+ }[];
738
+ }[];
739
+ environmentHash: string;
740
+ privateDatasetClosureHash: string;
741
+ criterion: {
742
+ minimumPassRate: number;
743
+ requireComplete: boolean;
669
744
  };
745
+ contentHash: string;
670
746
  } | undefined;
671
- split: "frozen_eval" | "test" | "train" | "validation";
672
- population: {
673
- taskId: string;
674
- seed: string;
675
- fixtureId: string | null;
676
- }[];
677
- evaluators: {
678
- release: {
747
+ sourceCandidate?: {
748
+ candidateId: string;
749
+ frozenRevision: number;
750
+ frozenHash: string;
751
+ harnessRelease: {
679
752
  id: string;
680
753
  contentHash: string;
681
- revision: string | number;
682
754
  };
683
- feedbackKey: string;
684
- configurationHash: string;
685
- output: "boolean" | "category" | "score";
686
- categories: string[];
687
- }[];
688
- fieldMappingsHash: string;
689
- fieldMappings: {
690
- graderId: string;
691
- fields: {
692
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
693
- path: string[];
694
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
695
- sourcePath: string[];
696
- }[];
697
- }[];
698
- environmentHash: string;
699
- privateDatasetClosureHash: string;
700
- criterion: {
701
- minimumPassRate: number;
702
- requireComplete: boolean;
703
- };
704
- contentHash: string;
705
- } | undefined;
706
- sourceCandidate?: {
707
- candidateId: string;
708
- frozenRevision: number;
709
- frozenHash: string;
755
+ protectedClosureHash: string;
756
+ } | undefined;
757
+ };
758
+ model: {
759
+ modelId: string;
760
+ configurationHash: string;
761
+ };
762
+ } | {
763
+ kind: "agent";
764
+ source: {
710
765
  harnessRelease: {
711
766
  id: string;
712
767
  contentHash: string;
713
768
  };
714
- protectedClosureHash: string;
715
- } | undefined;
716
- };
717
- model: {
718
- modelId: string;
769
+ agentSnapshot: {
770
+ id: string;
771
+ contentHash: string;
772
+ };
773
+ sourcePackageHash: string;
774
+ };
775
+ model: {
776
+ modelId: string;
777
+ configurationHash: string;
778
+ };
779
+ } | {
780
+ kind: "fixture";
719
781
  configurationHash: string;
720
782
  };
721
- } | {
722
- kind: "agent";
723
- source: {
724
- harnessRelease: {
783
+ execution?: {
784
+ packageHash: string;
785
+ runtimeTargetHash: string;
786
+ metricPolicyHash: string;
787
+ compatibility?: {
788
+ protocol: "openpond.evaluation-execution.v1";
789
+ targetKind: "agent" | "fixture" | "harness" | "model";
790
+ } | undefined;
791
+ } | undefined;
792
+ lineage?: {
793
+ definition: {
725
794
  id: string;
726
795
  contentHash: string;
727
- };
728
- agentSnapshot: {
796
+ revision: number;
797
+ } | null;
798
+ execution: {
729
799
  id: string;
730
800
  contentHash: string;
731
801
  };
732
- sourcePackageHash: string;
733
- };
734
- model: {
735
- modelId: string;
736
- configurationHash: string;
737
- };
738
- } | {
739
- kind: "fixture";
740
- configurationHash: string;
741
- };
742
- execution?: {
743
- packageHash: string;
744
- runtimeTargetHash: string;
745
- metricPolicyHash: string;
746
- compatibility?: {
747
- protocol: "openpond.evaluation-execution.v1";
748
- targetKind: "agent" | "fixture" | "harness" | "model";
802
+ scoringPassId: string | null;
803
+ humanProjection?: {
804
+ originalManifest: {
805
+ id: string;
806
+ contentHash: string;
807
+ };
808
+ selectionHash: string;
809
+ rulesHash: string;
810
+ } | undefined;
749
811
  } | undefined;
750
- } | undefined;
751
- lineage?: {
752
- definition: {
753
- id: string;
754
- contentHash: string;
755
- revision: number;
756
- } | null;
757
- execution: {
758
- id: string;
759
- contentHash: string;
760
- };
761
- scoringPassId: string | null;
762
- humanProjection?: {
763
- originalManifest: {
812
+ evaluators: {
813
+ release: {
764
814
  id: string;
765
815
  contentHash: string;
816
+ revision: string | number;
766
817
  };
767
- selectionHash: string;
768
- rulesHash: string;
769
- } | undefined;
770
- } | undefined;
771
- evaluators: {
772
- release: {
773
- id: string;
774
- contentHash: string;
775
- revision: string | number;
776
- };
777
- feedbackKey: string;
778
- configurationHash?: string | undefined;
779
- output: "boolean" | "category" | "score";
780
- categories: string[];
781
- }[];
782
- population: {
783
- caseId: string;
784
- seed: string;
785
- fixtureId: string | null;
786
- }[];
787
- createdAt: string;
788
- contentHash: string;
789
- };
790
- result: {
791
- schemaVersion: "openpond.experimentResult.v1";
792
- manifest: {
793
- id: string;
794
- contentHash: string;
795
- };
796
- status: "cancelled" | "completed" | "failed";
797
- cases: {
798
- identity: {
818
+ feedbackKey: string;
819
+ configurationHash?: string | undefined;
820
+ output: "boolean" | "category" | "score";
821
+ categories: string[];
822
+ }[];
823
+ population: {
799
824
  caseId: string;
800
825
  seed: string;
801
826
  fixtureId: string | null;
827
+ }[];
828
+ createdAt: string;
829
+ contentHash: string;
830
+ };
831
+ result: {
832
+ schemaVersion: "openpond.experimentResult.v1";
833
+ manifest: {
834
+ id: string;
835
+ contentHash: string;
802
836
  };
803
- status: "cancelled" | "completed" | "failed" | "unavailable";
804
- output: unknown;
805
- error: {
806
- code: string;
807
- message: string;
808
- } | null;
809
- feedback: {
810
- feedbackKey: string;
811
- evaluator: {
812
- id: string;
813
- contentHash: string;
814
- revision: string | number;
837
+ status: "cancelled" | "completed" | "failed";
838
+ cases: {
839
+ identity: {
840
+ caseId: string;
841
+ seed: string;
842
+ fixtureId: string | null;
815
843
  };
816
- status: "failed" | "pending" | "scored" | "unavailable";
817
- value: string | number | boolean | null;
818
- passed?: boolean | null | undefined;
819
- reasoning: string | null;
820
- evidenceRefs: {
844
+ status: "cancelled" | "completed" | "failed" | "unavailable";
845
+ output: unknown;
846
+ error: {
847
+ code: string;
848
+ message: string;
849
+ } | null;
850
+ feedback: {
851
+ feedbackKey: string;
852
+ evaluator: {
853
+ id: string;
854
+ contentHash: string;
855
+ revision: string | number;
856
+ };
857
+ status: "failed" | "pending" | "scored" | "unavailable";
858
+ value: string | number | boolean | null;
859
+ passed?: boolean | null | undefined;
860
+ reasoning: string | null;
861
+ evidenceRefs: {
862
+ id: string;
863
+ contentHash: string;
864
+ mediaType: string | null;
865
+ sizeBytes: number | null;
866
+ }[];
867
+ }[];
868
+ usage: {
869
+ inputTokens: number | null;
870
+ outputTokens: number | null;
871
+ totalTokens: number | null;
872
+ costUsd: number | null;
873
+ latencyMs: number | null;
874
+ };
875
+ traceRef: {
821
876
  id: string;
822
877
  contentHash: string;
823
878
  mediaType: string | null;
824
879
  sizeBytes: number | null;
825
- }[];
880
+ } | null;
881
+ startedAt: string | null;
882
+ completedAt: string | null;
826
883
  }[];
827
- usage: {
828
- inputTokens: number | null;
829
- outputTokens: number | null;
830
- totalTokens: number | null;
831
- costUsd: number | null;
832
- latencyMs: number | null;
833
- };
834
- traceRef: {
884
+ completedAt: string;
885
+ contentHash: string;
886
+ };
887
+ };
888
+ candidate: {
889
+ manifest: {
890
+ schemaVersion: "openpond.experimentManifest.v1";
891
+ id: string;
892
+ name: string;
893
+ teamId: string;
894
+ operationId: string;
895
+ maximumCostUsd: number | null;
896
+ dataset: {
835
897
  id: string;
836
898
  contentHash: string;
837
- mediaType: string | null;
838
- sizeBytes: number | null;
839
- } | null;
840
- startedAt: string | null;
841
- completedAt: string | null;
842
- }[];
843
- completedAt: string;
844
- contentHash: string;
845
- };
846
- };
847
- contentHash: string;
848
- } | {
849
- candidateId: string;
850
- candidateRevision: number;
851
- path: string;
852
- text: string;
853
- textHash: string;
854
- contentHash: string;
855
- } | {
856
- schemaVersion: "openpond.experimentImprovement.v1";
857
- id: string;
858
- operationId: string;
859
- requestHash: string;
860
- actorId: string;
861
- teamId: string;
862
- projectId: string | null;
863
- ownerWorkspaceId: string;
864
- ownerRevision: number;
865
- ownerSourceRevision: string;
866
- baseRelease: {
867
- id: string;
868
- contentHash: string;
869
- };
870
- profileRef: {
871
- source: "github" | "local" | "openpond_git";
872
- repositoryId: string;
873
- profileId: string;
874
- };
875
- baseProfileSourceRevision: string;
876
- evidence: {
877
- manifest: {
878
- schemaVersion: "openpond.experimentManifest.v1";
879
- id: string;
880
- name: string;
881
- teamId: string;
882
- operationId: string;
883
- maximumCostUsd: number | null;
884
- dataset: {
885
- id: string;
886
- contentHash: string;
887
- revision: number;
888
- };
889
- target: {
890
- kind: "model";
891
- modelId: string;
892
- configurationHash: string;
893
- } | {
894
- kind: "harness";
895
- source: {
896
- profileId: string;
897
- sourceRevision: string;
898
- harnessRelease: {
899
- id: string;
900
- contentHash: string;
901
- };
902
- catalogHash: string;
903
- definitionId: string;
904
- definitionHash: string;
905
- target: {
906
- kind: "profile";
907
- } | {
908
- kind: "workflow";
909
- workflowId: string;
910
- } | {
911
- kind: "skill";
912
- skillPath: string;
913
- } | {
914
- kind: "agent_action";
915
- actionId: string;
916
- };
917
- environmentHash: string;
918
- externalDatasetBinding?: {
919
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
920
- dataset: {
899
+ revision: number;
900
+ };
901
+ target: {
902
+ kind: "model";
903
+ modelId: string;
904
+ configurationHash: string;
905
+ } | {
906
+ kind: "harness";
907
+ source: {
908
+ profileId: string;
909
+ sourceRevision: string;
910
+ harnessRelease: {
921
911
  id: string;
922
912
  contentHash: string;
923
- revision: number;
924
913
  };
925
- packageHash: string;
926
- profileId: string;
914
+ catalogHash: string;
915
+ definitionId: string;
916
+ definitionHash: string;
927
917
  target: {
928
918
  kind: "profile";
929
919
  } | {
@@ -936,309 +926,158 @@ export declare class ExperimentImprovementClient {
936
926
  kind: "agent_action";
937
927
  actionId: string;
938
928
  };
939
- declaredProfileCatalogHash: string | null;
940
- protectedProfileClosureHash: string;
941
- recordedOrigin?: {
942
- execution: {
929
+ environmentHash: string;
930
+ externalDatasetBinding?: {
931
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
932
+ dataset: {
943
933
  id: string;
944
934
  contentHash: string;
935
+ revision: number;
945
936
  };
946
- originalEvidence: {
947
- id: string;
948
- contentHash: string;
937
+ packageHash: string;
938
+ profileId: string;
939
+ target: {
940
+ kind: "profile";
941
+ } | {
942
+ kind: "workflow";
943
+ workflowId: string;
944
+ } | {
945
+ kind: "skill";
946
+ skillPath: string;
947
+ } | {
948
+ kind: "agent_action";
949
+ actionId: string;
949
950
  };
950
- members: {
951
- caseId: string;
951
+ declaredProfileCatalogHash: string | null;
952
+ protectedProfileClosureHash: string;
953
+ recordedOrigin?: {
954
+ execution: {
955
+ id: string;
956
+ contentHash: string;
957
+ };
958
+ originalEvidence: {
959
+ id: string;
960
+ contentHash: string;
961
+ };
962
+ members: {
963
+ caseId: string;
964
+ seed: string;
965
+ fixtureId: string;
966
+ sourceId: string;
967
+ snapshotHash: string;
968
+ boundaryRevisionHash: string;
969
+ }[];
970
+ } | undefined;
971
+ gradingSource?: {
972
+ pass: {
973
+ id: string;
974
+ contentHash: string;
975
+ };
976
+ evidence: {
977
+ id: string;
978
+ contentHash: string;
979
+ };
980
+ } | undefined;
981
+ split: "frozen_eval" | "test" | "train" | "validation";
982
+ population: {
983
+ taskId: string;
952
984
  seed: string;
953
- fixtureId: string;
954
- sourceId: string;
955
- snapshotHash: string;
956
- boundaryRevisionHash: string;
985
+ fixtureId: string | null;
957
986
  }[];
958
- } | undefined;
959
- gradingSource?: {
960
- pass: {
961
- id: string;
962
- contentHash: string;
963
- };
964
- evidence: {
965
- id: string;
966
- contentHash: string;
987
+ evaluators: {
988
+ release: {
989
+ id: string;
990
+ contentHash: string;
991
+ revision: string | number;
992
+ };
993
+ feedbackKey: string;
994
+ configurationHash: string;
995
+ output: "boolean" | "category" | "score";
996
+ categories: string[];
997
+ }[];
998
+ fieldMappingsHash: string;
999
+ fieldMappings: {
1000
+ graderId: string;
1001
+ fields: {
1002
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1003
+ path: string[];
1004
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1005
+ sourcePath: string[];
1006
+ }[];
1007
+ }[];
1008
+ environmentHash: string;
1009
+ privateDatasetClosureHash: string;
1010
+ criterion: {
1011
+ minimumPassRate: number;
1012
+ requireComplete: boolean;
967
1013
  };
1014
+ contentHash: string;
968
1015
  } | undefined;
969
- split: "frozen_eval" | "test" | "train" | "validation";
970
- population: {
971
- taskId: string;
972
- seed: string;
973
- fixtureId: string | null;
974
- }[];
975
- evaluators: {
976
- release: {
1016
+ sourceCandidate?: {
1017
+ candidateId: string;
1018
+ frozenRevision: number;
1019
+ frozenHash: string;
1020
+ harnessRelease: {
977
1021
  id: string;
978
1022
  contentHash: string;
979
- revision: string | number;
980
1023
  };
981
- feedbackKey: string;
982
- configurationHash: string;
983
- output: "boolean" | "category" | "score";
984
- categories: string[];
985
- }[];
986
- fieldMappingsHash: string;
987
- fieldMappings: {
988
- graderId: string;
989
- fields: {
990
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
991
- path: string[];
992
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
993
- sourcePath: string[];
994
- }[];
995
- }[];
996
- environmentHash: string;
997
- privateDatasetClosureHash: string;
998
- criterion: {
999
- minimumPassRate: number;
1000
- requireComplete: boolean;
1001
- };
1002
- contentHash: string;
1003
- } | undefined;
1004
- sourceCandidate?: {
1005
- candidateId: string;
1006
- frozenRevision: number;
1007
- frozenHash: string;
1024
+ protectedClosureHash: string;
1025
+ } | undefined;
1026
+ };
1027
+ model: {
1028
+ modelId: string;
1029
+ configurationHash: string;
1030
+ };
1031
+ } | {
1032
+ kind: "agent";
1033
+ source: {
1008
1034
  harnessRelease: {
1009
1035
  id: string;
1010
1036
  contentHash: string;
1011
1037
  };
1012
- protectedClosureHash: string;
1013
- } | undefined;
1014
- };
1015
- model: {
1016
- modelId: string;
1017
- configurationHash: string;
1018
- };
1019
- } | {
1020
- kind: "agent";
1021
- source: {
1022
- harnessRelease: {
1023
- id: string;
1024
- contentHash: string;
1038
+ agentSnapshot: {
1039
+ id: string;
1040
+ contentHash: string;
1041
+ };
1042
+ sourcePackageHash: string;
1025
1043
  };
1026
- agentSnapshot: {
1027
- id: string;
1028
- contentHash: string;
1044
+ model: {
1045
+ modelId: string;
1046
+ configurationHash: string;
1029
1047
  };
1030
- sourcePackageHash: string;
1048
+ } | {
1049
+ kind: "fixture";
1050
+ configurationHash: string;
1031
1051
  };
1032
- model: {
1033
- modelId: string;
1034
- configurationHash: string;
1035
- };
1036
- } | {
1037
- kind: "fixture";
1038
- configurationHash: string;
1039
- };
1040
- execution?: {
1041
- packageHash: string;
1042
- runtimeTargetHash: string;
1043
- metricPolicyHash: string;
1044
- compatibility?: {
1045
- protocol: "openpond.evaluation-execution.v1";
1046
- targetKind: "agent" | "fixture" | "harness" | "model";
1047
- } | undefined;
1048
- } | undefined;
1049
- lineage?: {
1050
- definition: {
1051
- id: string;
1052
- contentHash: string;
1053
- revision: number;
1054
- } | null;
1055
- execution: {
1056
- id: string;
1057
- contentHash: string;
1058
- };
1059
- scoringPassId: string | null;
1060
- humanProjection?: {
1061
- originalManifest: {
1062
- id: string;
1063
- contentHash: string;
1064
- };
1065
- selectionHash: string;
1066
- rulesHash: string;
1052
+ execution?: {
1053
+ packageHash: string;
1054
+ runtimeTargetHash: string;
1055
+ metricPolicyHash: string;
1056
+ compatibility?: {
1057
+ protocol: "openpond.evaluation-execution.v1";
1058
+ targetKind: "agent" | "fixture" | "harness" | "model";
1059
+ } | undefined;
1067
1060
  } | undefined;
1068
- } | undefined;
1069
- evaluators: {
1070
- release: {
1071
- id: string;
1072
- contentHash: string;
1073
- revision: string | number;
1074
- };
1075
- feedbackKey: string;
1076
- configurationHash?: string | undefined;
1077
- output: "boolean" | "category" | "score";
1078
- categories: string[];
1079
- }[];
1080
- population: {
1081
- caseId: string;
1082
- seed: string;
1083
- fixtureId: string | null;
1084
- }[];
1085
- createdAt: string;
1086
- contentHash: string;
1087
- };
1088
- result: {
1089
- schemaVersion: "openpond.experimentResult.v1";
1090
- manifest: {
1091
- id: string;
1092
- contentHash: string;
1093
- };
1094
- status: "cancelled" | "completed" | "failed";
1095
- cases: {
1096
- identity: {
1097
- caseId: string;
1098
- seed: string;
1099
- fixtureId: string | null;
1100
- };
1101
- status: "cancelled" | "completed" | "failed" | "unavailable";
1102
- output: unknown;
1103
- error: {
1104
- code: string;
1105
- message: string;
1106
- } | null;
1107
- feedback: {
1108
- feedbackKey: string;
1109
- evaluator: {
1110
- id: string;
1111
- contentHash: string;
1112
- revision: string | number;
1113
- };
1114
- status: "failed" | "pending" | "scored" | "unavailable";
1115
- value: string | number | boolean | null;
1116
- passed?: boolean | null | undefined;
1117
- reasoning: string | null;
1118
- evidenceRefs: {
1061
+ lineage?: {
1062
+ definition: {
1119
1063
  id: string;
1120
1064
  contentHash: string;
1121
- mediaType: string | null;
1122
- sizeBytes: number | null;
1123
- }[];
1124
- }[];
1125
- usage: {
1126
- inputTokens: number | null;
1127
- outputTokens: number | null;
1128
- totalTokens: number | null;
1129
- costUsd: number | null;
1130
- latencyMs: number | null;
1131
- };
1132
- traceRef: {
1133
- id: string;
1134
- contentHash: string;
1135
- mediaType: string | null;
1136
- sizeBytes: number | null;
1137
- } | null;
1138
- startedAt: string | null;
1139
- completedAt: string | null;
1140
- }[];
1141
- completedAt: string;
1142
- contentHash: string;
1143
- };
1144
- };
1145
- component: {
1146
- kind: "agent" | "instruction" | "skill" | "workflow";
1147
- path: string;
1148
- workflowId?: string | undefined;
1149
- };
1150
- mode: "llm_assisted" | "manual";
1151
- status: "accepted" | "adopting" | "authoring" | "authoring_queued" | "cancelled" | "declined" | "draft" | "failed" | "frozen" | "ready_for_review" | "rolled_back" | "testing";
1152
- revision: number;
1153
- limits: {
1154
- maximumCostUsd: number;
1155
- maximumAuthoringSteps: number;
1156
- maximumDurationMs: number;
1157
- };
1158
- authoringSteps: number;
1159
- work: {
1160
- sessionId: string;
1161
- turnId: string;
1162
- revision: number;
1163
- startedAt: string;
1164
- } | null;
1165
- frozen: {
1166
- release: {
1167
- id: string;
1168
- contentHash: string;
1169
- };
1170
- sourceRevision: string;
1171
- profileSourceRevision: string;
1172
- authoringHash: string;
1173
- partitionHash: string;
1174
- executableSourceHash: string;
1175
- protectedClosureHash: string;
1176
- candidateRevision: number;
1177
- diff: {
1178
- path: string;
1179
- before: string | null;
1180
- after: string | null;
1181
- }[];
1182
- } | null;
1183
- test: {
1184
- recipe: {
1185
- externalDatasetBinding: {
1186
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
1187
- dataset: {
1188
- id: string;
1189
- contentHash: string;
1190
- revision: number;
1191
- };
1192
- packageHash: string;
1193
- profileId: string;
1194
- target: {
1195
- kind: "profile";
1196
- } | {
1197
- kind: "workflow";
1198
- workflowId: string;
1199
- } | {
1200
- kind: "skill";
1201
- skillPath: string;
1202
- } | {
1203
- kind: "agent_action";
1204
- actionId: string;
1205
- };
1206
- declaredProfileCatalogHash: string | null;
1207
- protectedProfileClosureHash: string;
1208
- recordedOrigin?: {
1065
+ revision: number;
1066
+ } | null;
1209
1067
  execution: {
1210
1068
  id: string;
1211
1069
  contentHash: string;
1212
1070
  };
1213
- originalEvidence: {
1214
- id: string;
1215
- contentHash: string;
1216
- };
1217
- members: {
1218
- caseId: string;
1219
- seed: string;
1220
- fixtureId: string;
1221
- sourceId: string;
1222
- snapshotHash: string;
1223
- boundaryRevisionHash: string;
1224
- }[];
1225
- } | undefined;
1226
- gradingSource?: {
1227
- pass: {
1228
- id: string;
1229
- contentHash: string;
1230
- };
1231
- evidence: {
1232
- id: string;
1233
- contentHash: string;
1234
- };
1071
+ scoringPassId: string | null;
1072
+ humanProjection?: {
1073
+ originalManifest: {
1074
+ id: string;
1075
+ contentHash: string;
1076
+ };
1077
+ selectionHash: string;
1078
+ rulesHash: string;
1079
+ } | undefined;
1235
1080
  } | undefined;
1236
- split: "frozen_eval" | "test" | "train" | "validation";
1237
- population: {
1238
- taskId: string;
1239
- seed: string;
1240
- fixtureId: string | null;
1241
- }[];
1242
1081
  evaluators: {
1243
1082
  release: {
1244
1083
  id: string;
@@ -1246,280 +1085,21 @@ export declare class ExperimentImprovementClient {
1246
1085
  revision: string | number;
1247
1086
  };
1248
1087
  feedbackKey: string;
1249
- configurationHash: string;
1088
+ configurationHash?: string | undefined;
1250
1089
  output: "boolean" | "category" | "score";
1251
1090
  categories: string[];
1252
1091
  }[];
1253
- fieldMappingsHash: string;
1254
- fieldMappings: {
1255
- graderId: string;
1256
- fields: {
1257
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1258
- path: string[];
1259
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1260
- sourcePath: string[];
1261
- }[];
1092
+ population: {
1093
+ caseId: string;
1094
+ seed: string;
1095
+ fixtureId: string | null;
1262
1096
  }[];
1263
- environmentHash: string;
1264
- privateDatasetClosureHash: string;
1265
- criterion: {
1266
- minimumPassRate: number;
1267
- requireComplete: boolean;
1268
- };
1097
+ createdAt: string;
1269
1098
  contentHash: string;
1270
1099
  };
1271
- scoring?: {
1272
- sourcePass: {
1273
- id: string;
1274
- contentHash: string;
1275
- };
1276
- graders: {
1277
- id: string;
1278
- contentHash: string;
1279
- revision: number;
1280
- }[];
1281
- mappings?: {
1282
- graderId: string;
1283
- fields: {
1284
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1285
- path: string[];
1286
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1287
- sourcePath: string[];
1288
- }[];
1289
- }[] | undefined;
1290
- maximumCostUsd: number;
1291
- } | undefined;
1292
- modelId: string;
1293
- maximumCostUsd: number;
1294
- };
1295
- recipeHash: string;
1296
- baselineOperationId: string;
1297
- candidateOperationId: string;
1298
- cancelRequestedAt: string | null;
1299
- dispatching: {
1300
- kind: "baseline" | "candidate";
1301
- token: string;
1302
- pid: number;
1303
- processStart: string | null;
1304
- startedAt: string;
1305
- } | null;
1306
- } | null;
1307
- comparison: {
1308
- baseline: {
1309
- manifest: {
1310
- schemaVersion: "openpond.experimentManifest.v1";
1311
- id: string;
1312
- name: string;
1313
- teamId: string;
1314
- operationId: string;
1315
- maximumCostUsd: number | null;
1316
- dataset: {
1317
- id: string;
1318
- contentHash: string;
1319
- revision: number;
1320
- };
1321
- target: {
1322
- kind: "model";
1323
- modelId: string;
1324
- configurationHash: string;
1325
- } | {
1326
- kind: "harness";
1327
- source: {
1328
- profileId: string;
1329
- sourceRevision: string;
1330
- harnessRelease: {
1331
- id: string;
1332
- contentHash: string;
1333
- };
1334
- catalogHash: string;
1335
- definitionId: string;
1336
- definitionHash: string;
1337
- target: {
1338
- kind: "profile";
1339
- } | {
1340
- kind: "workflow";
1341
- workflowId: string;
1342
- } | {
1343
- kind: "skill";
1344
- skillPath: string;
1345
- } | {
1346
- kind: "agent_action";
1347
- actionId: string;
1348
- };
1349
- environmentHash: string;
1350
- externalDatasetBinding?: {
1351
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
1352
- dataset: {
1353
- id: string;
1354
- contentHash: string;
1355
- revision: number;
1356
- };
1357
- packageHash: string;
1358
- profileId: string;
1359
- target: {
1360
- kind: "profile";
1361
- } | {
1362
- kind: "workflow";
1363
- workflowId: string;
1364
- } | {
1365
- kind: "skill";
1366
- skillPath: string;
1367
- } | {
1368
- kind: "agent_action";
1369
- actionId: string;
1370
- };
1371
- declaredProfileCatalogHash: string | null;
1372
- protectedProfileClosureHash: string;
1373
- recordedOrigin?: {
1374
- execution: {
1375
- id: string;
1376
- contentHash: string;
1377
- };
1378
- originalEvidence: {
1379
- id: string;
1380
- contentHash: string;
1381
- };
1382
- members: {
1383
- caseId: string;
1384
- seed: string;
1385
- fixtureId: string;
1386
- sourceId: string;
1387
- snapshotHash: string;
1388
- boundaryRevisionHash: string;
1389
- }[];
1390
- } | undefined;
1391
- gradingSource?: {
1392
- pass: {
1393
- id: string;
1394
- contentHash: string;
1395
- };
1396
- evidence: {
1397
- id: string;
1398
- contentHash: string;
1399
- };
1400
- } | undefined;
1401
- split: "frozen_eval" | "test" | "train" | "validation";
1402
- population: {
1403
- taskId: string;
1404
- seed: string;
1405
- fixtureId: string | null;
1406
- }[];
1407
- evaluators: {
1408
- release: {
1409
- id: string;
1410
- contentHash: string;
1411
- revision: string | number;
1412
- };
1413
- feedbackKey: string;
1414
- configurationHash: string;
1415
- output: "boolean" | "category" | "score";
1416
- categories: string[];
1417
- }[];
1418
- fieldMappingsHash: string;
1419
- fieldMappings: {
1420
- graderId: string;
1421
- fields: {
1422
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1423
- path: string[];
1424
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1425
- sourcePath: string[];
1426
- }[];
1427
- }[];
1428
- environmentHash: string;
1429
- privateDatasetClosureHash: string;
1430
- criterion: {
1431
- minimumPassRate: number;
1432
- requireComplete: boolean;
1433
- };
1434
- contentHash: string;
1435
- } | undefined;
1436
- sourceCandidate?: {
1437
- candidateId: string;
1438
- frozenRevision: number;
1439
- frozenHash: string;
1440
- harnessRelease: {
1441
- id: string;
1442
- contentHash: string;
1443
- };
1444
- protectedClosureHash: string;
1445
- } | undefined;
1446
- };
1447
- model: {
1448
- modelId: string;
1449
- configurationHash: string;
1450
- };
1451
- } | {
1452
- kind: "agent";
1453
- source: {
1454
- harnessRelease: {
1455
- id: string;
1456
- contentHash: string;
1457
- };
1458
- agentSnapshot: {
1459
- id: string;
1460
- contentHash: string;
1461
- };
1462
- sourcePackageHash: string;
1463
- };
1464
- model: {
1465
- modelId: string;
1466
- configurationHash: string;
1467
- };
1468
- } | {
1469
- kind: "fixture";
1470
- configurationHash: string;
1471
- };
1472
- execution?: {
1473
- packageHash: string;
1474
- runtimeTargetHash: string;
1475
- metricPolicyHash: string;
1476
- compatibility?: {
1477
- protocol: "openpond.evaluation-execution.v1";
1478
- targetKind: "agent" | "fixture" | "harness" | "model";
1479
- } | undefined;
1480
- } | undefined;
1481
- lineage?: {
1482
- definition: {
1483
- id: string;
1484
- contentHash: string;
1485
- revision: number;
1486
- } | null;
1487
- execution: {
1488
- id: string;
1489
- contentHash: string;
1490
- };
1491
- scoringPassId: string | null;
1492
- humanProjection?: {
1493
- originalManifest: {
1494
- id: string;
1495
- contentHash: string;
1496
- };
1497
- selectionHash: string;
1498
- rulesHash: string;
1499
- } | undefined;
1500
- } | undefined;
1501
- evaluators: {
1502
- release: {
1503
- id: string;
1504
- contentHash: string;
1505
- revision: string | number;
1506
- };
1507
- feedbackKey: string;
1508
- configurationHash?: string | undefined;
1509
- output: "boolean" | "category" | "score";
1510
- categories: string[];
1511
- }[];
1512
- population: {
1513
- caseId: string;
1514
- seed: string;
1515
- fixtureId: string | null;
1516
- }[];
1517
- createdAt: string;
1518
- contentHash: string;
1519
- };
1520
- result: {
1521
- schemaVersion: "openpond.experimentResult.v1";
1522
- manifest: {
1100
+ result: {
1101
+ schemaVersion: "openpond.experimentResult.v1";
1102
+ manifest: {
1523
1103
  id: string;
1524
1104
  contentHash: string;
1525
1105
  };
@@ -1574,57 +1154,207 @@ export declare class ExperimentImprovementClient {
1574
1154
  contentHash: string;
1575
1155
  };
1576
1156
  };
1577
- candidate: {
1578
- manifest: {
1579
- schemaVersion: "openpond.experimentManifest.v1";
1580
- id: string;
1581
- name: string;
1582
- teamId: string;
1583
- operationId: string;
1584
- maximumCostUsd: number | null;
1585
- dataset: {
1586
- id: string;
1587
- contentHash: string;
1588
- revision: number;
1589
- };
1590
- target: {
1591
- kind: "model";
1592
- modelId: string;
1593
- configurationHash: string;
1157
+ frozenHash: string;
1158
+ humanProjection?: {
1159
+ schemaVersion: "openpond.humanComparisonProjection.v1";
1160
+ teamId: string;
1161
+ projectId: string;
1162
+ selections: {
1163
+ rules: ({
1164
+ feedbackKey: string;
1165
+ criterionId: string;
1166
+ kind: "normalized_score";
1167
+ minimum: number;
1168
+ maximum: number;
1169
+ passThreshold?: number | undefined;
1594
1170
  } | {
1595
- kind: "harness";
1596
- source: {
1597
- profileId: string;
1598
- sourceRevision: string;
1599
- harnessRelease: {
1171
+ feedbackKey: string;
1172
+ criterionId: string;
1173
+ kind: "boolean";
1174
+ } | {
1175
+ feedbackKey: string;
1176
+ criterionId: string;
1177
+ kind: "category";
1178
+ } | {
1179
+ feedbackKey: string;
1180
+ criterionId: string;
1181
+ kind: "preference_score";
1182
+ tieScore: number | null;
1183
+ })[];
1184
+ baseline: {
1185
+ execution: {
1186
+ id: string;
1187
+ contentHash: string;
1188
+ };
1189
+ resultHash: string;
1190
+ reviews: {
1191
+ review: {
1600
1192
  id: string;
1193
+ revision: number;
1601
1194
  contentHash: string;
1602
1195
  };
1603
- catalogHash: string;
1604
- definitionId: string;
1605
- definitionHash: string;
1606
- target: {
1607
- kind: "profile";
1608
- } | {
1609
- kind: "workflow";
1610
- workflowId: string;
1611
- } | {
1612
- kind: "skill";
1613
- skillPath: string;
1614
- } | {
1615
- kind: "agent_action";
1616
- actionId: string;
1196
+ decision: {
1197
+ id: string;
1198
+ revision: number;
1199
+ contentHash: string;
1617
1200
  };
1618
- environmentHash: string;
1619
- externalDatasetBinding?: {
1620
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
1621
- dataset: {
1201
+ }[];
1202
+ };
1203
+ candidate: {
1204
+ execution: {
1205
+ id: string;
1206
+ contentHash: string;
1207
+ };
1208
+ resultHash: string;
1209
+ reviews: {
1210
+ review: {
1211
+ id: string;
1212
+ revision: number;
1213
+ contentHash: string;
1214
+ };
1215
+ decision: {
1216
+ id: string;
1217
+ revision: number;
1218
+ contentHash: string;
1219
+ };
1220
+ }[];
1221
+ };
1222
+ };
1223
+ original: {
1224
+ baseline: {
1225
+ id: string;
1226
+ manifestHash: string;
1227
+ resultHash: string;
1228
+ };
1229
+ candidate: {
1230
+ id: string;
1231
+ manifestHash: string;
1232
+ resultHash: string;
1233
+ };
1234
+ };
1235
+ humanResults: {
1236
+ baseline: {
1237
+ schemaVersion: "openpond.humanResultView.v1";
1238
+ executionId: string;
1239
+ executionHash: string;
1240
+ selection: {
1241
+ id: string;
1242
+ revision: number;
1243
+ contentHash: string;
1244
+ }[];
1245
+ coverage: {
1246
+ pending: number;
1247
+ submitted: number;
1248
+ accepted: number;
1249
+ rejected: number;
1250
+ cancelled: number;
1251
+ };
1252
+ results: {
1253
+ review: {
1254
+ id: string;
1255
+ revision: number;
1256
+ contentHash: string;
1257
+ };
1258
+ decision: {
1259
+ id: string;
1260
+ revision: number;
1261
+ contentHash: string;
1262
+ } | null;
1263
+ attemptIds: string[];
1264
+ mode: "individual" | "pairwise";
1265
+ origin: "hosted_retained" | "native_local" | "owner_attested_local" | "recorded_evidence";
1266
+ status: "accepted" | "cancelled" | "pending" | "rejected" | "submitted";
1267
+ grader: {
1268
+ id: string;
1269
+ revision: number;
1270
+ contentHash: string;
1271
+ };
1272
+ answers: {
1273
+ criterionId: string;
1274
+ value: string | number | boolean | null;
1275
+ abstain: boolean;
1276
+ note: string;
1277
+ strength?: "moderate" | "slight" | "strong" | undefined;
1278
+ }[];
1279
+ }[];
1280
+ contentHash: string;
1281
+ };
1282
+ candidate: {
1283
+ schemaVersion: "openpond.humanResultView.v1";
1284
+ executionId: string;
1285
+ executionHash: string;
1286
+ selection: {
1287
+ id: string;
1288
+ revision: number;
1289
+ contentHash: string;
1290
+ }[];
1291
+ coverage: {
1292
+ pending: number;
1293
+ submitted: number;
1294
+ accepted: number;
1295
+ rejected: number;
1296
+ cancelled: number;
1297
+ };
1298
+ results: {
1299
+ review: {
1300
+ id: string;
1301
+ revision: number;
1302
+ contentHash: string;
1303
+ };
1304
+ decision: {
1305
+ id: string;
1306
+ revision: number;
1307
+ contentHash: string;
1308
+ } | null;
1309
+ attemptIds: string[];
1310
+ mode: "individual" | "pairwise";
1311
+ origin: "hosted_retained" | "native_local" | "owner_attested_local" | "recorded_evidence";
1312
+ status: "accepted" | "cancelled" | "pending" | "rejected" | "submitted";
1313
+ grader: {
1314
+ id: string;
1315
+ revision: number;
1316
+ contentHash: string;
1317
+ };
1318
+ answers: {
1319
+ criterionId: string;
1320
+ value: string | number | boolean | null;
1321
+ abstain: boolean;
1322
+ note: string;
1323
+ strength?: "moderate" | "slight" | "strong" | undefined;
1324
+ }[];
1325
+ }[];
1326
+ contentHash: string;
1327
+ };
1328
+ };
1329
+ baseline: {
1330
+ manifest: {
1331
+ schemaVersion: "openpond.experimentManifest.v1";
1332
+ id: string;
1333
+ name: string;
1334
+ teamId: string;
1335
+ operationId: string;
1336
+ maximumCostUsd: number | null;
1337
+ dataset: {
1338
+ id: string;
1339
+ contentHash: string;
1340
+ revision: number;
1341
+ };
1342
+ target: {
1343
+ kind: "model";
1344
+ modelId: string;
1345
+ configurationHash: string;
1346
+ } | {
1347
+ kind: "harness";
1348
+ source: {
1349
+ profileId: string;
1350
+ sourceRevision: string;
1351
+ harnessRelease: {
1622
1352
  id: string;
1623
1353
  contentHash: string;
1624
- revision: number;
1625
1354
  };
1626
- packageHash: string;
1627
- profileId: string;
1355
+ catalogHash: string;
1356
+ definitionId: string;
1357
+ definitionHash: string;
1628
1358
  target: {
1629
1359
  kind: "profile";
1630
1360
  } | {
@@ -1637,654 +1367,235 @@ export declare class ExperimentImprovementClient {
1637
1367
  kind: "agent_action";
1638
1368
  actionId: string;
1639
1369
  };
1640
- declaredProfileCatalogHash: string | null;
1641
- protectedProfileClosureHash: string;
1642
- recordedOrigin?: {
1643
- execution: {
1370
+ environmentHash: string;
1371
+ externalDatasetBinding?: {
1372
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
1373
+ dataset: {
1644
1374
  id: string;
1645
1375
  contentHash: string;
1376
+ revision: number;
1646
1377
  };
1647
- originalEvidence: {
1648
- id: string;
1649
- contentHash: string;
1378
+ packageHash: string;
1379
+ profileId: string;
1380
+ target: {
1381
+ kind: "profile";
1382
+ } | {
1383
+ kind: "workflow";
1384
+ workflowId: string;
1385
+ } | {
1386
+ kind: "skill";
1387
+ skillPath: string;
1388
+ } | {
1389
+ kind: "agent_action";
1390
+ actionId: string;
1650
1391
  };
1651
- members: {
1652
- caseId: string;
1653
- seed: string;
1654
- fixtureId: string;
1655
- sourceId: string;
1656
- snapshotHash: string;
1657
- boundaryRevisionHash: string;
1392
+ declaredProfileCatalogHash: string | null;
1393
+ protectedProfileClosureHash: string;
1394
+ recordedOrigin?: {
1395
+ execution: {
1396
+ id: string;
1397
+ contentHash: string;
1398
+ };
1399
+ originalEvidence: {
1400
+ id: string;
1401
+ contentHash: string;
1402
+ };
1403
+ members: {
1404
+ caseId: string;
1405
+ seed: string;
1406
+ fixtureId: string;
1407
+ sourceId: string;
1408
+ snapshotHash: string;
1409
+ boundaryRevisionHash: string;
1410
+ }[];
1411
+ } | undefined;
1412
+ gradingSource?: {
1413
+ pass: {
1414
+ id: string;
1415
+ contentHash: string;
1416
+ };
1417
+ evidence: {
1418
+ id: string;
1419
+ contentHash: string;
1420
+ };
1421
+ } | undefined;
1422
+ split: "frozen_eval" | "test" | "train" | "validation";
1423
+ population: {
1424
+ taskId: string;
1425
+ seed: string;
1426
+ fixtureId: string | null;
1658
1427
  }[];
1659
- } | undefined;
1660
- gradingSource?: {
1661
- pass: {
1662
- id: string;
1663
- contentHash: string;
1664
- };
1665
- evidence: {
1666
- id: string;
1667
- contentHash: string;
1428
+ evaluators: {
1429
+ release: {
1430
+ id: string;
1431
+ contentHash: string;
1432
+ revision: string | number;
1433
+ };
1434
+ feedbackKey: string;
1435
+ configurationHash: string;
1436
+ output: "boolean" | "category" | "score";
1437
+ categories: string[];
1438
+ }[];
1439
+ fieldMappingsHash: string;
1440
+ fieldMappings: {
1441
+ graderId: string;
1442
+ fields: {
1443
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1444
+ path: string[];
1445
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1446
+ sourcePath: string[];
1447
+ }[];
1448
+ }[];
1449
+ environmentHash: string;
1450
+ privateDatasetClosureHash: string;
1451
+ criterion: {
1452
+ minimumPassRate: number;
1453
+ requireComplete: boolean;
1668
1454
  };
1455
+ contentHash: string;
1669
1456
  } | undefined;
1670
- split: "frozen_eval" | "test" | "train" | "validation";
1671
- population: {
1672
- taskId: string;
1673
- seed: string;
1674
- fixtureId: string | null;
1675
- }[];
1676
- evaluators: {
1677
- release: {
1457
+ sourceCandidate?: {
1458
+ candidateId: string;
1459
+ frozenRevision: number;
1460
+ frozenHash: string;
1461
+ harnessRelease: {
1678
1462
  id: string;
1679
1463
  contentHash: string;
1680
- revision: string | number;
1681
1464
  };
1682
- feedbackKey: string;
1683
- configurationHash: string;
1684
- output: "boolean" | "category" | "score";
1685
- categories: string[];
1686
- }[];
1687
- fieldMappingsHash: string;
1688
- fieldMappings: {
1689
- graderId: string;
1690
- fields: {
1691
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1692
- path: string[];
1693
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1694
- sourcePath: string[];
1695
- }[];
1696
- }[];
1697
- environmentHash: string;
1698
- privateDatasetClosureHash: string;
1699
- criterion: {
1700
- minimumPassRate: number;
1701
- requireComplete: boolean;
1702
- };
1703
- contentHash: string;
1704
- } | undefined;
1705
- sourceCandidate?: {
1706
- candidateId: string;
1707
- frozenRevision: number;
1708
- frozenHash: string;
1465
+ protectedClosureHash: string;
1466
+ } | undefined;
1467
+ };
1468
+ model: {
1469
+ modelId: string;
1470
+ configurationHash: string;
1471
+ };
1472
+ } | {
1473
+ kind: "agent";
1474
+ source: {
1709
1475
  harnessRelease: {
1710
1476
  id: string;
1711
1477
  contentHash: string;
1712
1478
  };
1713
- protectedClosureHash: string;
1714
- } | undefined;
1715
- };
1716
- model: {
1717
- modelId: string;
1479
+ agentSnapshot: {
1480
+ id: string;
1481
+ contentHash: string;
1482
+ };
1483
+ sourcePackageHash: string;
1484
+ };
1485
+ model: {
1486
+ modelId: string;
1487
+ configurationHash: string;
1488
+ };
1489
+ } | {
1490
+ kind: "fixture";
1718
1491
  configurationHash: string;
1719
1492
  };
1720
- } | {
1721
- kind: "agent";
1722
- source: {
1723
- harnessRelease: {
1493
+ execution?: {
1494
+ packageHash: string;
1495
+ runtimeTargetHash: string;
1496
+ metricPolicyHash: string;
1497
+ compatibility?: {
1498
+ protocol: "openpond.evaluation-execution.v1";
1499
+ targetKind: "agent" | "fixture" | "harness" | "model";
1500
+ } | undefined;
1501
+ } | undefined;
1502
+ lineage?: {
1503
+ definition: {
1724
1504
  id: string;
1725
1505
  contentHash: string;
1726
- };
1727
- agentSnapshot: {
1506
+ revision: number;
1507
+ } | null;
1508
+ execution: {
1728
1509
  id: string;
1729
1510
  contentHash: string;
1730
1511
  };
1731
- sourcePackageHash: string;
1732
- };
1733
- model: {
1734
- modelId: string;
1735
- configurationHash: string;
1736
- };
1737
- } | {
1738
- kind: "fixture";
1739
- configurationHash: string;
1740
- };
1741
- execution?: {
1742
- packageHash: string;
1743
- runtimeTargetHash: string;
1744
- metricPolicyHash: string;
1745
- compatibility?: {
1746
- protocol: "openpond.evaluation-execution.v1";
1747
- targetKind: "agent" | "fixture" | "harness" | "model";
1512
+ scoringPassId: string | null;
1513
+ humanProjection?: {
1514
+ originalManifest: {
1515
+ id: string;
1516
+ contentHash: string;
1517
+ };
1518
+ selectionHash: string;
1519
+ rulesHash: string;
1520
+ } | undefined;
1748
1521
  } | undefined;
1749
- } | undefined;
1750
- lineage?: {
1751
- definition: {
1752
- id: string;
1753
- contentHash: string;
1754
- revision: number;
1755
- } | null;
1756
- execution: {
1757
- id: string;
1758
- contentHash: string;
1759
- };
1760
- scoringPassId: string | null;
1761
- humanProjection?: {
1762
- originalManifest: {
1522
+ evaluators: {
1523
+ release: {
1763
1524
  id: string;
1764
1525
  contentHash: string;
1526
+ revision: string | number;
1765
1527
  };
1766
- selectionHash: string;
1767
- rulesHash: string;
1768
- } | undefined;
1769
- } | undefined;
1770
- evaluators: {
1771
- release: {
1772
- id: string;
1773
- contentHash: string;
1774
- revision: string | number;
1775
- };
1776
- feedbackKey: string;
1777
- configurationHash?: string | undefined;
1778
- output: "boolean" | "category" | "score";
1779
- categories: string[];
1780
- }[];
1781
- population: {
1782
- caseId: string;
1783
- seed: string;
1784
- fixtureId: string | null;
1785
- }[];
1786
- createdAt: string;
1787
- contentHash: string;
1788
- };
1789
- result: {
1790
- schemaVersion: "openpond.experimentResult.v1";
1791
- manifest: {
1792
- id: string;
1793
- contentHash: string;
1794
- };
1795
- status: "cancelled" | "completed" | "failed";
1796
- cases: {
1797
- identity: {
1528
+ feedbackKey: string;
1529
+ configurationHash?: string | undefined;
1530
+ output: "boolean" | "category" | "score";
1531
+ categories: string[];
1532
+ }[];
1533
+ population: {
1798
1534
  caseId: string;
1799
1535
  seed: string;
1800
1536
  fixtureId: string | null;
1537
+ }[];
1538
+ createdAt: string;
1539
+ contentHash: string;
1540
+ };
1541
+ result: {
1542
+ schemaVersion: "openpond.experimentResult.v1";
1543
+ manifest: {
1544
+ id: string;
1545
+ contentHash: string;
1801
1546
  };
1802
- status: "cancelled" | "completed" | "failed" | "unavailable";
1803
- output: unknown;
1804
- error: {
1805
- code: string;
1806
- message: string;
1807
- } | null;
1808
- feedback: {
1809
- feedbackKey: string;
1810
- evaluator: {
1811
- id: string;
1812
- contentHash: string;
1813
- revision: string | number;
1814
- };
1815
- status: "failed" | "pending" | "scored" | "unavailable";
1816
- value: string | number | boolean | null;
1817
- passed?: boolean | null | undefined;
1818
- reasoning: string | null;
1819
- evidenceRefs: {
1820
- id: string;
1821
- contentHash: string;
1822
- mediaType: string | null;
1823
- sizeBytes: number | null;
1824
- }[];
1825
- }[];
1826
- usage: {
1827
- inputTokens: number | null;
1828
- outputTokens: number | null;
1829
- totalTokens: number | null;
1830
- costUsd: number | null;
1831
- latencyMs: number | null;
1832
- };
1833
- traceRef: {
1834
- id: string;
1835
- contentHash: string;
1836
- mediaType: string | null;
1837
- sizeBytes: number | null;
1838
- } | null;
1839
- startedAt: string | null;
1840
- completedAt: string | null;
1841
- }[];
1842
- completedAt: string;
1843
- contentHash: string;
1844
- };
1845
- };
1846
- frozenHash: string;
1847
- humanProjection?: {
1848
- schemaVersion: "openpond.humanComparisonProjection.v1";
1849
- teamId: string;
1850
- projectId: string;
1851
- selections: {
1852
- rules: ({
1853
- feedbackKey: string;
1854
- criterionId: string;
1855
- kind: "normalized_score";
1856
- minimum: number;
1857
- maximum: number;
1858
- passThreshold?: number | undefined;
1859
- } | {
1860
- feedbackKey: string;
1861
- criterionId: string;
1862
- kind: "boolean";
1863
- } | {
1864
- feedbackKey: string;
1865
- criterionId: string;
1866
- kind: "category";
1867
- } | {
1868
- feedbackKey: string;
1869
- criterionId: string;
1870
- kind: "preference_score";
1871
- tieScore: number | null;
1872
- })[];
1873
- baseline: {
1874
- execution: {
1875
- id: string;
1876
- contentHash: string;
1877
- };
1878
- resultHash: string;
1879
- reviews: {
1880
- review: {
1881
- id: string;
1882
- revision: number;
1883
- contentHash: string;
1884
- };
1885
- decision: {
1886
- id: string;
1887
- revision: number;
1888
- contentHash: string;
1889
- };
1890
- }[];
1891
- };
1892
- candidate: {
1893
- execution: {
1894
- id: string;
1895
- contentHash: string;
1896
- };
1897
- resultHash: string;
1898
- reviews: {
1899
- review: {
1900
- id: string;
1901
- revision: number;
1902
- contentHash: string;
1903
- };
1904
- decision: {
1905
- id: string;
1906
- revision: number;
1907
- contentHash: string;
1908
- };
1909
- }[];
1910
- };
1911
- };
1912
- original: {
1913
- baseline: {
1914
- id: string;
1915
- manifestHash: string;
1916
- resultHash: string;
1917
- };
1918
- candidate: {
1919
- id: string;
1920
- manifestHash: string;
1921
- resultHash: string;
1922
- };
1923
- };
1924
- humanResults: {
1925
- baseline: {
1926
- schemaVersion: "openpond.humanResultView.v1";
1927
- executionId: string;
1928
- executionHash: string;
1929
- selection: {
1930
- id: string;
1931
- revision: number;
1932
- contentHash: string;
1933
- }[];
1934
- coverage: {
1935
- pending: number;
1936
- submitted: number;
1937
- accepted: number;
1938
- rejected: number;
1939
- cancelled: number;
1940
- };
1941
- results: {
1942
- review: {
1943
- id: string;
1944
- revision: number;
1945
- contentHash: string;
1547
+ status: "cancelled" | "completed" | "failed";
1548
+ cases: {
1549
+ identity: {
1550
+ caseId: string;
1551
+ seed: string;
1552
+ fixtureId: string | null;
1946
1553
  };
1947
- decision: {
1948
- id: string;
1949
- revision: number;
1950
- contentHash: string;
1554
+ status: "cancelled" | "completed" | "failed" | "unavailable";
1555
+ output: unknown;
1556
+ error: {
1557
+ code: string;
1558
+ message: string;
1951
1559
  } | null;
1952
- attemptIds: string[];
1953
- mode: "individual" | "pairwise";
1954
- origin: "hosted_retained" | "native_local" | "owner_attested_local" | "recorded_evidence";
1955
- status: "accepted" | "cancelled" | "pending" | "rejected" | "submitted";
1956
- grader: {
1957
- id: string;
1958
- revision: number;
1959
- contentHash: string;
1960
- };
1961
- answers: {
1962
- criterionId: string;
1560
+ feedback: {
1561
+ feedbackKey: string;
1562
+ evaluator: {
1563
+ id: string;
1564
+ contentHash: string;
1565
+ revision: string | number;
1566
+ };
1567
+ status: "failed" | "pending" | "scored" | "unavailable";
1963
1568
  value: string | number | boolean | null;
1964
- abstain: boolean;
1965
- note: string;
1966
- strength?: "moderate" | "slight" | "strong" | undefined;
1569
+ passed?: boolean | null | undefined;
1570
+ reasoning: string | null;
1571
+ evidenceRefs: {
1572
+ id: string;
1573
+ contentHash: string;
1574
+ mediaType: string | null;
1575
+ sizeBytes: number | null;
1576
+ }[];
1967
1577
  }[];
1968
- }[];
1969
- contentHash: string;
1970
- };
1971
- candidate: {
1972
- schemaVersion: "openpond.humanResultView.v1";
1973
- executionId: string;
1974
- executionHash: string;
1975
- selection: {
1976
- id: string;
1977
- revision: number;
1978
- contentHash: string;
1979
- }[];
1980
- coverage: {
1981
- pending: number;
1982
- submitted: number;
1983
- accepted: number;
1984
- rejected: number;
1985
- cancelled: number;
1986
- };
1987
- results: {
1988
- review: {
1989
- id: string;
1990
- revision: number;
1991
- contentHash: string;
1578
+ usage: {
1579
+ inputTokens: number | null;
1580
+ outputTokens: number | null;
1581
+ totalTokens: number | null;
1582
+ costUsd: number | null;
1583
+ latencyMs: number | null;
1992
1584
  };
1993
- decision: {
1585
+ traceRef: {
1994
1586
  id: string;
1995
- revision: number;
1996
1587
  contentHash: string;
1588
+ mediaType: string | null;
1589
+ sizeBytes: number | null;
1997
1590
  } | null;
1998
- attemptIds: string[];
1999
- mode: "individual" | "pairwise";
2000
- origin: "hosted_retained" | "native_local" | "owner_attested_local" | "recorded_evidence";
2001
- status: "accepted" | "cancelled" | "pending" | "rejected" | "submitted";
2002
- grader: {
2003
- id: string;
2004
- revision: number;
2005
- contentHash: string;
2006
- };
2007
- answers: {
2008
- criterionId: string;
2009
- value: string | number | boolean | null;
2010
- abstain: boolean;
2011
- note: string;
2012
- strength?: "moderate" | "slight" | "strong" | undefined;
2013
- }[];
1591
+ startedAt: string | null;
1592
+ completedAt: string | null;
2014
1593
  }[];
1594
+ completedAt: string;
2015
1595
  contentHash: string;
2016
1596
  };
2017
1597
  };
2018
- baseline: {
2019
- manifest: {
2020
- schemaVersion: "openpond.experimentManifest.v1";
2021
- id: string;
2022
- name: string;
2023
- teamId: string;
2024
- operationId: string;
2025
- maximumCostUsd: number | null;
2026
- dataset: {
2027
- id: string;
2028
- contentHash: string;
2029
- revision: number;
2030
- };
2031
- target: {
2032
- kind: "model";
2033
- modelId: string;
2034
- configurationHash: string;
2035
- } | {
2036
- kind: "harness";
2037
- source: {
2038
- profileId: string;
2039
- sourceRevision: string;
2040
- harnessRelease: {
2041
- id: string;
2042
- contentHash: string;
2043
- };
2044
- catalogHash: string;
2045
- definitionId: string;
2046
- definitionHash: string;
2047
- target: {
2048
- kind: "profile";
2049
- } | {
2050
- kind: "workflow";
2051
- workflowId: string;
2052
- } | {
2053
- kind: "skill";
2054
- skillPath: string;
2055
- } | {
2056
- kind: "agent_action";
2057
- actionId: string;
2058
- };
2059
- environmentHash: string;
2060
- externalDatasetBinding?: {
2061
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
2062
- dataset: {
2063
- id: string;
2064
- contentHash: string;
2065
- revision: number;
2066
- };
2067
- packageHash: string;
2068
- profileId: string;
2069
- target: {
2070
- kind: "profile";
2071
- } | {
2072
- kind: "workflow";
2073
- workflowId: string;
2074
- } | {
2075
- kind: "skill";
2076
- skillPath: string;
2077
- } | {
2078
- kind: "agent_action";
2079
- actionId: string;
2080
- };
2081
- declaredProfileCatalogHash: string | null;
2082
- protectedProfileClosureHash: string;
2083
- recordedOrigin?: {
2084
- execution: {
2085
- id: string;
2086
- contentHash: string;
2087
- };
2088
- originalEvidence: {
2089
- id: string;
2090
- contentHash: string;
2091
- };
2092
- members: {
2093
- caseId: string;
2094
- seed: string;
2095
- fixtureId: string;
2096
- sourceId: string;
2097
- snapshotHash: string;
2098
- boundaryRevisionHash: string;
2099
- }[];
2100
- } | undefined;
2101
- gradingSource?: {
2102
- pass: {
2103
- id: string;
2104
- contentHash: string;
2105
- };
2106
- evidence: {
2107
- id: string;
2108
- contentHash: string;
2109
- };
2110
- } | undefined;
2111
- split: "frozen_eval" | "test" | "train" | "validation";
2112
- population: {
2113
- taskId: string;
2114
- seed: string;
2115
- fixtureId: string | null;
2116
- }[];
2117
- evaluators: {
2118
- release: {
2119
- id: string;
2120
- contentHash: string;
2121
- revision: string | number;
2122
- };
2123
- feedbackKey: string;
2124
- configurationHash: string;
2125
- output: "boolean" | "category" | "score";
2126
- categories: string[];
2127
- }[];
2128
- fieldMappingsHash: string;
2129
- fieldMappings: {
2130
- graderId: string;
2131
- fields: {
2132
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
2133
- path: string[];
2134
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
2135
- sourcePath: string[];
2136
- }[];
2137
- }[];
2138
- environmentHash: string;
2139
- privateDatasetClosureHash: string;
2140
- criterion: {
2141
- minimumPassRate: number;
2142
- requireComplete: boolean;
2143
- };
2144
- contentHash: string;
2145
- } | undefined;
2146
- sourceCandidate?: {
2147
- candidateId: string;
2148
- frozenRevision: number;
2149
- frozenHash: string;
2150
- harnessRelease: {
2151
- id: string;
2152
- contentHash: string;
2153
- };
2154
- protectedClosureHash: string;
2155
- } | undefined;
2156
- };
2157
- model: {
2158
- modelId: string;
2159
- configurationHash: string;
2160
- };
2161
- } | {
2162
- kind: "agent";
2163
- source: {
2164
- harnessRelease: {
2165
- id: string;
2166
- contentHash: string;
2167
- };
2168
- agentSnapshot: {
2169
- id: string;
2170
- contentHash: string;
2171
- };
2172
- sourcePackageHash: string;
2173
- };
2174
- model: {
2175
- modelId: string;
2176
- configurationHash: string;
2177
- };
2178
- } | {
2179
- kind: "fixture";
2180
- configurationHash: string;
2181
- };
2182
- execution?: {
2183
- packageHash: string;
2184
- runtimeTargetHash: string;
2185
- metricPolicyHash: string;
2186
- compatibility?: {
2187
- protocol: "openpond.evaluation-execution.v1";
2188
- targetKind: "agent" | "fixture" | "harness" | "model";
2189
- } | undefined;
2190
- } | undefined;
2191
- lineage?: {
2192
- definition: {
2193
- id: string;
2194
- contentHash: string;
2195
- revision: number;
2196
- } | null;
2197
- execution: {
2198
- id: string;
2199
- contentHash: string;
2200
- };
2201
- scoringPassId: string | null;
2202
- humanProjection?: {
2203
- originalManifest: {
2204
- id: string;
2205
- contentHash: string;
2206
- };
2207
- selectionHash: string;
2208
- rulesHash: string;
2209
- } | undefined;
2210
- } | undefined;
2211
- evaluators: {
2212
- release: {
2213
- id: string;
2214
- contentHash: string;
2215
- revision: string | number;
2216
- };
2217
- feedbackKey: string;
2218
- configurationHash?: string | undefined;
2219
- output: "boolean" | "category" | "score";
2220
- categories: string[];
2221
- }[];
2222
- population: {
2223
- caseId: string;
2224
- seed: string;
2225
- fixtureId: string | null;
2226
- }[];
2227
- createdAt: string;
2228
- contentHash: string;
2229
- };
2230
- result: {
2231
- schemaVersion: "openpond.experimentResult.v1";
2232
- manifest: {
2233
- id: string;
2234
- contentHash: string;
2235
- };
2236
- status: "cancelled" | "completed" | "failed";
2237
- cases: {
2238
- identity: {
2239
- caseId: string;
2240
- seed: string;
2241
- fixtureId: string | null;
2242
- };
2243
- status: "cancelled" | "completed" | "failed" | "unavailable";
2244
- output: unknown;
2245
- error: {
2246
- code: string;
2247
- message: string;
2248
- } | null;
2249
- feedback: {
2250
- feedbackKey: string;
2251
- evaluator: {
2252
- id: string;
2253
- contentHash: string;
2254
- revision: string | number;
2255
- };
2256
- status: "failed" | "pending" | "scored" | "unavailable";
2257
- value: string | number | boolean | null;
2258
- passed?: boolean | null | undefined;
2259
- reasoning: string | null;
2260
- evidenceRefs: {
2261
- id: string;
2262
- contentHash: string;
2263
- mediaType: string | null;
2264
- sizeBytes: number | null;
2265
- }[];
2266
- }[];
2267
- usage: {
2268
- inputTokens: number | null;
2269
- outputTokens: number | null;
2270
- totalTokens: number | null;
2271
- costUsd: number | null;
2272
- latencyMs: number | null;
2273
- };
2274
- traceRef: {
2275
- id: string;
2276
- contentHash: string;
2277
- mediaType: string | null;
2278
- sizeBytes: number | null;
2279
- } | null;
2280
- startedAt: string | null;
2281
- completedAt: string | null;
2282
- }[];
2283
- completedAt: string;
2284
- contentHash: string;
2285
- };
2286
- };
2287
- candidate: {
1598
+ candidate: {
2288
1599
  manifest: {
2289
1600
  schemaVersion: "openpond.experimentManifest.v1";
2290
1601
  id: string;
@@ -3284,6 +2595,695 @@ export declare class ExperimentImprovementClient {
3284
2595
  createdAt: string;
3285
2596
  updatedAt: string;
3286
2597
  contentHash: string;
2598
+ } | {
2599
+ candidateId: string;
2600
+ actorId: string;
2601
+ teamId: string;
2602
+ projectId: string | null;
2603
+ candidateRevision: number;
2604
+ recipeHash: string;
2605
+ baseline: {
2606
+ manifest: {
2607
+ schemaVersion: "openpond.experimentManifest.v1";
2608
+ id: string;
2609
+ name: string;
2610
+ teamId: string;
2611
+ operationId: string;
2612
+ maximumCostUsd: number | null;
2613
+ dataset: {
2614
+ id: string;
2615
+ contentHash: string;
2616
+ revision: number;
2617
+ };
2618
+ target: {
2619
+ kind: "model";
2620
+ modelId: string;
2621
+ configurationHash: string;
2622
+ } | {
2623
+ kind: "harness";
2624
+ source: {
2625
+ profileId: string;
2626
+ sourceRevision: string;
2627
+ harnessRelease: {
2628
+ id: string;
2629
+ contentHash: string;
2630
+ };
2631
+ catalogHash: string;
2632
+ definitionId: string;
2633
+ definitionHash: string;
2634
+ target: {
2635
+ kind: "profile";
2636
+ } | {
2637
+ kind: "workflow";
2638
+ workflowId: string;
2639
+ } | {
2640
+ kind: "skill";
2641
+ skillPath: string;
2642
+ } | {
2643
+ kind: "agent_action";
2644
+ actionId: string;
2645
+ };
2646
+ environmentHash: string;
2647
+ externalDatasetBinding?: {
2648
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
2649
+ dataset: {
2650
+ id: string;
2651
+ contentHash: string;
2652
+ revision: number;
2653
+ };
2654
+ packageHash: string;
2655
+ profileId: string;
2656
+ target: {
2657
+ kind: "profile";
2658
+ } | {
2659
+ kind: "workflow";
2660
+ workflowId: string;
2661
+ } | {
2662
+ kind: "skill";
2663
+ skillPath: string;
2664
+ } | {
2665
+ kind: "agent_action";
2666
+ actionId: string;
2667
+ };
2668
+ declaredProfileCatalogHash: string | null;
2669
+ protectedProfileClosureHash: string;
2670
+ recordedOrigin?: {
2671
+ execution: {
2672
+ id: string;
2673
+ contentHash: string;
2674
+ };
2675
+ originalEvidence: {
2676
+ id: string;
2677
+ contentHash: string;
2678
+ };
2679
+ members: {
2680
+ caseId: string;
2681
+ seed: string;
2682
+ fixtureId: string;
2683
+ sourceId: string;
2684
+ snapshotHash: string;
2685
+ boundaryRevisionHash: string;
2686
+ }[];
2687
+ } | undefined;
2688
+ gradingSource?: {
2689
+ pass: {
2690
+ id: string;
2691
+ contentHash: string;
2692
+ };
2693
+ evidence: {
2694
+ id: string;
2695
+ contentHash: string;
2696
+ };
2697
+ } | undefined;
2698
+ split: "frozen_eval" | "test" | "train" | "validation";
2699
+ population: {
2700
+ taskId: string;
2701
+ seed: string;
2702
+ fixtureId: string | null;
2703
+ }[];
2704
+ evaluators: {
2705
+ release: {
2706
+ id: string;
2707
+ contentHash: string;
2708
+ revision: string | number;
2709
+ };
2710
+ feedbackKey: string;
2711
+ configurationHash: string;
2712
+ output: "boolean" | "category" | "score";
2713
+ categories: string[];
2714
+ }[];
2715
+ fieldMappingsHash: string;
2716
+ fieldMappings: {
2717
+ graderId: string;
2718
+ fields: {
2719
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
2720
+ path: string[];
2721
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
2722
+ sourcePath: string[];
2723
+ }[];
2724
+ }[];
2725
+ environmentHash: string;
2726
+ privateDatasetClosureHash: string;
2727
+ criterion: {
2728
+ minimumPassRate: number;
2729
+ requireComplete: boolean;
2730
+ };
2731
+ contentHash: string;
2732
+ } | undefined;
2733
+ sourceCandidate?: {
2734
+ candidateId: string;
2735
+ frozenRevision: number;
2736
+ frozenHash: string;
2737
+ harnessRelease: {
2738
+ id: string;
2739
+ contentHash: string;
2740
+ };
2741
+ protectedClosureHash: string;
2742
+ } | undefined;
2743
+ };
2744
+ model: {
2745
+ modelId: string;
2746
+ configurationHash: string;
2747
+ };
2748
+ } | {
2749
+ kind: "agent";
2750
+ source: {
2751
+ harnessRelease: {
2752
+ id: string;
2753
+ contentHash: string;
2754
+ };
2755
+ agentSnapshot: {
2756
+ id: string;
2757
+ contentHash: string;
2758
+ };
2759
+ sourcePackageHash: string;
2760
+ };
2761
+ model: {
2762
+ modelId: string;
2763
+ configurationHash: string;
2764
+ };
2765
+ } | {
2766
+ kind: "fixture";
2767
+ configurationHash: string;
2768
+ };
2769
+ execution?: {
2770
+ packageHash: string;
2771
+ runtimeTargetHash: string;
2772
+ metricPolicyHash: string;
2773
+ compatibility?: {
2774
+ protocol: "openpond.evaluation-execution.v1";
2775
+ targetKind: "agent" | "fixture" | "harness" | "model";
2776
+ } | undefined;
2777
+ } | undefined;
2778
+ lineage?: {
2779
+ definition: {
2780
+ id: string;
2781
+ contentHash: string;
2782
+ revision: number;
2783
+ } | null;
2784
+ execution: {
2785
+ id: string;
2786
+ contentHash: string;
2787
+ };
2788
+ scoringPassId: string | null;
2789
+ humanProjection?: {
2790
+ originalManifest: {
2791
+ id: string;
2792
+ contentHash: string;
2793
+ };
2794
+ selectionHash: string;
2795
+ rulesHash: string;
2796
+ } | undefined;
2797
+ } | undefined;
2798
+ evaluators: {
2799
+ release: {
2800
+ id: string;
2801
+ contentHash: string;
2802
+ revision: string | number;
2803
+ };
2804
+ feedbackKey: string;
2805
+ configurationHash?: string | undefined;
2806
+ output: "boolean" | "category" | "score";
2807
+ categories: string[];
2808
+ }[];
2809
+ population: {
2810
+ caseId: string;
2811
+ seed: string;
2812
+ fixtureId: string | null;
2813
+ }[];
2814
+ createdAt: string;
2815
+ contentHash: string;
2816
+ };
2817
+ result: {
2818
+ schemaVersion: "openpond.experimentResult.v1";
2819
+ manifest: {
2820
+ id: string;
2821
+ contentHash: string;
2822
+ };
2823
+ status: "cancelled" | "completed" | "failed";
2824
+ cases: {
2825
+ identity: {
2826
+ caseId: string;
2827
+ seed: string;
2828
+ fixtureId: string | null;
2829
+ };
2830
+ status: "cancelled" | "completed" | "failed" | "unavailable";
2831
+ output: unknown;
2832
+ error: {
2833
+ code: string;
2834
+ message: string;
2835
+ } | null;
2836
+ feedback: {
2837
+ feedbackKey: string;
2838
+ evaluator: {
2839
+ id: string;
2840
+ contentHash: string;
2841
+ revision: string | number;
2842
+ };
2843
+ status: "failed" | "pending" | "scored" | "unavailable";
2844
+ value: string | number | boolean | null;
2845
+ passed?: boolean | null | undefined;
2846
+ reasoning: string | null;
2847
+ evidenceRefs: {
2848
+ id: string;
2849
+ contentHash: string;
2850
+ mediaType: string | null;
2851
+ sizeBytes: number | null;
2852
+ }[];
2853
+ }[];
2854
+ usage: {
2855
+ inputTokens: number | null;
2856
+ outputTokens: number | null;
2857
+ totalTokens: number | null;
2858
+ costUsd: number | null;
2859
+ latencyMs: number | null;
2860
+ };
2861
+ traceRef: {
2862
+ id: string;
2863
+ contentHash: string;
2864
+ mediaType: string | null;
2865
+ sizeBytes: number | null;
2866
+ } | null;
2867
+ startedAt: string | null;
2868
+ completedAt: string | null;
2869
+ }[];
2870
+ completedAt: string;
2871
+ contentHash: string;
2872
+ };
2873
+ };
2874
+ candidate: {
2875
+ manifest: {
2876
+ schemaVersion: "openpond.experimentManifest.v1";
2877
+ id: string;
2878
+ name: string;
2879
+ teamId: string;
2880
+ operationId: string;
2881
+ maximumCostUsd: number | null;
2882
+ dataset: {
2883
+ id: string;
2884
+ contentHash: string;
2885
+ revision: number;
2886
+ };
2887
+ target: {
2888
+ kind: "model";
2889
+ modelId: string;
2890
+ configurationHash: string;
2891
+ } | {
2892
+ kind: "harness";
2893
+ source: {
2894
+ profileId: string;
2895
+ sourceRevision: string;
2896
+ harnessRelease: {
2897
+ id: string;
2898
+ contentHash: string;
2899
+ };
2900
+ catalogHash: string;
2901
+ definitionId: string;
2902
+ definitionHash: string;
2903
+ target: {
2904
+ kind: "profile";
2905
+ } | {
2906
+ kind: "workflow";
2907
+ workflowId: string;
2908
+ } | {
2909
+ kind: "skill";
2910
+ skillPath: string;
2911
+ } | {
2912
+ kind: "agent_action";
2913
+ actionId: string;
2914
+ };
2915
+ environmentHash: string;
2916
+ externalDatasetBinding?: {
2917
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
2918
+ dataset: {
2919
+ id: string;
2920
+ contentHash: string;
2921
+ revision: number;
2922
+ };
2923
+ packageHash: string;
2924
+ profileId: string;
2925
+ target: {
2926
+ kind: "profile";
2927
+ } | {
2928
+ kind: "workflow";
2929
+ workflowId: string;
2930
+ } | {
2931
+ kind: "skill";
2932
+ skillPath: string;
2933
+ } | {
2934
+ kind: "agent_action";
2935
+ actionId: string;
2936
+ };
2937
+ declaredProfileCatalogHash: string | null;
2938
+ protectedProfileClosureHash: string;
2939
+ recordedOrigin?: {
2940
+ execution: {
2941
+ id: string;
2942
+ contentHash: string;
2943
+ };
2944
+ originalEvidence: {
2945
+ id: string;
2946
+ contentHash: string;
2947
+ };
2948
+ members: {
2949
+ caseId: string;
2950
+ seed: string;
2951
+ fixtureId: string;
2952
+ sourceId: string;
2953
+ snapshotHash: string;
2954
+ boundaryRevisionHash: string;
2955
+ }[];
2956
+ } | undefined;
2957
+ gradingSource?: {
2958
+ pass: {
2959
+ id: string;
2960
+ contentHash: string;
2961
+ };
2962
+ evidence: {
2963
+ id: string;
2964
+ contentHash: string;
2965
+ };
2966
+ } | undefined;
2967
+ split: "frozen_eval" | "test" | "train" | "validation";
2968
+ population: {
2969
+ taskId: string;
2970
+ seed: string;
2971
+ fixtureId: string | null;
2972
+ }[];
2973
+ evaluators: {
2974
+ release: {
2975
+ id: string;
2976
+ contentHash: string;
2977
+ revision: string | number;
2978
+ };
2979
+ feedbackKey: string;
2980
+ configurationHash: string;
2981
+ output: "boolean" | "category" | "score";
2982
+ categories: string[];
2983
+ }[];
2984
+ fieldMappingsHash: string;
2985
+ fieldMappings: {
2986
+ graderId: string;
2987
+ fields: {
2988
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
2989
+ path: string[];
2990
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
2991
+ sourcePath: string[];
2992
+ }[];
2993
+ }[];
2994
+ environmentHash: string;
2995
+ privateDatasetClosureHash: string;
2996
+ criterion: {
2997
+ minimumPassRate: number;
2998
+ requireComplete: boolean;
2999
+ };
3000
+ contentHash: string;
3001
+ } | undefined;
3002
+ sourceCandidate?: {
3003
+ candidateId: string;
3004
+ frozenRevision: number;
3005
+ frozenHash: string;
3006
+ harnessRelease: {
3007
+ id: string;
3008
+ contentHash: string;
3009
+ };
3010
+ protectedClosureHash: string;
3011
+ } | undefined;
3012
+ };
3013
+ model: {
3014
+ modelId: string;
3015
+ configurationHash: string;
3016
+ };
3017
+ } | {
3018
+ kind: "agent";
3019
+ source: {
3020
+ harnessRelease: {
3021
+ id: string;
3022
+ contentHash: string;
3023
+ };
3024
+ agentSnapshot: {
3025
+ id: string;
3026
+ contentHash: string;
3027
+ };
3028
+ sourcePackageHash: string;
3029
+ };
3030
+ model: {
3031
+ modelId: string;
3032
+ configurationHash: string;
3033
+ };
3034
+ } | {
3035
+ kind: "fixture";
3036
+ configurationHash: string;
3037
+ };
3038
+ execution?: {
3039
+ packageHash: string;
3040
+ runtimeTargetHash: string;
3041
+ metricPolicyHash: string;
3042
+ compatibility?: {
3043
+ protocol: "openpond.evaluation-execution.v1";
3044
+ targetKind: "agent" | "fixture" | "harness" | "model";
3045
+ } | undefined;
3046
+ } | undefined;
3047
+ lineage?: {
3048
+ definition: {
3049
+ id: string;
3050
+ contentHash: string;
3051
+ revision: number;
3052
+ } | null;
3053
+ execution: {
3054
+ id: string;
3055
+ contentHash: string;
3056
+ };
3057
+ scoringPassId: string | null;
3058
+ humanProjection?: {
3059
+ originalManifest: {
3060
+ id: string;
3061
+ contentHash: string;
3062
+ };
3063
+ selectionHash: string;
3064
+ rulesHash: string;
3065
+ } | undefined;
3066
+ } | undefined;
3067
+ evaluators: {
3068
+ release: {
3069
+ id: string;
3070
+ contentHash: string;
3071
+ revision: string | number;
3072
+ };
3073
+ feedbackKey: string;
3074
+ configurationHash?: string | undefined;
3075
+ output: "boolean" | "category" | "score";
3076
+ categories: string[];
3077
+ }[];
3078
+ population: {
3079
+ caseId: string;
3080
+ seed: string;
3081
+ fixtureId: string | null;
3082
+ }[];
3083
+ createdAt: string;
3084
+ contentHash: string;
3085
+ };
3086
+ result: {
3087
+ schemaVersion: "openpond.experimentResult.v1";
3088
+ manifest: {
3089
+ id: string;
3090
+ contentHash: string;
3091
+ };
3092
+ status: "cancelled" | "completed" | "failed";
3093
+ cases: {
3094
+ identity: {
3095
+ caseId: string;
3096
+ seed: string;
3097
+ fixtureId: string | null;
3098
+ };
3099
+ status: "cancelled" | "completed" | "failed" | "unavailable";
3100
+ output: unknown;
3101
+ error: {
3102
+ code: string;
3103
+ message: string;
3104
+ } | null;
3105
+ feedback: {
3106
+ feedbackKey: string;
3107
+ evaluator: {
3108
+ id: string;
3109
+ contentHash: string;
3110
+ revision: string | number;
3111
+ };
3112
+ status: "failed" | "pending" | "scored" | "unavailable";
3113
+ value: string | number | boolean | null;
3114
+ passed?: boolean | null | undefined;
3115
+ reasoning: string | null;
3116
+ evidenceRefs: {
3117
+ id: string;
3118
+ contentHash: string;
3119
+ mediaType: string | null;
3120
+ sizeBytes: number | null;
3121
+ }[];
3122
+ }[];
3123
+ usage: {
3124
+ inputTokens: number | null;
3125
+ outputTokens: number | null;
3126
+ totalTokens: number | null;
3127
+ costUsd: number | null;
3128
+ latencyMs: number | null;
3129
+ };
3130
+ traceRef: {
3131
+ id: string;
3132
+ contentHash: string;
3133
+ mediaType: string | null;
3134
+ sizeBytes: number | null;
3135
+ } | null;
3136
+ startedAt: string | null;
3137
+ completedAt: string | null;
3138
+ }[];
3139
+ completedAt: string;
3140
+ contentHash: string;
3141
+ };
3142
+ };
3143
+ contentHash: string;
3144
+ } | {
3145
+ teamId: string;
3146
+ evidence: {
3147
+ id: string;
3148
+ contentHash: string;
3149
+ };
3150
+ scoring?: {
3151
+ sourcePass: {
3152
+ id: string;
3153
+ contentHash: string;
3154
+ };
3155
+ graders: {
3156
+ id: string;
3157
+ contentHash: string;
3158
+ revision: number;
3159
+ }[];
3160
+ mappings?: {
3161
+ graderId: string;
3162
+ fields: {
3163
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
3164
+ path: string[];
3165
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
3166
+ sourcePath: string[];
3167
+ }[];
3168
+ }[] | undefined;
3169
+ maximumCostUsd: number;
3170
+ } | undefined;
3171
+ profiles: {
3172
+ id: string;
3173
+ name: string;
3174
+ profileRef: {
3175
+ source: "github" | "local" | "openpond_git";
3176
+ repositoryId: string;
3177
+ profileId: string;
3178
+ };
3179
+ ownerWorkspaceId: string;
3180
+ ownerRevision: number;
3181
+ baseRelease: {
3182
+ id: string;
3183
+ contentHash: string;
3184
+ };
3185
+ profileSourceRevision: string;
3186
+ components: {
3187
+ kind: "agent" | "instruction" | "skill" | "workflow";
3188
+ path: string;
3189
+ agentId?: string | undefined;
3190
+ workflowId?: string | undefined;
3191
+ label: string;
3192
+ binding: {
3193
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
3194
+ dataset: {
3195
+ id: string;
3196
+ contentHash: string;
3197
+ revision: number;
3198
+ };
3199
+ packageHash: string;
3200
+ profileId: string;
3201
+ target: {
3202
+ kind: "profile";
3203
+ } | {
3204
+ kind: "workflow";
3205
+ workflowId: string;
3206
+ } | {
3207
+ kind: "skill";
3208
+ skillPath: string;
3209
+ } | {
3210
+ kind: "agent_action";
3211
+ actionId: string;
3212
+ };
3213
+ declaredProfileCatalogHash: string | null;
3214
+ protectedProfileClosureHash: string;
3215
+ recordedOrigin?: {
3216
+ execution: {
3217
+ id: string;
3218
+ contentHash: string;
3219
+ };
3220
+ originalEvidence: {
3221
+ id: string;
3222
+ contentHash: string;
3223
+ };
3224
+ members: {
3225
+ caseId: string;
3226
+ seed: string;
3227
+ fixtureId: string;
3228
+ sourceId: string;
3229
+ snapshotHash: string;
3230
+ boundaryRevisionHash: string;
3231
+ }[];
3232
+ } | undefined;
3233
+ gradingSource?: {
3234
+ pass: {
3235
+ id: string;
3236
+ contentHash: string;
3237
+ };
3238
+ evidence: {
3239
+ id: string;
3240
+ contentHash: string;
3241
+ };
3242
+ } | undefined;
3243
+ split: "frozen_eval" | "test" | "train" | "validation";
3244
+ population: {
3245
+ taskId: string;
3246
+ seed: string;
3247
+ fixtureId: string | null;
3248
+ }[];
3249
+ evaluators: {
3250
+ release: {
3251
+ id: string;
3252
+ contentHash: string;
3253
+ revision: string | number;
3254
+ };
3255
+ feedbackKey: string;
3256
+ configurationHash: string;
3257
+ output: "boolean" | "category" | "score";
3258
+ categories: string[];
3259
+ }[];
3260
+ fieldMappingsHash: string;
3261
+ fieldMappings: {
3262
+ graderId: string;
3263
+ fields: {
3264
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
3265
+ path: string[];
3266
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
3267
+ sourcePath: string[];
3268
+ }[];
3269
+ }[];
3270
+ environmentHash: string;
3271
+ privateDatasetClosureHash: string;
3272
+ criterion: {
3273
+ minimumPassRate: number;
3274
+ requireComplete: boolean;
3275
+ };
3276
+ contentHash: string;
3277
+ };
3278
+ }[];
3279
+ }[];
3280
+ } | {
3281
+ candidateId: string;
3282
+ candidateRevision: number;
3283
+ path: string;
3284
+ text: string;
3285
+ textHash: string;
3286
+ contentHash: string;
3287
3287
  } | {
3288
3288
  states: {
3289
3289
  schemaVersion: "openpond.experimentImprovement.v1";