openpond-sdk 0.12.17 → 0.12.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/TRAINING_WORKFLOWS.md +15 -0
  2. package/dist/{chunk-VPK3SM45.js → chunk-BJMENXR4.js} +2 -1
  3. package/dist/{chunk-VPK3SM45.js.map → chunk-BJMENXR4.js.map} +1 -1
  4. package/dist/learning.js +2 -2
  5. package/dist/learning.js.map +1 -1
  6. package/dist/post-training.js +1 -1
  7. package/dist/training-bundle.js +1 -1
  8. package/dist/training.js +1102 -971
  9. package/dist/training.js.map +1 -1
  10. package/dist/types/packages/sdk/src/dataset-training-review.d.ts.map +1 -1
  11. package/dist/types/packages/sdk/src/experiment-draft-storage.d.ts +1057 -57
  12. package/dist/types/packages/sdk/src/experiment-draft-storage.d.ts.map +1 -1
  13. package/dist/types/packages/sdk/src/experiment-evaluation-schedules.d.ts +216 -216
  14. package/dist/types/packages/sdk/src/experiment-improvement-client.d.ts +1889 -1889
  15. package/dist/types/packages/sdk/src/managed-training-preparation.d.ts +25 -25
  16. package/dist/types/packages/sdk/src/model-taskset-runs.d.ts +216 -216
  17. package/dist/types/packages/sdk/src/post-training.d.ts +1 -0
  18. package/dist/types/packages/sdk/src/post-training.d.ts.map +1 -1
  19. package/dist/types/packages/sdk/src/steering-model-serving.d.ts +1349 -1349
  20. package/dist/types/packages/sdk/src/taskset-draft-client.d.ts +120 -120
  21. package/dist/types/packages/sdk/src/training-experiment-learning-intent.d.ts +26 -0
  22. package/dist/types/packages/sdk/src/training-experiment-learning-intent.d.ts.map +1 -0
  23. package/dist/types/packages/sdk/src/training-experiment-learning.d.ts +699 -0
  24. package/dist/types/packages/sdk/src/training-experiment-learning.d.ts.map +1 -0
  25. package/dist/types/packages/sdk/src/training-experiment-setup.d.ts +573 -0
  26. package/dist/types/packages/sdk/src/training-experiment-setup.d.ts.map +1 -1
  27. package/dist/types/packages/sdk/src/training-harness-sources.d.ts +2 -2
  28. package/dist/types/packages/sdk/src/training-learning-batch.d.ts +40 -40
  29. package/dist/types/packages/sdk/src/training-qualification-preparation.d.ts +25 -25
  30. package/dist/types/packages/sdk/src/training.d.ts +1178 -1064
  31. package/dist/types/packages/sdk/src/training.d.ts.map +1 -1
  32. package/package.json +1 -1
package/dist/types/packages/sdk/src/experiment-improvement-client.d.ts CHANGED
@@ -164,27 +164,149 @@ export declare class ExperimentImprovementClient {
164
164
  contentHash: string;
165
165
  }>;
166
166
  command(raw: ExperimentImprovementCommand, signal?: AbortSignal): Promise<{
167
- schemaVersion: "openpond.experimentImprovement.v1";
168
- id: string;
169
- operationId: string;
170
- requestHash: string;
171
- actorId: string;
172
167
  teamId: string;
173
- projectId: string | null;
174
- ownerWorkspaceId: string;
175
- ownerRevision: number;
176
- ownerSourceRevision: string;
177
- baseRelease: {
168
+ evidence: {
178
169
  id: string;
179
170
  contentHash: string;
180
171
  };
181
- profileRef: {
182
- source: "github" | "local" | "openpond_git";
183
- repositoryId: string;
184
- profileId: string;
185
- };
186
- baseProfileSourceRevision: string;
187
- evidence: {
172
+ scoring?: {
173
+ sourcePass: {
174
+ id: string;
175
+ contentHash: string;
176
+ };
177
+ graders: {
178
+ id: string;
179
+ contentHash: string;
180
+ revision: number;
181
+ }[];
182
+ mappings?: {
183
+ graderId: string;
184
+ fields: {
185
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
186
+ path: string[];
187
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
188
+ sourcePath: string[];
189
+ }[];
190
+ }[] | undefined;
191
+ maximumCostUsd: number;
192
+ } | undefined;
193
+ profiles: {
194
+ id: string;
195
+ name: string;
196
+ profileRef: {
197
+ source: "github" | "local" | "openpond_git";
198
+ repositoryId: string;
199
+ profileId: string;
200
+ };
201
+ ownerWorkspaceId: string;
202
+ ownerRevision: number;
203
+ baseRelease: {
204
+ id: string;
205
+ contentHash: string;
206
+ };
207
+ profileSourceRevision: string;
208
+ components: {
209
+ kind: "agent" | "instruction" | "skill" | "workflow";
210
+ path: string;
211
+ agentId?: string | undefined;
212
+ workflowId?: string | undefined;
213
+ label: string;
214
+ binding: {
215
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
216
+ dataset: {
217
+ id: string;
218
+ contentHash: string;
219
+ revision: number;
220
+ };
221
+ packageHash: string;
222
+ profileId: string;
223
+ target: {
224
+ kind: "profile";
225
+ } | {
226
+ kind: "workflow";
227
+ workflowId: string;
228
+ } | {
229
+ kind: "skill";
230
+ skillPath: string;
231
+ } | {
232
+ kind: "agent_action";
233
+ actionId: string;
234
+ };
235
+ declaredProfileCatalogHash: string | null;
236
+ protectedProfileClosureHash: string;
237
+ recordedOrigin?: {
238
+ execution: {
239
+ id: string;
240
+ contentHash: string;
241
+ };
242
+ originalEvidence: {
243
+ id: string;
244
+ contentHash: string;
245
+ };
246
+ members: {
247
+ caseId: string;
248
+ seed: string;
249
+ fixtureId: string;
250
+ sourceId: string;
251
+ snapshotHash: string;
252
+ boundaryRevisionHash: string;
253
+ }[];
254
+ } | undefined;
255
+ gradingSource?: {
256
+ pass: {
257
+ id: string;
258
+ contentHash: string;
259
+ };
260
+ evidence: {
261
+ id: string;
262
+ contentHash: string;
263
+ };
264
+ } | undefined;
265
+ split: "frozen_eval" | "test" | "train" | "validation";
266
+ population: {
267
+ taskId: string;
268
+ seed: string;
269
+ fixtureId: string | null;
270
+ }[];
271
+ evaluators: {
272
+ release: {
273
+ id: string;
274
+ contentHash: string;
275
+ revision: string | number;
276
+ };
277
+ feedbackKey: string;
278
+ configurationHash: string;
279
+ output: "boolean" | "category" | "score";
280
+ categories: string[];
281
+ }[];
282
+ fieldMappingsHash: string;
283
+ fieldMappings: {
284
+ graderId: string;
285
+ fields: {
286
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
287
+ path: string[];
288
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
289
+ sourcePath: string[];
290
+ }[];
291
+ }[];
292
+ environmentHash: string;
293
+ privateDatasetClosureHash: string;
294
+ criterion: {
295
+ minimumPassRate: number;
296
+ requireComplete: boolean;
297
+ };
298
+ contentHash: string;
299
+ };
300
+ }[];
301
+ }[];
302
+ } | {
303
+ candidateId: string;
304
+ actorId: string;
305
+ teamId: string;
306
+ projectId: string | null;
307
+ candidateRevision: number;
308
+ recipeHash: string;
309
+ baseline: {
188
310
  manifest: {
189
311
  schemaVersion: "openpond.experimentManifest.v1";
190
312
  id: string;
@@ -453,198 +575,57 @@ export declare class ExperimentImprovementClient {
453
575
  contentHash: string;
454
576
  };
455
577
  };
456
- component: {
457
- kind: "agent" | "instruction" | "skill" | "workflow";
458
- path: string;
459
- workflowId?: string | undefined;
460
- };
461
- mode: "llm_assisted" | "manual";
462
- status: "accepted" | "adopting" | "authoring" | "authoring_queued" | "cancelled" | "declined" | "draft" | "failed" | "frozen" | "ready_for_review" | "rolled_back" | "testing";
463
- revision: number;
464
- limits: {
465
- maximumCostUsd: number;
466
- maximumAuthoringSteps: number;
467
- maximumDurationMs: number;
468
- };
469
- authoringSteps: number;
470
- work: {
471
- sessionId: string;
472
- turnId: string;
473
- revision: number;
474
- startedAt: string;
475
- } | null;
476
- frozen: {
477
- release: {
578
+ candidate: {
579
+ manifest: {
580
+ schemaVersion: "openpond.experimentManifest.v1";
478
581
  id: string;
479
- contentHash: string;
480
- };
481
- sourceRevision: string;
482
- profileSourceRevision: string;
483
- authoringHash: string;
484
- partitionHash: string;
485
- executableSourceHash: string;
486
- protectedClosureHash: string;
487
- candidateRevision: number;
488
- diff: {
489
- path: string;
490
- before: string | null;
491
- after: string | null;
492
- }[];
493
- } | null;
494
- test: {
495
- recipe: {
496
- externalDatasetBinding: {
497
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
498
- dataset: {
499
- id: string;
500
- contentHash: string;
501
- revision: number;
502
- };
503
- packageHash: string;
504
- profileId: string;
505
- target: {
506
- kind: "profile";
507
- } | {
508
- kind: "workflow";
509
- workflowId: string;
510
- } | {
511
- kind: "skill";
512
- skillPath: string;
513
- } | {
514
- kind: "agent_action";
515
- actionId: string;
516
- };
517
- declaredProfileCatalogHash: string | null;
518
- protectedProfileClosureHash: string;
519
- recordedOrigin?: {
520
- execution: {
521
- id: string;
522
- contentHash: string;
523
- };
524
- originalEvidence: {
582
+ name: string;
583
+ teamId: string;
584
+ operationId: string;
585
+ maximumCostUsd: number | null;
586
+ dataset: {
587
+ id: string;
588
+ contentHash: string;
589
+ revision: number;
590
+ };
591
+ target: {
592
+ kind: "model";
593
+ modelId: string;
594
+ configurationHash: string;
595
+ } | {
596
+ kind: "harness";
597
+ source: {
598
+ profileId: string;
599
+ sourceRevision: string;
600
+ harnessRelease: {
525
601
  id: string;
526
602
  contentHash: string;
527
603
  };
528
- members: {
529
- caseId: string;
530
- seed: string;
531
- fixtureId: string;
532
- sourceId: string;
533
- snapshotHash: string;
534
- boundaryRevisionHash: string;
535
- }[];
536
- } | undefined;
537
- gradingSource?: {
538
- pass: {
539
- id: string;
540
- contentHash: string;
541
- };
542
- evidence: {
543
- id: string;
544
- contentHash: string;
545
- };
546
- } | undefined;
547
- split: "frozen_eval" | "test" | "train" | "validation";
548
- population: {
549
- taskId: string;
550
- seed: string;
551
- fixtureId: string | null;
552
- }[];
553
- evaluators: {
554
- release: {
555
- id: string;
556
- contentHash: string;
557
- revision: string | number;
604
+ catalogHash: string;
605
+ definitionId: string;
606
+ definitionHash: string;
607
+ target: {
608
+ kind: "profile";
609
+ } | {
610
+ kind: "workflow";
611
+ workflowId: string;
612
+ } | {
613
+ kind: "skill";
614
+ skillPath: string;
615
+ } | {
616
+ kind: "agent_action";
617
+ actionId: string;
558
618
  };
559
- feedbackKey: string;
560
- configurationHash: string;
561
- output: "boolean" | "category" | "score";
562
- categories: string[];
563
- }[];
564
- fieldMappingsHash: string;
565
- fieldMappings: {
566
- graderId: string;
567
- fields: {
568
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
569
- path: string[];
570
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
571
- sourcePath: string[];
572
- }[];
573
- }[];
574
- environmentHash: string;
575
- privateDatasetClosureHash: string;
576
- criterion: {
577
- minimumPassRate: number;
578
- requireComplete: boolean;
579
- };
580
- contentHash: string;
581
- };
582
- scoring?: {
583
- sourcePass: {
584
- id: string;
585
- contentHash: string;
586
- };
587
- graders: {
588
- id: string;
589
- contentHash: string;
590
- revision: number;
591
- }[];
592
- mappings?: {
593
- graderId: string;
594
- fields: {
595
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
596
- path: string[];
597
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
598
- sourcePath: string[];
599
- }[];
600
- }[] | undefined;
601
- maximumCostUsd: number;
602
- } | undefined;
603
- modelId: string;
604
- maximumCostUsd: number;
605
- };
606
- recipeHash: string;
607
- baselineOperationId: string;
608
- candidateOperationId: string;
609
- cancelRequestedAt: string | null;
610
- dispatching: {
611
- kind: "baseline" | "candidate";
612
- token: string;
613
- pid: number;
614
- processStart: string | null;
615
- startedAt: string;
616
- } | null;
617
- } | null;
618
- comparison: {
619
- baseline: {
620
- manifest: {
621
- schemaVersion: "openpond.experimentManifest.v1";
622
- id: string;
623
- name: string;
624
- teamId: string;
625
- operationId: string;
626
- maximumCostUsd: number | null;
627
- dataset: {
628
- id: string;
629
- contentHash: string;
630
- revision: number;
631
- };
632
- target: {
633
- kind: "model";
634
- modelId: string;
635
- configurationHash: string;
636
- } | {
637
- kind: "harness";
638
- source: {
639
- profileId: string;
640
- sourceRevision: string;
641
- harnessRelease: {
619
+ environmentHash: string;
620
+ externalDatasetBinding?: {
621
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
622
+ dataset: {
642
623
  id: string;
643
624
  contentHash: string;
625
+ revision: number;
644
626
  };
645
- catalogHash: string;
646
- definitionId: string;
647
- definitionHash: string;
627
+ packageHash: string;
628
+ profileId: string;
648
629
  target: {
649
630
  kind: "profile";
650
631
  } | {
@@ -657,263 +638,292 @@ export declare class ExperimentImprovementClient {
657
638
  kind: "agent_action";
658
639
  actionId: string;
659
640
  };
660
- environmentHash: string;
661
- externalDatasetBinding?: {
662
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
663
- dataset: {
641
+ declaredProfileCatalogHash: string | null;
642
+ protectedProfileClosureHash: string;
643
+ recordedOrigin?: {
644
+ execution: {
664
645
  id: string;
665
646
  contentHash: string;
666
- revision: number;
667
647
  };
668
- packageHash: string;
669
- profileId: string;
670
- target: {
671
- kind: "profile";
672
- } | {
673
- kind: "workflow";
674
- workflowId: string;
675
- } | {
676
- kind: "skill";
677
- skillPath: string;
678
- } | {
679
- kind: "agent_action";
680
- actionId: string;
648
+ originalEvidence: {
649
+ id: string;
650
+ contentHash: string;
681
651
  };
682
- declaredProfileCatalogHash: string | null;
683
- protectedProfileClosureHash: string;
684
- recordedOrigin?: {
685
- execution: {
686
- id: string;
687
- contentHash: string;
688
- };
689
- originalEvidence: {
690
- id: string;
691
- contentHash: string;
692
- };
693
- members: {
694
- caseId: string;
695
- seed: string;
696
- fixtureId: string;
697
- sourceId: string;
698
- snapshotHash: string;
699
- boundaryRevisionHash: string;
700
- }[];
701
- } | undefined;
702
- gradingSource?: {
703
- pass: {
704
- id: string;
705
- contentHash: string;
706
- };
707
- evidence: {
708
- id: string;
709
- contentHash: string;
710
- };
711
- } | undefined;
712
- split: "frozen_eval" | "test" | "train" | "validation";
713
- population: {
714
- taskId: string;
652
+ members: {
653
+ caseId: string;
715
654
  seed: string;
716
- fixtureId: string | null;
717
- }[];
718
- evaluators: {
719
- release: {
720
- id: string;
721
- contentHash: string;
722
- revision: string | number;
723
- };
724
- feedbackKey: string;
725
- configurationHash: string;
726
- output: "boolean" | "category" | "score";
727
- categories: string[];
728
- }[];
729
- fieldMappingsHash: string;
730
- fieldMappings: {
731
- graderId: string;
732
- fields: {
733
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
734
- path: string[];
735
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
736
- sourcePath: string[];
737
- }[];
655
+ fixtureId: string;
656
+ sourceId: string;
657
+ snapshotHash: string;
658
+ boundaryRevisionHash: string;
738
659
  }[];
739
- environmentHash: string;
740
- privateDatasetClosureHash: string;
741
- criterion: {
742
- minimumPassRate: number;
743
- requireComplete: boolean;
744
- };
745
- contentHash: string;
746
660
  } | undefined;
747
- sourceCandidate?: {
748
- candidateId: string;
749
- frozenRevision: number;
750
- frozenHash: string;
751
- harnessRelease: {
661
+ gradingSource?: {
662
+ pass: {
663
+ id: string;
664
+ contentHash: string;
665
+ };
666
+ evidence: {
752
667
  id: string;
753
668
  contentHash: string;
754
669
  };
755
- protectedClosureHash: string;
756
670
  } | undefined;
757
- };
758
- model: {
759
- modelId: string;
760
- configurationHash: string;
761
- };
762
- } | {
763
- kind: "agent";
764
- source: {
765
- harnessRelease: {
766
- id: string;
767
- contentHash: string;
768
- };
769
- agentSnapshot: {
770
- id: string;
771
- contentHash: string;
772
- };
773
- sourcePackageHash: string;
774
- };
775
- model: {
776
- modelId: string;
777
- configurationHash: string;
778
- };
779
- } | {
780
- kind: "fixture";
671
+ split: "frozen_eval" | "test" | "train" | "validation";
672
+ population: {
673
+ taskId: string;
674
+ seed: string;
675
+ fixtureId: string | null;
676
+ }[];
677
+ evaluators: {
678
+ release: {
679
+ id: string;
680
+ contentHash: string;
681
+ revision: string | number;
682
+ };
683
+ feedbackKey: string;
684
+ configurationHash: string;
685
+ output: "boolean" | "category" | "score";
686
+ categories: string[];
687
+ }[];
688
+ fieldMappingsHash: string;
689
+ fieldMappings: {
690
+ graderId: string;
691
+ fields: {
692
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
693
+ path: string[];
694
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
695
+ sourcePath: string[];
696
+ }[];
697
+ }[];
698
+ environmentHash: string;
699
+ privateDatasetClosureHash: string;
700
+ criterion: {
701
+ minimumPassRate: number;
702
+ requireComplete: boolean;
703
+ };
704
+ contentHash: string;
705
+ } | undefined;
706
+ sourceCandidate?: {
707
+ candidateId: string;
708
+ frozenRevision: number;
709
+ frozenHash: string;
710
+ harnessRelease: {
711
+ id: string;
712
+ contentHash: string;
713
+ };
714
+ protectedClosureHash: string;
715
+ } | undefined;
716
+ };
717
+ model: {
718
+ modelId: string;
781
719
  configurationHash: string;
782
720
  };
783
- execution?: {
784
- packageHash: string;
785
- runtimeTargetHash: string;
786
- metricPolicyHash: string;
787
- compatibility?: {
788
- protocol: "openpond.evaluation-execution.v1";
789
- targetKind: "agent" | "fixture" | "harness" | "model";
790
- } | undefined;
791
- } | undefined;
792
- lineage?: {
793
- definition: {
721
+ } | {
722
+ kind: "agent";
723
+ source: {
724
+ harnessRelease: {
794
725
  id: string;
795
726
  contentHash: string;
796
- revision: number;
797
- } | null;
798
- execution: {
727
+ };
728
+ agentSnapshot: {
799
729
  id: string;
800
730
  contentHash: string;
801
731
  };
802
- scoringPassId: string | null;
803
- humanProjection?: {
804
- originalManifest: {
805
- id: string;
806
- contentHash: string;
807
- };
808
- selectionHash: string;
809
- rulesHash: string;
810
- } | undefined;
732
+ sourcePackageHash: string;
733
+ };
734
+ model: {
735
+ modelId: string;
736
+ configurationHash: string;
737
+ };
738
+ } | {
739
+ kind: "fixture";
740
+ configurationHash: string;
741
+ };
742
+ execution?: {
743
+ packageHash: string;
744
+ runtimeTargetHash: string;
745
+ metricPolicyHash: string;
746
+ compatibility?: {
747
+ protocol: "openpond.evaluation-execution.v1";
748
+ targetKind: "agent" | "fixture" | "harness" | "model";
811
749
  } | undefined;
812
- evaluators: {
813
- release: {
750
+ } | undefined;
751
+ lineage?: {
752
+ definition: {
753
+ id: string;
754
+ contentHash: string;
755
+ revision: number;
756
+ } | null;
757
+ execution: {
758
+ id: string;
759
+ contentHash: string;
760
+ };
761
+ scoringPassId: string | null;
762
+ humanProjection?: {
763
+ originalManifest: {
814
764
  id: string;
815
765
  contentHash: string;
816
- revision: string | number;
817
766
  };
818
- feedbackKey: string;
819
- configurationHash?: string | undefined;
820
- output: "boolean" | "category" | "score";
821
- categories: string[];
822
- }[];
823
- population: {
767
+ selectionHash: string;
768
+ rulesHash: string;
769
+ } | undefined;
770
+ } | undefined;
771
+ evaluators: {
772
+ release: {
773
+ id: string;
774
+ contentHash: string;
775
+ revision: string | number;
776
+ };
777
+ feedbackKey: string;
778
+ configurationHash?: string | undefined;
779
+ output: "boolean" | "category" | "score";
780
+ categories: string[];
781
+ }[];
782
+ population: {
783
+ caseId: string;
784
+ seed: string;
785
+ fixtureId: string | null;
786
+ }[];
787
+ createdAt: string;
788
+ contentHash: string;
789
+ };
790
+ result: {
791
+ schemaVersion: "openpond.experimentResult.v1";
792
+ manifest: {
793
+ id: string;
794
+ contentHash: string;
795
+ };
796
+ status: "cancelled" | "completed" | "failed";
797
+ cases: {
798
+ identity: {
824
799
  caseId: string;
825
800
  seed: string;
826
801
  fixtureId: string | null;
827
- }[];
828
- createdAt: string;
829
- contentHash: string;
830
- };
831
- result: {
832
- schemaVersion: "openpond.experimentResult.v1";
833
- manifest: {
834
- id: string;
835
- contentHash: string;
836
802
  };
837
- status: "cancelled" | "completed" | "failed";
838
- cases: {
839
- identity: {
840
- caseId: string;
841
- seed: string;
842
- fixtureId: string | null;
843
- };
844
- status: "cancelled" | "completed" | "failed" | "unavailable";
845
- output: unknown;
846
- error: {
847
- code: string;
848
- message: string;
849
- } | null;
850
- feedback: {
851
- feedbackKey: string;
852
- evaluator: {
853
- id: string;
854
- contentHash: string;
855
- revision: string | number;
856
- };
857
- status: "failed" | "pending" | "scored" | "unavailable";
858
- value: string | number | boolean | null;
859
- passed?: boolean | null | undefined;
860
- reasoning: string | null;
861
- evidenceRefs: {
862
- id: string;
863
- contentHash: string;
864
- mediaType: string | null;
865
- sizeBytes: number | null;
866
- }[];
867
- }[];
868
- usage: {
869
- inputTokens: number | null;
870
- outputTokens: number | null;
871
- totalTokens: number | null;
872
- costUsd: number | null;
873
- latencyMs: number | null;
803
+ status: "cancelled" | "completed" | "failed" | "unavailable";
804
+ output: unknown;
805
+ error: {
806
+ code: string;
807
+ message: string;
808
+ } | null;
809
+ feedback: {
810
+ feedbackKey: string;
811
+ evaluator: {
812
+ id: string;
813
+ contentHash: string;
814
+ revision: string | number;
874
815
  };
875
- traceRef: {
816
+ status: "failed" | "pending" | "scored" | "unavailable";
817
+ value: string | number | boolean | null;
818
+ passed?: boolean | null | undefined;
819
+ reasoning: string | null;
820
+ evidenceRefs: {
876
821
  id: string;
877
822
  contentHash: string;
878
823
  mediaType: string | null;
879
824
  sizeBytes: number | null;
880
- } | null;
881
- startedAt: string | null;
882
- completedAt: string | null;
825
+ }[];
883
826
  }[];
884
- completedAt: string;
885
- contentHash: string;
886
- };
887
- };
888
- candidate: {
889
- manifest: {
890
- schemaVersion: "openpond.experimentManifest.v1";
891
- id: string;
892
- name: string;
893
- teamId: string;
894
- operationId: string;
895
- maximumCostUsd: number | null;
896
- dataset: {
827
+ usage: {
828
+ inputTokens: number | null;
829
+ outputTokens: number | null;
830
+ totalTokens: number | null;
831
+ costUsd: number | null;
832
+ latencyMs: number | null;
833
+ };
834
+ traceRef: {
897
835
  id: string;
898
836
  contentHash: string;
899
- revision: number;
900
- };
901
- target: {
902
- kind: "model";
903
- modelId: string;
904
- configurationHash: string;
905
- } | {
906
- kind: "harness";
907
- source: {
908
- profileId: string;
909
- sourceRevision: string;
910
- harnessRelease: {
911
- id: string;
912
- contentHash: string;
913
- };
914
- catalogHash: string;
915
- definitionId: string;
916
- definitionHash: string;
837
+ mediaType: string | null;
838
+ sizeBytes: number | null;
839
+ } | null;
840
+ startedAt: string | null;
841
+ completedAt: string | null;
842
+ }[];
843
+ completedAt: string;
844
+ contentHash: string;
845
+ };
846
+ };
847
+ contentHash: string;
848
+ } | {
849
+ candidateId: string;
850
+ candidateRevision: number;
851
+ path: string;
852
+ text: string;
853
+ textHash: string;
854
+ contentHash: string;
855
+ } | {
856
+ schemaVersion: "openpond.experimentImprovement.v1";
857
+ id: string;
858
+ operationId: string;
859
+ requestHash: string;
860
+ actorId: string;
861
+ teamId: string;
862
+ projectId: string | null;
863
+ ownerWorkspaceId: string;
864
+ ownerRevision: number;
865
+ ownerSourceRevision: string;
866
+ baseRelease: {
867
+ id: string;
868
+ contentHash: string;
869
+ };
870
+ profileRef: {
871
+ source: "github" | "local" | "openpond_git";
872
+ repositoryId: string;
873
+ profileId: string;
874
+ };
875
+ baseProfileSourceRevision: string;
876
+ evidence: {
877
+ manifest: {
878
+ schemaVersion: "openpond.experimentManifest.v1";
879
+ id: string;
880
+ name: string;
881
+ teamId: string;
882
+ operationId: string;
883
+ maximumCostUsd: number | null;
884
+ dataset: {
885
+ id: string;
886
+ contentHash: string;
887
+ revision: number;
888
+ };
889
+ target: {
890
+ kind: "model";
891
+ modelId: string;
892
+ configurationHash: string;
893
+ } | {
894
+ kind: "harness";
895
+ source: {
896
+ profileId: string;
897
+ sourceRevision: string;
898
+ harnessRelease: {
899
+ id: string;
900
+ contentHash: string;
901
+ };
902
+ catalogHash: string;
903
+ definitionId: string;
904
+ definitionHash: string;
905
+ target: {
906
+ kind: "profile";
907
+ } | {
908
+ kind: "workflow";
909
+ workflowId: string;
910
+ } | {
911
+ kind: "skill";
912
+ skillPath: string;
913
+ } | {
914
+ kind: "agent_action";
915
+ actionId: string;
916
+ };
917
+ environmentHash: string;
918
+ externalDatasetBinding?: {
919
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
920
+ dataset: {
921
+ id: string;
922
+ contentHash: string;
923
+ revision: number;
924
+ };
925
+ packageHash: string;
926
+ profileId: string;
917
927
  target: {
918
928
  kind: "profile";
919
929
  } | {
@@ -926,950 +936,913 @@ export declare class ExperimentImprovementClient {
926
936
  kind: "agent_action";
927
937
  actionId: string;
928
938
  };
929
- environmentHash: string;
930
- externalDatasetBinding?: {
931
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
932
- dataset: {
939
+ declaredProfileCatalogHash: string | null;
940
+ protectedProfileClosureHash: string;
941
+ recordedOrigin?: {
942
+ execution: {
933
943
  id: string;
934
944
  contentHash: string;
935
- revision: number;
936
945
  };
937
- packageHash: string;
938
- profileId: string;
939
- target: {
940
- kind: "profile";
941
- } | {
942
- kind: "workflow";
943
- workflowId: string;
944
- } | {
945
- kind: "skill";
946
- skillPath: string;
947
- } | {
948
- kind: "agent_action";
949
- actionId: string;
946
+ originalEvidence: {
947
+ id: string;
948
+ contentHash: string;
950
949
  };
951
- declaredProfileCatalogHash: string | null;
952
- protectedProfileClosureHash: string;
953
- recordedOrigin?: {
954
- execution: {
955
- id: string;
956
- contentHash: string;
957
- };
958
- originalEvidence: {
959
- id: string;
960
- contentHash: string;
961
- };
962
- members: {
963
- caseId: string;
964
- seed: string;
965
- fixtureId: string;
966
- sourceId: string;
967
- snapshotHash: string;
968
- boundaryRevisionHash: string;
969
- }[];
970
- } | undefined;
971
- gradingSource?: {
972
- pass: {
973
- id: string;
974
- contentHash: string;
975
- };
976
- evidence: {
977
- id: string;
978
- contentHash: string;
979
- };
980
- } | undefined;
981
- split: "frozen_eval" | "test" | "train" | "validation";
982
- population: {
983
- taskId: string;
950
+ members: {
951
+ caseId: string;
984
952
  seed: string;
985
- fixtureId: string | null;
986
- }[];
987
- evaluators: {
988
- release: {
989
- id: string;
990
- contentHash: string;
991
- revision: string | number;
992
- };
993
- feedbackKey: string;
994
- configurationHash: string;
995
- output: "boolean" | "category" | "score";
996
- categories: string[];
997
- }[];
998
- fieldMappingsHash: string;
999
- fieldMappings: {
1000
- graderId: string;
1001
- fields: {
1002
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1003
- path: string[];
1004
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1005
- sourcePath: string[];
1006
- }[];
953
+ fixtureId: string;
954
+ sourceId: string;
955
+ snapshotHash: string;
956
+ boundaryRevisionHash: string;
1007
957
  }[];
1008
- environmentHash: string;
1009
- privateDatasetClosureHash: string;
1010
- criterion: {
1011
- minimumPassRate: number;
1012
- requireComplete: boolean;
1013
- };
1014
- contentHash: string;
1015
958
  } | undefined;
1016
- sourceCandidate?: {
1017
- candidateId: string;
1018
- frozenRevision: number;
1019
- frozenHash: string;
1020
- harnessRelease: {
959
+ gradingSource?: {
960
+ pass: {
961
+ id: string;
962
+ contentHash: string;
963
+ };
964
+ evidence: {
1021
965
  id: string;
1022
966
  contentHash: string;
1023
967
  };
1024
- protectedClosureHash: string;
1025
968
  } | undefined;
1026
- };
1027
- model: {
1028
- modelId: string;
1029
- configurationHash: string;
1030
- };
1031
- } | {
1032
- kind: "agent";
1033
- source: {
1034
- harnessRelease: {
1035
- id: string;
1036
- contentHash: string;
969
+ split: "frozen_eval" | "test" | "train" | "validation";
970
+ population: {
971
+ taskId: string;
972
+ seed: string;
973
+ fixtureId: string | null;
974
+ }[];
975
+ evaluators: {
976
+ release: {
977
+ id: string;
978
+ contentHash: string;
979
+ revision: string | number;
980
+ };
981
+ feedbackKey: string;
982
+ configurationHash: string;
983
+ output: "boolean" | "category" | "score";
984
+ categories: string[];
985
+ }[];
986
+ fieldMappingsHash: string;
987
+ fieldMappings: {
988
+ graderId: string;
989
+ fields: {
990
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
991
+ path: string[];
992
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
993
+ sourcePath: string[];
994
+ }[];
995
+ }[];
996
+ environmentHash: string;
997
+ privateDatasetClosureHash: string;
998
+ criterion: {
999
+ minimumPassRate: number;
1000
+ requireComplete: boolean;
1037
1001
  };
1038
- agentSnapshot: {
1002
+ contentHash: string;
1003
+ } | undefined;
1004
+ sourceCandidate?: {
1005
+ candidateId: string;
1006
+ frozenRevision: number;
1007
+ frozenHash: string;
1008
+ harnessRelease: {
1039
1009
  id: string;
1040
1010
  contentHash: string;
1041
1011
  };
1042
- sourcePackageHash: string;
1043
- };
1044
- model: {
1045
- modelId: string;
1046
- configurationHash: string;
1047
- };
1048
- } | {
1049
- kind: "fixture";
1050
- configurationHash: string;
1051
- };
1052
- execution?: {
1053
- packageHash: string;
1054
- runtimeTargetHash: string;
1055
- metricPolicyHash: string;
1056
- compatibility?: {
1057
- protocol: "openpond.evaluation-execution.v1";
1058
- targetKind: "agent" | "fixture" | "harness" | "model";
1012
+ protectedClosureHash: string;
1059
1013
  } | undefined;
1060
- } | undefined;
1061
- lineage?: {
1062
- definition: {
1063
- id: string;
1064
- contentHash: string;
1065
- revision: number;
1066
- } | null;
1067
- execution: {
1014
+ };
1015
+ model: {
1016
+ modelId: string;
1017
+ configurationHash: string;
1018
+ };
1019
+ } | {
1020
+ kind: "agent";
1021
+ source: {
1022
+ harnessRelease: {
1068
1023
  id: string;
1069
1024
  contentHash: string;
1070
1025
  };
1071
- scoringPassId: string | null;
1072
- humanProjection?: {
1073
- originalManifest: {
1074
- id: string;
1075
- contentHash: string;
1076
- };
1077
- selectionHash: string;
1078
- rulesHash: string;
1079
- } | undefined;
1080
- } | undefined;
1081
- evaluators: {
1082
- release: {
1026
+ agentSnapshot: {
1083
1027
  id: string;
1084
1028
  contentHash: string;
1085
- revision: string | number;
1086
1029
  };
1087
- feedbackKey: string;
1088
- configurationHash?: string | undefined;
1089
- output: "boolean" | "category" | "score";
1090
- categories: string[];
1091
- }[];
1092
- population: {
1093
- caseId: string;
1094
- seed: string;
1095
- fixtureId: string | null;
1096
- }[];
1097
- createdAt: string;
1098
- contentHash: string;
1030
+ sourcePackageHash: string;
1031
+ };
1032
+ model: {
1033
+ modelId: string;
1034
+ configurationHash: string;
1035
+ };
1036
+ } | {
1037
+ kind: "fixture";
1038
+ configurationHash: string;
1099
1039
  };
1100
- result: {
1101
- schemaVersion: "openpond.experimentResult.v1";
1102
- manifest: {
1040
+ execution?: {
1041
+ packageHash: string;
1042
+ runtimeTargetHash: string;
1043
+ metricPolicyHash: string;
1044
+ compatibility?: {
1045
+ protocol: "openpond.evaluation-execution.v1";
1046
+ targetKind: "agent" | "fixture" | "harness" | "model";
1047
+ } | undefined;
1048
+ } | undefined;
1049
+ lineage?: {
1050
+ definition: {
1051
+ id: string;
1052
+ contentHash: string;
1053
+ revision: number;
1054
+ } | null;
1055
+ execution: {
1103
1056
  id: string;
1104
1057
  contentHash: string;
1105
1058
  };
1106
- status: "cancelled" | "completed" | "failed";
1107
- cases: {
1108
- identity: {
1109
- caseId: string;
1110
- seed: string;
1111
- fixtureId: string | null;
1112
- };
1113
- status: "cancelled" | "completed" | "failed" | "unavailable";
1114
- output: unknown;
1115
- error: {
1116
- code: string;
1117
- message: string;
1118
- } | null;
1119
- feedback: {
1120
- feedbackKey: string;
1121
- evaluator: {
1122
- id: string;
1123
- contentHash: string;
1124
- revision: string | number;
1125
- };
1126
- status: "failed" | "pending" | "scored" | "unavailable";
1127
- value: string | number | boolean | null;
1128
- passed?: boolean | null | undefined;
1129
- reasoning: string | null;
1130
- evidenceRefs: {
1131
- id: string;
1132
- contentHash: string;
1133
- mediaType: string | null;
1134
- sizeBytes: number | null;
1135
- }[];
1136
- }[];
1137
- usage: {
1138
- inputTokens: number | null;
1139
- outputTokens: number | null;
1140
- totalTokens: number | null;
1141
- costUsd: number | null;
1142
- latencyMs: number | null;
1143
- };
1144
- traceRef: {
1059
+ scoringPassId: string | null;
1060
+ humanProjection?: {
1061
+ originalManifest: {
1145
1062
  id: string;
1146
1063
  contentHash: string;
1147
- mediaType: string | null;
1148
- sizeBytes: number | null;
1149
- } | null;
1150
- startedAt: string | null;
1151
- completedAt: string | null;
1152
- }[];
1153
- completedAt: string;
1064
+ };
1065
+ selectionHash: string;
1066
+ rulesHash: string;
1067
+ } | undefined;
1068
+ } | undefined;
1069
+ evaluators: {
1070
+ release: {
1071
+ id: string;
1072
+ contentHash: string;
1073
+ revision: string | number;
1074
+ };
1075
+ feedbackKey: string;
1076
+ configurationHash?: string | undefined;
1077
+ output: "boolean" | "category" | "score";
1078
+ categories: string[];
1079
+ }[];
1080
+ population: {
1081
+ caseId: string;
1082
+ seed: string;
1083
+ fixtureId: string | null;
1084
+ }[];
1085
+ createdAt: string;
1086
+ contentHash: string;
1087
+ };
1088
+ result: {
1089
+ schemaVersion: "openpond.experimentResult.v1";
1090
+ manifest: {
1091
+ id: string;
1154
1092
  contentHash: string;
1155
1093
  };
1156
- };
1157
- frozenHash: string;
1158
- humanProjection?: {
1159
- schemaVersion: "openpond.humanComparisonProjection.v1";
1160
- teamId: string;
1161
- projectId: string;
1162
- selections: {
1163
- rules: ({
1164
- feedbackKey: string;
1165
- criterionId: string;
1166
- kind: "normalized_score";
1167
- minimum: number;
1168
- maximum: number;
1169
- passThreshold?: number | undefined;
1170
- } | {
1171
- feedbackKey: string;
1172
- criterionId: string;
1173
- kind: "boolean";
1174
- } | {
1175
- feedbackKey: string;
1176
- criterionId: string;
1177
- kind: "category";
1178
- } | {
1094
+ status: "cancelled" | "completed" | "failed";
1095
+ cases: {
1096
+ identity: {
1097
+ caseId: string;
1098
+ seed: string;
1099
+ fixtureId: string | null;
1100
+ };
1101
+ status: "cancelled" | "completed" | "failed" | "unavailable";
1102
+ output: unknown;
1103
+ error: {
1104
+ code: string;
1105
+ message: string;
1106
+ } | null;
1107
+ feedback: {
1179
1108
  feedbackKey: string;
1180
- criterionId: string;
1181
- kind: "preference_score";
1182
- tieScore: number | null;
1183
- })[];
1184
- baseline: {
1185
- execution: {
1109
+ evaluator: {
1186
1110
  id: string;
1187
1111
  contentHash: string;
1112
+ revision: string | number;
1188
1113
  };
1189
- resultHash: string;
1190
- reviews: {
1191
- review: {
1192
- id: string;
1193
- revision: number;
1194
- contentHash: string;
1195
- };
1196
- decision: {
1197
- id: string;
1198
- revision: number;
1199
- contentHash: string;
1200
- };
1201
- }[];
1202
- };
1203
- candidate: {
1204
- execution: {
1114
+ status: "failed" | "pending" | "scored" | "unavailable";
1115
+ value: string | number | boolean | null;
1116
+ passed?: boolean | null | undefined;
1117
+ reasoning: string | null;
1118
+ evidenceRefs: {
1205
1119
  id: string;
1206
1120
  contentHash: string;
1207
- };
1208
- resultHash: string;
1209
- reviews: {
1210
- review: {
1211
- id: string;
1212
- revision: number;
1213
- contentHash: string;
1214
- };
1215
- decision: {
1216
- id: string;
1217
- revision: number;
1218
- contentHash: string;
1219
- };
1121
+ mediaType: string | null;
1122
+ sizeBytes: number | null;
1220
1123
  }[];
1124
+ }[];
1125
+ usage: {
1126
+ inputTokens: number | null;
1127
+ outputTokens: number | null;
1128
+ totalTokens: number | null;
1129
+ costUsd: number | null;
1130
+ latencyMs: number | null;
1221
1131
  };
1222
- };
1223
- original: {
1224
- baseline: {
1225
- id: string;
1226
- manifestHash: string;
1227
- resultHash: string;
1228
- };
1229
- candidate: {
1132
+ traceRef: {
1230
1133
  id: string;
1231
- manifestHash: string;
1232
- resultHash: string;
1233
- };
1234
- };
1235
- humanResults: {
1236
- baseline: {
1237
- schemaVersion: "openpond.humanResultView.v1";
1238
- executionId: string;
1239
- executionHash: string;
1240
- selection: {
1134
+ contentHash: string;
1135
+ mediaType: string | null;
1136
+ sizeBytes: number | null;
1137
+ } | null;
1138
+ startedAt: string | null;
1139
+ completedAt: string | null;
1140
+ }[];
1141
+ completedAt: string;
1142
+ contentHash: string;
1143
+ };
1144
+ };
1145
+ component: {
1146
+ kind: "agent" | "instruction" | "skill" | "workflow";
1147
+ path: string;
1148
+ workflowId?: string | undefined;
1149
+ };
1150
+ mode: "llm_assisted" | "manual";
1151
+ status: "accepted" | "adopting" | "authoring" | "authoring_queued" | "cancelled" | "declined" | "draft" | "failed" | "frozen" | "ready_for_review" | "rolled_back" | "testing";
1152
+ revision: number;
1153
+ limits: {
1154
+ maximumCostUsd: number;
1155
+ maximumAuthoringSteps: number;
1156
+ maximumDurationMs: number;
1157
+ };
1158
+ authoringSteps: number;
1159
+ work: {
1160
+ sessionId: string;
1161
+ turnId: string;
1162
+ revision: number;
1163
+ startedAt: string;
1164
+ } | null;
1165
+ frozen: {
1166
+ release: {
1167
+ id: string;
1168
+ contentHash: string;
1169
+ };
1170
+ sourceRevision: string;
1171
+ profileSourceRevision: string;
1172
+ authoringHash: string;
1173
+ partitionHash: string;
1174
+ executableSourceHash: string;
1175
+ protectedClosureHash: string;
1176
+ candidateRevision: number;
1177
+ diff: {
1178
+ path: string;
1179
+ before: string | null;
1180
+ after: string | null;
1181
+ }[];
1182
+ } | null;
1183
+ test: {
1184
+ recipe: {
1185
+ externalDatasetBinding: {
1186
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
1187
+ dataset: {
1188
+ id: string;
1189
+ contentHash: string;
1190
+ revision: number;
1191
+ };
1192
+ packageHash: string;
1193
+ profileId: string;
1194
+ target: {
1195
+ kind: "profile";
1196
+ } | {
1197
+ kind: "workflow";
1198
+ workflowId: string;
1199
+ } | {
1200
+ kind: "skill";
1201
+ skillPath: string;
1202
+ } | {
1203
+ kind: "agent_action";
1204
+ actionId: string;
1205
+ };
1206
+ declaredProfileCatalogHash: string | null;
1207
+ protectedProfileClosureHash: string;
1208
+ recordedOrigin?: {
1209
+ execution: {
1241
1210
  id: string;
1242
- revision: number;
1243
1211
  contentHash: string;
1244
- }[];
1245
- coverage: {
1246
- pending: number;
1247
- submitted: number;
1248
- accepted: number;
1249
- rejected: number;
1250
- cancelled: number;
1251
1212
  };
1252
- results: {
1253
- review: {
1254
- id: string;
1255
- revision: number;
1256
- contentHash: string;
1257
- };
1258
- decision: {
1259
- id: string;
1260
- revision: number;
1261
- contentHash: string;
1262
- } | null;
1263
- attemptIds: string[];
1264
- mode: "individual" | "pairwise";
1265
- origin: "hosted_retained" | "native_local" | "owner_attested_local" | "recorded_evidence";
1266
- status: "accepted" | "cancelled" | "pending" | "rejected" | "submitted";
1267
- grader: {
1268
- id: string;
1269
- revision: number;
1270
- contentHash: string;
1271
- };
1272
- answers: {
1273
- criterionId: string;
1274
- value: string | number | boolean | null;
1275
- abstain: boolean;
1276
- note: string;
1277
- strength?: "moderate" | "slight" | "strong" | undefined;
1278
- }[];
1279
- }[];
1280
- contentHash: string;
1281
- };
1282
- candidate: {
1283
- schemaVersion: "openpond.humanResultView.v1";
1284
- executionId: string;
1285
- executionHash: string;
1286
- selection: {
1213
+ originalEvidence: {
1287
1214
  id: string;
1288
- revision: number;
1289
1215
  contentHash: string;
1216
+ };
1217
+ members: {
1218
+ caseId: string;
1219
+ seed: string;
1220
+ fixtureId: string;
1221
+ sourceId: string;
1222
+ snapshotHash: string;
1223
+ boundaryRevisionHash: string;
1290
1224
  }[];
1291
- coverage: {
1292
- pending: number;
1293
- submitted: number;
1294
- accepted: number;
1295
- rejected: number;
1296
- cancelled: number;
1225
+ } | undefined;
1226
+ gradingSource?: {
1227
+ pass: {
1228
+ id: string;
1229
+ contentHash: string;
1297
1230
  };
1298
- results: {
1299
- review: {
1300
- id: string;
1301
- revision: number;
1302
- contentHash: string;
1303
- };
1304
- decision: {
1305
- id: string;
1306
- revision: number;
1307
- contentHash: string;
1308
- } | null;
1309
- attemptIds: string[];
1310
- mode: "individual" | "pairwise";
1311
- origin: "hosted_retained" | "native_local" | "owner_attested_local" | "recorded_evidence";
1312
- status: "accepted" | "cancelled" | "pending" | "rejected" | "submitted";
1313
- grader: {
1314
- id: string;
1315
- revision: number;
1316
- contentHash: string;
1317
- };
1318
- answers: {
1319
- criterionId: string;
1320
- value: string | number | boolean | null;
1321
- abstain: boolean;
1322
- note: string;
1323
- strength?: "moderate" | "slight" | "strong" | undefined;
1324
- }[];
1231
+ evidence: {
1232
+ id: string;
1233
+ contentHash: string;
1234
+ };
1235
+ } | undefined;
1236
+ split: "frozen_eval" | "test" | "train" | "validation";
1237
+ population: {
1238
+ taskId: string;
1239
+ seed: string;
1240
+ fixtureId: string | null;
1241
+ }[];
1242
+ evaluators: {
1243
+ release: {
1244
+ id: string;
1245
+ contentHash: string;
1246
+ revision: string | number;
1247
+ };
1248
+ feedbackKey: string;
1249
+ configurationHash: string;
1250
+ output: "boolean" | "category" | "score";
1251
+ categories: string[];
1252
+ }[];
1253
+ fieldMappingsHash: string;
1254
+ fieldMappings: {
1255
+ graderId: string;
1256
+ fields: {
1257
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1258
+ path: string[];
1259
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1260
+ sourcePath: string[];
1325
1261
  }[];
1326
- contentHash: string;
1262
+ }[];
1263
+ environmentHash: string;
1264
+ privateDatasetClosureHash: string;
1265
+ criterion: {
1266
+ minimumPassRate: number;
1267
+ requireComplete: boolean;
1327
1268
  };
1269
+ contentHash: string;
1328
1270
  };
1329
- baseline: {
1330
- manifest: {
1331
- schemaVersion: "openpond.experimentManifest.v1";
1271
+ scoring?: {
1272
+ sourcePass: {
1332
1273
  id: string;
1333
- name: string;
1334
- teamId: string;
1335
- operationId: string;
1336
- maximumCostUsd: number | null;
1337
- dataset: {
1338
- id: string;
1339
- contentHash: string;
1340
- revision: number;
1341
- };
1342
- target: {
1343
- kind: "model";
1344
- modelId: string;
1345
- configurationHash: string;
1346
- } | {
1347
- kind: "harness";
1348
- source: {
1349
- profileId: string;
1350
- sourceRevision: string;
1351
- harnessRelease: {
1352
- id: string;
1353
- contentHash: string;
1354
- };
1355
- catalogHash: string;
1356
- definitionId: string;
1357
- definitionHash: string;
1358
- target: {
1359
- kind: "profile";
1360
- } | {
1361
- kind: "workflow";
1362
- workflowId: string;
1363
- } | {
1364
- kind: "skill";
1365
- skillPath: string;
1366
- } | {
1367
- kind: "agent_action";
1368
- actionId: string;
1369
- };
1370
- environmentHash: string;
1371
- externalDatasetBinding?: {
1372
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
1373
- dataset: {
1274
+ contentHash: string;
1275
+ };
1276
+ graders: {
1277
+ id: string;
1278
+ contentHash: string;
1279
+ revision: number;
1280
+ }[];
1281
+ mappings?: {
1282
+ graderId: string;
1283
+ fields: {
1284
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1285
+ path: string[];
1286
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1287
+ sourcePath: string[];
1288
+ }[];
1289
+ }[] | undefined;
1290
+ maximumCostUsd: number;
1291
+ } | undefined;
1292
+ modelId: string;
1293
+ maximumCostUsd: number;
1294
+ };
1295
+ recipeHash: string;
1296
+ baselineOperationId: string;
1297
+ candidateOperationId: string;
1298
+ cancelRequestedAt: string | null;
1299
+ dispatching: {
1300
+ kind: "baseline" | "candidate";
1301
+ token: string;
1302
+ pid: number;
1303
+ processStart: string | null;
1304
+ startedAt: string;
1305
+ } | null;
1306
+ } | null;
1307
+ comparison: {
1308
+ baseline: {
1309
+ manifest: {
1310
+ schemaVersion: "openpond.experimentManifest.v1";
1311
+ id: string;
1312
+ name: string;
1313
+ teamId: string;
1314
+ operationId: string;
1315
+ maximumCostUsd: number | null;
1316
+ dataset: {
1317
+ id: string;
1318
+ contentHash: string;
1319
+ revision: number;
1320
+ };
1321
+ target: {
1322
+ kind: "model";
1323
+ modelId: string;
1324
+ configurationHash: string;
1325
+ } | {
1326
+ kind: "harness";
1327
+ source: {
1328
+ profileId: string;
1329
+ sourceRevision: string;
1330
+ harnessRelease: {
1331
+ id: string;
1332
+ contentHash: string;
1333
+ };
1334
+ catalogHash: string;
1335
+ definitionId: string;
1336
+ definitionHash: string;
1337
+ target: {
1338
+ kind: "profile";
1339
+ } | {
1340
+ kind: "workflow";
1341
+ workflowId: string;
1342
+ } | {
1343
+ kind: "skill";
1344
+ skillPath: string;
1345
+ } | {
1346
+ kind: "agent_action";
1347
+ actionId: string;
1348
+ };
1349
+ environmentHash: string;
1350
+ externalDatasetBinding?: {
1351
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
1352
+ dataset: {
1353
+ id: string;
1354
+ contentHash: string;
1355
+ revision: number;
1356
+ };
1357
+ packageHash: string;
1358
+ profileId: string;
1359
+ target: {
1360
+ kind: "profile";
1361
+ } | {
1362
+ kind: "workflow";
1363
+ workflowId: string;
1364
+ } | {
1365
+ kind: "skill";
1366
+ skillPath: string;
1367
+ } | {
1368
+ kind: "agent_action";
1369
+ actionId: string;
1370
+ };
1371
+ declaredProfileCatalogHash: string | null;
1372
+ protectedProfileClosureHash: string;
1373
+ recordedOrigin?: {
1374
+ execution: {
1374
1375
  id: string;
1375
1376
  contentHash: string;
1376
- revision: number;
1377
1377
  };
1378
- packageHash: string;
1379
- profileId: string;
1380
- target: {
1381
- kind: "profile";
1382
- } | {
1383
- kind: "workflow";
1384
- workflowId: string;
1385
- } | {
1386
- kind: "skill";
1387
- skillPath: string;
1388
- } | {
1389
- kind: "agent_action";
1390
- actionId: string;
1378
+ originalEvidence: {
1379
+ id: string;
1380
+ contentHash: string;
1391
1381
  };
1392
- declaredProfileCatalogHash: string | null;
1393
- protectedProfileClosureHash: string;
1394
- recordedOrigin?: {
1395
- execution: {
1396
- id: string;
1397
- contentHash: string;
1398
- };
1399
- originalEvidence: {
1400
- id: string;
1401
- contentHash: string;
1402
- };
1403
- members: {
1404
- caseId: string;
1405
- seed: string;
1406
- fixtureId: string;
1407
- sourceId: string;
1408
- snapshotHash: string;
1409
- boundaryRevisionHash: string;
1410
- }[];
1411
- } | undefined;
1412
- gradingSource?: {
1413
- pass: {
1414
- id: string;
1415
- contentHash: string;
1416
- };
1417
- evidence: {
1418
- id: string;
1419
- contentHash: string;
1420
- };
1421
- } | undefined;
1422
- split: "frozen_eval" | "test" | "train" | "validation";
1423
- population: {
1424
- taskId: string;
1382
+ members: {
1383
+ caseId: string;
1425
1384
  seed: string;
1426
- fixtureId: string | null;
1427
- }[];
1428
- evaluators: {
1429
- release: {
1430
- id: string;
1431
- contentHash: string;
1432
- revision: string | number;
1433
- };
1434
- feedbackKey: string;
1435
- configurationHash: string;
1436
- output: "boolean" | "category" | "score";
1437
- categories: string[];
1438
- }[];
1439
- fieldMappingsHash: string;
1440
- fieldMappings: {
1441
- graderId: string;
1442
- fields: {
1443
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1444
- path: string[];
1445
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1446
- sourcePath: string[];
1447
- }[];
1385
+ fixtureId: string;
1386
+ sourceId: string;
1387
+ snapshotHash: string;
1388
+ boundaryRevisionHash: string;
1448
1389
  }[];
1449
- environmentHash: string;
1450
- privateDatasetClosureHash: string;
1451
- criterion: {
1452
- minimumPassRate: number;
1453
- requireComplete: boolean;
1454
- };
1455
- contentHash: string;
1456
1390
  } | undefined;
1457
- sourceCandidate?: {
1458
- candidateId: string;
1459
- frozenRevision: number;
1460
- frozenHash: string;
1461
- harnessRelease: {
1391
+ gradingSource?: {
1392
+ pass: {
1393
+ id: string;
1394
+ contentHash: string;
1395
+ };
1396
+ evidence: {
1462
1397
  id: string;
1463
1398
  contentHash: string;
1464
1399
  };
1465
- protectedClosureHash: string;
1466
1400
  } | undefined;
1467
- };
1468
- model: {
1469
- modelId: string;
1470
- configurationHash: string;
1471
- };
1472
- } | {
1473
- kind: "agent";
1474
- source: {
1475
- harnessRelease: {
1476
- id: string;
1477
- contentHash: string;
1401
+ split: "frozen_eval" | "test" | "train" | "validation";
1402
+ population: {
1403
+ taskId: string;
1404
+ seed: string;
1405
+ fixtureId: string | null;
1406
+ }[];
1407
+ evaluators: {
1408
+ release: {
1409
+ id: string;
1410
+ contentHash: string;
1411
+ revision: string | number;
1412
+ };
1413
+ feedbackKey: string;
1414
+ configurationHash: string;
1415
+ output: "boolean" | "category" | "score";
1416
+ categories: string[];
1417
+ }[];
1418
+ fieldMappingsHash: string;
1419
+ fieldMappings: {
1420
+ graderId: string;
1421
+ fields: {
1422
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1423
+ path: string[];
1424
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1425
+ sourcePath: string[];
1426
+ }[];
1427
+ }[];
1428
+ environmentHash: string;
1429
+ privateDatasetClosureHash: string;
1430
+ criterion: {
1431
+ minimumPassRate: number;
1432
+ requireComplete: boolean;
1478
1433
  };
1479
- agentSnapshot: {
1434
+ contentHash: string;
1435
+ } | undefined;
1436
+ sourceCandidate?: {
1437
+ candidateId: string;
1438
+ frozenRevision: number;
1439
+ frozenHash: string;
1440
+ harnessRelease: {
1480
1441
  id: string;
1481
1442
  contentHash: string;
1482
1443
  };
1483
- sourcePackageHash: string;
1484
- };
1485
- model: {
1486
- modelId: string;
1487
- configurationHash: string;
1488
- };
1489
- } | {
1490
- kind: "fixture";
1444
+ protectedClosureHash: string;
1445
+ } | undefined;
1446
+ };
1447
+ model: {
1448
+ modelId: string;
1491
1449
  configurationHash: string;
1492
1450
  };
1493
- execution?: {
1494
- packageHash: string;
1495
- runtimeTargetHash: string;
1496
- metricPolicyHash: string;
1497
- compatibility?: {
1498
- protocol: "openpond.evaluation-execution.v1";
1499
- targetKind: "agent" | "fixture" | "harness" | "model";
1500
- } | undefined;
1501
- } | undefined;
1502
- lineage?: {
1503
- definition: {
1451
+ } | {
1452
+ kind: "agent";
1453
+ source: {
1454
+ harnessRelease: {
1504
1455
  id: string;
1505
1456
  contentHash: string;
1506
- revision: number;
1507
- } | null;
1508
- execution: {
1457
+ };
1458
+ agentSnapshot: {
1509
1459
  id: string;
1510
1460
  contentHash: string;
1511
1461
  };
1512
- scoringPassId: string | null;
1513
- humanProjection?: {
1514
- originalManifest: {
1515
- id: string;
1516
- contentHash: string;
1517
- };
1518
- selectionHash: string;
1519
- rulesHash: string;
1520
- } | undefined;
1462
+ sourcePackageHash: string;
1463
+ };
1464
+ model: {
1465
+ modelId: string;
1466
+ configurationHash: string;
1467
+ };
1468
+ } | {
1469
+ kind: "fixture";
1470
+ configurationHash: string;
1471
+ };
1472
+ execution?: {
1473
+ packageHash: string;
1474
+ runtimeTargetHash: string;
1475
+ metricPolicyHash: string;
1476
+ compatibility?: {
1477
+ protocol: "openpond.evaluation-execution.v1";
1478
+ targetKind: "agent" | "fixture" | "harness" | "model";
1521
1479
  } | undefined;
1522
- evaluators: {
1523
- release: {
1480
+ } | undefined;
1481
+ lineage?: {
1482
+ definition: {
1483
+ id: string;
1484
+ contentHash: string;
1485
+ revision: number;
1486
+ } | null;
1487
+ execution: {
1488
+ id: string;
1489
+ contentHash: string;
1490
+ };
1491
+ scoringPassId: string | null;
1492
+ humanProjection?: {
1493
+ originalManifest: {
1524
1494
  id: string;
1525
1495
  contentHash: string;
1526
- revision: string | number;
1527
1496
  };
1528
- feedbackKey: string;
1529
- configurationHash?: string | undefined;
1530
- output: "boolean" | "category" | "score";
1531
- categories: string[];
1532
- }[];
1533
- population: {
1497
+ selectionHash: string;
1498
+ rulesHash: string;
1499
+ } | undefined;
1500
+ } | undefined;
1501
+ evaluators: {
1502
+ release: {
1503
+ id: string;
1504
+ contentHash: string;
1505
+ revision: string | number;
1506
+ };
1507
+ feedbackKey: string;
1508
+ configurationHash?: string | undefined;
1509
+ output: "boolean" | "category" | "score";
1510
+ categories: string[];
1511
+ }[];
1512
+ population: {
1513
+ caseId: string;
1514
+ seed: string;
1515
+ fixtureId: string | null;
1516
+ }[];
1517
+ createdAt: string;
1518
+ contentHash: string;
1519
+ };
1520
+ result: {
1521
+ schemaVersion: "openpond.experimentResult.v1";
1522
+ manifest: {
1523
+ id: string;
1524
+ contentHash: string;
1525
+ };
1526
+ status: "cancelled" | "completed" | "failed";
1527
+ cases: {
1528
+ identity: {
1534
1529
  caseId: string;
1535
1530
  seed: string;
1536
1531
  fixtureId: string | null;
1537
- }[];
1538
- createdAt: string;
1539
- contentHash: string;
1540
- };
1541
- result: {
1542
- schemaVersion: "openpond.experimentResult.v1";
1543
- manifest: {
1544
- id: string;
1545
- contentHash: string;
1546
1532
  };
1547
- status: "cancelled" | "completed" | "failed";
1548
- cases: {
1549
- identity: {
1550
- caseId: string;
1551
- seed: string;
1552
- fixtureId: string | null;
1553
- };
1554
- status: "cancelled" | "completed" | "failed" | "unavailable";
1555
- output: unknown;
1556
- error: {
1557
- code: string;
1558
- message: string;
1559
- } | null;
1560
- feedback: {
1561
- feedbackKey: string;
1562
- evaluator: {
1563
- id: string;
1564
- contentHash: string;
1565
- revision: string | number;
1566
- };
1567
- status: "failed" | "pending" | "scored" | "unavailable";
1568
- value: string | number | boolean | null;
1569
- passed?: boolean | null | undefined;
1570
- reasoning: string | null;
1571
- evidenceRefs: {
1572
- id: string;
1573
- contentHash: string;
1574
- mediaType: string | null;
1575
- sizeBytes: number | null;
1576
- }[];
1577
- }[];
1578
- usage: {
1579
- inputTokens: number | null;
1580
- outputTokens: number | null;
1581
- totalTokens: number | null;
1582
- costUsd: number | null;
1583
- latencyMs: number | null;
1533
+ status: "cancelled" | "completed" | "failed" | "unavailable";
1534
+ output: unknown;
1535
+ error: {
1536
+ code: string;
1537
+ message: string;
1538
+ } | null;
1539
+ feedback: {
1540
+ feedbackKey: string;
1541
+ evaluator: {
1542
+ id: string;
1543
+ contentHash: string;
1544
+ revision: string | number;
1584
1545
  };
1585
- traceRef: {
1546
+ status: "failed" | "pending" | "scored" | "unavailable";
1547
+ value: string | number | boolean | null;
1548
+ passed?: boolean | null | undefined;
1549
+ reasoning: string | null;
1550
+ evidenceRefs: {
1586
1551
  id: string;
1587
1552
  contentHash: string;
1588
1553
  mediaType: string | null;
1589
1554
  sizeBytes: number | null;
1590
- } | null;
1591
- startedAt: string | null;
1592
- completedAt: string | null;
1555
+ }[];
1593
1556
  }[];
1594
- completedAt: string;
1595
- contentHash: string;
1596
- };
1597
- };
1598
- candidate: {
1599
- manifest: {
1600
- schemaVersion: "openpond.experimentManifest.v1";
1601
- id: string;
1602
- name: string;
1603
- teamId: string;
1604
- operationId: string;
1605
- maximumCostUsd: number | null;
1606
- dataset: {
1557
+ usage: {
1558
+ inputTokens: number | null;
1559
+ outputTokens: number | null;
1560
+ totalTokens: number | null;
1561
+ costUsd: number | null;
1562
+ latencyMs: number | null;
1563
+ };
1564
+ traceRef: {
1607
1565
  id: string;
1608
1566
  contentHash: string;
1609
- revision: number;
1610
- };
1611
- target: {
1612
- kind: "model";
1613
- modelId: string;
1614
- configurationHash: string;
1615
- } | {
1616
- kind: "harness";
1617
- source: {
1618
- profileId: string;
1619
- sourceRevision: string;
1620
- harnessRelease: {
1621
- id: string;
1622
- contentHash: string;
1623
- };
1624
- catalogHash: string;
1625
- definitionId: string;
1626
- definitionHash: string;
1627
- target: {
1628
- kind: "profile";
1629
- } | {
1630
- kind: "workflow";
1631
- workflowId: string;
1632
- } | {
1633
- kind: "skill";
1634
- skillPath: string;
1635
- } | {
1636
- kind: "agent_action";
1637
- actionId: string;
1638
- };
1639
- environmentHash: string;
1640
- externalDatasetBinding?: {
1641
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
1642
- dataset: {
1643
- id: string;
1644
- contentHash: string;
1645
- revision: number;
1646
- };
1647
- packageHash: string;
1648
- profileId: string;
1649
- target: {
1650
- kind: "profile";
1651
- } | {
1652
- kind: "workflow";
1653
- workflowId: string;
1654
- } | {
1655
- kind: "skill";
1656
- skillPath: string;
1657
- } | {
1658
- kind: "agent_action";
1659
- actionId: string;
1660
- };
1661
- declaredProfileCatalogHash: string | null;
1662
- protectedProfileClosureHash: string;
1663
- recordedOrigin?: {
1664
- execution: {
1665
- id: string;
1666
- contentHash: string;
1667
- };
1668
- originalEvidence: {
1669
- id: string;
1670
- contentHash: string;
1671
- };
1672
- members: {
1673
- caseId: string;
1674
- seed: string;
1675
- fixtureId: string;
1676
- sourceId: string;
1677
- snapshotHash: string;
1678
- boundaryRevisionHash: string;
1679
- }[];
1680
- } | undefined;
1681
- gradingSource?: {
1682
- pass: {
1683
- id: string;
1684
- contentHash: string;
1685
- };
1686
- evidence: {
1687
- id: string;
1688
- contentHash: string;
1689
- };
1690
- } | undefined;
1691
- split: "frozen_eval" | "test" | "train" | "validation";
1692
- population: {
1693
- taskId: string;
1567
+ mediaType: string | null;
1568
+ sizeBytes: number | null;
1569
+ } | null;
1570
+ startedAt: string | null;
1571
+ completedAt: string | null;
1572
+ }[];
1573
+ completedAt: string;
1574
+ contentHash: string;
1575
+ };
1576
+ };
1577
+ candidate: {
1578
+ manifest: {
1579
+ schemaVersion: "openpond.experimentManifest.v1";
1580
+ id: string;
1581
+ name: string;
1582
+ teamId: string;
1583
+ operationId: string;
1584
+ maximumCostUsd: number | null;
1585
+ dataset: {
1586
+ id: string;
1587
+ contentHash: string;
1588
+ revision: number;
1589
+ };
1590
+ target: {
1591
+ kind: "model";
1592
+ modelId: string;
1593
+ configurationHash: string;
1594
+ } | {
1595
+ kind: "harness";
1596
+ source: {
1597
+ profileId: string;
1598
+ sourceRevision: string;
1599
+ harnessRelease: {
1600
+ id: string;
1601
+ contentHash: string;
1602
+ };
1603
+ catalogHash: string;
1604
+ definitionId: string;
1605
+ definitionHash: string;
1606
+ target: {
1607
+ kind: "profile";
1608
+ } | {
1609
+ kind: "workflow";
1610
+ workflowId: string;
1611
+ } | {
1612
+ kind: "skill";
1613
+ skillPath: string;
1614
+ } | {
1615
+ kind: "agent_action";
1616
+ actionId: string;
1617
+ };
1618
+ environmentHash: string;
1619
+ externalDatasetBinding?: {
1620
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
1621
+ dataset: {
1622
+ id: string;
1623
+ contentHash: string;
1624
+ revision: number;
1625
+ };
1626
+ packageHash: string;
1627
+ profileId: string;
1628
+ target: {
1629
+ kind: "profile";
1630
+ } | {
1631
+ kind: "workflow";
1632
+ workflowId: string;
1633
+ } | {
1634
+ kind: "skill";
1635
+ skillPath: string;
1636
+ } | {
1637
+ kind: "agent_action";
1638
+ actionId: string;
1639
+ };
1640
+ declaredProfileCatalogHash: string | null;
1641
+ protectedProfileClosureHash: string;
1642
+ recordedOrigin?: {
1643
+ execution: {
1644
+ id: string;
1645
+ contentHash: string;
1646
+ };
1647
+ originalEvidence: {
1648
+ id: string;
1649
+ contentHash: string;
1650
+ };
1651
+ members: {
1652
+ caseId: string;
1694
1653
  seed: string;
1695
- fixtureId: string | null;
1696
- }[];
1697
- evaluators: {
1698
- release: {
1699
- id: string;
1700
- contentHash: string;
1701
- revision: string | number;
1702
- };
1703
- feedbackKey: string;
1704
- configurationHash: string;
1705
- output: "boolean" | "category" | "score";
1706
- categories: string[];
1707
- }[];
1708
- fieldMappingsHash: string;
1709
- fieldMappings: {
1710
- graderId: string;
1711
- fields: {
1712
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1713
- path: string[];
1714
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1715
- sourcePath: string[];
1716
- }[];
1654
+ fixtureId: string;
1655
+ sourceId: string;
1656
+ snapshotHash: string;
1657
+ boundaryRevisionHash: string;
1717
1658
  }[];
1718
- environmentHash: string;
1719
- privateDatasetClosureHash: string;
1720
- criterion: {
1721
- minimumPassRate: number;
1722
- requireComplete: boolean;
1723
- };
1724
- contentHash: string;
1725
1659
  } | undefined;
1726
- sourceCandidate?: {
1727
- candidateId: string;
1728
- frozenRevision: number;
1729
- frozenHash: string;
1730
- harnessRelease: {
1660
+ gradingSource?: {
1661
+ pass: {
1662
+ id: string;
1663
+ contentHash: string;
1664
+ };
1665
+ evidence: {
1731
1666
  id: string;
1732
1667
  contentHash: string;
1733
1668
  };
1734
- protectedClosureHash: string;
1735
1669
  } | undefined;
1736
- };
1737
- model: {
1738
- modelId: string;
1739
- configurationHash: string;
1740
- };
1741
- } | {
1742
- kind: "agent";
1743
- source: {
1744
- harnessRelease: {
1745
- id: string;
1746
- contentHash: string;
1670
+ split: "frozen_eval" | "test" | "train" | "validation";
1671
+ population: {
1672
+ taskId: string;
1673
+ seed: string;
1674
+ fixtureId: string | null;
1675
+ }[];
1676
+ evaluators: {
1677
+ release: {
1678
+ id: string;
1679
+ contentHash: string;
1680
+ revision: string | number;
1681
+ };
1682
+ feedbackKey: string;
1683
+ configurationHash: string;
1684
+ output: "boolean" | "category" | "score";
1685
+ categories: string[];
1686
+ }[];
1687
+ fieldMappingsHash: string;
1688
+ fieldMappings: {
1689
+ graderId: string;
1690
+ fields: {
1691
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
1692
+ path: string[];
1693
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
1694
+ sourcePath: string[];
1695
+ }[];
1696
+ }[];
1697
+ environmentHash: string;
1698
+ privateDatasetClosureHash: string;
1699
+ criterion: {
1700
+ minimumPassRate: number;
1701
+ requireComplete: boolean;
1747
1702
  };
1748
- agentSnapshot: {
1703
+ contentHash: string;
1704
+ } | undefined;
1705
+ sourceCandidate?: {
1706
+ candidateId: string;
1707
+ frozenRevision: number;
1708
+ frozenHash: string;
1709
+ harnessRelease: {
1749
1710
  id: string;
1750
1711
  contentHash: string;
1751
1712
  };
1752
- sourcePackageHash: string;
1753
- };
1754
- model: {
1755
- modelId: string;
1756
- configurationHash: string;
1757
- };
1758
- } | {
1759
- kind: "fixture";
1713
+ protectedClosureHash: string;
1714
+ } | undefined;
1715
+ };
1716
+ model: {
1717
+ modelId: string;
1760
1718
  configurationHash: string;
1761
1719
  };
1762
- execution?: {
1763
- packageHash: string;
1764
- runtimeTargetHash: string;
1765
- metricPolicyHash: string;
1766
- compatibility?: {
1767
- protocol: "openpond.evaluation-execution.v1";
1768
- targetKind: "agent" | "fixture" | "harness" | "model";
1769
- } | undefined;
1770
- } | undefined;
1771
- lineage?: {
1772
- definition: {
1773
- id: string;
1774
- contentHash: string;
1775
- revision: number;
1776
- } | null;
1777
- execution: {
1720
+ } | {
1721
+ kind: "agent";
1722
+ source: {
1723
+ harnessRelease: {
1778
1724
  id: string;
1779
1725
  contentHash: string;
1780
1726
  };
1781
- scoringPassId: string | null;
1782
- humanProjection?: {
1783
- originalManifest: {
1784
- id: string;
1785
- contentHash: string;
1786
- };
1787
- selectionHash: string;
1788
- rulesHash: string;
1789
- } | undefined;
1790
- } | undefined;
1791
- evaluators: {
1792
- release: {
1727
+ agentSnapshot: {
1793
1728
  id: string;
1794
1729
  contentHash: string;
1795
- revision: string | number;
1796
1730
  };
1797
- feedbackKey: string;
1798
- configurationHash?: string | undefined;
1799
- output: "boolean" | "category" | "score";
1800
- categories: string[];
1801
- }[];
1802
- population: {
1803
- caseId: string;
1804
- seed: string;
1805
- fixtureId: string | null;
1806
- }[];
1807
- createdAt: string;
1808
- contentHash: string;
1731
+ sourcePackageHash: string;
1732
+ };
1733
+ model: {
1734
+ modelId: string;
1735
+ configurationHash: string;
1736
+ };
1737
+ } | {
1738
+ kind: "fixture";
1739
+ configurationHash: string;
1809
1740
  };
1810
- result: {
1811
- schemaVersion: "openpond.experimentResult.v1";
1812
- manifest: {
1741
+ execution?: {
1742
+ packageHash: string;
1743
+ runtimeTargetHash: string;
1744
+ metricPolicyHash: string;
1745
+ compatibility?: {
1746
+ protocol: "openpond.evaluation-execution.v1";
1747
+ targetKind: "agent" | "fixture" | "harness" | "model";
1748
+ } | undefined;
1749
+ } | undefined;
1750
+ lineage?: {
1751
+ definition: {
1752
+ id: string;
1753
+ contentHash: string;
1754
+ revision: number;
1755
+ } | null;
1756
+ execution: {
1813
1757
  id: string;
1814
1758
  contentHash: string;
1815
1759
  };
1816
- status: "cancelled" | "completed" | "failed";
1817
- cases: {
1818
- identity: {
1819
- caseId: string;
1820
- seed: string;
1821
- fixtureId: string | null;
1760
+ scoringPassId: string | null;
1761
+ humanProjection?: {
1762
+ originalManifest: {
1763
+ id: string;
1764
+ contentHash: string;
1822
1765
  };
1823
- status: "cancelled" | "completed" | "failed" | "unavailable";
1824
- output: unknown;
1825
- error: {
1826
- code: string;
1827
- message: string;
1828
- } | null;
1829
- feedback: {
1830
- feedbackKey: string;
1831
- evaluator: {
1832
- id: string;
1833
- contentHash: string;
1834
- revision: string | number;
1835
- };
1836
- status: "failed" | "pending" | "scored" | "unavailable";
1837
- value: string | number | boolean | null;
1838
- passed?: boolean | null | undefined;
1839
- reasoning: string | null;
1840
- evidenceRefs: {
1841
- id: string;
1842
- contentHash: string;
1843
- mediaType: string | null;
1844
- sizeBytes: number | null;
1845
- }[];
1846
- }[];
1847
- usage: {
1848
- inputTokens: number | null;
1849
- outputTokens: number | null;
1850
- totalTokens: number | null;
1851
- costUsd: number | null;
1852
- latencyMs: number | null;
1766
+ selectionHash: string;
1767
+ rulesHash: string;
1768
+ } | undefined;
1769
+ } | undefined;
1770
+ evaluators: {
1771
+ release: {
1772
+ id: string;
1773
+ contentHash: string;
1774
+ revision: string | number;
1775
+ };
1776
+ feedbackKey: string;
1777
+ configurationHash?: string | undefined;
1778
+ output: "boolean" | "category" | "score";
1779
+ categories: string[];
1780
+ }[];
1781
+ population: {
1782
+ caseId: string;
1783
+ seed: string;
1784
+ fixtureId: string | null;
1785
+ }[];
1786
+ createdAt: string;
1787
+ contentHash: string;
1788
+ };
1789
+ result: {
1790
+ schemaVersion: "openpond.experimentResult.v1";
1791
+ manifest: {
1792
+ id: string;
1793
+ contentHash: string;
1794
+ };
1795
+ status: "cancelled" | "completed" | "failed";
1796
+ cases: {
1797
+ identity: {
1798
+ caseId: string;
1799
+ seed: string;
1800
+ fixtureId: string | null;
1801
+ };
1802
+ status: "cancelled" | "completed" | "failed" | "unavailable";
1803
+ output: unknown;
1804
+ error: {
1805
+ code: string;
1806
+ message: string;
1807
+ } | null;
1808
+ feedback: {
1809
+ feedbackKey: string;
1810
+ evaluator: {
1811
+ id: string;
1812
+ contentHash: string;
1813
+ revision: string | number;
1853
1814
  };
1854
- traceRef: {
1815
+ status: "failed" | "pending" | "scored" | "unavailable";
1816
+ value: string | number | boolean | null;
1817
+ passed?: boolean | null | undefined;
1818
+ reasoning: string | null;
1819
+ evidenceRefs: {
1855
1820
  id: string;
1856
1821
  contentHash: string;
1857
1822
  mediaType: string | null;
1858
1823
  sizeBytes: number | null;
1859
- } | null;
1860
- startedAt: string | null;
1861
- completedAt: string | null;
1824
+ }[];
1862
1825
  }[];
1863
- completedAt: string;
1864
- contentHash: string;
1865
- };
1826
+ usage: {
1827
+ inputTokens: number | null;
1828
+ outputTokens: number | null;
1829
+ totalTokens: number | null;
1830
+ costUsd: number | null;
1831
+ latencyMs: number | null;
1832
+ };
1833
+ traceRef: {
1834
+ id: string;
1835
+ contentHash: string;
1836
+ mediaType: string | null;
1837
+ sizeBytes: number | null;
1838
+ } | null;
1839
+ startedAt: string | null;
1840
+ completedAt: string | null;
1841
+ }[];
1842
+ completedAt: string;
1843
+ contentHash: string;
1866
1844
  };
1867
- contentHash: string;
1868
- } | undefined;
1869
- } | null;
1870
- adoptionIntent: {
1871
- operationId: string;
1872
- expectedOwnerRevision: number;
1845
+ };
1873
1846
  frozenHash: string;
1874
1847
  humanProjection?: {
1875
1848
  schemaVersion: "openpond.humanComparisonProjection.v1";
@@ -2583,706 +2556,733 @@ export declare class ExperimentImprovementClient {
2583
2556
  contentHash: string;
2584
2557
  } | undefined;
2585
2558
  } | null;
2586
- acceptance: {
2587
- id: string;
2588
- contentHash: string;
2589
- } | null;
2590
- rollback: {
2591
- id: string;
2592
- contentHash: string;
2593
- } | null;
2594
- error: string | null;
2595
- createdAt: string;
2596
- updatedAt: string;
2597
- contentHash: string;
2598
- } | {
2599
- candidateId: string;
2600
- actorId: string;
2601
- teamId: string;
2602
- projectId: string | null;
2603
- candidateRevision: number;
2604
- recipeHash: string;
2605
- baseline: {
2606
- manifest: {
2607
- schemaVersion: "openpond.experimentManifest.v1";
2608
- id: string;
2609
- name: string;
2559
+ adoptionIntent: {
2560
+ operationId: string;
2561
+ expectedOwnerRevision: number;
2562
+ frozenHash: string;
2563
+ humanProjection?: {
2564
+ schemaVersion: "openpond.humanComparisonProjection.v1";
2610
2565
  teamId: string;
2611
- operationId: string;
2612
- maximumCostUsd: number | null;
2613
- dataset: {
2614
- id: string;
2615
- contentHash: string;
2616
- revision: number;
2617
- };
2618
- target: {
2619
- kind: "model";
2620
- modelId: string;
2621
- configurationHash: string;
2622
- } | {
2623
- kind: "harness";
2624
- source: {
2625
- profileId: string;
2626
- sourceRevision: string;
2627
- harnessRelease: {
2566
+ projectId: string;
2567
+ selections: {
2568
+ rules: ({
2569
+ feedbackKey: string;
2570
+ criterionId: string;
2571
+ kind: "normalized_score";
2572
+ minimum: number;
2573
+ maximum: number;
2574
+ passThreshold?: number | undefined;
2575
+ } | {
2576
+ feedbackKey: string;
2577
+ criterionId: string;
2578
+ kind: "boolean";
2579
+ } | {
2580
+ feedbackKey: string;
2581
+ criterionId: string;
2582
+ kind: "category";
2583
+ } | {
2584
+ feedbackKey: string;
2585
+ criterionId: string;
2586
+ kind: "preference_score";
2587
+ tieScore: number | null;
2588
+ })[];
2589
+ baseline: {
2590
+ execution: {
2628
2591
  id: string;
2629
2592
  contentHash: string;
2630
2593
  };
2631
- catalogHash: string;
2632
- definitionId: string;
2633
- definitionHash: string;
2634
- target: {
2635
- kind: "profile";
2636
- } | {
2637
- kind: "workflow";
2638
- workflowId: string;
2639
- } | {
2640
- kind: "skill";
2641
- skillPath: string;
2642
- } | {
2643
- kind: "agent_action";
2644
- actionId: string;
2645
- };
2646
- environmentHash: string;
2647
- externalDatasetBinding?: {
2648
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
2649
- dataset: {
2594
+ resultHash: string;
2595
+ reviews: {
2596
+ review: {
2650
2597
  id: string;
2651
- contentHash: string;
2652
2598
  revision: number;
2599
+ contentHash: string;
2653
2600
  };
2654
- packageHash: string;
2655
- profileId: string;
2656
- target: {
2657
- kind: "profile";
2658
- } | {
2659
- kind: "workflow";
2660
- workflowId: string;
2661
- } | {
2662
- kind: "skill";
2663
- skillPath: string;
2664
- } | {
2665
- kind: "agent_action";
2666
- actionId: string;
2667
- };
2668
- declaredProfileCatalogHash: string | null;
2669
- protectedProfileClosureHash: string;
2670
- recordedOrigin?: {
2671
- execution: {
2672
- id: string;
2673
- contentHash: string;
2674
- };
2675
- originalEvidence: {
2676
- id: string;
2677
- contentHash: string;
2678
- };
2679
- members: {
2680
- caseId: string;
2681
- seed: string;
2682
- fixtureId: string;
2683
- sourceId: string;
2684
- snapshotHash: string;
2685
- boundaryRevisionHash: string;
2686
- }[];
2687
- } | undefined;
2688
- gradingSource?: {
2689
- pass: {
2690
- id: string;
2691
- contentHash: string;
2692
- };
2693
- evidence: {
2694
- id: string;
2695
- contentHash: string;
2696
- };
2697
- } | undefined;
2698
- split: "frozen_eval" | "test" | "train" | "validation";
2699
- population: {
2700
- taskId: string;
2701
- seed: string;
2702
- fixtureId: string | null;
2703
- }[];
2704
- evaluators: {
2705
- release: {
2706
- id: string;
2707
- contentHash: string;
2708
- revision: string | number;
2709
- };
2710
- feedbackKey: string;
2711
- configurationHash: string;
2712
- output: "boolean" | "category" | "score";
2713
- categories: string[];
2714
- }[];
2715
- fieldMappingsHash: string;
2716
- fieldMappings: {
2717
- graderId: string;
2718
- fields: {
2719
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
2720
- path: string[];
2721
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
2722
- sourcePath: string[];
2723
- }[];
2724
- }[];
2725
- environmentHash: string;
2726
- privateDatasetClosureHash: string;
2727
- criterion: {
2728
- minimumPassRate: number;
2729
- requireComplete: boolean;
2730
- };
2731
- contentHash: string;
2732
- } | undefined;
2733
- sourceCandidate?: {
2734
- candidateId: string;
2735
- frozenRevision: number;
2736
- frozenHash: string;
2737
- harnessRelease: {
2601
+ decision: {
2738
2602
  id: string;
2603
+ revision: number;
2739
2604
  contentHash: string;
2740
2605
  };
2741
- protectedClosureHash: string;
2742
- } | undefined;
2743
- };
2744
- model: {
2745
- modelId: string;
2746
- configurationHash: string;
2606
+ }[];
2747
2607
  };
2748
- } | {
2749
- kind: "agent";
2750
- source: {
2751
- harnessRelease: {
2752
- id: string;
2753
- contentHash: string;
2754
- };
2755
- agentSnapshot: {
2608
+ candidate: {
2609
+ execution: {
2756
2610
  id: string;
2757
2611
  contentHash: string;
2758
2612
  };
2759
- sourcePackageHash: string;
2760
- };
2761
- model: {
2762
- modelId: string;
2763
- configurationHash: string;
2613
+ resultHash: string;
2614
+ reviews: {
2615
+ review: {
2616
+ id: string;
2617
+ revision: number;
2618
+ contentHash: string;
2619
+ };
2620
+ decision: {
2621
+ id: string;
2622
+ revision: number;
2623
+ contentHash: string;
2624
+ };
2625
+ }[];
2764
2626
  };
2765
- } | {
2766
- kind: "fixture";
2767
- configurationHash: string;
2768
2627
  };
2769
- execution?: {
2770
- packageHash: string;
2771
- runtimeTargetHash: string;
2772
- metricPolicyHash: string;
2773
- compatibility?: {
2774
- protocol: "openpond.evaluation-execution.v1";
2775
- targetKind: "agent" | "fixture" | "harness" | "model";
2776
- } | undefined;
2777
- } | undefined;
2778
- lineage?: {
2779
- definition: {
2628
+ original: {
2629
+ baseline: {
2780
2630
  id: string;
2781
- contentHash: string;
2782
- revision: number;
2783
- } | null;
2784
- execution: {
2631
+ manifestHash: string;
2632
+ resultHash: string;
2633
+ };
2634
+ candidate: {
2785
2635
  id: string;
2786
- contentHash: string;
2636
+ manifestHash: string;
2637
+ resultHash: string;
2787
2638
  };
2788
- scoringPassId: string | null;
2789
- humanProjection?: {
2790
- originalManifest: {
2639
+ };
2640
+ humanResults: {
2641
+ baseline: {
2642
+ schemaVersion: "openpond.humanResultView.v1";
2643
+ executionId: string;
2644
+ executionHash: string;
2645
+ selection: {
2791
2646
  id: string;
2647
+ revision: number;
2792
2648
  contentHash: string;
2649
+ }[];
2650
+ coverage: {
2651
+ pending: number;
2652
+ submitted: number;
2653
+ accepted: number;
2654
+ rejected: number;
2655
+ cancelled: number;
2793
2656
  };
2794
- selectionHash: string;
2795
- rulesHash: string;
2796
- } | undefined;
2797
- } | undefined;
2798
- evaluators: {
2799
- release: {
2800
- id: string;
2657
+ results: {
2658
+ review: {
2659
+ id: string;
2660
+ revision: number;
2661
+ contentHash: string;
2662
+ };
2663
+ decision: {
2664
+ id: string;
2665
+ revision: number;
2666
+ contentHash: string;
2667
+ } | null;
2668
+ attemptIds: string[];
2669
+ mode: "individual" | "pairwise";
2670
+ origin: "hosted_retained" | "native_local" | "owner_attested_local" | "recorded_evidence";
2671
+ status: "accepted" | "cancelled" | "pending" | "rejected" | "submitted";
2672
+ grader: {
2673
+ id: string;
2674
+ revision: number;
2675
+ contentHash: string;
2676
+ };
2677
+ answers: {
2678
+ criterionId: string;
2679
+ value: string | number | boolean | null;
2680
+ abstain: boolean;
2681
+ note: string;
2682
+ strength?: "moderate" | "slight" | "strong" | undefined;
2683
+ }[];
2684
+ }[];
2801
2685
  contentHash: string;
2802
- revision: string | number;
2803
- };
2804
- feedbackKey: string;
2805
- configurationHash?: string | undefined;
2806
- output: "boolean" | "category" | "score";
2807
- categories: string[];
2808
- }[];
2809
- population: {
2810
- caseId: string;
2811
- seed: string;
2812
- fixtureId: string | null;
2813
- }[];
2814
- createdAt: string;
2815
- contentHash: string;
2816
- };
2817
- result: {
2818
- schemaVersion: "openpond.experimentResult.v1";
2819
- manifest: {
2820
- id: string;
2821
- contentHash: string;
2822
- };
2823
- status: "cancelled" | "completed" | "failed";
2824
- cases: {
2825
- identity: {
2826
- caseId: string;
2827
- seed: string;
2828
- fixtureId: string | null;
2829
2686
  };
2830
- status: "cancelled" | "completed" | "failed" | "unavailable";
2831
- output: unknown;
2832
- error: {
2833
- code: string;
2834
- message: string;
2835
- } | null;
2836
- feedback: {
2837
- feedbackKey: string;
2838
- evaluator: {
2687
+ candidate: {
2688
+ schemaVersion: "openpond.humanResultView.v1";
2689
+ executionId: string;
2690
+ executionHash: string;
2691
+ selection: {
2839
2692
  id: string;
2693
+ revision: number;
2840
2694
  contentHash: string;
2841
- revision: string | number;
2842
- };
2843
- status: "failed" | "pending" | "scored" | "unavailable";
2844
- value: string | number | boolean | null;
2845
- passed?: boolean | null | undefined;
2846
- reasoning: string | null;
2847
- evidenceRefs: {
2848
- id: string;
2849
- contentHash: string;
2850
- mediaType: string | null;
2851
- sizeBytes: number | null;
2852
2695
  }[];
2853
- }[];
2854
- usage: {
2855
- inputTokens: number | null;
2856
- outputTokens: number | null;
2857
- totalTokens: number | null;
2858
- costUsd: number | null;
2859
- latencyMs: number | null;
2860
- };
2861
- traceRef: {
2862
- id: string;
2863
- contentHash: string;
2864
- mediaType: string | null;
2865
- sizeBytes: number | null;
2866
- } | null;
2867
- startedAt: string | null;
2868
- completedAt: string | null;
2869
- }[];
2870
- completedAt: string;
2871
- contentHash: string;
2872
- };
2873
- };
2874
- candidate: {
2875
- manifest: {
2876
- schemaVersion: "openpond.experimentManifest.v1";
2877
- id: string;
2878
- name: string;
2879
- teamId: string;
2880
- operationId: string;
2881
- maximumCostUsd: number | null;
2882
- dataset: {
2883
- id: string;
2884
- contentHash: string;
2885
- revision: number;
2886
- };
2887
- target: {
2888
- kind: "model";
2889
- modelId: string;
2890
- configurationHash: string;
2891
- } | {
2892
- kind: "harness";
2893
- source: {
2894
- profileId: string;
2895
- sourceRevision: string;
2896
- harnessRelease: {
2897
- id: string;
2898
- contentHash: string;
2899
- };
2900
- catalogHash: string;
2901
- definitionId: string;
2902
- definitionHash: string;
2903
- target: {
2904
- kind: "profile";
2905
- } | {
2906
- kind: "workflow";
2907
- workflowId: string;
2908
- } | {
2909
- kind: "skill";
2910
- skillPath: string;
2911
- } | {
2912
- kind: "agent_action";
2913
- actionId: string;
2696
+ coverage: {
2697
+ pending: number;
2698
+ submitted: number;
2699
+ accepted: number;
2700
+ rejected: number;
2701
+ cancelled: number;
2914
2702
  };
2915
- environmentHash: string;
2916
- externalDatasetBinding?: {
2917
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
2918
- dataset: {
2703
+ results: {
2704
+ review: {
2919
2705
  id: string;
2920
- contentHash: string;
2921
2706
  revision: number;
2707
+ contentHash: string;
2922
2708
  };
2923
- packageHash: string;
2924
- profileId: string;
2925
- target: {
2926
- kind: "profile";
2927
- } | {
2928
- kind: "workflow";
2929
- workflowId: string;
2930
- } | {
2931
- kind: "skill";
2932
- skillPath: string;
2933
- } | {
2934
- kind: "agent_action";
2935
- actionId: string;
2936
- };
2937
- declaredProfileCatalogHash: string | null;
2938
- protectedProfileClosureHash: string;
2939
- recordedOrigin?: {
2940
- execution: {
2941
- id: string;
2942
- contentHash: string;
2943
- };
2944
- originalEvidence: {
2945
- id: string;
2946
- contentHash: string;
2947
- };
2948
- members: {
2949
- caseId: string;
2950
- seed: string;
2951
- fixtureId: string;
2952
- sourceId: string;
2953
- snapshotHash: string;
2954
- boundaryRevisionHash: string;
2955
- }[];
2956
- } | undefined;
2957
- gradingSource?: {
2958
- pass: {
2959
- id: string;
2960
- contentHash: string;
2961
- };
2962
- evidence: {
2963
- id: string;
2964
- contentHash: string;
2965
- };
2966
- } | undefined;
2967
- split: "frozen_eval" | "test" | "train" | "validation";
2968
- population: {
2969
- taskId: string;
2970
- seed: string;
2971
- fixtureId: string | null;
2972
- }[];
2973
- evaluators: {
2974
- release: {
2975
- id: string;
2976
- contentHash: string;
2977
- revision: string | number;
2978
- };
2979
- feedbackKey: string;
2980
- configurationHash: string;
2981
- output: "boolean" | "category" | "score";
2982
- categories: string[];
2983
- }[];
2984
- fieldMappingsHash: string;
2985
- fieldMappings: {
2986
- graderId: string;
2987
- fields: {
2988
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
2989
- path: string[];
2990
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
2991
- sourcePath: string[];
2992
- }[];
2993
- }[];
2994
- environmentHash: string;
2995
- privateDatasetClosureHash: string;
2996
- criterion: {
2997
- minimumPassRate: number;
2998
- requireComplete: boolean;
2999
- };
3000
- contentHash: string;
3001
- } | undefined;
3002
- sourceCandidate?: {
3003
- candidateId: string;
3004
- frozenRevision: number;
3005
- frozenHash: string;
3006
- harnessRelease: {
2709
+ decision: {
2710
+ id: string;
2711
+ revision: number;
2712
+ contentHash: string;
2713
+ } | null;
2714
+ attemptIds: string[];
2715
+ mode: "individual" | "pairwise";
2716
+ origin: "hosted_retained" | "native_local" | "owner_attested_local" | "recorded_evidence";
2717
+ status: "accepted" | "cancelled" | "pending" | "rejected" | "submitted";
2718
+ grader: {
3007
2719
  id: string;
2720
+ revision: number;
3008
2721
  contentHash: string;
3009
2722
  };
3010
- protectedClosureHash: string;
3011
- } | undefined;
3012
- };
3013
- model: {
3014
- modelId: string;
3015
- configurationHash: string;
3016
- };
3017
- } | {
3018
- kind: "agent";
3019
- source: {
3020
- harnessRelease: {
3021
- id: string;
3022
- contentHash: string;
3023
- };
3024
- agentSnapshot: {
3025
- id: string;
3026
- contentHash: string;
3027
- };
3028
- sourcePackageHash: string;
3029
- };
3030
- model: {
3031
- modelId: string;
3032
- configurationHash: string;
2723
+ answers: {
2724
+ criterionId: string;
2725
+ value: string | number | boolean | null;
2726
+ abstain: boolean;
2727
+ note: string;
2728
+ strength?: "moderate" | "slight" | "strong" | undefined;
2729
+ }[];
2730
+ }[];
2731
+ contentHash: string;
3033
2732
  };
3034
- } | {
3035
- kind: "fixture";
3036
- configurationHash: string;
3037
2733
  };
3038
- execution?: {
3039
- packageHash: string;
3040
- runtimeTargetHash: string;
3041
- metricPolicyHash: string;
3042
- compatibility?: {
3043
- protocol: "openpond.evaluation-execution.v1";
3044
- targetKind: "agent" | "fixture" | "harness" | "model";
3045
- } | undefined;
3046
- } | undefined;
3047
- lineage?: {
3048
- definition: {
3049
- id: string;
3050
- contentHash: string;
3051
- revision: number;
3052
- } | null;
3053
- execution: {
2734
+ baseline: {
2735
+ manifest: {
2736
+ schemaVersion: "openpond.experimentManifest.v1";
3054
2737
  id: string;
3055
- contentHash: string;
3056
- };
3057
- scoringPassId: string | null;
3058
- humanProjection?: {
3059
- originalManifest: {
2738
+ name: string;
2739
+ teamId: string;
2740
+ operationId: string;
2741
+ maximumCostUsd: number | null;
2742
+ dataset: {
3060
2743
  id: string;
3061
2744
  contentHash: string;
2745
+ revision: number;
3062
2746
  };
3063
- selectionHash: string;
3064
- rulesHash: string;
3065
- } | undefined;
3066
- } | undefined;
3067
- evaluators: {
3068
- release: {
3069
- id: string;
3070
- contentHash: string;
3071
- revision: string | number;
3072
- };
3073
- feedbackKey: string;
3074
- configurationHash?: string | undefined;
3075
- output: "boolean" | "category" | "score";
3076
- categories: string[];
3077
- }[];
3078
- population: {
3079
- caseId: string;
3080
- seed: string;
3081
- fixtureId: string | null;
3082
- }[];
3083
- createdAt: string;
3084
- contentHash: string;
3085
- };
3086
- result: {
3087
- schemaVersion: "openpond.experimentResult.v1";
3088
- manifest: {
3089
- id: string;
3090
- contentHash: string;
3091
- };
3092
- status: "cancelled" | "completed" | "failed";
3093
- cases: {
3094
- identity: {
3095
- caseId: string;
3096
- seed: string;
3097
- fixtureId: string | null;
2747
+ target: {
2748
+ kind: "model";
2749
+ modelId: string;
2750
+ configurationHash: string;
2751
+ } | {
2752
+ kind: "harness";
2753
+ source: {
2754
+ profileId: string;
2755
+ sourceRevision: string;
2756
+ harnessRelease: {
2757
+ id: string;
2758
+ contentHash: string;
2759
+ };
2760
+ catalogHash: string;
2761
+ definitionId: string;
2762
+ definitionHash: string;
2763
+ target: {
2764
+ kind: "profile";
2765
+ } | {
2766
+ kind: "workflow";
2767
+ workflowId: string;
2768
+ } | {
2769
+ kind: "skill";
2770
+ skillPath: string;
2771
+ } | {
2772
+ kind: "agent_action";
2773
+ actionId: string;
2774
+ };
2775
+ environmentHash: string;
2776
+ externalDatasetBinding?: {
2777
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
2778
+ dataset: {
2779
+ id: string;
2780
+ contentHash: string;
2781
+ revision: number;
2782
+ };
2783
+ packageHash: string;
2784
+ profileId: string;
2785
+ target: {
2786
+ kind: "profile";
2787
+ } | {
2788
+ kind: "workflow";
2789
+ workflowId: string;
2790
+ } | {
2791
+ kind: "skill";
2792
+ skillPath: string;
2793
+ } | {
2794
+ kind: "agent_action";
2795
+ actionId: string;
2796
+ };
2797
+ declaredProfileCatalogHash: string | null;
2798
+ protectedProfileClosureHash: string;
2799
+ recordedOrigin?: {
2800
+ execution: {
2801
+ id: string;
2802
+ contentHash: string;
2803
+ };
2804
+ originalEvidence: {
2805
+ id: string;
2806
+ contentHash: string;
2807
+ };
2808
+ members: {
2809
+ caseId: string;
2810
+ seed: string;
2811
+ fixtureId: string;
2812
+ sourceId: string;
2813
+ snapshotHash: string;
2814
+ boundaryRevisionHash: string;
2815
+ }[];
2816
+ } | undefined;
2817
+ gradingSource?: {
2818
+ pass: {
2819
+ id: string;
2820
+ contentHash: string;
2821
+ };
2822
+ evidence: {
2823
+ id: string;
2824
+ contentHash: string;
2825
+ };
2826
+ } | undefined;
2827
+ split: "frozen_eval" | "test" | "train" | "validation";
2828
+ population: {
2829
+ taskId: string;
2830
+ seed: string;
2831
+ fixtureId: string | null;
2832
+ }[];
2833
+ evaluators: {
2834
+ release: {
2835
+ id: string;
2836
+ contentHash: string;
2837
+ revision: string | number;
2838
+ };
2839
+ feedbackKey: string;
2840
+ configurationHash: string;
2841
+ output: "boolean" | "category" | "score";
2842
+ categories: string[];
2843
+ }[];
2844
+ fieldMappingsHash: string;
2845
+ fieldMappings: {
2846
+ graderId: string;
2847
+ fields: {
2848
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
2849
+ path: string[];
2850
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
2851
+ sourcePath: string[];
2852
+ }[];
2853
+ }[];
2854
+ environmentHash: string;
2855
+ privateDatasetClosureHash: string;
2856
+ criterion: {
2857
+ minimumPassRate: number;
2858
+ requireComplete: boolean;
2859
+ };
2860
+ contentHash: string;
2861
+ } | undefined;
2862
+ sourceCandidate?: {
2863
+ candidateId: string;
2864
+ frozenRevision: number;
2865
+ frozenHash: string;
2866
+ harnessRelease: {
2867
+ id: string;
2868
+ contentHash: string;
2869
+ };
2870
+ protectedClosureHash: string;
2871
+ } | undefined;
2872
+ };
2873
+ model: {
2874
+ modelId: string;
2875
+ configurationHash: string;
2876
+ };
2877
+ } | {
2878
+ kind: "agent";
2879
+ source: {
2880
+ harnessRelease: {
2881
+ id: string;
2882
+ contentHash: string;
2883
+ };
2884
+ agentSnapshot: {
2885
+ id: string;
2886
+ contentHash: string;
2887
+ };
2888
+ sourcePackageHash: string;
2889
+ };
2890
+ model: {
2891
+ modelId: string;
2892
+ configurationHash: string;
2893
+ };
2894
+ } | {
2895
+ kind: "fixture";
2896
+ configurationHash: string;
2897
+ };
2898
+ execution?: {
2899
+ packageHash: string;
2900
+ runtimeTargetHash: string;
2901
+ metricPolicyHash: string;
2902
+ compatibility?: {
2903
+ protocol: "openpond.evaluation-execution.v1";
2904
+ targetKind: "agent" | "fixture" | "harness" | "model";
2905
+ } | undefined;
2906
+ } | undefined;
2907
+ lineage?: {
2908
+ definition: {
2909
+ id: string;
2910
+ contentHash: string;
2911
+ revision: number;
2912
+ } | null;
2913
+ execution: {
2914
+ id: string;
2915
+ contentHash: string;
2916
+ };
2917
+ scoringPassId: string | null;
2918
+ humanProjection?: {
2919
+ originalManifest: {
2920
+ id: string;
2921
+ contentHash: string;
2922
+ };
2923
+ selectionHash: string;
2924
+ rulesHash: string;
2925
+ } | undefined;
2926
+ } | undefined;
2927
+ evaluators: {
2928
+ release: {
2929
+ id: string;
2930
+ contentHash: string;
2931
+ revision: string | number;
2932
+ };
2933
+ feedbackKey: string;
2934
+ configurationHash?: string | undefined;
2935
+ output: "boolean" | "category" | "score";
2936
+ categories: string[];
2937
+ }[];
2938
+ population: {
2939
+ caseId: string;
2940
+ seed: string;
2941
+ fixtureId: string | null;
2942
+ }[];
2943
+ createdAt: string;
2944
+ contentHash: string;
3098
2945
  };
3099
- status: "cancelled" | "completed" | "failed" | "unavailable";
3100
- output: unknown;
3101
- error: {
3102
- code: string;
3103
- message: string;
3104
- } | null;
3105
- feedback: {
3106
- feedbackKey: string;
3107
- evaluator: {
2946
+ result: {
2947
+ schemaVersion: "openpond.experimentResult.v1";
2948
+ manifest: {
3108
2949
  id: string;
3109
2950
  contentHash: string;
3110
- revision: string | number;
3111
2951
  };
3112
- status: "failed" | "pending" | "scored" | "unavailable";
3113
- value: string | number | boolean | null;
3114
- passed?: boolean | null | undefined;
3115
- reasoning: string | null;
3116
- evidenceRefs: {
3117
- id: string;
3118
- contentHash: string;
3119
- mediaType: string | null;
3120
- sizeBytes: number | null;
2952
+ status: "cancelled" | "completed" | "failed";
2953
+ cases: {
2954
+ identity: {
2955
+ caseId: string;
2956
+ seed: string;
2957
+ fixtureId: string | null;
2958
+ };
2959
+ status: "cancelled" | "completed" | "failed" | "unavailable";
2960
+ output: unknown;
2961
+ error: {
2962
+ code: string;
2963
+ message: string;
2964
+ } | null;
2965
+ feedback: {
2966
+ feedbackKey: string;
2967
+ evaluator: {
2968
+ id: string;
2969
+ contentHash: string;
2970
+ revision: string | number;
2971
+ };
2972
+ status: "failed" | "pending" | "scored" | "unavailable";
2973
+ value: string | number | boolean | null;
2974
+ passed?: boolean | null | undefined;
2975
+ reasoning: string | null;
2976
+ evidenceRefs: {
2977
+ id: string;
2978
+ contentHash: string;
2979
+ mediaType: string | null;
2980
+ sizeBytes: number | null;
2981
+ }[];
2982
+ }[];
2983
+ usage: {
2984
+ inputTokens: number | null;
2985
+ outputTokens: number | null;
2986
+ totalTokens: number | null;
2987
+ costUsd: number | null;
2988
+ latencyMs: number | null;
2989
+ };
2990
+ traceRef: {
2991
+ id: string;
2992
+ contentHash: string;
2993
+ mediaType: string | null;
2994
+ sizeBytes: number | null;
2995
+ } | null;
2996
+ startedAt: string | null;
2997
+ completedAt: string | null;
3121
2998
  }[];
3122
- }[];
3123
- usage: {
3124
- inputTokens: number | null;
3125
- outputTokens: number | null;
3126
- totalTokens: number | null;
3127
- costUsd: number | null;
3128
- latencyMs: number | null;
3129
- };
3130
- traceRef: {
3131
- id: string;
3132
- contentHash: string;
3133
- mediaType: string | null;
3134
- sizeBytes: number | null;
3135
- } | null;
3136
- startedAt: string | null;
3137
- completedAt: string | null;
3138
- }[];
3139
- completedAt: string;
3140
- contentHash: string;
3141
- };
3142
- };
3143
- contentHash: string;
3144
- } | {
3145
- teamId: string;
3146
- evidence: {
3147
- id: string;
3148
- contentHash: string;
3149
- };
3150
- scoring?: {
3151
- sourcePass: {
3152
- id: string;
3153
- contentHash: string;
3154
- };
3155
- graders: {
3156
- id: string;
3157
- contentHash: string;
3158
- revision: number;
3159
- }[];
3160
- mappings?: {
3161
- graderId: string;
3162
- fields: {
3163
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
3164
- path: string[];
3165
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
3166
- sourcePath: string[];
3167
- }[];
3168
- }[] | undefined;
3169
- maximumCostUsd: number;
3170
- } | undefined;
3171
- profiles: {
3172
- id: string;
3173
- name: string;
3174
- profileRef: {
3175
- source: "github" | "local" | "openpond_git";
3176
- repositoryId: string;
3177
- profileId: string;
3178
- };
3179
- ownerWorkspaceId: string;
3180
- ownerRevision: number;
3181
- baseRelease: {
3182
- id: string;
3183
- contentHash: string;
3184
- };
3185
- profileSourceRevision: string;
3186
- components: {
3187
- kind: "agent" | "instruction" | "skill" | "workflow";
3188
- path: string;
3189
- agentId?: string | undefined;
3190
- workflowId?: string | undefined;
3191
- label: string;
3192
- binding: {
3193
- schemaVersion: "openpond.profileExternalDatasetBinding.v1";
3194
- dataset: {
3195
- id: string;
2999
+ completedAt: string;
3196
3000
  contentHash: string;
3197
- revision: number;
3198
- };
3199
- packageHash: string;
3200
- profileId: string;
3201
- target: {
3202
- kind: "profile";
3203
- } | {
3204
- kind: "workflow";
3205
- workflowId: string;
3206
- } | {
3207
- kind: "skill";
3208
- skillPath: string;
3209
- } | {
3210
- kind: "agent_action";
3211
- actionId: string;
3212
3001
  };
3213
- declaredProfileCatalogHash: string | null;
3214
- protectedProfileClosureHash: string;
3215
- recordedOrigin?: {
3216
- execution: {
3002
+ };
3003
+ candidate: {
3004
+ manifest: {
3005
+ schemaVersion: "openpond.experimentManifest.v1";
3006
+ id: string;
3007
+ name: string;
3008
+ teamId: string;
3009
+ operationId: string;
3010
+ maximumCostUsd: number | null;
3011
+ dataset: {
3217
3012
  id: string;
3218
3013
  contentHash: string;
3014
+ revision: number;
3219
3015
  };
3220
- originalEvidence: {
3221
- id: string;
3222
- contentHash: string;
3016
+ target: {
3017
+ kind: "model";
3018
+ modelId: string;
3019
+ configurationHash: string;
3020
+ } | {
3021
+ kind: "harness";
3022
+ source: {
3023
+ profileId: string;
3024
+ sourceRevision: string;
3025
+ harnessRelease: {
3026
+ id: string;
3027
+ contentHash: string;
3028
+ };
3029
+ catalogHash: string;
3030
+ definitionId: string;
3031
+ definitionHash: string;
3032
+ target: {
3033
+ kind: "profile";
3034
+ } | {
3035
+ kind: "workflow";
3036
+ workflowId: string;
3037
+ } | {
3038
+ kind: "skill";
3039
+ skillPath: string;
3040
+ } | {
3041
+ kind: "agent_action";
3042
+ actionId: string;
3043
+ };
3044
+ environmentHash: string;
3045
+ externalDatasetBinding?: {
3046
+ schemaVersion: "openpond.profileExternalDatasetBinding.v1";
3047
+ dataset: {
3048
+ id: string;
3049
+ contentHash: string;
3050
+ revision: number;
3051
+ };
3052
+ packageHash: string;
3053
+ profileId: string;
3054
+ target: {
3055
+ kind: "profile";
3056
+ } | {
3057
+ kind: "workflow";
3058
+ workflowId: string;
3059
+ } | {
3060
+ kind: "skill";
3061
+ skillPath: string;
3062
+ } | {
3063
+ kind: "agent_action";
3064
+ actionId: string;
3065
+ };
3066
+ declaredProfileCatalogHash: string | null;
3067
+ protectedProfileClosureHash: string;
3068
+ recordedOrigin?: {
3069
+ execution: {
3070
+ id: string;
3071
+ contentHash: string;
3072
+ };
3073
+ originalEvidence: {
3074
+ id: string;
3075
+ contentHash: string;
3076
+ };
3077
+ members: {
3078
+ caseId: string;
3079
+ seed: string;
3080
+ fixtureId: string;
3081
+ sourceId: string;
3082
+ snapshotHash: string;
3083
+ boundaryRevisionHash: string;
3084
+ }[];
3085
+ } | undefined;
3086
+ gradingSource?: {
3087
+ pass: {
3088
+ id: string;
3089
+ contentHash: string;
3090
+ };
3091
+ evidence: {
3092
+ id: string;
3093
+ contentHash: string;
3094
+ };
3095
+ } | undefined;
3096
+ split: "frozen_eval" | "test" | "train" | "validation";
3097
+ population: {
3098
+ taskId: string;
3099
+ seed: string;
3100
+ fixtureId: string | null;
3101
+ }[];
3102
+ evaluators: {
3103
+ release: {
3104
+ id: string;
3105
+ contentHash: string;
3106
+ revision: string | number;
3107
+ };
3108
+ feedbackKey: string;
3109
+ configurationHash: string;
3110
+ output: "boolean" | "category" | "score";
3111
+ categories: string[];
3112
+ }[];
3113
+ fieldMappingsHash: string;
3114
+ fieldMappings: {
3115
+ graderId: string;
3116
+ fields: {
3117
+ destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
3118
+ path: string[];
3119
+ source: "evaluatorContext" | "expectedOutput" | "input" | "output";
3120
+ sourcePath: string[];
3121
+ }[];
3122
+ }[];
3123
+ environmentHash: string;
3124
+ privateDatasetClosureHash: string;
3125
+ criterion: {
3126
+ minimumPassRate: number;
3127
+ requireComplete: boolean;
3128
+ };
3129
+ contentHash: string;
3130
+ } | undefined;
3131
+ sourceCandidate?: {
3132
+ candidateId: string;
3133
+ frozenRevision: number;
3134
+ frozenHash: string;
3135
+ harnessRelease: {
3136
+ id: string;
3137
+ contentHash: string;
3138
+ };
3139
+ protectedClosureHash: string;
3140
+ } | undefined;
3141
+ };
3142
+ model: {
3143
+ modelId: string;
3144
+ configurationHash: string;
3145
+ };
3146
+ } | {
3147
+ kind: "agent";
3148
+ source: {
3149
+ harnessRelease: {
3150
+ id: string;
3151
+ contentHash: string;
3152
+ };
3153
+ agentSnapshot: {
3154
+ id: string;
3155
+ contentHash: string;
3156
+ };
3157
+ sourcePackageHash: string;
3158
+ };
3159
+ model: {
3160
+ modelId: string;
3161
+ configurationHash: string;
3162
+ };
3163
+ } | {
3164
+ kind: "fixture";
3165
+ configurationHash: string;
3223
3166
  };
3224
- members: {
3167
+ execution?: {
3168
+ packageHash: string;
3169
+ runtimeTargetHash: string;
3170
+ metricPolicyHash: string;
3171
+ compatibility?: {
3172
+ protocol: "openpond.evaluation-execution.v1";
3173
+ targetKind: "agent" | "fixture" | "harness" | "model";
3174
+ } | undefined;
3175
+ } | undefined;
3176
+ lineage?: {
3177
+ definition: {
3178
+ id: string;
3179
+ contentHash: string;
3180
+ revision: number;
3181
+ } | null;
3182
+ execution: {
3183
+ id: string;
3184
+ contentHash: string;
3185
+ };
3186
+ scoringPassId: string | null;
3187
+ humanProjection?: {
3188
+ originalManifest: {
3189
+ id: string;
3190
+ contentHash: string;
3191
+ };
3192
+ selectionHash: string;
3193
+ rulesHash: string;
3194
+ } | undefined;
3195
+ } | undefined;
3196
+ evaluators: {
3197
+ release: {
3198
+ id: string;
3199
+ contentHash: string;
3200
+ revision: string | number;
3201
+ };
3202
+ feedbackKey: string;
3203
+ configurationHash?: string | undefined;
3204
+ output: "boolean" | "category" | "score";
3205
+ categories: string[];
3206
+ }[];
3207
+ population: {
3225
3208
  caseId: string;
3226
3209
  seed: string;
3227
- fixtureId: string;
3228
- sourceId: string;
3229
- snapshotHash: string;
3230
- boundaryRevisionHash: string;
3210
+ fixtureId: string | null;
3231
3211
  }[];
3232
- } | undefined;
3233
- gradingSource?: {
3234
- pass: {
3235
- id: string;
3236
- contentHash: string;
3237
- };
3238
- evidence: {
3239
- id: string;
3240
- contentHash: string;
3241
- };
3242
- } | undefined;
3243
- split: "frozen_eval" | "test" | "train" | "validation";
3244
- population: {
3245
- taskId: string;
3246
- seed: string;
3247
- fixtureId: string | null;
3248
- }[];
3249
- evaluators: {
3250
- release: {
3212
+ createdAt: string;
3213
+ contentHash: string;
3214
+ };
3215
+ result: {
3216
+ schemaVersion: "openpond.experimentResult.v1";
3217
+ manifest: {
3251
3218
  id: string;
3252
3219
  contentHash: string;
3253
- revision: string | number;
3254
3220
  };
3255
- feedbackKey: string;
3256
- configurationHash: string;
3257
- output: "boolean" | "category" | "score";
3258
- categories: string[];
3259
- }[];
3260
- fieldMappingsHash: string;
3261
- fieldMappings: {
3262
- graderId: string;
3263
- fields: {
3264
- destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
3265
- path: string[];
3266
- source: "evaluatorContext" | "expectedOutput" | "input" | "output";
3267
- sourcePath: string[];
3221
+ status: "cancelled" | "completed" | "failed";
3222
+ cases: {
3223
+ identity: {
3224
+ caseId: string;
3225
+ seed: string;
3226
+ fixtureId: string | null;
3227
+ };
3228
+ status: "cancelled" | "completed" | "failed" | "unavailable";
3229
+ output: unknown;
3230
+ error: {
3231
+ code: string;
3232
+ message: string;
3233
+ } | null;
3234
+ feedback: {
3235
+ feedbackKey: string;
3236
+ evaluator: {
3237
+ id: string;
3238
+ contentHash: string;
3239
+ revision: string | number;
3240
+ };
3241
+ status: "failed" | "pending" | "scored" | "unavailable";
3242
+ value: string | number | boolean | null;
3243
+ passed?: boolean | null | undefined;
3244
+ reasoning: string | null;
3245
+ evidenceRefs: {
3246
+ id: string;
3247
+ contentHash: string;
3248
+ mediaType: string | null;
3249
+ sizeBytes: number | null;
3250
+ }[];
3251
+ }[];
3252
+ usage: {
3253
+ inputTokens: number | null;
3254
+ outputTokens: number | null;
3255
+ totalTokens: number | null;
3256
+ costUsd: number | null;
3257
+ latencyMs: number | null;
3258
+ };
3259
+ traceRef: {
3260
+ id: string;
3261
+ contentHash: string;
3262
+ mediaType: string | null;
3263
+ sizeBytes: number | null;
3264
+ } | null;
3265
+ startedAt: string | null;
3266
+ completedAt: string | null;
3268
3267
  }[];
3269
- }[];
3270
- environmentHash: string;
3271
- privateDatasetClosureHash: string;
3272
- criterion: {
3273
- minimumPassRate: number;
3274
- requireComplete: boolean;
3268
+ completedAt: string;
3269
+ contentHash: string;
3275
3270
  };
3276
- contentHash: string;
3277
3271
  };
3278
- }[];
3279
- }[];
3280
- } | {
3281
- candidateId: string;
3282
- candidateRevision: number;
3283
- path: string;
3284
- text: string;
3285
- textHash: string;
3272
+ contentHash: string;
3273
+ } | undefined;
3274
+ } | null;
3275
+ acceptance: {
3276
+ id: string;
3277
+ contentHash: string;
3278
+ } | null;
3279
+ rollback: {
3280
+ id: string;
3281
+ contentHash: string;
3282
+ } | null;
3283
+ error: string | null;
3284
+ createdAt: string;
3285
+ updatedAt: string;
3286
3286
  contentHash: string;
3287
3287
  } | {
3288
3288
  states: {