@openpond/evals 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/CONTRACT.md +13 -2
  2. package/RELEASING.md +6 -5
  3. package/dist/compatibility.js +45 -0
  4. package/dist/conformance.js +6 -9
  5. package/dist/harness-improvements.js +329 -0
  6. package/dist/harness-workspaces.js +368 -0
  7. package/dist/harness.js +4 -6
  8. package/dist/index.js +3 -0
  9. package/dist/runs.js +33 -5
  10. package/dist/tasksets.js +2 -3
  11. package/dist/types/compatibility.d.ts +11 -0
  12. package/dist/types/compatibility.d.ts.map +1 -0
  13. package/dist/types/conformance.d.ts +8 -48
  14. package/dist/types/conformance.d.ts.map +1 -1
  15. package/dist/types/evidence/conformance.d.ts +12 -12
  16. package/dist/types/evidence/contracts.d.ts +10 -10
  17. package/dist/types/harness-improvements.d.ts +523 -0
  18. package/dist/types/harness-improvements.d.ts.map +1 -0
  19. package/dist/types/harness-workspaces.d.ts +802 -0
  20. package/dist/types/harness-workspaces.d.ts.map +1 -0
  21. package/dist/types/harness.d.ts +6 -68
  22. package/dist/types/harness.d.ts.map +1 -1
  23. package/dist/types/index.d.ts +3 -0
  24. package/dist/types/index.d.ts.map +1 -1
  25. package/dist/types/runs.d.ts +46 -1
  26. package/dist/types/runs.d.ts.map +1 -1
  27. package/dist/types/tasksets.d.ts +2 -10
  28. package/dist/types/tasksets.d.ts.map +1 -1
  29. package/package.json +9 -1
package/dist/types/evidence/conformance.d.ts CHANGED
@@ -10,7 +10,7 @@ export declare const completeWorkProcessTraceFixture: {
10
10
  timestamp: string;
11
11
  layer: "agent" | "environment";
12
12
  kind: "validation" | "artifact" | "approval" | "tool" | "state_transition" | "question" | "cleanup";
13
- action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
13
+ action: "turn_completed" | "turn_started" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
14
14
  status: "cancelled" | "failed" | "started" | "completed";
15
15
  inputHash: string | null;
16
16
  outputHash: string | null;
@@ -23,7 +23,7 @@ export declare const completeWorkProcessTraceFixture: {
23
23
  id: string;
24
24
  }[];
25
25
  attributes: {
26
- toolCategory: "agent" | "filesystem" | "source_control" | "command" | "browser" | "connected_app" | "sandbox" | "other" | null;
26
+ toolCategory: "agent" | "connected_app" | "filesystem" | "source_control" | "command" | "browser" | "sandbox" | "other" | null;
27
27
  validationKind: "test" | "other" | "structural" | "visual" | "user_review" | null;
28
28
  transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
29
29
  interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
@@ -41,7 +41,7 @@ export declare const completeWorkEvidenceFixture: {
41
41
  schemaVersion: "openpond.workEvidenceReceipt.v1";
42
42
  id: string;
43
43
  source: {
44
- surface: "desktop" | "hosted";
44
+ surface: "hosted" | "desktop";
45
45
  experience: "work" | "development";
46
46
  opaqueRef: string;
47
47
  revisionHash: string;
@@ -187,7 +187,7 @@ export declare const incompleteWorkProcessTraceFixture: {
187
187
  timestamp: string;
188
188
  layer: "agent" | "environment";
189
189
  kind: "validation" | "artifact" | "approval" | "tool" | "state_transition" | "question" | "cleanup";
190
- action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
190
+ action: "turn_completed" | "turn_started" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
191
191
  status: "cancelled" | "failed" | "started" | "completed";
192
192
  inputHash: string | null;
193
193
  outputHash: string | null;
@@ -200,7 +200,7 @@ export declare const incompleteWorkProcessTraceFixture: {
200
200
  id: string;
201
201
  }[];
202
202
  attributes: {
203
- toolCategory: "agent" | "filesystem" | "source_control" | "command" | "browser" | "connected_app" | "sandbox" | "other" | null;
203
+ toolCategory: "agent" | "connected_app" | "filesystem" | "source_control" | "command" | "browser" | "sandbox" | "other" | null;
204
204
  validationKind: "test" | "other" | "structural" | "visual" | "user_review" | null;
205
205
  transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
206
206
  interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
@@ -273,7 +273,7 @@ export declare const revokedWorkEvidenceEligibilityFixture: {
273
273
  export declare const invalidRawEvidenceFixture: {
274
274
  source: {
275
275
  opaqueRef: string;
276
- surface: "desktop" | "hosted";
276
+ surface: "hosted" | "desktop";
277
277
  experience: "work" | "development";
278
278
  revisionHash: string;
279
279
  };
@@ -393,7 +393,7 @@ export declare const workEvidenceConformance: {
393
393
  timestamp: string;
394
394
  layer: "agent" | "environment";
395
395
  kind: "validation" | "artifact" | "approval" | "tool" | "state_transition" | "question" | "cleanup";
396
- action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
396
+ action: "turn_completed" | "turn_started" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
397
397
  status: "cancelled" | "failed" | "started" | "completed";
398
398
  inputHash: string | null;
399
399
  outputHash: string | null;
@@ -406,7 +406,7 @@ export declare const workEvidenceConformance: {
406
406
  id: string;
407
407
  }[];
408
408
  attributes: {
409
- toolCategory: "agent" | "filesystem" | "source_control" | "command" | "browser" | "connected_app" | "sandbox" | "other" | null;
409
+ toolCategory: "agent" | "connected_app" | "filesystem" | "source_control" | "command" | "browser" | "sandbox" | "other" | null;
410
410
  validationKind: "test" | "other" | "structural" | "visual" | "user_review" | null;
411
411
  transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
412
412
  interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
@@ -424,7 +424,7 @@ export declare const workEvidenceConformance: {
424
424
  schemaVersion: "openpond.workEvidenceReceipt.v1";
425
425
  id: string;
426
426
  source: {
427
- surface: "desktop" | "hosted";
427
+ surface: "hosted" | "desktop";
428
428
  experience: "work" | "development";
429
429
  opaqueRef: string;
430
430
  revisionHash: string;
@@ -626,7 +626,7 @@ export declare const workEvidenceConformance: {
626
626
  timestamp: string;
627
627
  layer: "agent" | "environment";
628
628
  kind: "validation" | "artifact" | "approval" | "tool" | "state_transition" | "question" | "cleanup";
629
- action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
629
+ action: "turn_completed" | "turn_started" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
630
630
  status: "cancelled" | "failed" | "started" | "completed";
631
631
  inputHash: string | null;
632
632
  outputHash: string | null;
@@ -639,7 +639,7 @@ export declare const workEvidenceConformance: {
639
639
  id: string;
640
640
  }[];
641
641
  attributes: {
642
- toolCategory: "agent" | "filesystem" | "source_control" | "command" | "browser" | "connected_app" | "sandbox" | "other" | null;
642
+ toolCategory: "agent" | "connected_app" | "filesystem" | "source_control" | "command" | "browser" | "sandbox" | "other" | null;
643
643
  validationKind: "test" | "other" | "structural" | "visual" | "user_review" | null;
644
644
  transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
645
645
  interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
@@ -656,7 +656,7 @@ export declare const workEvidenceConformance: {
656
656
  readonly invalidRawEvidence: {
657
657
  source: {
658
658
  opaqueRef: string;
659
- surface: "desktop" | "hosted";
659
+ surface: "hosted" | "desktop";
660
660
  experience: "work" | "development";
661
661
  revisionHash: string;
662
662
  };
package/dist/types/evidence/contracts.d.ts CHANGED
@@ -26,8 +26,8 @@ export declare const WorkProcessLayerSchema: z.ZodEnum<{
26
26
  environment: "environment";
27
27
  }>;
28
28
  export declare const WorkProcessActionSchema: z.ZodEnum<{
29
- turn_started: "turn_started";
30
29
  turn_completed: "turn_completed";
30
+ turn_started: "turn_started";
31
31
  turn_failed: "turn_failed";
32
32
  turn_cancelled: "turn_cancelled";
33
33
  turn_timed_out: "turn_timed_out";
@@ -57,11 +57,11 @@ export declare const WorkProcessStepStatusSchema: z.ZodEnum<{
57
57
  }>;
58
58
  export declare const WorkToolCategorySchema: z.ZodEnum<{
59
59
  agent: "agent";
60
+ connected_app: "connected_app";
60
61
  filesystem: "filesystem";
61
62
  source_control: "source_control";
62
63
  command: "command";
63
64
  browser: "browser";
64
- connected_app: "connected_app";
65
65
  sandbox: "sandbox";
66
66
  other: "other";
67
67
  }>;
@@ -120,8 +120,8 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
120
120
  cleanup: "cleanup";
121
121
  }>;
122
122
  action: z.ZodEnum<{
123
- turn_started: "turn_started";
124
123
  turn_completed: "turn_completed";
124
+ turn_started: "turn_started";
125
125
  turn_failed: "turn_failed";
126
126
  turn_cancelled: "turn_cancelled";
127
127
  turn_timed_out: "turn_timed_out";
@@ -162,11 +162,11 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
162
162
  attributes: z.ZodObject<{
163
163
  toolCategory: z.ZodNullable<z.ZodEnum<{
164
164
  agent: "agent";
165
+ connected_app: "connected_app";
165
166
  filesystem: "filesystem";
166
167
  source_control: "source_control";
167
168
  command: "command";
168
169
  browser: "browser";
169
- connected_app: "connected_app";
170
170
  sandbox: "sandbox";
171
171
  other: "other";
172
172
  }>>;
@@ -238,8 +238,8 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
238
238
  cleanup: "cleanup";
239
239
  }>;
240
240
  action: z.ZodEnum<{
241
- turn_started: "turn_started";
242
241
  turn_completed: "turn_completed";
242
+ turn_started: "turn_started";
243
243
  turn_failed: "turn_failed";
244
244
  turn_cancelled: "turn_cancelled";
245
245
  turn_timed_out: "turn_timed_out";
@@ -280,11 +280,11 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
280
280
  attributes: z.ZodObject<{
281
281
  toolCategory: z.ZodNullable<z.ZodEnum<{
282
282
  agent: "agent";
283
+ connected_app: "connected_app";
283
284
  filesystem: "filesystem";
284
285
  source_control: "source_control";
285
286
  command: "command";
286
287
  browser: "browser";
287
- connected_app: "connected_app";
288
288
  sandbox: "sandbox";
289
289
  other: "other";
290
290
  }>>;
@@ -357,8 +357,8 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
357
357
  cleanup: "cleanup";
358
358
  }>;
359
359
  action: z.ZodEnum<{
360
- turn_started: "turn_started";
361
360
  turn_completed: "turn_completed";
361
+ turn_started: "turn_started";
362
362
  turn_failed: "turn_failed";
363
363
  turn_cancelled: "turn_cancelled";
364
364
  turn_timed_out: "turn_timed_out";
@@ -399,11 +399,11 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
399
399
  attributes: z.ZodObject<{
400
400
  toolCategory: z.ZodNullable<z.ZodEnum<{
401
401
  agent: "agent";
402
+ connected_app: "connected_app";
402
403
  filesystem: "filesystem";
403
404
  source_control: "source_control";
404
405
  command: "command";
405
406
  browser: "browser";
406
- connected_app: "connected_app";
407
407
  sandbox: "sandbox";
408
408
  other: "other";
409
409
  }>>;
@@ -479,8 +479,8 @@ export declare const WorkEvidenceReceiptContentSchema: z.ZodObject<{
479
479
  id: z.ZodString;
480
480
  source: z.ZodObject<{
481
481
  surface: z.ZodEnum<{
482
- desktop: "desktop";
483
482
  hosted: "hosted";
483
+ desktop: "desktop";
484
484
  }>;
485
485
  experience: z.ZodEnum<{
486
486
  work: "work";
@@ -609,8 +609,8 @@ export declare const WorkEvidenceReceiptSchema: z.ZodObject<{
609
609
  id: z.ZodString;
610
610
  source: z.ZodObject<{
611
611
  surface: z.ZodEnum<{
612
- desktop: "desktop";
613
612
  hosted: "hosted";
613
+ desktop: "desktop";
614
614
  }>;
615
615
  experience: z.ZodEnum<{
616
616
  work: "work";