@openpond/evals 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/CONTRACT.md +13 -1
  2. package/LEARNING.md +106 -0
  3. package/README.md +28 -2
  4. package/conformance/telemetry/v1/invalid-batch.json +17 -0
  5. package/conformance/telemetry/v1/valid-batch.json +58 -0
  6. package/dist/graders.js +26 -5
  7. package/dist/index.js +8 -0
  8. package/dist/javascript-verifier-contract.js +7 -0
  9. package/dist/javascript-verifier-node.js +56 -0
  10. package/dist/javascript-verifier-worker-source.js +1 -0
  11. package/dist/javascript-verifier-worker.js +6 -0
  12. package/dist/javascript-verifier.js +64 -0
  13. package/dist/learned-preference.js +334 -0
  14. package/dist/learning/admission.js +209 -0
  15. package/dist/learning/assets.js +37 -0
  16. package/dist/learning/contracts.js +267 -0
  17. package/dist/learning/errors.js +12 -0
  18. package/dist/learning/grade-worker.js +99 -0
  19. package/dist/learning/index.js +9 -0
  20. package/dist/learning/operations.js +33 -0
  21. package/dist/learning/repository.js +48 -0
  22. package/dist/learning/service.js +367 -0
  23. package/dist/learning/transport.js +25 -0
  24. package/dist/preferences.js +77 -11
  25. package/dist/rewards.js +230 -0
  26. package/dist/task-schema.js +142 -0
  27. package/dist/tasksets.js +3 -0
  28. package/dist/telemetry/index.js +4 -0
  29. package/dist/telemetry-analysis.js +183 -0
  30. package/dist/telemetry-bundle.js +60 -0
  31. package/dist/telemetry-catalog.js +50 -0
  32. package/dist/telemetry.js +112 -0
  33. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +7 -0
  34. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  35. package/dist/types/conformance.d.ts +14 -0
  36. package/dist/types/conformance.d.ts.map +1 -1
  37. package/dist/types/execution-contracts.d.ts +14 -0
  38. package/dist/types/execution-contracts.d.ts.map +1 -1
  39. package/dist/types/graders.d.ts.map +1 -1
  40. package/dist/types/index.d.ts +8 -0
  41. package/dist/types/index.d.ts.map +1 -1
  42. package/dist/types/javascript-verifier-contract.d.ts +9 -0
  43. package/dist/types/javascript-verifier-contract.d.ts.map +1 -0
  44. package/dist/types/javascript-verifier-node.d.ts +5 -0
  45. package/dist/types/javascript-verifier-node.d.ts.map +1 -0
  46. package/dist/types/javascript-verifier-worker-source.d.ts +2 -0
  47. package/dist/types/javascript-verifier-worker.d.ts +2 -0
  48. package/dist/types/javascript-verifier-worker.d.ts.map +1 -0
  49. package/dist/types/javascript-verifier.d.ts +15 -0
  50. package/dist/types/javascript-verifier.d.ts.map +1 -0
  51. package/dist/types/learned-preference.d.ts +282 -0
  52. package/dist/types/learned-preference.d.ts.map +1 -0
  53. package/dist/types/learning/admission.d.ts +335 -0
  54. package/dist/types/learning/admission.d.ts.map +1 -0
  55. package/dist/types/learning/assets.d.ts +49 -0
  56. package/dist/types/learning/assets.d.ts.map +1 -0
  57. package/dist/types/learning/contracts.d.ts +2149 -0
  58. package/dist/types/learning/contracts.d.ts.map +1 -0
  59. package/dist/types/learning/errors.d.ts +8 -0
  60. package/dist/types/learning/errors.d.ts.map +1 -0
  61. package/dist/types/learning/grade-worker.d.ts +31 -0
  62. package/dist/types/learning/grade-worker.d.ts.map +1 -0
  63. package/dist/types/learning/index.d.ts +10 -0
  64. package/dist/types/learning/index.d.ts.map +1 -0
  65. package/dist/types/learning/operations.d.ts +2061 -0
  66. package/dist/types/learning/operations.d.ts.map +1 -0
  67. package/dist/types/learning/repository.d.ts +1143 -0
  68. package/dist/types/learning/repository.d.ts.map +1 -0
  69. package/dist/types/learning/service.d.ts +22 -0
  70. package/dist/types/learning/service.d.ts.map +1 -0
  71. package/dist/types/learning/transport.d.ts +2139 -0
  72. package/dist/types/learning/transport.d.ts.map +1 -0
  73. package/dist/types/preferences.d.ts +55 -7
  74. package/dist/types/preferences.d.ts.map +1 -1
  75. package/dist/types/review-conformance.d.ts +5 -5
  76. package/dist/types/rewards.d.ts +611 -0
  77. package/dist/types/rewards.d.ts.map +1 -0
  78. package/dist/types/task-schema.d.ts +18 -0
  79. package/dist/types/task-schema.d.ts.map +1 -0
  80. package/dist/types/tasksets.d.ts +28 -0
  81. package/dist/types/tasksets.d.ts.map +1 -1
  82. package/dist/types/telemetry/index.d.ts +5 -0
  83. package/dist/types/telemetry/index.d.ts.map +1 -0
  84. package/dist/types/telemetry-analysis.d.ts +116 -0
  85. package/dist/types/telemetry-analysis.d.ts.map +1 -0
  86. package/dist/types/telemetry-bundle.d.ts +309 -0
  87. package/dist/types/telemetry-bundle.d.ts.map +1 -0
  88. package/dist/types/telemetry-catalog.d.ts +269 -0
  89. package/dist/types/telemetry-catalog.d.ts.map +1 -0
  90. package/dist/types/telemetry.d.ts +266 -0
  91. package/dist/types/telemetry.d.ts.map +1 -0
  92. package/package.json +40 -3
  93. package/schemas/learning/v1/asset.schema.json +89 -0
  94. package/schemas/learning/v1/batch.schema.json +132 -0
  95. package/schemas/learning/v1/binding.schema.json +207 -0
  96. package/schemas/learning/v1/command-request.schema.json +2414 -0
  97. package/schemas/learning/v1/decision.schema.json +440 -0
  98. package/schemas/learning/v1/definition.schema.json +397 -0
  99. package/schemas/learning/v1/evidence.schema.json +317 -0
  100. package/schemas/learning/v1/example-submission.schema.json +253 -0
  101. package/schemas/learning/v1/feedback-submission.schema.json +114 -0
  102. package/schemas/learning/v1/feedback.schema.json +233 -0
  103. package/schemas/learning/v1/grade.schema.json +412 -0
  104. package/schemas/learning/v1/iteration.schema.json +236 -0
  105. package/schemas/learning/v1/package.schema.json +776 -0
  106. package/schemas/learning/v1/policy.schema.json +373 -0
  107. package/schemas/learning/v1/read-request.schema.json +101 -0
  108. package/schemas/learning/v1/reward.schema.json +302 -0
  109. package/schemas/learning/v1/source.schema.json +198 -0
  110. package/schemas/telemetry/v1/evidence-completeness.schema.json +82 -0
  111. package/schemas/telemetry/v1/evidence-reference.schema.json +41 -0
  112. package/schemas/telemetry/v1/metric-definition.schema.json +96 -0
  113. package/schemas/telemetry/v1/metric-observation.schema.json +182 -0
  114. package/schemas/telemetry/v1/run-metric-summary.schema.json +98 -0
  115. package/schemas/telemetry/v1/run-telemetry-batch.schema.json +405 -0
  116. package/schemas/telemetry/v1/run-telemetry-event.schema.json +201 -0
  117. package/schemas/telemetry/v1/telemetry-cohort.schema.json +100 -0
  118. package/schemas/telemetry/v1/telemetry-export-bundle.schema.json +655 -0
package/dist/types/learning/repository.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"repository.d.ts","sourceRoot":"","sources":["../../../../../src/learning/repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAkO,KAAK,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAG1R,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAGlD,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAM1B,CAAC;AACX,MAAM,MAAM,oBAAoB,GAAG,MAAM,OAAO,uBAAuB,CAAC;AACxE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;EAAoG,CAAC;AAC5I,MAAM,MAAM,mBAAmB,CAAC,CAAC,SAAS,oBAAoB,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,uBAAuB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/G,MAAM,MAAM,sBAAsB,GAAG,mBAAmB,CAAC,oBAAoB,CAAC,CAAC;AAE/E,MAAM,MAAM,uBAAuB,GAAG;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AACnG,MAAM,MAAM,wBAAwB,GAAG;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,uBAAuB,EAAE,CAAA;CAAE,CAAC;AACrG,MAAM,MAAM,qBAAqB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAC5G,MAAM,MAAM,oBAAoB,CAAC,CAAC,SAAS,oBAAoB,IAAI;IAAE,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC;AAElI,qFAAqF;AACrF,MAAM,WAAW,mBAAmB;IAClC,GAAG,CAAC,CAAC,SAAS,oBAAoB,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IACpH,GAAG,CAAC,CAAC,SAAS,oBAAoB,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,mBAAmB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxK,IAAI,CAAC,CAAC,SAAS,oBAAoB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,qBAAqB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9G,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,GAAG,IAAI,CAAC,CAAC;IAChE,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC9F,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5G;AACD,MAAM,WAAW,kBAAkB;IACjC,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,WAAW,EAAE,mBAAmB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACvG;AAED,qBAAa,qBAAsB,SAAQ,mBAAmB;IAChD,QAAQ,CAAC,IAAI,EAAE,MAAM;IAAE,QAAQ,CAAC,EAAE,EAAE,MAAM;IAAE,QAAQ,CAAC,gBAAgB,EAAE,MAAM;IAAE,QAAQ,CAAC,eAAe,EAAE,MAAM;gBAAtG,IAAI,EAAE,MAAM,EAAW,EAAE,EAAE,MAAM,EAAW,gBAAgB,EAAE,MAAM,EAAW,eAAe,EAAE,MAAM;CAI5H;AAED,wBAAsB,uBAAuB,CAAC,CAAC,SAAS,oBAAoB,EAAE,WAAW,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAIvL;AACD,wBAAsB,sBAAsB,CAAC,CAAC,SAAS,oBAAoB,EAAE,WAAW,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAKjL;AACD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjG;AACD,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAEhF"}
package/dist/types/learning/service.d.ts ADDED
@@ -0,0 +1,22 @@
1
+ import { type LearningRepository, type LearningResourceFor, type LearningResourceKind, type LearningResourcePage, type LearningResourceQuery, type LearningStoredResource } from "./repository.js";
2
+ export type LearningActor = {
3
+ id: string;
4
+ role: "editor" | "reviewer" | "source";
5
+ sourceId?: string;
6
+ };
7
+ export type LearningServiceContext = {
8
+ scope: string;
9
+ actor: LearningActor;
10
+ };
11
+ export type LearningOperationResult = {
12
+ operationId: string;
13
+ resources: LearningStoredResource[];
14
+ };
15
+ export declare function createLearningService(repository: LearningRepository, options?: {
16
+ now?: () => string;
17
+ }): {
18
+ command: (context: LearningServiceContext, raw: unknown) => Promise<LearningOperationResult>;
19
+ get<K extends LearningResourceKind>(context: LearningServiceContext, kind: K, id: string, revision?: number): Promise<LearningResourceFor<K>>;
20
+ list<K extends LearningResourceKind>(context: LearningServiceContext, kind: K, query?: Partial<LearningResourceQuery>): Promise<LearningResourcePage<K>>;
21
+ };
22
+ //# sourceMappingURL=service.d.ts.map
package/dist/types/learning/service.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../../../../src/learning/service.ts"],"names":[],"mappings":"AAQA,OAAO,EAAmH,KAAK,kBAAkB,EAAE,KAAK,mBAAmB,EAAE,KAAK,oBAAoB,EAAE,KAAK,oBAAoB,EAAgC,KAAK,qBAAqB,EAAE,KAAK,sBAAsB,EAA4B,MAAM,iBAAiB,CAAC;AAI5W,MAAM,MAAM,aAAa,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,GAAG,UAAU,GAAG,QAAQ,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AACtG,MAAM,MAAM,sBAAsB,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,aAAa,CAAA;CAAE,CAAC;AAC7E,MAAM,MAAM,uBAAuB,GAAG;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,sBAAsB,EAAE,CAAA;CAAE,CAAC;AAEnG,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,kBAAkB,EAAE,OAAO,GAAE;IAAE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CAAO;uBAGxE,sBAAsB,OAAO,OAAO,KAAG,OAAO,CAAC,uBAAuB,CAAC;QA0Q3F,CAAC,SAAS,oBAAoB,WAAW,sBAAsB,QAAQ,CAAC,MAAM,MAAM,aAAa,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;SAIxI,CAAC,SAAS,oBAAoB,WAAW,sBAAsB,QAAQ,CAAC,UAAS,OAAO,CAAC,qBAAqB,CAAC,GAAQ,OAAO,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;EAMrK"}