@openpond/evals 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LEARNING.md +106 -0
- package/README.md +2 -0
- package/dist/graders.js +26 -5
- package/dist/index.js +3 -0
- package/dist/javascript-verifier-contract.js +7 -0
- package/dist/javascript-verifier-node.js +56 -0
- package/dist/javascript-verifier-worker-source.js +1 -0
- package/dist/javascript-verifier-worker.js +6 -0
- package/dist/javascript-verifier.js +64 -0
- package/dist/learning/admission.js +209 -0
- package/dist/learning/assets.js +37 -0
- package/dist/learning/contracts.js +267 -0
- package/dist/learning/errors.js +12 -0
- package/dist/learning/grade-worker.js +99 -0
- package/dist/learning/index.js +9 -0
- package/dist/learning/operations.js +33 -0
- package/dist/learning/repository.js +48 -0
- package/dist/learning/service.js +367 -0
- package/dist/learning/transport.js +25 -0
- package/dist/rewards.js +230 -0
- package/dist/task-schema.js +142 -0
- package/dist/tasksets.js +3 -0
- package/dist/telemetry-catalog.js +1 -1
- package/dist/types/builtin-benchmarks/harness-refiner.d.ts +7 -0
- package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
- package/dist/types/conformance.d.ts +14 -0
- package/dist/types/conformance.d.ts.map +1 -1
- package/dist/types/execution-contracts.d.ts +14 -0
- package/dist/types/execution-contracts.d.ts.map +1 -1
- package/dist/types/graders.d.ts.map +1 -1
- package/dist/types/index.d.ts +3 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/javascript-verifier-contract.d.ts +9 -0
- package/dist/types/javascript-verifier-contract.d.ts.map +1 -0
- package/dist/types/javascript-verifier-node.d.ts +5 -0
- package/dist/types/javascript-verifier-node.d.ts.map +1 -0
- package/dist/types/javascript-verifier-worker-source.d.ts +2 -0
- package/dist/types/javascript-verifier-worker.d.ts +2 -0
- package/dist/types/javascript-verifier-worker.d.ts.map +1 -0
- package/dist/types/javascript-verifier.d.ts +15 -0
- package/dist/types/javascript-verifier.d.ts.map +1 -0
- package/dist/types/learning/admission.d.ts +335 -0
- package/dist/types/learning/admission.d.ts.map +1 -0
- package/dist/types/learning/assets.d.ts +49 -0
- package/dist/types/learning/assets.d.ts.map +1 -0
- package/dist/types/learning/contracts.d.ts +2149 -0
- package/dist/types/learning/contracts.d.ts.map +1 -0
- package/dist/types/learning/errors.d.ts +8 -0
- package/dist/types/learning/errors.d.ts.map +1 -0
- package/dist/types/learning/grade-worker.d.ts +31 -0
- package/dist/types/learning/grade-worker.d.ts.map +1 -0
- package/dist/types/learning/index.d.ts +10 -0
- package/dist/types/learning/index.d.ts.map +1 -0
- package/dist/types/learning/operations.d.ts +2061 -0
- package/dist/types/learning/operations.d.ts.map +1 -0
- package/dist/types/learning/repository.d.ts +1143 -0
- package/dist/types/learning/repository.d.ts.map +1 -0
- package/dist/types/learning/service.d.ts +22 -0
- package/dist/types/learning/service.d.ts.map +1 -0
- package/dist/types/learning/transport.d.ts +2139 -0
- package/dist/types/learning/transport.d.ts.map +1 -0
- package/dist/types/preferences.d.ts +3 -3
- package/dist/types/review-conformance.d.ts +5 -5
- package/dist/types/rewards.d.ts +611 -0
- package/dist/types/rewards.d.ts.map +1 -0
- package/dist/types/task-schema.d.ts +18 -0
- package/dist/types/task-schema.d.ts.map +1 -0
- package/dist/types/tasksets.d.ts +28 -0
- package/dist/types/tasksets.d.ts.map +1 -1
- package/package.json +30 -3
- package/schemas/learning/v1/asset.schema.json +89 -0
- package/schemas/learning/v1/batch.schema.json +132 -0
- package/schemas/learning/v1/binding.schema.json +207 -0
- package/schemas/learning/v1/command-request.schema.json +2414 -0
- package/schemas/learning/v1/decision.schema.json +440 -0
- package/schemas/learning/v1/definition.schema.json +397 -0
- package/schemas/learning/v1/evidence.schema.json +317 -0
- package/schemas/learning/v1/example-submission.schema.json +253 -0
- package/schemas/learning/v1/feedback-submission.schema.json +114 -0
- package/schemas/learning/v1/feedback.schema.json +233 -0
- package/schemas/learning/v1/grade.schema.json +412 -0
- package/schemas/learning/v1/iteration.schema.json +236 -0
- package/schemas/learning/v1/package.schema.json +776 -0
- package/schemas/learning/v1/policy.schema.json +373 -0
- package/schemas/learning/v1/read-request.schema.json +101 -0
- package/schemas/learning/v1/reward.schema.json +302 -0
- package/schemas/learning/v1/source.schema.json +198 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"contracts.d.ts","sourceRoot":"","sources":["../../../../../src/learning/contracts.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,eAAO,MAAM,wBAAwB,2CAAiC,CAAC;AACvE,eAAO,MAAM,yBAAyB;;;;kBAAyB,CAAC;AAChE,eAAO,MAAM,yBAAyB,aAAiJ,CAAC;AAExL,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAsBtC,CAAC;AACH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAExH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;kBAWzB,CAAC;AACZ,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAetC,CAAC;AACH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAExH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAgB7B,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AACZ,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAgF,CAAC;AAEhH,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;kBAW9B,CAAC;AACZ,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBASpB,CAAC;AAEZ,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAqB7C,CAAC;AACH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA6F,CAAC;AAEtI,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAoBjC,CAAC;AACH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAiF,CAAC;AAE9G,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA8BtC,CAAC;AACH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AACxH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;EAAkL,CAAC;AAC7N,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAqBzB,CAAC;AAEZ,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkBpB,CAAC;AAEZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAA2P,CAAC;AAE/R,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAChF,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAChF,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,wBAAgB,WAAW,CAAC,QAAQ,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GAAG,mBAAmB,CAEhH;AACD,wBAAgB,eAAe,CAAC,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAE9F;AACD,wBAAgB,mBAAmB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,GAAG;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,CAG9G;AACD,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAIjF"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** Stable domain failures that hosts may expose without treating rejected work
|
|
2
|
+
* as an infrastructure failure. Unexpected exceptions remain server errors. */
|
|
3
|
+
export declare class LearningDomainError extends Error {
|
|
4
|
+
readonly code: string;
|
|
5
|
+
readonly status: 400 | 403 | 404 | 409 | 422;
|
|
6
|
+
constructor(code: string, status?: 400 | 403 | 404 | 409 | 422, description?: string);
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../../../src/learning/errors.ts"],"names":[],"mappings":"AAAA;+EAC+E;AAC/E,qBAAa,mBAAoB,SAAQ,KAAK;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM;IAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG;gBAA1D,IAAI,EAAE,MAAM,EAAW,MAAM,GAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAS,EAAE,WAAW,CAAC,EAAE,MAAM;CAI5G"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { type RewardBinding, type RewardComposition, type RewardRelease } from "../rewards.js";
|
|
2
|
+
import { type TaskDefinition, type TaskEvidence, type TaskGradeRun } from "./contracts.js";
|
|
3
|
+
import { type LearningRepository } from "./repository.js";
|
|
4
|
+
export type TaskGradeExecution = {
|
|
5
|
+
scope: string;
|
|
6
|
+
run: TaskGradeRun;
|
|
7
|
+
evidence: TaskEvidence;
|
|
8
|
+
definition: TaskDefinition;
|
|
9
|
+
binding: RewardBinding;
|
|
10
|
+
rewards: RewardRelease[];
|
|
11
|
+
signal: AbortSignal;
|
|
12
|
+
};
|
|
13
|
+
export interface TaskGradeExecutor {
|
|
14
|
+
/** Host adapters enforce run timeout/spend and deduplicate remote dispatch by run.id. */
|
|
15
|
+
execute(input: TaskGradeExecution): Promise<RewardComposition>;
|
|
16
|
+
/** True means the execution owner has confirmed cleanup/terminal cancellation. */
|
|
17
|
+
cancel(input: {
|
|
18
|
+
scope: string;
|
|
19
|
+
run: TaskGradeRun;
|
|
20
|
+
}): Promise<boolean>;
|
|
21
|
+
}
|
|
22
|
+
export declare function createBuiltinTaskGradeExecutor(): TaskGradeExecutor;
|
|
23
|
+
export declare function createTaskGradeWorker(repository: LearningRepository, executor: TaskGradeExecutor, options: {
|
|
24
|
+
workerId: string;
|
|
25
|
+
now?: () => string;
|
|
26
|
+
}): {
|
|
27
|
+
run: (scope: string, gradeId: string) => Promise<TaskGradeRun>;
|
|
28
|
+
requestCancellation(scope: string, gradeId: string): void;
|
|
29
|
+
drain(scope: string, limit?: number): Promise<TaskGradeRun[]>;
|
|
30
|
+
};
|
|
31
|
+
//# sourceMappingURL=grade-worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"grade-worker.d.ts","sourceRoot":"","sources":["../../../../../src/learning/grade-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAwB,KAAK,aAAa,EAAE,KAAK,iBAAiB,EAAE,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AAErH,OAAO,EAAsB,KAAK,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC/G,OAAO,EAAmD,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAE3G,MAAM,MAAM,kBAAkB,GAAG;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,YAAY,CAAC;IAClB,QAAQ,EAAE,YAAY,CAAC;IACvB,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,aAAa,CAAC;IACvB,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,MAAM,EAAE,WAAW,CAAC;CACrB,CAAC;AACF,MAAM,WAAW,iBAAiB;IAChC,yFAAyF;IACzF,OAAO,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC/D,kFAAkF;IAClF,MAAM,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,YAAY,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACvE;AAED,wBAAgB,8BAA8B,IAAI,iBAAiB,CAQlE;AAED,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,OAAO,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CAAE;iBAKxH,MAAM,WAAW,MAAM,KAAG,OAAO,CAAC,YAAY,CAAC;+BA6D5C,MAAM,WAAW,MAAM;iBAC/B,MAAM,mBAAe,OAAO,CAAC,YAAY,EAAE,CAAC;EAWlE"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export * from "./contracts.js";
|
|
2
|
+
export * from "./assets.js";
|
|
3
|
+
export * from "./admission.js";
|
|
4
|
+
export * from "./operations.js";
|
|
5
|
+
export * from "./repository.js";
|
|
6
|
+
export * from "./service.js";
|
|
7
|
+
export * from "./grade-worker.js";
|
|
8
|
+
export * from "./transport.js";
|
|
9
|
+
export * from "./errors.js";
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/learning/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC"}
|