openpond-sdk 0.5.27 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LEARNING.md +18 -0
- package/TASKSETS.md +20 -4
- package/dist/{chunk-SQCACAC7.js → chunk-2FKPFZO5.js} +2 -2
- package/dist/{chunk-S7I45WJQ.js → chunk-2U54EOXJ.js} +3 -2
- package/dist/chunk-2U54EOXJ.js.map +7 -0
- package/dist/chunk-3KXDH6OB.js +16 -0
- package/dist/chunk-3KXDH6OB.js.map +7 -0
- package/dist/{chunk-GZOXT7YE.js → chunk-E5IFONUB.js} +2 -2
- package/dist/{chunk-ULQC4SQF.js → chunk-GJ62G6BO.js} +4 -4
- package/dist/{chunk-5M3PPP7D.js → chunk-K4C5QOSV.js} +9 -1
- package/dist/chunk-K4C5QOSV.js.map +7 -0
- package/dist/{chunk-IDLMILN4.js → chunk-T5JMZVQR.js} +22 -15
- package/dist/chunk-T5JMZVQR.js.map +7 -0
- package/dist/{chunk-AUQOAMHL.js → chunk-TB33HKGV.js} +2 -2
- package/dist/{chunk-DV7XI3UE.js → chunk-VW2SNUWB.js} +8 -4
- package/dist/chunk-VW2SNUWB.js.map +7 -0
- package/dist/{chunk-ZNFOVV7B.js → chunk-XANQWTTP.js} +16 -7
- package/dist/chunk-XANQWTTP.js.map +7 -0
- package/dist/index.js +1 -1
- package/dist/learning.js +51 -8
- package/dist/learning.js.map +2 -2
- package/dist/model-projects.js +5 -1
- package/dist/model-starter-attempts.js +3 -2
- package/dist/model-starter-catalog.js +6 -5
- package/dist/model-starter-catalog.js.map +1 -1
- package/dist/model-starters.js +6 -5
- package/dist/model-taskset-authoring.js +1 -1
- package/dist/model-taskset-runs.js +3 -2
- package/dist/model-taskset-runs.js.map +1 -1
- package/dist/taskset-drafts.js +6 -5
- package/dist/taskset-drafts.js.map +1 -1
- package/dist/taskset-packages.js +209 -46
- package/dist/taskset-packages.js.map +4 -4
- package/dist/training-bundle.js +6 -5
- package/dist/training-bundle.js.map +1 -1
- package/dist/training.js +3 -2
- package/dist/types/packages/sdk/src/learning-client.d.ts +11 -0
- package/dist/types/packages/sdk/src/learning-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-bound-taskset-authoring.d.ts +10 -0
- package/dist/types/packages/sdk/src/model-bound-taskset-authoring.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-learning-intent.d.ts +16 -0
- package/dist/types/packages/sdk/src/model-learning-intent.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-learning-policy.d.ts +155 -1
- package/dist/types/packages/sdk/src/model-learning-policy.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-ordinary-taskset-reward.d.ts +16 -0
- package/dist/types/packages/sdk/src/model-ordinary-taskset-reward.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-projects.d.ts +14 -1
- package/dist/types/packages/sdk/src/model-projects.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-starters.d.ts +30 -0
- package/dist/types/packages/sdk/src/model-starters.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-authoring-contracts.d.ts +1 -0
- package/dist/types/packages/sdk/src/model-taskset-authoring-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-authoring.d.ts +1 -0
- package/dist/types/packages/sdk/src/model-taskset-authoring.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-derivation.d.ts +11 -0
- package/dist/types/packages/sdk/src/model-taskset-derivation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-api-contracts.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-draft-api-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-client.d.ts +6 -0
- package/dist/types/packages/sdk/src/taskset-draft-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-document.d.ts +1 -0
- package/dist/types/packages/sdk/src/taskset-draft-document.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-package-compiler.d.ts +3 -0
- package/dist/types/packages/sdk/src/taskset-draft-package-compiler.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-source.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-draft-source.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-workspace.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-draft-workspace.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts +3 -0
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-projection.d.ts +1 -0
- package/dist/types/packages/sdk/src/taskset-package-projection.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-packages.d.ts +1 -0
- package/dist/types/packages/sdk/src/taskset-packages.d.ts.map +1 -1
- package/package.json +2 -2
- package/dist/chunk-5M3PPP7D.js.map +0 -7
- package/dist/chunk-DV7XI3UE.js.map +0 -7
- package/dist/chunk-IDLMILN4.js.map +0 -7
- package/dist/chunk-S7I45WJQ.js.map +0 -7
- package/dist/chunk-ZNFOVV7B.js.map +0 -7
- /package/dist/{chunk-SQCACAC7.js.map → chunk-2FKPFZO5.js.map} +0 -0
- /package/dist/{chunk-GZOXT7YE.js.map → chunk-E5IFONUB.js.map} +0 -0
- /package/dist/{chunk-ULQC4SQF.js.map → chunk-GJ62G6BO.js.map} +0 -0
- /package/dist/{chunk-AUQOAMHL.js.map → chunk-TB33HKGV.js.map} +0 -0
package/dist/taskset-packages.js
CHANGED
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
projectPortableTaskRecord,
|
|
14
14
|
projectPreferenceComparisonRelease,
|
|
15
15
|
resolvePortableTasksetRewardExecution
|
|
16
|
-
} from "./chunk-
|
|
16
|
+
} from "./chunk-TB33HKGV.js";
|
|
17
17
|
import {
|
|
18
18
|
OpenPondTasksetPackageClient,
|
|
19
19
|
OpenPondTasksetPackageError,
|
|
@@ -30,7 +30,7 @@ import {
|
|
|
30
30
|
publishTasksetDraft,
|
|
31
31
|
renderTasksetDraftManifests,
|
|
32
32
|
validateTasksetDraftWorkspace
|
|
33
|
-
} from "./chunk-
|
|
33
|
+
} from "./chunk-GJ62G6BO.js";
|
|
34
34
|
import {
|
|
35
35
|
GraderFixtureSchema,
|
|
36
36
|
TasksetEnvironmentResourceSchema,
|
|
@@ -41,14 +41,17 @@ import {
|
|
|
41
41
|
import {
|
|
42
42
|
MAX_TASKSET_PACKAGE_BYTES,
|
|
43
43
|
ModelStarterExecutionSchema,
|
|
44
|
+
ModelTasksetDerivationSchema,
|
|
44
45
|
TasksetPackageContentSchema,
|
|
45
46
|
TasksetPackageFileSchema,
|
|
46
47
|
TasksetPackageLearningResourcesSchema,
|
|
47
48
|
TasksetPackageSchema,
|
|
48
49
|
assertModelTasksetAuthoring,
|
|
50
|
+
compileModelTasksetReward,
|
|
49
51
|
createTasksetPackage,
|
|
50
52
|
createTasksetPackageExecutionFile,
|
|
51
53
|
decodeTasksetPackageFile,
|
|
54
|
+
deriveModelTaskset,
|
|
52
55
|
learningPackageContextFiles,
|
|
53
56
|
modelAuthoredTasksetId,
|
|
54
57
|
modelStarterExecutionAssetId,
|
|
@@ -57,7 +60,7 @@ import {
|
|
|
57
60
|
tasksetPackageRewardBinding,
|
|
58
61
|
validateTasksetLearningResources,
|
|
59
62
|
validateTasksetPackage
|
|
60
|
-
} from "./chunk-
|
|
63
|
+
} from "./chunk-T5JMZVQR.js";
|
|
61
64
|
import {
|
|
62
65
|
ModelTasksetAuthoringOwnerSchema,
|
|
63
66
|
ModelTasksetAuthoringSchema,
|
|
@@ -68,8 +71,8 @@ import {
|
|
|
68
71
|
TasksetDraftFileMutationSchema,
|
|
69
72
|
TasksetDraftFilePathSchema,
|
|
70
73
|
TasksetDraftFileSchema
|
|
71
|
-
} from "./chunk-
|
|
72
|
-
import "./chunk-
|
|
74
|
+
} from "./chunk-2U54EOXJ.js";
|
|
75
|
+
import "./chunk-E5IFONUB.js";
|
|
73
76
|
import "./chunk-A4ZDOZKP.js";
|
|
74
77
|
import "./chunk-5FBPNWT4.js";
|
|
75
78
|
import {
|
|
@@ -77,12 +80,72 @@ import {
|
|
|
77
80
|
contentHash,
|
|
78
81
|
sha256
|
|
79
82
|
} from "./chunk-QUOW6URQ.js";
|
|
80
|
-
import "./chunk-
|
|
83
|
+
import "./chunk-XANQWTTP.js";
|
|
84
|
+
import "./chunk-3KXDH6OB.js";
|
|
85
|
+
|
|
86
|
+
// src/model-ordinary-taskset-reward.ts
|
|
87
|
+
import { LearningTextAssetSchema, TaskDefinitionSchema, learningRef, sealLearningContent } from "@openpond/evals/learning";
|
|
88
|
+
function bindOrdinaryModelTasksetReward(input) {
|
|
89
|
+
const source = validateTasksetPackage(input.source);
|
|
90
|
+
if (source.modelResources || source.learningResources) throw new Error("Binding an ordinary Reward requires an ordinary source package.");
|
|
91
|
+
const instructions = resolveTasksetPackageInstructions(source);
|
|
92
|
+
if (!instructions.trim()) throw new Error("Add collection instructions before selecting a saved Reward.");
|
|
93
|
+
const assets = /* @__PURE__ */ new Map();
|
|
94
|
+
const execution = resolveTasksetPackageExecution(source);
|
|
95
|
+
for (const asset of execution?.assets ?? []) assets.set(asset.id, asset);
|
|
96
|
+
const contextIds = /* @__PURE__ */ new Set([
|
|
97
|
+
...source.taskset.tasks.flatMap((task) => task.privilegedContextRef ? [task.privilegedContextRef] : []),
|
|
98
|
+
...[source.environment.actionSchemaRef, source.environment.observationSchemaRef, source.environment.stateSchemaRef].flatMap((ref) => ref ? [ref.id] : [])
|
|
99
|
+
]);
|
|
100
|
+
for (const id of contextIds) {
|
|
101
|
+
if (assets.has(id)) continue;
|
|
102
|
+
const file = source.files.find((file2) => file2.asset.id === id);
|
|
103
|
+
if (!file) throw new Error(`Source Taskset context is missing: ${id}.`);
|
|
104
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(decodeTasksetPackageFile(file));
|
|
105
|
+
assets.set(id, LearningTextAssetSchema.parse(sealLearningContent({ schemaVersion: "openpond.learningTextAsset.v1", id, revision: 1, asset: file.asset, text })));
|
|
106
|
+
}
|
|
107
|
+
for (const asset of input.assets) {
|
|
108
|
+
const original = source.files.find((file) => file.asset.id === asset.id);
|
|
109
|
+
if (original && sealLearningContent({ asset: original.asset }).contentHash !== sealLearningContent({ asset: asset.asset }).contentHash) throw new Error(`Selected Reward conflicts with a source asset: ${asset.id}.`);
|
|
110
|
+
assets.set(asset.id, asset);
|
|
111
|
+
}
|
|
112
|
+
const { policy, environment, environmentRelease, tools, capabilities, verifierSetRelease } = source.taskset;
|
|
113
|
+
const taskDefinition = TaskDefinitionSchema.parse(sealLearningContent({
|
|
114
|
+
schemaVersion: "openpond.taskDefinition.v1",
|
|
115
|
+
id: `${source.taskset.id}-definition`,
|
|
116
|
+
revision: 1,
|
|
117
|
+
name: source.taskset.id,
|
|
118
|
+
description: "",
|
|
119
|
+
instructions,
|
|
120
|
+
category: "custom",
|
|
121
|
+
familyNamespace: source.taskset.id,
|
|
122
|
+
inputSchema: { type: "object" },
|
|
123
|
+
outputSchema: { type: "object" },
|
|
124
|
+
rewardBinding: learningRef(input.rewardBinding),
|
|
125
|
+
harness: null,
|
|
126
|
+
execution: { policy, environment, environmentRelease, tools, capabilities, verifierSetRelease }
|
|
127
|
+
}));
|
|
128
|
+
const derived = compileModelTasksetReward({ ...input, assets: [...assets.values()], source: {
|
|
129
|
+
taskset: source.taskset,
|
|
130
|
+
taskDefinition,
|
|
131
|
+
instructionMode: "per_task",
|
|
132
|
+
executionResources: { environment: source.environment, verifierSet: source.verifierSet },
|
|
133
|
+
...execution ? { execution: execution.execution } : {}
|
|
134
|
+
} });
|
|
135
|
+
const files = new Map(source.files.map((file) => [file.asset.id, file]));
|
|
136
|
+
for (const asset of derived.assets) {
|
|
137
|
+
let raw = "";
|
|
138
|
+
for (const byte of new TextEncoder().encode(asset.text)) raw += String.fromCharCode(byte);
|
|
139
|
+
files.set(asset.id, { asset: asset.asset, base64: btoa(raw) });
|
|
140
|
+
}
|
|
141
|
+
const { taskset, executionResources, ...modelResources } = derived;
|
|
142
|
+
return createTasksetPackage({ schemaVersion: "openpond.tasksetPackage.v1", taskset, ...executionResources, modelResources, files: [...files.values()] });
|
|
143
|
+
}
|
|
81
144
|
|
|
82
145
|
// src/model-batch-review.ts
|
|
83
146
|
import {
|
|
84
147
|
LearningDomainError as LearningDomainError2,
|
|
85
|
-
learningRef as
|
|
148
|
+
learningRef as learningRef3,
|
|
86
149
|
sameLearningRef as sameLearningRef2,
|
|
87
150
|
requireLearningRelease,
|
|
88
151
|
requireLearningResource,
|
|
@@ -93,14 +156,14 @@ import {
|
|
|
93
156
|
import {
|
|
94
157
|
LearningDomainError,
|
|
95
158
|
LearningSourceSchema,
|
|
96
|
-
TaskDefinitionSchema,
|
|
159
|
+
TaskDefinitionSchema as TaskDefinitionSchema2,
|
|
97
160
|
TaskEvidenceSchema,
|
|
98
161
|
TaskFeedbackSchema,
|
|
99
162
|
assertLearningContentHash,
|
|
100
163
|
learningEvidenceId,
|
|
101
|
-
learningRef,
|
|
164
|
+
learningRef as learningRef2,
|
|
102
165
|
sameLearningRef,
|
|
103
|
-
sealLearningContent,
|
|
166
|
+
sealLearningContent as sealLearningContent2,
|
|
104
167
|
taskBatchPackageMetadata,
|
|
105
168
|
verifyLearningTextAsset
|
|
106
169
|
} from "@openpond/evals/learning";
|
|
@@ -112,7 +175,7 @@ function prepareModelBatchReview(input) {
|
|
|
112
175
|
const metadata = taskBatchPackageMetadata(value.taskset);
|
|
113
176
|
const id = `model-review-${contentHash([input.scope, request.modelId, request.operationId])}`;
|
|
114
177
|
const edits = new Map(request.examples.map((edit) => [contentHash(edit.evidence), edit]));
|
|
115
|
-
const parentRefs = new Set(learning.evidence.map((evidence2) => contentHash(
|
|
178
|
+
const parentRefs = new Set(learning.evidence.map((evidence2) => contentHash(learningRef2(evidence2))));
|
|
116
179
|
const decisions = new Map(learning.decisions.map((decision) => [contentHash(decision.evidence), decision]));
|
|
117
180
|
if (edits.size !== request.examples.length || request.examples.some((edit) => !parentRefs.has(contentHash(edit.evidence)))) {
|
|
118
181
|
throw new LearningDomainError("model_review_evidence_mismatch", 409);
|
|
@@ -121,7 +184,7 @@ function prepareModelBatchReview(input) {
|
|
|
121
184
|
resolveBoundRewards(input.binding, input.rewards);
|
|
122
185
|
const rewardByHash = /* @__PURE__ */ new Map();
|
|
123
186
|
for (const source2 of input.binding.sources) {
|
|
124
|
-
const original = input.rewards.find((reward) => sameLearningRef(
|
|
187
|
+
const original = input.rewards.find((reward) => sameLearningRef(learningRef2(reward), source2.reward));
|
|
125
188
|
const { contentHash: _hash, ...content } = original;
|
|
126
189
|
rewardByHash.set(original.contentHash, createRewardRelease({ ...content, id: `${id}-reward-${original.contentHash}`, revision: 1 }));
|
|
127
190
|
}
|
|
@@ -131,7 +194,7 @@ function prepareModelBatchReview(input) {
|
|
|
131
194
|
...bindingContent,
|
|
132
195
|
id: `${id}-binding`,
|
|
133
196
|
revision: 1,
|
|
134
|
-
sources: input.binding.sources.map((source2) => ({ ...source2, reward:
|
|
197
|
+
sources: input.binding.sources.map((source2) => ({ ...source2, reward: learningRef2(rewardByHash.get(source2.reward.contentHash)) }))
|
|
135
198
|
}, rewards);
|
|
136
199
|
const assetRefs = rewards.flatMap((reward) => [
|
|
137
200
|
...reward.assets,
|
|
@@ -148,24 +211,24 @@ function prepareModelBatchReview(input) {
|
|
|
148
211
|
})).values()];
|
|
149
212
|
const { environmentRelease: _environment, verifierSetRelease: _verifiers, ...execution } = metadata.definition.execution;
|
|
150
213
|
const { contentHash: _definitionHash, ...definitionContent } = metadata.definition;
|
|
151
|
-
const definition =
|
|
214
|
+
const definition = TaskDefinitionSchema2.parse(sealLearningContent2({
|
|
152
215
|
...definitionContent,
|
|
153
216
|
...request.definition,
|
|
154
217
|
id: `${id}-definition`,
|
|
155
218
|
revision: 1,
|
|
156
|
-
rewardBinding:
|
|
219
|
+
rewardBinding: learningRef2(binding),
|
|
157
220
|
execution: {
|
|
158
221
|
...execution,
|
|
159
222
|
policy: { ...execution.policy, hiddenGraderRefs: compileBoundGraders(binding, rewards).filter((grader) => grader.privileged).map((grader) => grader.id) }
|
|
160
223
|
}
|
|
161
224
|
}));
|
|
162
|
-
const source = LearningSourceSchema.parse(
|
|
225
|
+
const source = LearningSourceSchema.parse(sealLearningContent2({
|
|
163
226
|
schemaVersion: "openpond.learningSource.v1",
|
|
164
227
|
id,
|
|
165
228
|
revision: 1,
|
|
166
229
|
name: `${definition.name} review`,
|
|
167
230
|
kind: "direct",
|
|
168
|
-
taskDefinition:
|
|
231
|
+
taskDefinition: learningRef2(definition),
|
|
169
232
|
enabled: true,
|
|
170
233
|
allowedSplits: [...new Set(learning.evidence.map((item) => item.submission.split))],
|
|
171
234
|
mapping: null,
|
|
@@ -173,13 +236,13 @@ function prepareModelBatchReview(input) {
|
|
|
173
236
|
reviewOrigin: {
|
|
174
237
|
modelId: request.modelId,
|
|
175
238
|
packageHash: value.contentHash,
|
|
176
|
-
taskset:
|
|
177
|
-
batch:
|
|
178
|
-
rewardBinding:
|
|
239
|
+
taskset: learningRef2(value.taskset),
|
|
240
|
+
batch: learningRef2(learning.batch),
|
|
241
|
+
rewardBinding: learningRef2(input.binding)
|
|
179
242
|
}
|
|
180
243
|
}));
|
|
181
244
|
const evidence = learning.evidence.map((parent) => {
|
|
182
|
-
const edit = edits.get(contentHash(
|
|
245
|
+
const edit = edits.get(contentHash(learningRef2(parent)));
|
|
183
246
|
const attemptId = `review-${parent.contentHash}`;
|
|
184
247
|
const policyChanged = contentHash({
|
|
185
248
|
instructions: definition.instructions,
|
|
@@ -194,15 +257,15 @@ function prepareModelBatchReview(input) {
|
|
|
194
257
|
requiredOutputs: metadata.definition.requiredOutputs ?? [],
|
|
195
258
|
input: parent.submission.input
|
|
196
259
|
});
|
|
197
|
-
return TaskEvidenceSchema.parse(
|
|
260
|
+
return TaskEvidenceSchema.parse(sealLearningContent2({
|
|
198
261
|
schemaVersion: "openpond.taskEvidence.v1",
|
|
199
262
|
id: learningEvidenceId(source.id, parent.submission.exampleId, attemptId),
|
|
200
263
|
revision: 1,
|
|
201
|
-
source:
|
|
264
|
+
source: learningRef2(source),
|
|
202
265
|
submission: {
|
|
203
266
|
...parent.submission,
|
|
204
267
|
sourceId: source.id,
|
|
205
|
-
taskDefinition:
|
|
268
|
+
taskDefinition: learningRef2(definition),
|
|
206
269
|
attemptId,
|
|
207
270
|
// The old response was observed under the old policy-visible task. Its
|
|
208
271
|
// parent snapshot remains available, but it is not a run of new inputs.
|
|
@@ -212,7 +275,7 @@ function prepareModelBatchReview(input) {
|
|
|
212
275
|
...edit?.expected === void 0 ? {} : { expected: edit.expected },
|
|
213
276
|
...edit?.evaluatorContext === void 0 ? {} : { evaluatorContext: edit.evaluatorContext }
|
|
214
277
|
},
|
|
215
|
-
supersedes:
|
|
278
|
+
supersedes: learningRef2(parent),
|
|
216
279
|
correctionFeedbackId: null,
|
|
217
280
|
receivedAt: input.now
|
|
218
281
|
}));
|
|
@@ -238,7 +301,7 @@ function prepareModelBatchReview(input) {
|
|
|
238
301
|
note: "Proposed target for the revised task. Grade and review before admission."
|
|
239
302
|
},
|
|
240
303
|
status: "pending_review",
|
|
241
|
-
evidence:
|
|
304
|
+
evidence: learningRef2(item),
|
|
242
305
|
createdAt: input.now,
|
|
243
306
|
review: null
|
|
244
307
|
})];
|
|
@@ -277,7 +340,7 @@ async function beginModelBatchReview(transaction, input) {
|
|
|
277
340
|
if (!value.learningResources) throw new LearningDomainError2("model_review_batch_required", 422);
|
|
278
341
|
const metadata = taskBatchPackageMetadata2(value.taskset);
|
|
279
342
|
const selected = request.rewardBindingRef;
|
|
280
|
-
const retained = !selected || sameLearningRef2(selected,
|
|
343
|
+
const retained = !selected || sameLearningRef2(selected, learningRef3(metadata.binding));
|
|
281
344
|
const binding = retained ? metadata.binding : await requireLearningRelease(transaction, "binding", selected);
|
|
282
345
|
const rewards = retained ? metadata.rewards : await Promise.all(binding.sources.map((source) => requireLearningRelease(transaction, "reward", source.reward)));
|
|
283
346
|
const assetIds = new Set(rewards.flatMap((reward) => [
|
|
@@ -324,23 +387,111 @@ async function readReceipt(transaction, request, pointers) {
|
|
|
324
387
|
|
|
325
388
|
// src/model-taskset-authoring.ts
|
|
326
389
|
import { createVerifierSetRelease } from "@openpond/evals";
|
|
327
|
-
import { learningRef as
|
|
390
|
+
import { learningRef as learningRef5, sameLearningRef as sameLearningRef4, sealLearningContent as sealLearningContent4 } from "@openpond/evals/learning";
|
|
391
|
+
import { TasksetReleaseSchema as TasksetReleaseSchema2 } from "@openpond/evals/tasksets";
|
|
392
|
+
|
|
393
|
+
// src/model-bound-taskset-authoring.ts
|
|
394
|
+
import { LearningTextAssetSchema as LearningTextAssetSchema2, TaskDefinitionSchema as TaskDefinitionSchema3, learningRef as learningRef4, sameLearningRef as sameLearningRef3, sealLearningContent as sealLearningContent3 } from "@openpond/evals/learning";
|
|
328
395
|
import { TasksetReleaseSchema } from "@openpond/evals/tasksets";
|
|
396
|
+
function publishBoundModelTasksetDraftPackage(input) {
|
|
397
|
+
const source = validateTasksetPackage(input.source);
|
|
398
|
+
const edited = validateTasksetPackage(input.edited);
|
|
399
|
+
const prepared = input.preparation;
|
|
400
|
+
if (prepared.authoringGraph !== "bound" || !source.modelResources || source.learningResources || edited.modelResources || edited.learningResources) throw new Error("Bound task editing requires its source Reward and an ordinary edited workspace.");
|
|
401
|
+
if (source.contentHash !== prepared.sourcePackageHash || !sameLearningRef3(learningRef4(source.taskset), prepared.sourceTasksetRef)) throw new Error("Bound draft differs from its retained source package.");
|
|
402
|
+
const normalizedGraders = (value) => value.taskset.graders.map((grader) => grader.kind === "custom_verifier" ? { ...grader, exportName: grader.exportName ?? "verify" } : grader);
|
|
403
|
+
if (contentHash(normalizedGraders(source)) !== contentHash(normalizedGraders(edited))) throw new Error("Edit the saved Reward in Rewards before changing this Taskset's graders.");
|
|
404
|
+
const resources = source.modelResources;
|
|
405
|
+
const assets = new Map(resources.assets.map((asset) => [asset.id, asset]));
|
|
406
|
+
const execution = resolveTasksetPackageExecution(edited);
|
|
407
|
+
const sourceExecution = resolveTasksetPackageExecution(source);
|
|
408
|
+
const environmentContent = (value) => {
|
|
409
|
+
const { id: _id, revision: _revision, contentHash: _hash, metadata, ...contract } = value;
|
|
410
|
+
const { javascriptEnvironment: _javascript, ...retained } = metadata;
|
|
411
|
+
return { ...contract, metadata: { ...retained, resources: retained.resources ?? [] } };
|
|
412
|
+
};
|
|
413
|
+
const javascriptContent = (value) => {
|
|
414
|
+
if (!value) return null;
|
|
415
|
+
const { id: _id, revision: _revision, contentHash: _hash, ...contract } = value.execution.javascript;
|
|
416
|
+
return contract;
|
|
417
|
+
};
|
|
418
|
+
const unchangedEnvironment = contentHash(environmentContent(source.environment)) === contentHash(environmentContent(edited.environment)) && contentHash(javascriptContent(sourceExecution)) === contentHash(javascriptContent(execution));
|
|
419
|
+
const selectedEnvironment = unchangedEnvironment ? source.environment : edited.environment;
|
|
420
|
+
const selectedExecution = unchangedEnvironment ? sourceExecution : execution;
|
|
421
|
+
const requiredIds = /* @__PURE__ */ new Set([
|
|
422
|
+
...resources.assets.map((asset) => asset.id),
|
|
423
|
+
...edited.taskset.tasks.flatMap((task) => task.privilegedContextRef ? [task.privilegedContextRef] : []),
|
|
424
|
+
...[edited.environment.actionSchemaRef, edited.environment.observationSchemaRef, edited.environment.stateSchemaRef].flatMap((ref) => ref ? [ref.id] : []),
|
|
425
|
+
...execution?.assets.map((asset) => asset.id) ?? []
|
|
426
|
+
]);
|
|
427
|
+
for (const id of requiredIds) {
|
|
428
|
+
const file = edited.files.find((file2) => file2.asset.id === id);
|
|
429
|
+
if (!file) throw new Error(`Bound Taskset asset is missing: ${id}.`);
|
|
430
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(decodeTasksetPackageFile(file));
|
|
431
|
+
assets.set(id, LearningTextAssetSchema2.parse(sealLearningContent3({ schemaVersion: "openpond.learningTextAsset.v1", id, revision: 1, asset: file.asset, text })));
|
|
432
|
+
}
|
|
433
|
+
const instructions = resolveTasksetPackageInstructions(edited);
|
|
434
|
+
const { contentHash: _definitionHash, ...definitionContent } = resources.taskDefinition;
|
|
435
|
+
const { policy, environment, environmentRelease, tools, capabilities, verifierSetRelease } = edited.taskset;
|
|
436
|
+
const taskDefinition = TaskDefinitionSchema3.parse(sealLearningContent3({
|
|
437
|
+
...definitionContent,
|
|
438
|
+
instructions,
|
|
439
|
+
execution: { policy, environment, environmentRelease, tools, capabilities, verifierSetRelease }
|
|
440
|
+
}));
|
|
441
|
+
const { contentHash: _taskHash, ...taskContent } = edited.taskset;
|
|
442
|
+
const taskset = TasksetReleaseSchema.parse(sealLearningContent3({ ...taskContent, tasks: edited.taskset.tasks.map((task) => resources.instructionMode === "per_task" ? task : { ...task, policyVisibleContext: { ...task.policyVisibleContext, instructions } }) }));
|
|
443
|
+
const derived = compileModelTasksetReward({
|
|
444
|
+
owner: prepared.lineage.owner,
|
|
445
|
+
source: { ...resources, taskset: source.taskset, executionResources: { environment: source.environment, verifierSet: source.verifierSet } },
|
|
446
|
+
rewardBinding: resources.rewardBinding,
|
|
447
|
+
rewards: resources.rewards,
|
|
448
|
+
assets: [...assets.values()],
|
|
449
|
+
edits: { taskset, taskDefinition, executionResources: { environment: selectedEnvironment, verifierSet: edited.verifierSet }, ...selectedExecution ? { execution: selectedExecution.execution } : {} }
|
|
450
|
+
});
|
|
451
|
+
const lineage = ModelTasksetDerivationSchema.parse(derived.taskset.metadata.modelTasksetDerivation);
|
|
452
|
+
if (derived.taskset.id !== prepared.tasksetId || derived.taskset.revision !== prepared.tasksetRevision || contentHash({ ...lineage, schemaVersion: prepared.lineage.schemaVersion }) !== contentHash(prepared.lineage)) throw new Error("Bound draft differs from its retained ownership lineage.");
|
|
453
|
+
const { taskset: published, executionResources, ...modelResources } = derived;
|
|
454
|
+
return createTasksetPackage({ schemaVersion: "openpond.tasksetPackage.v1", taskset: published, ...executionResources, modelResources, files: edited.files });
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// src/model-taskset-authoring.ts
|
|
329
458
|
function prepareModelTasksetDraft(input) {
|
|
330
459
|
const request = ModelTasksetDraftRequestSchema.parse(input.request);
|
|
331
460
|
const owner = ModelTasksetAuthoringOwnerSchema.parse(input.owner);
|
|
332
|
-
const source =
|
|
461
|
+
const source = validateTasksetPackage(input.source);
|
|
462
|
+
if (source.learningResources) throw new Error("Reviewed Tasksets require their reviewed authoring workflow.");
|
|
333
463
|
if (source.contentHash !== request.sourcePackageHash) throw new Error("Taskset draft source package changed.");
|
|
464
|
+
if (source.modelResources) {
|
|
465
|
+
const derived = deriveModelTaskset({
|
|
466
|
+
owner,
|
|
467
|
+
source: { ...source.modelResources, taskset: source.taskset, executionResources: { environment: source.environment, verifierSet: source.verifierSet } },
|
|
468
|
+
rewardBinding: source.modelResources.rewardBinding,
|
|
469
|
+
rewards: source.modelResources.rewards,
|
|
470
|
+
assets: source.modelResources.assets
|
|
471
|
+
});
|
|
472
|
+
const lineage2 = derived.taskset.metadata.modelTasksetDerivation;
|
|
473
|
+
return ModelTasksetDraftPreparationSchema.parse({
|
|
474
|
+
schemaVersion: "openpond.modelTasksetDraftPreparation.v1",
|
|
475
|
+
draftId: `model-taskset-draft-${contentHash({ owner, operationId: request.operationId })}`,
|
|
476
|
+
requestHash: contentHash(request),
|
|
477
|
+
sourcePackageHash: source.contentHash,
|
|
478
|
+
sourceTasksetRef: learningRef5(source.taskset),
|
|
479
|
+
tasksetId: derived.taskset.id,
|
|
480
|
+
tasksetRevision: derived.taskset.revision,
|
|
481
|
+
lineage: { ...lineage2, schemaVersion: "openpond.modelTasksetAuthoring.v1" },
|
|
482
|
+
authoringGraph: "bound"
|
|
483
|
+
});
|
|
484
|
+
}
|
|
334
485
|
const previous = assertModelTasksetAuthoring(source.taskset);
|
|
335
486
|
const owned = previous && contentHash(previous.owner) === contentHash(owner);
|
|
336
|
-
const root = owned ? previous.root :
|
|
337
|
-
const lineage = { schemaVersion: "openpond.modelTasksetAuthoring.v1", owner, root, parent:
|
|
487
|
+
const root = owned ? previous.root : learningRef5(source.taskset);
|
|
488
|
+
const lineage = { schemaVersion: "openpond.modelTasksetAuthoring.v1", owner, root, parent: learningRef5(source.taskset) };
|
|
338
489
|
return ModelTasksetDraftPreparationSchema.parse({
|
|
339
490
|
schemaVersion: "openpond.modelTasksetDraftPreparation.v1",
|
|
340
491
|
draftId: `model-taskset-draft-${contentHash({ owner, operationId: request.operationId })}`,
|
|
341
492
|
requestHash: contentHash(request),
|
|
342
493
|
sourcePackageHash: source.contentHash,
|
|
343
|
-
sourceTasksetRef:
|
|
494
|
+
sourceTasksetRef: learningRef5(source.taskset),
|
|
344
495
|
tasksetId: modelAuthoredTasksetId(lineage),
|
|
345
496
|
tasksetRevision: owned ? source.taskset.revision + 1 : 1,
|
|
346
497
|
lineage
|
|
@@ -348,7 +499,11 @@ function prepareModelTasksetDraft(input) {
|
|
|
348
499
|
}
|
|
349
500
|
function publishModelTasksetDraftPackage(input) {
|
|
350
501
|
const prepared = ModelTasksetDraftPreparationSchema.parse(input.preparation);
|
|
351
|
-
if (!
|
|
502
|
+
if (!sameLearningRef4(prepared.sourceTasksetRef, prepared.lineage.parent)) throw new Error("Taskset draft preparation differs from its source lineage.");
|
|
503
|
+
if (prepared.authoringGraph === "bound") {
|
|
504
|
+
if (!input.source) throw new Error("Bound Taskset authoring requires its retained source package.");
|
|
505
|
+
return publishBoundModelTasksetDraftPackage({ preparation: prepared, source: input.source, edited: input.edited });
|
|
506
|
+
}
|
|
352
507
|
const edited = requireOrdinaryPackage(input.edited);
|
|
353
508
|
const { contentHash: _verifierHash, ...verifierContent } = edited.verifierSet;
|
|
354
509
|
const verifierSet = createVerifierSetRelease({
|
|
@@ -356,10 +511,10 @@ function publishModelTasksetDraftPackage(input) {
|
|
|
356
511
|
id: `${prepared.tasksetId}-verifiers`,
|
|
357
512
|
revision: prepared.tasksetRevision,
|
|
358
513
|
calibrationReceiptRefs: [],
|
|
359
|
-
metadata: { sourceVerifierSet:
|
|
514
|
+
metadata: { sourceVerifierSet: learningRef5(edited.verifierSet) }
|
|
360
515
|
});
|
|
361
516
|
const { contentHash: _hash, ...content } = edited.taskset;
|
|
362
|
-
const taskset =
|
|
517
|
+
const taskset = TasksetReleaseSchema2.parse(sealLearningContent4({
|
|
363
518
|
...content,
|
|
364
519
|
id: prepared.tasksetId,
|
|
365
520
|
revision: prepared.tasksetRevision,
|
|
@@ -384,7 +539,7 @@ function requireOrdinaryPackage(value) {
|
|
|
384
539
|
}
|
|
385
540
|
|
|
386
541
|
// src/taskset-package-projection.ts
|
|
387
|
-
import { learningRef as
|
|
542
|
+
import { learningRef as learningRef6, taskBatchPackageMetadata as taskBatchPackageMetadata3, verifyLearningTextAsset as verifyLearningTextAsset2 } from "@openpond/evals/learning";
|
|
388
543
|
function prepareImportedTasksetPackage(input) {
|
|
389
544
|
const value = validateTasksetPackage(input.package);
|
|
390
545
|
const release = value.taskset;
|
|
@@ -507,8 +662,8 @@ function prepareImportedTasksetPackage(input) {
|
|
|
507
662
|
...learning ? { learning } : {},
|
|
508
663
|
portableFileInventory: value.files.map((file) => ({ asset: file.asset, sourcePath: file.asset.path })),
|
|
509
664
|
...resources ? {
|
|
510
|
-
taskDefinition:
|
|
511
|
-
rewardBinding:
|
|
665
|
+
taskDefinition: learningRef6(resources.taskDefinition),
|
|
666
|
+
rewardBinding: learningRef6(resources.rewardBinding),
|
|
512
667
|
rewardExecution: { binding: resources.rewardBinding, rewards: resources.rewards }
|
|
513
668
|
} : {},
|
|
514
669
|
portableCapabilities: release.capabilities,
|
|
@@ -528,7 +683,7 @@ import { z } from "zod";
|
|
|
528
683
|
// src/taskset-authored-tool-environment.ts
|
|
529
684
|
import { createEnvironmentRelease } from "@openpond/evals";
|
|
530
685
|
import { JavaScriptEnvironmentDefinitionSchema } from "@openpond/evals/javascript-environment";
|
|
531
|
-
import { learningRef as
|
|
686
|
+
import { learningRef as learningRef7, sealLearningContent as sealLearningContent5 } from "@openpond/evals/learning";
|
|
532
687
|
function prepareAuthoredToolEnvironment(taskset, bytes, inventory, adapterId) {
|
|
533
688
|
if (taskset.environment.entrypoint !== "openpond.javascript-environment.v1") return { taskset, inventory, generatedFiles: [] };
|
|
534
689
|
const declarationPath = "environment/execution.json";
|
|
@@ -542,7 +697,7 @@ function prepareAuthoredToolEnvironment(taskset, bytes, inventory, adapterId) {
|
|
|
542
697
|
return file2.asset;
|
|
543
698
|
};
|
|
544
699
|
const { contentHash: _javascriptHash, ...javascriptContent } = previous.javascript;
|
|
545
|
-
const javascript = JavaScriptEnvironmentDefinitionSchema.parse(
|
|
700
|
+
const javascript = JavaScriptEnvironmentDefinitionSchema.parse(sealLearningContent5({
|
|
546
701
|
...javascriptContent,
|
|
547
702
|
id: `javascript-${taskset.id}`,
|
|
548
703
|
revision: taskset.revision,
|
|
@@ -559,7 +714,7 @@ function prepareAuthoredToolEnvironment(taskset, bytes, inventory, adapterId) {
|
|
|
559
714
|
stateSchemaRef: repin(previous.environment.stateSchemaRef),
|
|
560
715
|
artifactCollection: previous.environment.artifactCollection,
|
|
561
716
|
adapterConformanceHashes: {},
|
|
562
|
-
metadata: { javascriptEnvironment:
|
|
717
|
+
metadata: { javascriptEnvironment: learningRef7(javascript), resources: taskset.environment.resources ?? [] }
|
|
563
718
|
});
|
|
564
719
|
const prepared = TasksetSchema.parse({ ...taskset, environment: {
|
|
565
720
|
...taskset.environment,
|
|
@@ -690,12 +845,12 @@ function compileModelTasksetDraftWorkspace(input) {
|
|
|
690
845
|
return { asset: entry.asset, base64: Buffer.from(bytes).toString("base64") };
|
|
691
846
|
})
|
|
692
847
|
});
|
|
693
|
-
return preparation ? publishModelTasksetDraftPackage({ preparation, edited }) : edited;
|
|
848
|
+
return preparation ? publishModelTasksetDraftPackage({ preparation, edited, source: input.source }) : edited;
|
|
694
849
|
}
|
|
695
850
|
|
|
696
851
|
// src/taskset-draft-source.ts
|
|
697
852
|
import { z as z2 } from "zod";
|
|
698
|
-
import { learningRef as
|
|
853
|
+
import { learningRef as learningRef8, sameLearningRef as sameLearningRef5 } from "@openpond/evals/learning";
|
|
699
854
|
var TasksetDraftSourceInitializationSchema = z2.object({
|
|
700
855
|
draft: TasksetDraftSchema,
|
|
701
856
|
filePaths: z2.array(z2.object({ assetId: z2.string().min(1), paths: z2.array(TasksetDraftFilePathSchema).min(1).max(1e4) }).strict()).max(1e4)
|
|
@@ -704,8 +859,9 @@ function prepareTasksetDraftSource(input) {
|
|
|
704
859
|
const source = validateTasksetPackage(input.source);
|
|
705
860
|
const original = TasksetDraftSchema.parse(input.sourceDraft);
|
|
706
861
|
const preparation = ModelTasksetDraftPreparationSchema.parse(input.preparation);
|
|
707
|
-
if (source.contentHash !== preparation.sourcePackageHash || !
|
|
708
|
-
if (source.
|
|
862
|
+
if (source.contentHash !== preparation.sourcePackageHash || !sameLearningRef5(learningRef8(source.taskset), preparation.sourceTasksetRef)) throw new Error("Taskset draft preparation differs from its source package.");
|
|
863
|
+
if (source.learningResources) throw new Error("Reviewed Tasksets require their reviewed authoring workflow.");
|
|
864
|
+
if (Boolean(source.modelResources) !== (preparation.authoringGraph === "bound")) throw new Error("Taskset draft authoring graph differs from its source.");
|
|
709
865
|
const inventory = AuthoredTasksetFileInventorySchema.parse(original.metadata.portableFileInventory ?? []);
|
|
710
866
|
const sourcePath = (relative) => isManagedTasksetDraftFilePath(relative) ? `source-artifacts/${source.contentHash}/${contentHash(relative)}/${relative.split("/").at(-1)}` : relative;
|
|
711
867
|
const references = [
|
|
@@ -726,16 +882,22 @@ function prepareTasksetDraftSource(input) {
|
|
|
726
882
|
}
|
|
727
883
|
return { assetId: file.asset.id, paths: [...paths].map(sourcePath) };
|
|
728
884
|
});
|
|
885
|
+
const metadata = { ...original.metadata };
|
|
886
|
+
if (source.modelResources) for (const key of ["taskDefinition", "rewardBinding", "rewardExecution", "derivedPortableMetadata", "modelTasksetDerivation", "importedPackageHash"]) delete metadata[key];
|
|
729
887
|
const draft = TasksetDraftSchema.parse({
|
|
730
888
|
...original,
|
|
731
889
|
id: preparation.draftId,
|
|
732
890
|
revision: 1,
|
|
733
891
|
status: "draft",
|
|
892
|
+
graders: original.graders.map((grader) => ({ ...grader, metadata: {
|
|
893
|
+
...grader.metadata,
|
|
894
|
+
portableGrader: source.taskset.graders.find((candidate) => candidate.id === grader.id)
|
|
895
|
+
} })),
|
|
734
896
|
environment: { ...original.environment, metadata: { ...original.environment.metadata, portableExecutionResources: { environment: source.environment } } },
|
|
735
897
|
modelScope: { modelId: preparation.lineage.owner.modelId, expectedModelRevision: input.expectedModelRevision, source: preparation },
|
|
736
898
|
publishedTasksetRef: preparation.tasksetRevision > 1 ? preparation.sourceTasksetRef : null,
|
|
737
899
|
metadata: {
|
|
738
|
-
...
|
|
900
|
+
...metadata,
|
|
739
901
|
modelTasksetAuthoring: preparation.lineage,
|
|
740
902
|
portableFileInventory: inventory.map((entry) => ({ ...entry, sourcePath: sourcePath(entry.sourcePath), asset: {
|
|
741
903
|
...entry.asset,
|
|
@@ -801,6 +963,7 @@ export {
|
|
|
801
963
|
TasksetPackageReceiptSchema,
|
|
802
964
|
TasksetPackageSchema,
|
|
803
965
|
beginModelBatchReview,
|
|
966
|
+
bindOrdinaryModelTasksetReward,
|
|
804
967
|
compileModelTasksetDraftWorkspace,
|
|
805
968
|
createTasksetPackage,
|
|
806
969
|
createTasksetPackageExecutionFile,
|