openpond-sdk 0.5.27 → 0.5.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/TASKSETS.md +20 -4
- package/dist/{chunk-S7I45WJQ.js → chunk-2U54EOXJ.js} +3 -2
- package/dist/chunk-2U54EOXJ.js.map +7 -0
- package/dist/{chunk-IDLMILN4.js → chunk-B3CH4V35.js} +21 -14
- package/dist/chunk-B3CH4V35.js.map +7 -0
- package/dist/{chunk-ULQC4SQF.js → chunk-FFRYTXGF.js} +3 -3
- package/dist/{chunk-AUQOAMHL.js → chunk-O27OEXXB.js} +2 -2
- package/dist/{chunk-DV7XI3UE.js → chunk-ZIMHPYXL.js} +2 -2
- package/dist/model-starter-catalog.js +3 -3
- package/dist/model-starters.js +3 -3
- package/dist/model-taskset-authoring.js +1 -1
- package/dist/taskset-drafts.js +3 -3
- package/dist/taskset-packages.js +206 -44
- package/dist/taskset-packages.js.map +4 -4
- package/dist/training-bundle.js +3 -3
- package/dist/types/packages/sdk/src/model-bound-taskset-authoring.d.ts +10 -0
- package/dist/types/packages/sdk/src/model-bound-taskset-authoring.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-ordinary-taskset-reward.d.ts +16 -0
- package/dist/types/packages/sdk/src/model-ordinary-taskset-reward.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-taskset-authoring-contracts.d.ts +1 -0
- package/dist/types/packages/sdk/src/model-taskset-authoring-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-authoring.d.ts +1 -0
- package/dist/types/packages/sdk/src/model-taskset-authoring.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-derivation.d.ts +11 -0
- package/dist/types/packages/sdk/src/model-taskset-derivation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-api-contracts.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-draft-api-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-client.d.ts +6 -0
- package/dist/types/packages/sdk/src/taskset-draft-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-document.d.ts +1 -0
- package/dist/types/packages/sdk/src/taskset-draft-document.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-package-compiler.d.ts +3 -0
- package/dist/types/packages/sdk/src/taskset-draft-package-compiler.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-source.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-draft-source.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-workspace.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-draft-workspace.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts +3 -0
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-projection.d.ts +1 -0
- package/dist/types/packages/sdk/src/taskset-package-projection.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-packages.d.ts +1 -0
- package/dist/types/packages/sdk/src/taskset-packages.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-IDLMILN4.js.map +0 -7
- package/dist/chunk-S7I45WJQ.js.map +0 -7
- /package/dist/{chunk-ULQC4SQF.js.map → chunk-FFRYTXGF.js.map} +0 -0
- /package/dist/{chunk-AUQOAMHL.js.map → chunk-O27OEXXB.js.map} +0 -0
- /package/dist/{chunk-DV7XI3UE.js.map → chunk-ZIMHPYXL.js.map} +0 -0
package/dist/taskset-packages.js
CHANGED
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
projectPortableTaskRecord,
|
|
14
14
|
projectPreferenceComparisonRelease,
|
|
15
15
|
resolvePortableTasksetRewardExecution
|
|
16
|
-
} from "./chunk-
|
|
16
|
+
} from "./chunk-O27OEXXB.js";
|
|
17
17
|
import {
|
|
18
18
|
OpenPondTasksetPackageClient,
|
|
19
19
|
OpenPondTasksetPackageError,
|
|
@@ -30,7 +30,7 @@ import {
|
|
|
30
30
|
publishTasksetDraft,
|
|
31
31
|
renderTasksetDraftManifests,
|
|
32
32
|
validateTasksetDraftWorkspace
|
|
33
|
-
} from "./chunk-
|
|
33
|
+
} from "./chunk-FFRYTXGF.js";
|
|
34
34
|
import {
|
|
35
35
|
GraderFixtureSchema,
|
|
36
36
|
TasksetEnvironmentResourceSchema,
|
|
@@ -41,14 +41,17 @@ import {
|
|
|
41
41
|
import {
|
|
42
42
|
MAX_TASKSET_PACKAGE_BYTES,
|
|
43
43
|
ModelStarterExecutionSchema,
|
|
44
|
+
ModelTasksetDerivationSchema,
|
|
44
45
|
TasksetPackageContentSchema,
|
|
45
46
|
TasksetPackageFileSchema,
|
|
46
47
|
TasksetPackageLearningResourcesSchema,
|
|
47
48
|
TasksetPackageSchema,
|
|
48
49
|
assertModelTasksetAuthoring,
|
|
50
|
+
compileModelTasksetReward,
|
|
49
51
|
createTasksetPackage,
|
|
50
52
|
createTasksetPackageExecutionFile,
|
|
51
53
|
decodeTasksetPackageFile,
|
|
54
|
+
deriveModelTaskset,
|
|
52
55
|
learningPackageContextFiles,
|
|
53
56
|
modelAuthoredTasksetId,
|
|
54
57
|
modelStarterExecutionAssetId,
|
|
@@ -57,7 +60,7 @@ import {
|
|
|
57
60
|
tasksetPackageRewardBinding,
|
|
58
61
|
validateTasksetLearningResources,
|
|
59
62
|
validateTasksetPackage
|
|
60
|
-
} from "./chunk-
|
|
63
|
+
} from "./chunk-B3CH4V35.js";
|
|
61
64
|
import {
|
|
62
65
|
ModelTasksetAuthoringOwnerSchema,
|
|
63
66
|
ModelTasksetAuthoringSchema,
|
|
@@ -68,7 +71,7 @@ import {
|
|
|
68
71
|
TasksetDraftFileMutationSchema,
|
|
69
72
|
TasksetDraftFilePathSchema,
|
|
70
73
|
TasksetDraftFileSchema
|
|
71
|
-
} from "./chunk-
|
|
74
|
+
} from "./chunk-2U54EOXJ.js";
|
|
72
75
|
import "./chunk-GZOXT7YE.js";
|
|
73
76
|
import "./chunk-A4ZDOZKP.js";
|
|
74
77
|
import "./chunk-5FBPNWT4.js";
|
|
@@ -79,10 +82,69 @@ import {
|
|
|
79
82
|
} from "./chunk-QUOW6URQ.js";
|
|
80
83
|
import "./chunk-ZNFOVV7B.js";
|
|
81
84
|
|
|
85
|
+
// src/model-ordinary-taskset-reward.ts
|
|
86
|
+
import { LearningTextAssetSchema, TaskDefinitionSchema, learningRef, sealLearningContent } from "@openpond/evals/learning";
|
|
87
|
+
function bindOrdinaryModelTasksetReward(input) {
|
|
88
|
+
const source = validateTasksetPackage(input.source);
|
|
89
|
+
if (source.modelResources || source.learningResources) throw new Error("Binding an ordinary Reward requires an ordinary source package.");
|
|
90
|
+
const instructions = resolveTasksetPackageInstructions(source);
|
|
91
|
+
if (!instructions.trim()) throw new Error("Add collection instructions before selecting a saved Reward.");
|
|
92
|
+
const assets = /* @__PURE__ */ new Map();
|
|
93
|
+
const execution = resolveTasksetPackageExecution(source);
|
|
94
|
+
for (const asset of execution?.assets ?? []) assets.set(asset.id, asset);
|
|
95
|
+
const contextIds = /* @__PURE__ */ new Set([
|
|
96
|
+
...source.taskset.tasks.flatMap((task) => task.privilegedContextRef ? [task.privilegedContextRef] : []),
|
|
97
|
+
...[source.environment.actionSchemaRef, source.environment.observationSchemaRef, source.environment.stateSchemaRef].flatMap((ref) => ref ? [ref.id] : [])
|
|
98
|
+
]);
|
|
99
|
+
for (const id of contextIds) {
|
|
100
|
+
if (assets.has(id)) continue;
|
|
101
|
+
const file = source.files.find((file2) => file2.asset.id === id);
|
|
102
|
+
if (!file) throw new Error(`Source Taskset context is missing: ${id}.`);
|
|
103
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(decodeTasksetPackageFile(file));
|
|
104
|
+
assets.set(id, LearningTextAssetSchema.parse(sealLearningContent({ schemaVersion: "openpond.learningTextAsset.v1", id, revision: 1, asset: file.asset, text })));
|
|
105
|
+
}
|
|
106
|
+
for (const asset of input.assets) {
|
|
107
|
+
const original = source.files.find((file) => file.asset.id === asset.id);
|
|
108
|
+
if (original && sealLearningContent({ asset: original.asset }).contentHash !== sealLearningContent({ asset: asset.asset }).contentHash) throw new Error(`Selected Reward conflicts with a source asset: ${asset.id}.`);
|
|
109
|
+
assets.set(asset.id, asset);
|
|
110
|
+
}
|
|
111
|
+
const { policy, environment, environmentRelease, tools, capabilities, verifierSetRelease } = source.taskset;
|
|
112
|
+
const taskDefinition = TaskDefinitionSchema.parse(sealLearningContent({
|
|
113
|
+
schemaVersion: "openpond.taskDefinition.v1",
|
|
114
|
+
id: `${source.taskset.id}-definition`,
|
|
115
|
+
revision: 1,
|
|
116
|
+
name: source.taskset.id,
|
|
117
|
+
description: "",
|
|
118
|
+
instructions,
|
|
119
|
+
category: "custom",
|
|
120
|
+
familyNamespace: source.taskset.id,
|
|
121
|
+
inputSchema: { type: "object" },
|
|
122
|
+
outputSchema: { type: "object" },
|
|
123
|
+
rewardBinding: learningRef(input.rewardBinding),
|
|
124
|
+
harness: null,
|
|
125
|
+
execution: { policy, environment, environmentRelease, tools, capabilities, verifierSetRelease }
|
|
126
|
+
}));
|
|
127
|
+
const derived = compileModelTasksetReward({ ...input, assets: [...assets.values()], source: {
|
|
128
|
+
taskset: source.taskset,
|
|
129
|
+
taskDefinition,
|
|
130
|
+
instructionMode: "per_task",
|
|
131
|
+
executionResources: { environment: source.environment, verifierSet: source.verifierSet },
|
|
132
|
+
...execution ? { execution: execution.execution } : {}
|
|
133
|
+
} });
|
|
134
|
+
const files = new Map(source.files.map((file) => [file.asset.id, file]));
|
|
135
|
+
for (const asset of derived.assets) {
|
|
136
|
+
let raw = "";
|
|
137
|
+
for (const byte of new TextEncoder().encode(asset.text)) raw += String.fromCharCode(byte);
|
|
138
|
+
files.set(asset.id, { asset: asset.asset, base64: btoa(raw) });
|
|
139
|
+
}
|
|
140
|
+
const { taskset, executionResources, ...modelResources } = derived;
|
|
141
|
+
return createTasksetPackage({ schemaVersion: "openpond.tasksetPackage.v1", taskset, ...executionResources, modelResources, files: [...files.values()] });
|
|
142
|
+
}
|
|
143
|
+
|
|
82
144
|
// src/model-batch-review.ts
|
|
83
145
|
import {
|
|
84
146
|
LearningDomainError as LearningDomainError2,
|
|
85
|
-
learningRef as
|
|
147
|
+
learningRef as learningRef3,
|
|
86
148
|
sameLearningRef as sameLearningRef2,
|
|
87
149
|
requireLearningRelease,
|
|
88
150
|
requireLearningResource,
|
|
@@ -93,14 +155,14 @@ import {
|
|
|
93
155
|
import {
|
|
94
156
|
LearningDomainError,
|
|
95
157
|
LearningSourceSchema,
|
|
96
|
-
TaskDefinitionSchema,
|
|
158
|
+
TaskDefinitionSchema as TaskDefinitionSchema2,
|
|
97
159
|
TaskEvidenceSchema,
|
|
98
160
|
TaskFeedbackSchema,
|
|
99
161
|
assertLearningContentHash,
|
|
100
162
|
learningEvidenceId,
|
|
101
|
-
learningRef,
|
|
163
|
+
learningRef as learningRef2,
|
|
102
164
|
sameLearningRef,
|
|
103
|
-
sealLearningContent,
|
|
165
|
+
sealLearningContent as sealLearningContent2,
|
|
104
166
|
taskBatchPackageMetadata,
|
|
105
167
|
verifyLearningTextAsset
|
|
106
168
|
} from "@openpond/evals/learning";
|
|
@@ -112,7 +174,7 @@ function prepareModelBatchReview(input) {
|
|
|
112
174
|
const metadata = taskBatchPackageMetadata(value.taskset);
|
|
113
175
|
const id = `model-review-${contentHash([input.scope, request.modelId, request.operationId])}`;
|
|
114
176
|
const edits = new Map(request.examples.map((edit) => [contentHash(edit.evidence), edit]));
|
|
115
|
-
const parentRefs = new Set(learning.evidence.map((evidence2) => contentHash(
|
|
177
|
+
const parentRefs = new Set(learning.evidence.map((evidence2) => contentHash(learningRef2(evidence2))));
|
|
116
178
|
const decisions = new Map(learning.decisions.map((decision) => [contentHash(decision.evidence), decision]));
|
|
117
179
|
if (edits.size !== request.examples.length || request.examples.some((edit) => !parentRefs.has(contentHash(edit.evidence)))) {
|
|
118
180
|
throw new LearningDomainError("model_review_evidence_mismatch", 409);
|
|
@@ -121,7 +183,7 @@ function prepareModelBatchReview(input) {
|
|
|
121
183
|
resolveBoundRewards(input.binding, input.rewards);
|
|
122
184
|
const rewardByHash = /* @__PURE__ */ new Map();
|
|
123
185
|
for (const source2 of input.binding.sources) {
|
|
124
|
-
const original = input.rewards.find((reward) => sameLearningRef(
|
|
186
|
+
const original = input.rewards.find((reward) => sameLearningRef(learningRef2(reward), source2.reward));
|
|
125
187
|
const { contentHash: _hash, ...content } = original;
|
|
126
188
|
rewardByHash.set(original.contentHash, createRewardRelease({ ...content, id: `${id}-reward-${original.contentHash}`, revision: 1 }));
|
|
127
189
|
}
|
|
@@ -131,7 +193,7 @@ function prepareModelBatchReview(input) {
|
|
|
131
193
|
...bindingContent,
|
|
132
194
|
id: `${id}-binding`,
|
|
133
195
|
revision: 1,
|
|
134
|
-
sources: input.binding.sources.map((source2) => ({ ...source2, reward:
|
|
196
|
+
sources: input.binding.sources.map((source2) => ({ ...source2, reward: learningRef2(rewardByHash.get(source2.reward.contentHash)) }))
|
|
135
197
|
}, rewards);
|
|
136
198
|
const assetRefs = rewards.flatMap((reward) => [
|
|
137
199
|
...reward.assets,
|
|
@@ -148,24 +210,24 @@ function prepareModelBatchReview(input) {
|
|
|
148
210
|
})).values()];
|
|
149
211
|
const { environmentRelease: _environment, verifierSetRelease: _verifiers, ...execution } = metadata.definition.execution;
|
|
150
212
|
const { contentHash: _definitionHash, ...definitionContent } = metadata.definition;
|
|
151
|
-
const definition =
|
|
213
|
+
const definition = TaskDefinitionSchema2.parse(sealLearningContent2({
|
|
152
214
|
...definitionContent,
|
|
153
215
|
...request.definition,
|
|
154
216
|
id: `${id}-definition`,
|
|
155
217
|
revision: 1,
|
|
156
|
-
rewardBinding:
|
|
218
|
+
rewardBinding: learningRef2(binding),
|
|
157
219
|
execution: {
|
|
158
220
|
...execution,
|
|
159
221
|
policy: { ...execution.policy, hiddenGraderRefs: compileBoundGraders(binding, rewards).filter((grader) => grader.privileged).map((grader) => grader.id) }
|
|
160
222
|
}
|
|
161
223
|
}));
|
|
162
|
-
const source = LearningSourceSchema.parse(
|
|
224
|
+
const source = LearningSourceSchema.parse(sealLearningContent2({
|
|
163
225
|
schemaVersion: "openpond.learningSource.v1",
|
|
164
226
|
id,
|
|
165
227
|
revision: 1,
|
|
166
228
|
name: `${definition.name} review`,
|
|
167
229
|
kind: "direct",
|
|
168
|
-
taskDefinition:
|
|
230
|
+
taskDefinition: learningRef2(definition),
|
|
169
231
|
enabled: true,
|
|
170
232
|
allowedSplits: [...new Set(learning.evidence.map((item) => item.submission.split))],
|
|
171
233
|
mapping: null,
|
|
@@ -173,13 +235,13 @@ function prepareModelBatchReview(input) {
|
|
|
173
235
|
reviewOrigin: {
|
|
174
236
|
modelId: request.modelId,
|
|
175
237
|
packageHash: value.contentHash,
|
|
176
|
-
taskset:
|
|
177
|
-
batch:
|
|
178
|
-
rewardBinding:
|
|
238
|
+
taskset: learningRef2(value.taskset),
|
|
239
|
+
batch: learningRef2(learning.batch),
|
|
240
|
+
rewardBinding: learningRef2(input.binding)
|
|
179
241
|
}
|
|
180
242
|
}));
|
|
181
243
|
const evidence = learning.evidence.map((parent) => {
|
|
182
|
-
const edit = edits.get(contentHash(
|
|
244
|
+
const edit = edits.get(contentHash(learningRef2(parent)));
|
|
183
245
|
const attemptId = `review-${parent.contentHash}`;
|
|
184
246
|
const policyChanged = contentHash({
|
|
185
247
|
instructions: definition.instructions,
|
|
@@ -194,15 +256,15 @@ function prepareModelBatchReview(input) {
|
|
|
194
256
|
requiredOutputs: metadata.definition.requiredOutputs ?? [],
|
|
195
257
|
input: parent.submission.input
|
|
196
258
|
});
|
|
197
|
-
return TaskEvidenceSchema.parse(
|
|
259
|
+
return TaskEvidenceSchema.parse(sealLearningContent2({
|
|
198
260
|
schemaVersion: "openpond.taskEvidence.v1",
|
|
199
261
|
id: learningEvidenceId(source.id, parent.submission.exampleId, attemptId),
|
|
200
262
|
revision: 1,
|
|
201
|
-
source:
|
|
263
|
+
source: learningRef2(source),
|
|
202
264
|
submission: {
|
|
203
265
|
...parent.submission,
|
|
204
266
|
sourceId: source.id,
|
|
205
|
-
taskDefinition:
|
|
267
|
+
taskDefinition: learningRef2(definition),
|
|
206
268
|
attemptId,
|
|
207
269
|
// The old response was observed under the old policy-visible task. Its
|
|
208
270
|
// parent snapshot remains available, but it is not a run of new inputs.
|
|
@@ -212,7 +274,7 @@ function prepareModelBatchReview(input) {
|
|
|
212
274
|
...edit?.expected === void 0 ? {} : { expected: edit.expected },
|
|
213
275
|
...edit?.evaluatorContext === void 0 ? {} : { evaluatorContext: edit.evaluatorContext }
|
|
214
276
|
},
|
|
215
|
-
supersedes:
|
|
277
|
+
supersedes: learningRef2(parent),
|
|
216
278
|
correctionFeedbackId: null,
|
|
217
279
|
receivedAt: input.now
|
|
218
280
|
}));
|
|
@@ -238,7 +300,7 @@ function prepareModelBatchReview(input) {
|
|
|
238
300
|
note: "Proposed target for the revised task. Grade and review before admission."
|
|
239
301
|
},
|
|
240
302
|
status: "pending_review",
|
|
241
|
-
evidence:
|
|
303
|
+
evidence: learningRef2(item),
|
|
242
304
|
createdAt: input.now,
|
|
243
305
|
review: null
|
|
244
306
|
})];
|
|
@@ -277,7 +339,7 @@ async function beginModelBatchReview(transaction, input) {
|
|
|
277
339
|
if (!value.learningResources) throw new LearningDomainError2("model_review_batch_required", 422);
|
|
278
340
|
const metadata = taskBatchPackageMetadata2(value.taskset);
|
|
279
341
|
const selected = request.rewardBindingRef;
|
|
280
|
-
const retained = !selected || sameLearningRef2(selected,
|
|
342
|
+
const retained = !selected || sameLearningRef2(selected, learningRef3(metadata.binding));
|
|
281
343
|
const binding = retained ? metadata.binding : await requireLearningRelease(transaction, "binding", selected);
|
|
282
344
|
const rewards = retained ? metadata.rewards : await Promise.all(binding.sources.map((source) => requireLearningRelease(transaction, "reward", source.reward)));
|
|
283
345
|
const assetIds = new Set(rewards.flatMap((reward) => [
|
|
@@ -324,23 +386,111 @@ async function readReceipt(transaction, request, pointers) {
|
|
|
324
386
|
|
|
325
387
|
// src/model-taskset-authoring.ts
|
|
326
388
|
import { createVerifierSetRelease } from "@openpond/evals";
|
|
327
|
-
import { learningRef as
|
|
389
|
+
import { learningRef as learningRef5, sameLearningRef as sameLearningRef4, sealLearningContent as sealLearningContent4 } from "@openpond/evals/learning";
|
|
390
|
+
import { TasksetReleaseSchema as TasksetReleaseSchema2 } from "@openpond/evals/tasksets";
|
|
391
|
+
|
|
392
|
+
// src/model-bound-taskset-authoring.ts
|
|
393
|
+
import { LearningTextAssetSchema as LearningTextAssetSchema2, TaskDefinitionSchema as TaskDefinitionSchema3, learningRef as learningRef4, sameLearningRef as sameLearningRef3, sealLearningContent as sealLearningContent3 } from "@openpond/evals/learning";
|
|
328
394
|
import { TasksetReleaseSchema } from "@openpond/evals/tasksets";
|
|
395
|
+
function publishBoundModelTasksetDraftPackage(input) {
|
|
396
|
+
const source = validateTasksetPackage(input.source);
|
|
397
|
+
const edited = validateTasksetPackage(input.edited);
|
|
398
|
+
const prepared = input.preparation;
|
|
399
|
+
if (prepared.authoringGraph !== "bound" || !source.modelResources || source.learningResources || edited.modelResources || edited.learningResources) throw new Error("Bound task editing requires its source Reward and an ordinary edited workspace.");
|
|
400
|
+
if (source.contentHash !== prepared.sourcePackageHash || !sameLearningRef3(learningRef4(source.taskset), prepared.sourceTasksetRef)) throw new Error("Bound draft differs from its retained source package.");
|
|
401
|
+
const normalizedGraders = (value) => value.taskset.graders.map((grader) => grader.kind === "custom_verifier" ? { ...grader, exportName: grader.exportName ?? "verify" } : grader);
|
|
402
|
+
if (contentHash(normalizedGraders(source)) !== contentHash(normalizedGraders(edited))) throw new Error("Edit the saved Reward in Rewards before changing this Taskset's graders.");
|
|
403
|
+
const resources = source.modelResources;
|
|
404
|
+
const assets = new Map(resources.assets.map((asset) => [asset.id, asset]));
|
|
405
|
+
const execution = resolveTasksetPackageExecution(edited);
|
|
406
|
+
const sourceExecution = resolveTasksetPackageExecution(source);
|
|
407
|
+
const environmentContent = (value) => {
|
|
408
|
+
const { id: _id, revision: _revision, contentHash: _hash, metadata, ...contract } = value;
|
|
409
|
+
const { javascriptEnvironment: _javascript, ...retained } = metadata;
|
|
410
|
+
return { ...contract, metadata: { ...retained, resources: retained.resources ?? [] } };
|
|
411
|
+
};
|
|
412
|
+
const javascriptContent = (value) => {
|
|
413
|
+
if (!value) return null;
|
|
414
|
+
const { id: _id, revision: _revision, contentHash: _hash, ...contract } = value.execution.javascript;
|
|
415
|
+
return contract;
|
|
416
|
+
};
|
|
417
|
+
const unchangedEnvironment = contentHash(environmentContent(source.environment)) === contentHash(environmentContent(edited.environment)) && contentHash(javascriptContent(sourceExecution)) === contentHash(javascriptContent(execution));
|
|
418
|
+
const selectedEnvironment = unchangedEnvironment ? source.environment : edited.environment;
|
|
419
|
+
const selectedExecution = unchangedEnvironment ? sourceExecution : execution;
|
|
420
|
+
const requiredIds = /* @__PURE__ */ new Set([
|
|
421
|
+
...resources.assets.map((asset) => asset.id),
|
|
422
|
+
...edited.taskset.tasks.flatMap((task) => task.privilegedContextRef ? [task.privilegedContextRef] : []),
|
|
423
|
+
...[edited.environment.actionSchemaRef, edited.environment.observationSchemaRef, edited.environment.stateSchemaRef].flatMap((ref) => ref ? [ref.id] : []),
|
|
424
|
+
...execution?.assets.map((asset) => asset.id) ?? []
|
|
425
|
+
]);
|
|
426
|
+
for (const id of requiredIds) {
|
|
427
|
+
const file = edited.files.find((file2) => file2.asset.id === id);
|
|
428
|
+
if (!file) throw new Error(`Bound Taskset asset is missing: ${id}.`);
|
|
429
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(decodeTasksetPackageFile(file));
|
|
430
|
+
assets.set(id, LearningTextAssetSchema2.parse(sealLearningContent3({ schemaVersion: "openpond.learningTextAsset.v1", id, revision: 1, asset: file.asset, text })));
|
|
431
|
+
}
|
|
432
|
+
const instructions = resolveTasksetPackageInstructions(edited);
|
|
433
|
+
const { contentHash: _definitionHash, ...definitionContent } = resources.taskDefinition;
|
|
434
|
+
const { policy, environment, environmentRelease, tools, capabilities, verifierSetRelease } = edited.taskset;
|
|
435
|
+
const taskDefinition = TaskDefinitionSchema3.parse(sealLearningContent3({
|
|
436
|
+
...definitionContent,
|
|
437
|
+
instructions,
|
|
438
|
+
execution: { policy, environment, environmentRelease, tools, capabilities, verifierSetRelease }
|
|
439
|
+
}));
|
|
440
|
+
const { contentHash: _taskHash, ...taskContent } = edited.taskset;
|
|
441
|
+
const taskset = TasksetReleaseSchema.parse(sealLearningContent3({ ...taskContent, tasks: edited.taskset.tasks.map((task) => resources.instructionMode === "per_task" ? task : { ...task, policyVisibleContext: { ...task.policyVisibleContext, instructions } }) }));
|
|
442
|
+
const derived = compileModelTasksetReward({
|
|
443
|
+
owner: prepared.lineage.owner,
|
|
444
|
+
source: { ...resources, taskset: source.taskset, executionResources: { environment: source.environment, verifierSet: source.verifierSet } },
|
|
445
|
+
rewardBinding: resources.rewardBinding,
|
|
446
|
+
rewards: resources.rewards,
|
|
447
|
+
assets: [...assets.values()],
|
|
448
|
+
edits: { taskset, taskDefinition, executionResources: { environment: selectedEnvironment, verifierSet: edited.verifierSet }, ...selectedExecution ? { execution: selectedExecution.execution } : {} }
|
|
449
|
+
});
|
|
450
|
+
const lineage = ModelTasksetDerivationSchema.parse(derived.taskset.metadata.modelTasksetDerivation);
|
|
451
|
+
if (derived.taskset.id !== prepared.tasksetId || derived.taskset.revision !== prepared.tasksetRevision || contentHash({ ...lineage, schemaVersion: prepared.lineage.schemaVersion }) !== contentHash(prepared.lineage)) throw new Error("Bound draft differs from its retained ownership lineage.");
|
|
452
|
+
const { taskset: published, executionResources, ...modelResources } = derived;
|
|
453
|
+
return createTasksetPackage({ schemaVersion: "openpond.tasksetPackage.v1", taskset: published, ...executionResources, modelResources, files: edited.files });
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
// src/model-taskset-authoring.ts
|
|
329
457
|
function prepareModelTasksetDraft(input) {
|
|
330
458
|
const request = ModelTasksetDraftRequestSchema.parse(input.request);
|
|
331
459
|
const owner = ModelTasksetAuthoringOwnerSchema.parse(input.owner);
|
|
332
|
-
const source =
|
|
460
|
+
const source = validateTasksetPackage(input.source);
|
|
461
|
+
if (source.learningResources) throw new Error("Reviewed Tasksets require their reviewed authoring workflow.");
|
|
333
462
|
if (source.contentHash !== request.sourcePackageHash) throw new Error("Taskset draft source package changed.");
|
|
463
|
+
if (source.modelResources) {
|
|
464
|
+
const derived = deriveModelTaskset({
|
|
465
|
+
owner,
|
|
466
|
+
source: { ...source.modelResources, taskset: source.taskset, executionResources: { environment: source.environment, verifierSet: source.verifierSet } },
|
|
467
|
+
rewardBinding: source.modelResources.rewardBinding,
|
|
468
|
+
rewards: source.modelResources.rewards,
|
|
469
|
+
assets: source.modelResources.assets
|
|
470
|
+
});
|
|
471
|
+
const lineage2 = derived.taskset.metadata.modelTasksetDerivation;
|
|
472
|
+
return ModelTasksetDraftPreparationSchema.parse({
|
|
473
|
+
schemaVersion: "openpond.modelTasksetDraftPreparation.v1",
|
|
474
|
+
draftId: `model-taskset-draft-${contentHash({ owner, operationId: request.operationId })}`,
|
|
475
|
+
requestHash: contentHash(request),
|
|
476
|
+
sourcePackageHash: source.contentHash,
|
|
477
|
+
sourceTasksetRef: learningRef5(source.taskset),
|
|
478
|
+
tasksetId: derived.taskset.id,
|
|
479
|
+
tasksetRevision: derived.taskset.revision,
|
|
480
|
+
lineage: { ...lineage2, schemaVersion: "openpond.modelTasksetAuthoring.v1" },
|
|
481
|
+
authoringGraph: "bound"
|
|
482
|
+
});
|
|
483
|
+
}
|
|
334
484
|
const previous = assertModelTasksetAuthoring(source.taskset);
|
|
335
485
|
const owned = previous && contentHash(previous.owner) === contentHash(owner);
|
|
336
|
-
const root = owned ? previous.root :
|
|
337
|
-
const lineage = { schemaVersion: "openpond.modelTasksetAuthoring.v1", owner, root, parent:
|
|
486
|
+
const root = owned ? previous.root : learningRef5(source.taskset);
|
|
487
|
+
const lineage = { schemaVersion: "openpond.modelTasksetAuthoring.v1", owner, root, parent: learningRef5(source.taskset) };
|
|
338
488
|
return ModelTasksetDraftPreparationSchema.parse({
|
|
339
489
|
schemaVersion: "openpond.modelTasksetDraftPreparation.v1",
|
|
340
490
|
draftId: `model-taskset-draft-${contentHash({ owner, operationId: request.operationId })}`,
|
|
341
491
|
requestHash: contentHash(request),
|
|
342
492
|
sourcePackageHash: source.contentHash,
|
|
343
|
-
sourceTasksetRef:
|
|
493
|
+
sourceTasksetRef: learningRef5(source.taskset),
|
|
344
494
|
tasksetId: modelAuthoredTasksetId(lineage),
|
|
345
495
|
tasksetRevision: owned ? source.taskset.revision + 1 : 1,
|
|
346
496
|
lineage
|
|
@@ -348,7 +498,11 @@ function prepareModelTasksetDraft(input) {
|
|
|
348
498
|
}
|
|
349
499
|
function publishModelTasksetDraftPackage(input) {
|
|
350
500
|
const prepared = ModelTasksetDraftPreparationSchema.parse(input.preparation);
|
|
351
|
-
if (!
|
|
501
|
+
if (!sameLearningRef4(prepared.sourceTasksetRef, prepared.lineage.parent)) throw new Error("Taskset draft preparation differs from its source lineage.");
|
|
502
|
+
if (prepared.authoringGraph === "bound") {
|
|
503
|
+
if (!input.source) throw new Error("Bound Taskset authoring requires its retained source package.");
|
|
504
|
+
return publishBoundModelTasksetDraftPackage({ preparation: prepared, source: input.source, edited: input.edited });
|
|
505
|
+
}
|
|
352
506
|
const edited = requireOrdinaryPackage(input.edited);
|
|
353
507
|
const { contentHash: _verifierHash, ...verifierContent } = edited.verifierSet;
|
|
354
508
|
const verifierSet = createVerifierSetRelease({
|
|
@@ -356,10 +510,10 @@ function publishModelTasksetDraftPackage(input) {
|
|
|
356
510
|
id: `${prepared.tasksetId}-verifiers`,
|
|
357
511
|
revision: prepared.tasksetRevision,
|
|
358
512
|
calibrationReceiptRefs: [],
|
|
359
|
-
metadata: { sourceVerifierSet:
|
|
513
|
+
metadata: { sourceVerifierSet: learningRef5(edited.verifierSet) }
|
|
360
514
|
});
|
|
361
515
|
const { contentHash: _hash, ...content } = edited.taskset;
|
|
362
|
-
const taskset =
|
|
516
|
+
const taskset = TasksetReleaseSchema2.parse(sealLearningContent4({
|
|
363
517
|
...content,
|
|
364
518
|
id: prepared.tasksetId,
|
|
365
519
|
revision: prepared.tasksetRevision,
|
|
@@ -384,7 +538,7 @@ function requireOrdinaryPackage(value) {
|
|
|
384
538
|
}
|
|
385
539
|
|
|
386
540
|
// src/taskset-package-projection.ts
|
|
387
|
-
import { learningRef as
|
|
541
|
+
import { learningRef as learningRef6, taskBatchPackageMetadata as taskBatchPackageMetadata3, verifyLearningTextAsset as verifyLearningTextAsset2 } from "@openpond/evals/learning";
|
|
388
542
|
function prepareImportedTasksetPackage(input) {
|
|
389
543
|
const value = validateTasksetPackage(input.package);
|
|
390
544
|
const release = value.taskset;
|
|
@@ -507,8 +661,8 @@ function prepareImportedTasksetPackage(input) {
|
|
|
507
661
|
...learning ? { learning } : {},
|
|
508
662
|
portableFileInventory: value.files.map((file) => ({ asset: file.asset, sourcePath: file.asset.path })),
|
|
509
663
|
...resources ? {
|
|
510
|
-
taskDefinition:
|
|
511
|
-
rewardBinding:
|
|
664
|
+
taskDefinition: learningRef6(resources.taskDefinition),
|
|
665
|
+
rewardBinding: learningRef6(resources.rewardBinding),
|
|
512
666
|
rewardExecution: { binding: resources.rewardBinding, rewards: resources.rewards }
|
|
513
667
|
} : {},
|
|
514
668
|
portableCapabilities: release.capabilities,
|
|
@@ -528,7 +682,7 @@ import { z } from "zod";
|
|
|
528
682
|
// src/taskset-authored-tool-environment.ts
|
|
529
683
|
import { createEnvironmentRelease } from "@openpond/evals";
|
|
530
684
|
import { JavaScriptEnvironmentDefinitionSchema } from "@openpond/evals/javascript-environment";
|
|
531
|
-
import { learningRef as
|
|
685
|
+
import { learningRef as learningRef7, sealLearningContent as sealLearningContent5 } from "@openpond/evals/learning";
|
|
532
686
|
function prepareAuthoredToolEnvironment(taskset, bytes, inventory, adapterId) {
|
|
533
687
|
if (taskset.environment.entrypoint !== "openpond.javascript-environment.v1") return { taskset, inventory, generatedFiles: [] };
|
|
534
688
|
const declarationPath = "environment/execution.json";
|
|
@@ -542,7 +696,7 @@ function prepareAuthoredToolEnvironment(taskset, bytes, inventory, adapterId) {
|
|
|
542
696
|
return file2.asset;
|
|
543
697
|
};
|
|
544
698
|
const { contentHash: _javascriptHash, ...javascriptContent } = previous.javascript;
|
|
545
|
-
const javascript = JavaScriptEnvironmentDefinitionSchema.parse(
|
|
699
|
+
const javascript = JavaScriptEnvironmentDefinitionSchema.parse(sealLearningContent5({
|
|
546
700
|
...javascriptContent,
|
|
547
701
|
id: `javascript-${taskset.id}`,
|
|
548
702
|
revision: taskset.revision,
|
|
@@ -559,7 +713,7 @@ function prepareAuthoredToolEnvironment(taskset, bytes, inventory, adapterId) {
|
|
|
559
713
|
stateSchemaRef: repin(previous.environment.stateSchemaRef),
|
|
560
714
|
artifactCollection: previous.environment.artifactCollection,
|
|
561
715
|
adapterConformanceHashes: {},
|
|
562
|
-
metadata: { javascriptEnvironment:
|
|
716
|
+
metadata: { javascriptEnvironment: learningRef7(javascript), resources: taskset.environment.resources ?? [] }
|
|
563
717
|
});
|
|
564
718
|
const prepared = TasksetSchema.parse({ ...taskset, environment: {
|
|
565
719
|
...taskset.environment,
|
|
@@ -690,12 +844,12 @@ function compileModelTasksetDraftWorkspace(input) {
|
|
|
690
844
|
return { asset: entry.asset, base64: Buffer.from(bytes).toString("base64") };
|
|
691
845
|
})
|
|
692
846
|
});
|
|
693
|
-
return preparation ? publishModelTasksetDraftPackage({ preparation, edited }) : edited;
|
|
847
|
+
return preparation ? publishModelTasksetDraftPackage({ preparation, edited, source: input.source }) : edited;
|
|
694
848
|
}
|
|
695
849
|
|
|
696
850
|
// src/taskset-draft-source.ts
|
|
697
851
|
import { z as z2 } from "zod";
|
|
698
|
-
import { learningRef as
|
|
852
|
+
import { learningRef as learningRef8, sameLearningRef as sameLearningRef5 } from "@openpond/evals/learning";
|
|
699
853
|
var TasksetDraftSourceInitializationSchema = z2.object({
|
|
700
854
|
draft: TasksetDraftSchema,
|
|
701
855
|
filePaths: z2.array(z2.object({ assetId: z2.string().min(1), paths: z2.array(TasksetDraftFilePathSchema).min(1).max(1e4) }).strict()).max(1e4)
|
|
@@ -704,8 +858,9 @@ function prepareTasksetDraftSource(input) {
|
|
|
704
858
|
const source = validateTasksetPackage(input.source);
|
|
705
859
|
const original = TasksetDraftSchema.parse(input.sourceDraft);
|
|
706
860
|
const preparation = ModelTasksetDraftPreparationSchema.parse(input.preparation);
|
|
707
|
-
if (source.contentHash !== preparation.sourcePackageHash || !
|
|
708
|
-
if (source.
|
|
861
|
+
if (source.contentHash !== preparation.sourcePackageHash || !sameLearningRef5(learningRef8(source.taskset), preparation.sourceTasksetRef)) throw new Error("Taskset draft preparation differs from its source package.");
|
|
862
|
+
if (source.learningResources) throw new Error("Reviewed Tasksets require their reviewed authoring workflow.");
|
|
863
|
+
if (Boolean(source.modelResources) !== (preparation.authoringGraph === "bound")) throw new Error("Taskset draft authoring graph differs from its source.");
|
|
709
864
|
const inventory = AuthoredTasksetFileInventorySchema.parse(original.metadata.portableFileInventory ?? []);
|
|
710
865
|
const sourcePath = (relative) => isManagedTasksetDraftFilePath(relative) ? `source-artifacts/${source.contentHash}/${contentHash(relative)}/${relative.split("/").at(-1)}` : relative;
|
|
711
866
|
const references = [
|
|
@@ -726,16 +881,22 @@ function prepareTasksetDraftSource(input) {
|
|
|
726
881
|
}
|
|
727
882
|
return { assetId: file.asset.id, paths: [...paths].map(sourcePath) };
|
|
728
883
|
});
|
|
884
|
+
const metadata = { ...original.metadata };
|
|
885
|
+
if (source.modelResources) for (const key of ["taskDefinition", "rewardBinding", "rewardExecution", "derivedPortableMetadata", "modelTasksetDerivation", "importedPackageHash"]) delete metadata[key];
|
|
729
886
|
const draft = TasksetDraftSchema.parse({
|
|
730
887
|
...original,
|
|
731
888
|
id: preparation.draftId,
|
|
732
889
|
revision: 1,
|
|
733
890
|
status: "draft",
|
|
891
|
+
graders: original.graders.map((grader) => ({ ...grader, metadata: {
|
|
892
|
+
...grader.metadata,
|
|
893
|
+
portableGrader: source.taskset.graders.find((candidate) => candidate.id === grader.id)
|
|
894
|
+
} })),
|
|
734
895
|
environment: { ...original.environment, metadata: { ...original.environment.metadata, portableExecutionResources: { environment: source.environment } } },
|
|
735
896
|
modelScope: { modelId: preparation.lineage.owner.modelId, expectedModelRevision: input.expectedModelRevision, source: preparation },
|
|
736
897
|
publishedTasksetRef: preparation.tasksetRevision > 1 ? preparation.sourceTasksetRef : null,
|
|
737
898
|
metadata: {
|
|
738
|
-
...
|
|
899
|
+
...metadata,
|
|
739
900
|
modelTasksetAuthoring: preparation.lineage,
|
|
740
901
|
portableFileInventory: inventory.map((entry) => ({ ...entry, sourcePath: sourcePath(entry.sourcePath), asset: {
|
|
741
902
|
...entry.asset,
|
|
@@ -801,6 +962,7 @@ export {
|
|
|
801
962
|
TasksetPackageReceiptSchema,
|
|
802
963
|
TasksetPackageSchema,
|
|
803
964
|
beginModelBatchReview,
|
|
965
|
+
bindOrdinaryModelTasksetReward,
|
|
804
966
|
compileModelTasksetDraftWorkspace,
|
|
805
967
|
createTasksetPackage,
|
|
806
968
|
createTasksetPackageExecutionFile,
|