openpond-sdk 0.5.5 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/TRAINING_PROTOCOL.md +44 -0
- package/dist/chunk-ARUJ3FJT.js +447 -0
- package/dist/chunk-ARUJ3FJT.js.map +7 -0
- package/dist/{chunk-M6MM6IGG.js → chunk-F5JBFRAH.js} +300 -220
- package/dist/chunk-F5JBFRAH.js.map +7 -0
- package/dist/chunk-QFT2MCIC.js +1526 -0
- package/dist/chunk-QFT2MCIC.js.map +7 -0
- package/dist/{chunk-NF63YMOL.js → chunk-QUOW6URQ.js} +7 -9
- package/dist/{chunk-NF63YMOL.js.map → chunk-QUOW6URQ.js.map} +3 -3
- package/dist/{chunk-25TBEGXB.js → chunk-TKJU3BOW.js} +2 -2
- package/dist/chunk-UJZ3XEE3.js +902 -0
- package/dist/chunk-UJZ3XEE3.js.map +7 -0
- package/dist/{chunk-EV4PKJWN.js → chunk-UKVK7MRJ.js} +320 -208
- package/dist/chunk-UKVK7MRJ.js.map +7 -0
- package/dist/index.js +3 -3
- package/dist/learning.js +3 -3
- package/dist/model-taskset-runs.js +2 -2
- package/dist/refiner.js +1 -1
- package/dist/taskset-drafts.js +36 -34
- package/dist/taskset-drafts.js.map +1 -1
- package/dist/taskset-packages.js +52 -466
- package/dist/taskset-packages.js.map +4 -4
- package/dist/training-bundle.js +459 -0
- package/dist/training-bundle.js.map +7 -0
- package/dist/training.js +9 -1
- package/dist/types/packages/sdk/src/taskset-authored-contracts.d.ts +2 -2
- package/dist/types/packages/sdk/src/taskset-authored-files.d.ts +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-dataset-artifacts.d.ts +2 -2
- package/dist/types/packages/sdk/src/taskset-package-projection.d.ts +1 -1
- package/dist/types/packages/sdk/src/training-bundle-contracts.d.ts +287 -0
- package/dist/types/packages/sdk/src/training-bundle-contracts.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/training-bundle.d.ts +48 -0
- package/dist/types/packages/sdk/src/training-bundle.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/training-candidate-decisions.d.ts +90 -0
- package/dist/types/packages/sdk/src/training-candidate-decisions.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/training-learning-batch.d.ts +719 -0
- package/dist/types/packages/sdk/src/training-learning-batch.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/training.d.ts +8 -0
- package/dist/types/packages/sdk/src/training.d.ts.map +1 -1
- package/package.json +6 -2
- package/dist/chunk-EV4PKJWN.js.map +0 -7
- package/dist/chunk-M6MM6IGG.js.map +0 -7
- package/dist/chunk-PXB6YY36.js +0 -2395
- package/dist/chunk-PXB6YY36.js.map +0 -7
- /package/dist/{chunk-25TBEGXB.js.map → chunk-TKJU3BOW.js.map} +0 -0
|
@@ -0,0 +1,902 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DatasetArtifactManifestSchema,
|
|
3
|
+
GraderFixtureSchema,
|
|
4
|
+
GraderSpecSchema,
|
|
5
|
+
LearningSignalInventorySchema,
|
|
6
|
+
MAX_TASKSET_PACKAGE_BYTES,
|
|
7
|
+
TaskDataRecordSchema,
|
|
8
|
+
TaskPolicyBoundarySchema,
|
|
9
|
+
TasksetBenchmarkBindingSchema,
|
|
10
|
+
TasksetCapabilityManifestSchema,
|
|
11
|
+
TasksetEnvironmentContractSchema,
|
|
12
|
+
TasksetPackageSchema,
|
|
13
|
+
TasksetPreferenceComparisonBindingSchema,
|
|
14
|
+
TasksetPurposeSchema,
|
|
15
|
+
TasksetSchema,
|
|
16
|
+
TasksetSourceRefSchema,
|
|
17
|
+
computeTasksetHash,
|
|
18
|
+
tasksetPackageRewardBinding,
|
|
19
|
+
validateTaskset,
|
|
20
|
+
validateTasksetPackage
|
|
21
|
+
} from "./chunk-QFT2MCIC.js";
|
|
22
|
+
import {
|
|
23
|
+
ModelTasksetDraftPreparationSchema,
|
|
24
|
+
TasksetDraftFileContentSchema,
|
|
25
|
+
TasksetDraftFileMutationSchema,
|
|
26
|
+
TasksetDraftFilePathSchema
|
|
27
|
+
} from "./chunk-S7I45WJQ.js";
|
|
28
|
+
import {
|
|
29
|
+
TasksetCatalogReleaseRefSchema
|
|
30
|
+
} from "./chunk-A4ZDOZKP.js";
|
|
31
|
+
import {
|
|
32
|
+
canonicalJson,
|
|
33
|
+
contentHash,
|
|
34
|
+
sha256
|
|
35
|
+
} from "./chunk-QUOW6URQ.js";
|
|
36
|
+
import {
|
|
37
|
+
HostedModelProjectSummarySchema,
|
|
38
|
+
HostedModelProjectSyncSchema,
|
|
39
|
+
HostedModelProjectTrainingSetupSchema
|
|
40
|
+
} from "./chunk-ZNFOVV7B.js";
|
|
41
|
+
|
|
42
|
+
// src/taskset-package-client.ts
|
|
43
|
+
import { z } from "zod";
|
|
44
|
+
import { learningRef } from "@openpond/evals/learning";
|
|
45
|
+
var IdSchema = z.string().trim().min(1).max(500);
|
|
46
|
+
var HashSchema = z.string().regex(/^[a-f0-9]{64}$/);
|
|
47
|
+
var TasksetPackageModelConfigurationSchema = HostedModelProjectSyncSchema.pick({
|
|
48
|
+
portableProjectId: true,
|
|
49
|
+
name: true,
|
|
50
|
+
objective: true,
|
|
51
|
+
defaultBaseModel: true,
|
|
52
|
+
defaultDestinationId: true,
|
|
53
|
+
sourceRevision: true,
|
|
54
|
+
sourceUpdatedAt: true
|
|
55
|
+
}).extend({ trainingSetup: HostedModelProjectTrainingSetupSchema.omit({ tasksetRef: true, rewardBindingRef: true, tasksetRelease: true, recipe: true }) }).strict();
|
|
56
|
+
var TasksetPackagePublicationSchema = z.object({
|
|
57
|
+
schemaVersion: z.literal("openpond.tasksetPackagePublication.v1"),
|
|
58
|
+
operationId: IdSchema,
|
|
59
|
+
modelProjectId: IdSchema,
|
|
60
|
+
expectedProjectEtag: HashSchema.nullable(),
|
|
61
|
+
name: z.string().trim().min(1).max(200),
|
|
62
|
+
description: z.string().max(5e3),
|
|
63
|
+
buildIntent: z.enum(["demonstrations", "preferences", "verifiable_reward", "rubric", "discovery"]),
|
|
64
|
+
methodHint: z.enum(["sft", "dpo", "grpo", "ppo"]).nullable(),
|
|
65
|
+
package: TasksetPackageSchema,
|
|
66
|
+
modelConfiguration: TasksetPackageModelConfigurationSchema.optional(),
|
|
67
|
+
selection: z.enum(["select", "attach"]).optional()
|
|
68
|
+
}).strict().refine((value) => value.selection !== "attach" || value.modelConfiguration === void 0, "Attachment-only publication cannot replace Model configuration.");
|
|
69
|
+
var TasksetPackageReceiptSchema = z.object({
|
|
70
|
+
schemaVersion: z.literal("openpond.tasksetPackageReceipt.v1"),
|
|
71
|
+
teamId: IdSchema,
|
|
72
|
+
modelProjectId: IdSchema,
|
|
73
|
+
operationId: IdSchema,
|
|
74
|
+
taskset: TasksetCatalogReleaseRefSchema,
|
|
75
|
+
packageHash: HashSchema,
|
|
76
|
+
hostedTasksetId: IdSchema,
|
|
77
|
+
projectEtag: HashSchema,
|
|
78
|
+
project: HostedModelProjectSummarySchema.optional(),
|
|
79
|
+
selection: z.enum(["select", "attach"]).optional()
|
|
80
|
+
}).strict();
|
|
81
|
+
var TasksetPackageReadbackSchema = z.object({
|
|
82
|
+
schemaVersion: z.literal("openpond.tasksetPackageReadback.v1"),
|
|
83
|
+
teamId: IdSchema,
|
|
84
|
+
modelProjectId: IdSchema,
|
|
85
|
+
package: TasksetPackageSchema
|
|
86
|
+
}).strict();
|
|
87
|
+
var OpenPondTasksetPackageError = class extends Error {
|
|
88
|
+
constructor(status, code, message) {
|
|
89
|
+
super(message);
|
|
90
|
+
this.status = status;
|
|
91
|
+
this.code = code;
|
|
92
|
+
this.name = "OpenPondTasksetPackageError";
|
|
93
|
+
}
|
|
94
|
+
status;
|
|
95
|
+
code;
|
|
96
|
+
};
|
|
97
|
+
var OpenPondTasksetPackageClient = class {
|
|
98
|
+
#options;
|
|
99
|
+
constructor(options) {
|
|
100
|
+
const url = new URL(options.baseUrl);
|
|
101
|
+
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash || !options.apiKey.trim() || !options.teamId.trim()) throw new Error("Taskset package client requires a clean HTTP(S) endpoint and workspace credentials.");
|
|
102
|
+
this.#options = { ...options, baseUrl: url.toString().replace(/\/+$/, "") };
|
|
103
|
+
}
|
|
104
|
+
async publish(input, options = {}) {
|
|
105
|
+
const request = TasksetPackagePublicationSchema.parse(input);
|
|
106
|
+
validateTasksetPackage(request.package);
|
|
107
|
+
const receipt = TasksetPackageReceiptSchema.parse(await this.#request("", "POST", request, options.signal));
|
|
108
|
+
const release = request.package.taskset;
|
|
109
|
+
if (receipt.teamId !== this.#options.teamId || receipt.modelProjectId !== request.modelProjectId || receipt.operationId !== request.operationId || receipt.packageHash !== request.package.contentHash || receipt.taskset.id !== release.id || receipt.taskset.revision !== release.revision || receipt.taskset.contentHash !== release.contentHash) {
|
|
110
|
+
throw new OpenPondTasksetPackageError(502, "package_receipt_mismatch", "Taskset publication receipt did not match the requested package and owner.");
|
|
111
|
+
}
|
|
112
|
+
if (request.selection !== void 0 && receipt.selection !== request.selection) throw new OpenPondTasksetPackageError(502, "package_receipt_mismatch", "Taskset publication did not retain its selection mode.");
|
|
113
|
+
if (request.selection === "attach" && receipt.projectEtag !== request.expectedProjectEtag) throw new OpenPondTasksetPackageError(502, "package_receipt_mismatch", "Attaching a historical Taskset changed the Model configuration.");
|
|
114
|
+
if (request.modelConfiguration) {
|
|
115
|
+
const expected = request.modelConfiguration;
|
|
116
|
+
const project = receipt.project;
|
|
117
|
+
const binding = tasksetPackageRewardBinding(request.package);
|
|
118
|
+
const setup = HostedModelProjectTrainingSetupSchema.parse({
|
|
119
|
+
...expected.trainingSetup,
|
|
120
|
+
tasksetRef: learningRef(release),
|
|
121
|
+
rewardBindingRef: binding ? learningRef(binding) : null,
|
|
122
|
+
tasksetRelease: null,
|
|
123
|
+
recipe: null
|
|
124
|
+
});
|
|
125
|
+
if (!project || project.teamId !== this.#options.teamId || project.etag !== receipt.projectEtag || ![project.id, project.portableProjectId].includes(request.modelProjectId) || project.portableProjectId !== expected.portableProjectId || project.name !== expected.name || project.objective !== expected.objective || project.sourceRevision !== expected.sourceRevision || Date.parse(project.sourceUpdatedAt) !== Date.parse(expected.sourceUpdatedAt) || contentHash(project.defaultBaseModel) !== contentHash(expected.defaultBaseModel) || project.defaultDestinationId !== expected.defaultDestinationId || contentHash(project.trainingSetup) !== contentHash(setup)) {
|
|
126
|
+
throw new OpenPondTasksetPackageError(502, "package_receipt_mismatch", "Taskset publication did not retain the requested Model configuration.");
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return receipt;
|
|
130
|
+
}
|
|
131
|
+
async get(modelProjectId, reference, options = {}) {
|
|
132
|
+
const project = IdSchema.parse(modelProjectId);
|
|
133
|
+
const ref = TasksetCatalogReleaseRefSchema.parse(reference);
|
|
134
|
+
const expectedPackageHash = options.expectedPackageHash === void 0 ? void 0 : HashSchema.parse(options.expectedPackageHash);
|
|
135
|
+
const response = TasksetPackageReadbackSchema.parse(await this.#request(`/${encodeURIComponent(project)}/${encodeURIComponent(ref.id)}/${ref.revision}/${ref.contentHash}`, "GET", void 0, options.signal));
|
|
136
|
+
const taskset = response.package.taskset;
|
|
137
|
+
if (response.teamId !== this.#options.teamId || response.modelProjectId !== project || taskset.id !== ref.id || taskset.revision !== ref.revision || taskset.contentHash !== ref.contentHash) throw new OpenPondTasksetPackageError(502, "package_readback_mismatch", "Taskset package did not match the requested release and owner.");
|
|
138
|
+
if (expectedPackageHash !== void 0 && response.package.contentHash !== expectedPackageHash) throw new OpenPondTasksetPackageError(502, "package_readback_mismatch", "Taskset package did not match the saved publication receipt.");
|
|
139
|
+
return validateTasksetPackage(response.package);
|
|
140
|
+
}
|
|
141
|
+
async #request(path, method, body, signal) {
|
|
142
|
+
const serialized = body === void 0 ? void 0 : JSON.stringify(body);
|
|
143
|
+
if (serialized && new TextEncoder().encode(serialized).byteLength > MAX_TASKSET_PACKAGE_BYTES) throw new OpenPondTasksetPackageError(413, "package_too_large", "Taskset publication exceeds the 64 MiB envelope.");
|
|
144
|
+
const response = await (this.#options.fetch ?? globalThis.fetch)(`${this.#options.baseUrl}/v1/taskset-packages${path}`, {
|
|
145
|
+
method,
|
|
146
|
+
headers: { Authorization: `Bearer ${this.#options.apiKey}`, "X-OpenPond-Team-Id": this.#options.teamId, Accept: "application/json", ...serialized ? { "Content-Type": "application/json" } : {} },
|
|
147
|
+
body: serialized,
|
|
148
|
+
signal,
|
|
149
|
+
redirect: "error"
|
|
150
|
+
});
|
|
151
|
+
const reader = response.body?.getReader();
|
|
152
|
+
if (!reader) throw new OpenPondTasksetPackageError(response.status, "empty_response", "Taskset package response was empty.");
|
|
153
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
154
|
+
let bytes = 0;
|
|
155
|
+
let text = "";
|
|
156
|
+
try {
|
|
157
|
+
while (true) {
|
|
158
|
+
const chunk = await reader.read();
|
|
159
|
+
if (chunk.done) break;
|
|
160
|
+
bytes += chunk.value.byteLength;
|
|
161
|
+
if (bytes > MAX_TASKSET_PACKAGE_BYTES) throw new OpenPondTasksetPackageError(413, "package_too_large", "Taskset response exceeds the 64 MiB envelope.");
|
|
162
|
+
text += decoder.decode(chunk.value, { stream: true });
|
|
163
|
+
}
|
|
164
|
+
text += decoder.decode();
|
|
165
|
+
} finally {
|
|
166
|
+
await reader.cancel();
|
|
167
|
+
reader.releaseLock();
|
|
168
|
+
}
|
|
169
|
+
const payload = JSON.parse(text);
|
|
170
|
+
if (!response.ok) {
|
|
171
|
+
const error = z.object({ code: z.string(), message: z.string() }).safeParse(payload);
|
|
172
|
+
throw new OpenPondTasksetPackageError(response.status, error.success ? error.data.code : "package_request_failed", error.success ? error.data.message : `Taskset package request failed (${response.status}).`);
|
|
173
|
+
}
|
|
174
|
+
return payload;
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
// src/taskset-draft-document.ts
|
|
179
|
+
import { z as z2 } from "zod";
|
|
180
|
+
import { TasksetMetricPolicySchema } from "@openpond/evals/metrics";
|
|
181
|
+
var IdSchema2 = z2.string().trim().min(1).max(240);
|
|
182
|
+
var TimestampSchema = z2.string().trim().min(1);
|
|
183
|
+
var HashSchema2 = z2.string().trim().min(8).max(256);
|
|
184
|
+
var CodeIdentifierSchema = z2.string().trim().regex(/^[A-Za-z_$][A-Za-z0-9_$]*$/);
|
|
185
|
+
var MetadataSchema = z2.record(z2.string(), z2.unknown()).default({});
|
|
186
|
+
function safeRelativeFilePath(value) {
|
|
187
|
+
const normalized = value.trim().replaceAll("\\", "/");
|
|
188
|
+
if (!normalized || normalized.startsWith("/") || normalized === "." || normalized === "..") {
|
|
189
|
+
return false;
|
|
190
|
+
}
|
|
191
|
+
return !normalized.split("/").some(
|
|
192
|
+
(segment) => !segment || segment === "." || segment === ".."
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
var TaskDataDraftSchema = TaskDataRecordSchema.extend({
|
|
196
|
+
sourceRefs: z2.array(IdSchema2).max(100).default([])
|
|
197
|
+
});
|
|
198
|
+
var TasksetDraftStatusSchema = z2.enum([
|
|
199
|
+
"draft",
|
|
200
|
+
"validating",
|
|
201
|
+
"needs_review",
|
|
202
|
+
"published"
|
|
203
|
+
]);
|
|
204
|
+
var TasksetDraftReviewPolicySchema = z2.object({
|
|
205
|
+
enabled: z2.boolean(),
|
|
206
|
+
candidateCount: z2.number().int().min(2).max(4),
|
|
207
|
+
minimumSamples: z2.number().int().min(1).max(1e4).default(100),
|
|
208
|
+
allowTies: z2.boolean(),
|
|
209
|
+
allowRejectAll: z2.boolean(),
|
|
210
|
+
rubric: z2.string().trim().max(5e4),
|
|
211
|
+
criteria: z2.array(z2.object({
|
|
212
|
+
id: IdSchema2,
|
|
213
|
+
label: z2.string().trim().min(1).max(500),
|
|
214
|
+
description: z2.string().trim().max(1e4),
|
|
215
|
+
weight: z2.number().nonnegative().max(1e3)
|
|
216
|
+
})).max(100)
|
|
217
|
+
});
|
|
218
|
+
var TasksetDraftOutputContractSchema = z2.object({
|
|
219
|
+
mode: z2.enum(["text", "structured_json", "artifacts"]),
|
|
220
|
+
jsonSchema: z2.record(z2.string(), z2.unknown()).nullable(),
|
|
221
|
+
renderer: z2.object({
|
|
222
|
+
module: z2.string().trim().min(1).max(1e3).refine(safeRelativeFilePath, "Renderer modules must use a safe relative path."),
|
|
223
|
+
exportName: CodeIdentifierSchema,
|
|
224
|
+
configRef: z2.string().trim().min(1).max(1e3).refine(safeRelativeFilePath, "Renderer configuration must use a safe relative path.").nullable().default(null)
|
|
225
|
+
}).nullable()
|
|
226
|
+
}).superRefine((output, context) => {
|
|
227
|
+
if (output.mode === "structured_json" && !output.jsonSchema) {
|
|
228
|
+
context.addIssue({
|
|
229
|
+
code: "custom",
|
|
230
|
+
message: "Structured JSON output requires a JSON Schema.",
|
|
231
|
+
path: ["jsonSchema"]
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
if (output.mode === "text" && output.renderer) {
|
|
235
|
+
context.addIssue({
|
|
236
|
+
code: "custom",
|
|
237
|
+
message: "Text output cannot configure an artifact renderer.",
|
|
238
|
+
path: ["renderer"]
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
});
|
|
242
|
+
var TasksetDraftSchema = z2.object({
|
|
243
|
+
schemaVersion: z2.literal("openpond.tasksetDraft.v1"),
|
|
244
|
+
id: IdSchema2,
|
|
245
|
+
revision: z2.number().int().positive(),
|
|
246
|
+
profileId: IdSchema2,
|
|
247
|
+
modelScope: z2.object({
|
|
248
|
+
modelId: IdSchema2,
|
|
249
|
+
expectedModelRevision: z2.number().int().positive(),
|
|
250
|
+
source: ModelTasksetDraftPreparationSchema.optional()
|
|
251
|
+
}).strict().nullable().default(null),
|
|
252
|
+
name: z2.string().trim().max(500),
|
|
253
|
+
objective: z2.string().trim().max(2e4),
|
|
254
|
+
purpose: TasksetPurposeSchema.default("general"),
|
|
255
|
+
benchmark: TasksetBenchmarkBindingSchema.nullable().default(null),
|
|
256
|
+
preferenceComparison: TasksetPreferenceComparisonBindingSchema.nullable().default(null),
|
|
257
|
+
status: TasksetDraftStatusSchema,
|
|
258
|
+
sourceRefs: z2.array(TasksetSourceRefSchema).max(1e5).default([]),
|
|
259
|
+
datasetArtifact: DatasetArtifactManifestSchema.nullable().optional(),
|
|
260
|
+
policy: TaskPolicyBoundarySchema,
|
|
261
|
+
environment: TasksetEnvironmentContractSchema,
|
|
262
|
+
output: TasksetDraftOutputContractSchema.default({
|
|
263
|
+
mode: "text",
|
|
264
|
+
jsonSchema: null,
|
|
265
|
+
renderer: null
|
|
266
|
+
}),
|
|
267
|
+
capabilities: TasksetCapabilityManifestSchema,
|
|
268
|
+
metrics: TasksetMetricPolicySchema,
|
|
269
|
+
review: TasksetDraftReviewPolicySchema,
|
|
270
|
+
tasks: z2.array(TaskDataDraftSchema).max(1e6).default([]),
|
|
271
|
+
graders: z2.array(GraderSpecSchema).max(1e3).default([]),
|
|
272
|
+
graderFixtures: z2.array(GraderFixtureSchema).max(1e5).default([]),
|
|
273
|
+
learningSignals: LearningSignalInventorySchema,
|
|
274
|
+
publishedTasksetRef: z2.object({
|
|
275
|
+
id: IdSchema2,
|
|
276
|
+
revision: z2.number().int().positive(),
|
|
277
|
+
contentHash: HashSchema2
|
|
278
|
+
}).nullable(),
|
|
279
|
+
createdAt: TimestampSchema,
|
|
280
|
+
updatedAt: TimestampSchema,
|
|
281
|
+
metadata: MetadataSchema
|
|
282
|
+
}).superRefine((draft, context) => {
|
|
283
|
+
if (draft.purpose === "benchmark" && !draft.benchmark) {
|
|
284
|
+
context.addIssue({
|
|
285
|
+
code: "custom",
|
|
286
|
+
message: "Benchmark drafts require an immutable benchmark binding.",
|
|
287
|
+
path: ["benchmark"]
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
if (draft.purpose !== "benchmark" && draft.benchmark) {
|
|
291
|
+
context.addIssue({
|
|
292
|
+
code: "custom",
|
|
293
|
+
message: "Only benchmark drafts may carry a benchmark binding.",
|
|
294
|
+
path: ["benchmark"]
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
if (draft.datasetArtifact && draft.tasks.length > 0) {
|
|
298
|
+
context.addIssue({
|
|
299
|
+
code: "custom",
|
|
300
|
+
message: "Artifact-backed drafts may not duplicate canonical rows inline.",
|
|
301
|
+
path: ["tasks"]
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
if (draft.status === "published" && !draft.publishedTasksetRef) {
|
|
305
|
+
context.addIssue({
|
|
306
|
+
code: "custom",
|
|
307
|
+
message: "Published drafts require an immutable Taskset reference.",
|
|
308
|
+
path: ["publishedTasksetRef"]
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
// src/taskset-draft-authoring.ts
|
|
314
|
+
var TasksetDraftPublishError = class extends Error {
|
|
315
|
+
issues;
|
|
316
|
+
constructor(issues) {
|
|
317
|
+
super(issues.map((issue) => issue.message).join("\n"));
|
|
318
|
+
this.name = "TasksetDraftPublishError";
|
|
319
|
+
this.issues = issues;
|
|
320
|
+
}
|
|
321
|
+
};
|
|
322
|
+
function createTasksetDraft(input) {
|
|
323
|
+
const timestamp = input.now ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
324
|
+
const id = input.id?.trim() || `taskset-draft-${globalThis.crypto.randomUUID()}`;
|
|
325
|
+
return TasksetDraftSchema.parse({
|
|
326
|
+
schemaVersion: "openpond.tasksetDraft.v1",
|
|
327
|
+
id,
|
|
328
|
+
revision: 1,
|
|
329
|
+
profileId: input.profileId,
|
|
330
|
+
modelScope: input.modelScope ?? null,
|
|
331
|
+
name: input.name?.trim() ?? "",
|
|
332
|
+
objective: "",
|
|
333
|
+
purpose: "general",
|
|
334
|
+
benchmark: null,
|
|
335
|
+
preferenceComparison: null,
|
|
336
|
+
status: "draft",
|
|
337
|
+
sourceRefs: [],
|
|
338
|
+
datasetArtifact: null,
|
|
339
|
+
policy: {
|
|
340
|
+
policyVisibleFields: ["input"],
|
|
341
|
+
privilegedFields: ["expectedOutput"],
|
|
342
|
+
hiddenGraderRefs: [],
|
|
343
|
+
connectedAppScopes: []
|
|
344
|
+
},
|
|
345
|
+
environment: {
|
|
346
|
+
protocolVersion: "openpond.taskEnvironment.v1",
|
|
347
|
+
kind: "chat",
|
|
348
|
+
entrypoint: "openpond-chat-v1",
|
|
349
|
+
stateful: false,
|
|
350
|
+
deterministicSeeds: true,
|
|
351
|
+
toolNames: [],
|
|
352
|
+
lifecycle: ["create", "reset", "step", "grade", "cleanup"],
|
|
353
|
+
defaultTimeoutMs: 12e4,
|
|
354
|
+
networkPolicy: "none",
|
|
355
|
+
resources: [],
|
|
356
|
+
metadata: {}
|
|
357
|
+
},
|
|
358
|
+
output: {
|
|
359
|
+
mode: "text",
|
|
360
|
+
jsonSchema: null,
|
|
361
|
+
renderer: null
|
|
362
|
+
},
|
|
363
|
+
capabilities: {
|
|
364
|
+
schemaVersion: "openpond.tasksetCapabilities.v1",
|
|
365
|
+
taskKind: "chat",
|
|
366
|
+
supportedSignals: [
|
|
367
|
+
"demonstration",
|
|
368
|
+
"preference",
|
|
369
|
+
"correction",
|
|
370
|
+
"feedback",
|
|
371
|
+
"reward",
|
|
372
|
+
"label"
|
|
373
|
+
],
|
|
374
|
+
compatibleMethods: ["none"],
|
|
375
|
+
rewardKinds: ["none"],
|
|
376
|
+
requiresTools: false,
|
|
377
|
+
requiresState: false,
|
|
378
|
+
requiresPrivilegedGrading: true,
|
|
379
|
+
environmentPlacements: ["local", "remote", "colocated"],
|
|
380
|
+
exportable: true,
|
|
381
|
+
portabilityBlockers: []
|
|
382
|
+
},
|
|
383
|
+
metrics: {
|
|
384
|
+
schemaVersion: "openpond.tasksetMetricPolicy.v1",
|
|
385
|
+
primaryMetric: "score",
|
|
386
|
+
aggregation: "mean_score",
|
|
387
|
+
missingReward: "zero",
|
|
388
|
+
customAggregator: null
|
|
389
|
+
},
|
|
390
|
+
review: {
|
|
391
|
+
enabled: false,
|
|
392
|
+
candidateCount: 2,
|
|
393
|
+
minimumSamples: 100,
|
|
394
|
+
allowTies: true,
|
|
395
|
+
allowRejectAll: true,
|
|
396
|
+
rubric: "",
|
|
397
|
+
criteria: []
|
|
398
|
+
},
|
|
399
|
+
tasks: [],
|
|
400
|
+
graders: [],
|
|
401
|
+
graderFixtures: [],
|
|
402
|
+
learningSignals: {
|
|
403
|
+
demonstrations: [],
|
|
404
|
+
preferences: [],
|
|
405
|
+
corrections: [],
|
|
406
|
+
feedback: [],
|
|
407
|
+
rewards: [],
|
|
408
|
+
labels: []
|
|
409
|
+
},
|
|
410
|
+
publishedTasksetRef: null,
|
|
411
|
+
createdAt: timestamp,
|
|
412
|
+
updatedAt: timestamp,
|
|
413
|
+
metadata: {}
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
function draftPublishIssues(draft) {
|
|
417
|
+
const issues = [];
|
|
418
|
+
if (!draft.name.trim()) issues.push({ code: "name_missing", message: "Name is required.", path: "name" });
|
|
419
|
+
if (!draft.objective.trim()) issues.push({ code: "objective_missing", message: "Objective is required.", path: "objective" });
|
|
420
|
+
if (!draft.datasetArtifact && draft.tasks.length === 0) issues.push({ code: "tasks_missing", message: "Add at least one task or Dataset artifact.", path: "tasks" });
|
|
421
|
+
if (draft.graders.length === 0) issues.push({ code: "graders_missing", message: "Add at least one grader.", path: "graders" });
|
|
422
|
+
if (draft.graderFixtures.length === 0) issues.push({ code: "grader_fixtures_missing", message: "Add grader fixtures before publishing.", path: "graderFixtures" });
|
|
423
|
+
const sourceIds = new Set(draft.sourceRefs.map((source) => source.id));
|
|
424
|
+
for (const task of draft.tasks) {
|
|
425
|
+
for (const sourceId of task.sourceRefs) {
|
|
426
|
+
if (!sourceIds.has(sourceId)) issues.push({ code: "task_source_missing", message: `Task ${task.id} references missing source ${sourceId}.`, path: `tasks.${task.id}.sourceRefs` });
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
return issues;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
// src/taskset-draft-publication.ts
|
|
433
|
+
function tasksetDraftFromTaskset(tasksetInput, now = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
434
|
+
const taskset = TasksetSchema.parse(tasksetInput);
|
|
435
|
+
return TasksetDraftSchema.parse({
|
|
436
|
+
schemaVersion: "openpond.tasksetDraft.v1",
|
|
437
|
+
id: `${taskset.id}-draft`,
|
|
438
|
+
revision: 1,
|
|
439
|
+
profileId: taskset.profileId,
|
|
440
|
+
name: taskset.name,
|
|
441
|
+
objective: taskset.objective,
|
|
442
|
+
purpose: taskset.purpose,
|
|
443
|
+
benchmark: taskset.benchmark,
|
|
444
|
+
preferenceComparison: taskset.preferenceComparison,
|
|
445
|
+
status: "draft",
|
|
446
|
+
sourceRefs: taskset.sourceRefs,
|
|
447
|
+
datasetArtifact: taskset.datasetArtifact ?? null,
|
|
448
|
+
policy: taskset.policy,
|
|
449
|
+
environment: taskset.environment,
|
|
450
|
+
output: tasksetOutputContract(taskset.metadata),
|
|
451
|
+
capabilities: taskset.capabilities,
|
|
452
|
+
metrics: taskset.metrics ?? {
|
|
453
|
+
schemaVersion: "openpond.tasksetMetricPolicy.v1",
|
|
454
|
+
primaryMetric: "score",
|
|
455
|
+
aggregation: "mean_score",
|
|
456
|
+
missingReward: "zero",
|
|
457
|
+
customAggregator: null
|
|
458
|
+
},
|
|
459
|
+
review: {
|
|
460
|
+
enabled: Boolean(taskset.preferenceComparison),
|
|
461
|
+
candidateCount: 2,
|
|
462
|
+
minimumSamples: 100,
|
|
463
|
+
allowTies: true,
|
|
464
|
+
allowRejectAll: true,
|
|
465
|
+
rubric: "",
|
|
466
|
+
criteria: []
|
|
467
|
+
},
|
|
468
|
+
tasks: taskset.tasks,
|
|
469
|
+
graders: taskset.graders,
|
|
470
|
+
graderFixtures: taskset.graderFixtures,
|
|
471
|
+
learningSignals: taskset.learningSignals,
|
|
472
|
+
publishedTasksetRef: {
|
|
473
|
+
id: taskset.id,
|
|
474
|
+
revision: taskset.revision,
|
|
475
|
+
contentHash: taskset.contentHash
|
|
476
|
+
},
|
|
477
|
+
createdAt: now,
|
|
478
|
+
updatedAt: now,
|
|
479
|
+
metadata: {
|
|
480
|
+
...taskset.metadata,
|
|
481
|
+
importedFromTaskset: {
|
|
482
|
+
id: taskset.id,
|
|
483
|
+
revision: taskset.revision,
|
|
484
|
+
contentHash: taskset.contentHash
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
function publishTasksetDraft(input) {
|
|
490
|
+
const draft = TasksetDraftSchema.parse(input.draft);
|
|
491
|
+
const timestamp = input.now ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
492
|
+
const issues = draftPublishIssues(draft);
|
|
493
|
+
if (issues.length) throw new TasksetDraftPublishError(issues);
|
|
494
|
+
const preparedSource = draft.modelScope?.source;
|
|
495
|
+
if (preparedSource && input.tasksetId && input.tasksetId !== preparedSource.tasksetId) throw new Error("Taskset draft publication differs from its prepared identity.");
|
|
496
|
+
const tasksetId = preparedSource?.tasksetId || input.tasksetId?.trim() || draft.publishedTasksetRef?.id || draft.id.replace(/-draft$/, "");
|
|
497
|
+
const revision = preparedSource?.tasksetRevision ?? (draft.publishedTasksetRef?.id === tasksetId ? draft.publishedTasksetRef.revision + 1 : 1);
|
|
498
|
+
const sourceRefs = draft.sourceRefs.length ? draft.sourceRefs : [generatedDraftSource(draft, tasksetId, timestamp)];
|
|
499
|
+
const defaultSourceId = sourceRefs[0].id;
|
|
500
|
+
const tasks = draft.tasks.map((task) => TaskDataRecordSchema.parse({
|
|
501
|
+
...task,
|
|
502
|
+
sourceRefs: task.sourceRefs.length ? task.sourceRefs : [defaultSourceId],
|
|
503
|
+
metadata: {
|
|
504
|
+
exampleOrigin: "expert_authored",
|
|
505
|
+
...task.metadata
|
|
506
|
+
}
|
|
507
|
+
}));
|
|
508
|
+
const authoredMethod = draft.capabilities.compatibleMethods.find(
|
|
509
|
+
(method) => method !== "none" && method !== "retrieval"
|
|
510
|
+
) ?? null;
|
|
511
|
+
const buildIntent = buildIntentForDraft(draft);
|
|
512
|
+
const taskset = TasksetSchema.parse({
|
|
513
|
+
schemaVersion: "openpond.taskset.v1",
|
|
514
|
+
id: tasksetId,
|
|
515
|
+
revision,
|
|
516
|
+
profileId: draft.profileId,
|
|
517
|
+
profileRelease: null,
|
|
518
|
+
createImproveRunId: null,
|
|
519
|
+
name: draft.name,
|
|
520
|
+
objective: draft.objective,
|
|
521
|
+
purpose: draft.purpose,
|
|
522
|
+
benchmark: draft.benchmark,
|
|
523
|
+
preferenceComparison: draft.preferenceComparison,
|
|
524
|
+
status: "needs_review",
|
|
525
|
+
sourceRefs,
|
|
526
|
+
datasetArtifact: draft.datasetArtifact ?? null,
|
|
527
|
+
policy: draft.policy,
|
|
528
|
+
environment: draft.environment,
|
|
529
|
+
capabilities: draft.capabilities,
|
|
530
|
+
metrics: draft.metrics,
|
|
531
|
+
tasks,
|
|
532
|
+
graders: draft.graders,
|
|
533
|
+
graderFixtures: draft.graderFixtures,
|
|
534
|
+
learningSignals: draft.learningSignals,
|
|
535
|
+
authoringProvenance: {
|
|
536
|
+
schemaVersion: "openpond.taskAuthoringProvenance.v1",
|
|
537
|
+
model: null,
|
|
538
|
+
modelConfig: {},
|
|
539
|
+
skillHash: contentHash("openpond-taskset-draft-authoring-v1"),
|
|
540
|
+
promptTemplateVersion: "taskset-draft-v1",
|
|
541
|
+
buildIntent,
|
|
542
|
+
buildSpecification: null,
|
|
543
|
+
evidenceHashes: sourceRefs.map((source) => source.sourceHash),
|
|
544
|
+
tasksetSdkVersion: "draft-v1",
|
|
545
|
+
sourceCommit: null,
|
|
546
|
+
repairHistory: [],
|
|
547
|
+
createdAt: timestamp
|
|
548
|
+
},
|
|
549
|
+
readiness: null,
|
|
550
|
+
contentHash: "00000000",
|
|
551
|
+
createdAt: revision === 1 ? draft.createdAt : timestamp,
|
|
552
|
+
updatedAt: timestamp,
|
|
553
|
+
metadata: {
|
|
554
|
+
...draft.metadata,
|
|
555
|
+
...preparedSource ? { modelTasksetAuthoring: preparedSource.lineage } : {},
|
|
556
|
+
tasksetReviewPolicy: draft.review,
|
|
557
|
+
tasksetOutputContract: draft.output,
|
|
558
|
+
...input.sourcePackageHash ? { sourcePackageHash: input.sourcePackageHash } : {},
|
|
559
|
+
...authoredMethod ? { trainingMethod: authoredMethod } : {},
|
|
560
|
+
diagnosis: {
|
|
561
|
+
schemaVersion: "openpond.capabilityDiagnosis.v1",
|
|
562
|
+
summary: draft.objective,
|
|
563
|
+
stableBehavior: [draft.objective],
|
|
564
|
+
changingKnowledge: [],
|
|
565
|
+
requiredContext: [],
|
|
566
|
+
requiredTools: draft.environment.toolNames,
|
|
567
|
+
intervention: authoredMethod === "dpo" ? "preference" : authoredMethod === "grpo" || authoredMethod === "ppo" ? "grpo_rft" : authoredMethod === "sft" ? "sft" : "no_training",
|
|
568
|
+
trainingEligible: authoredMethod !== null,
|
|
569
|
+
rationale: authoredMethod ? [`The author selected ${authoredMethod.toUpperCase()} compatibility.`] : ["The Taskset is currently configured for evaluation only."],
|
|
570
|
+
confidence: 1
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
});
|
|
574
|
+
const hashed = TasksetSchema.parse({
|
|
575
|
+
...taskset,
|
|
576
|
+
contentHash: computeTasksetHash(taskset)
|
|
577
|
+
});
|
|
578
|
+
const report = validateTaskset(hashed);
|
|
579
|
+
const validationErrors = report.issues.filter((issue) => issue.severity === "error");
|
|
580
|
+
if (validationErrors.length) {
|
|
581
|
+
throw new TasksetDraftPublishError(validationErrors.map((issue) => ({
|
|
582
|
+
code: issue.code,
|
|
583
|
+
message: issue.message,
|
|
584
|
+
path: issue.path
|
|
585
|
+
})));
|
|
586
|
+
}
|
|
587
|
+
return hashed;
|
|
588
|
+
}
|
|
589
|
+
function generatedDraftSource(draft, tasksetId, timestamp) {
|
|
590
|
+
const generatorHash = contentHash({
|
|
591
|
+
draftId: draft.id,
|
|
592
|
+
draftRevision: draft.revision,
|
|
593
|
+
tasks: draft.tasks
|
|
594
|
+
});
|
|
595
|
+
return {
|
|
596
|
+
schemaVersion: "openpond.generatedDatasetSource.v1",
|
|
597
|
+
kind: "generated",
|
|
598
|
+
id: `source-${tasksetId}`,
|
|
599
|
+
profileId: draft.profileId,
|
|
600
|
+
title: `${draft.name} manual authoring`,
|
|
601
|
+
sourceHash: generatorHash,
|
|
602
|
+
occurredAt: timestamp,
|
|
603
|
+
licensingStatus: "approved",
|
|
604
|
+
secretScanStatus: "passed",
|
|
605
|
+
piiScanStatus: "passed",
|
|
606
|
+
generatorId: "openpond-taskset-draft",
|
|
607
|
+
generatorVersion: "1",
|
|
608
|
+
seed: 0,
|
|
609
|
+
generatorHash,
|
|
610
|
+
metadata: { draftId: draft.id, draftRevision: draft.revision }
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
function buildIntentForDraft(draft) {
|
|
614
|
+
if (draft.learningSignals.preferences.length) return "preferences";
|
|
615
|
+
if (draft.learningSignals.rewards.length) return "verifiable_reward";
|
|
616
|
+
if (draft.learningSignals.labels.length) return "rubric";
|
|
617
|
+
return "demonstrations";
|
|
618
|
+
}
|
|
619
|
+
function tasksetOutputContract(metadata) {
|
|
620
|
+
const value = metadata.tasksetOutputContract;
|
|
621
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
622
|
+
return value;
|
|
623
|
+
}
|
|
624
|
+
return { mode: "text", jsonSchema: null, renderer: null };
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
// src/taskset-draft-files.ts
|
|
628
|
+
var MAX_TASKSET_DRAFT_EDIT_FILE_BYTES = 6e6;
|
|
629
|
+
var MANAGED_FILES = /* @__PURE__ */ new Set(["taskset.json", "capabilities.json", "data/tasks.jsonl", "tasks/tasks.jsonl", "graders/graders.json", "fixtures/grader-fixtures.json", "metrics/policy.json", "assets/manifest.json", "environment/contract.json", "environment/taskset.ts", "rubrics/preference-review.md", "comparisons/policy.json"]);
|
|
630
|
+
function isWritableTasksetDraftFilePath(value) {
|
|
631
|
+
const path = TasksetDraftFilePathSchema.parse(value);
|
|
632
|
+
return !MANAGED_FILES.has(path) && !path.startsWith("source-artifacts/");
|
|
633
|
+
}
|
|
634
|
+
function isManagedTasksetDraftFilePath(value) {
|
|
635
|
+
return MANAGED_FILES.has(TasksetDraftFilePathSchema.parse(value));
|
|
636
|
+
}
|
|
637
|
+
function createTasksetDraftFile(path, bytes) {
|
|
638
|
+
const writable = isWritableTasksetDraftFilePath(path);
|
|
639
|
+
if (bytes.byteLength > MAX_TASKSET_DRAFT_EDIT_FILE_BYTES) throw new Error("Taskset draft file content is invalid or exceeds 6 MB.");
|
|
640
|
+
let content;
|
|
641
|
+
try {
|
|
642
|
+
const text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
|
|
643
|
+
content = /[\u0000-\u0008\u000e-\u001f]/.test(text) ? base64Content(bytes) : { encoding: "utf8", data: text };
|
|
644
|
+
} catch {
|
|
645
|
+
content = base64Content(bytes);
|
|
646
|
+
}
|
|
647
|
+
return { path, writable, content, sizeBytes: bytes.byteLength, contentHash: sha256(bytes) };
|
|
648
|
+
}
|
|
649
|
+
function decodeTasksetDraftFileContent(input) {
|
|
650
|
+
const content = TasksetDraftFileContentSchema.parse(input);
|
|
651
|
+
let bytes;
|
|
652
|
+
try {
|
|
653
|
+
if (content.encoding === "utf8") bytes = new TextEncoder().encode(content.data);
|
|
654
|
+
else {
|
|
655
|
+
const raw = atob(content.data);
|
|
656
|
+
if (btoa(raw) !== content.data) throw new Error("Noncanonical base64");
|
|
657
|
+
bytes = Uint8Array.from(raw, (character) => character.charCodeAt(0));
|
|
658
|
+
}
|
|
659
|
+
} catch {
|
|
660
|
+
throw new Error("Taskset draft file content is invalid or exceeds 6 MB.");
|
|
661
|
+
}
|
|
662
|
+
if (bytes.byteLength > MAX_TASKSET_DRAFT_EDIT_FILE_BYTES) throw new Error("Taskset draft file content is invalid or exceeds 6 MB.");
|
|
663
|
+
return bytes;
|
|
664
|
+
}
|
|
665
|
+
function base64Content(bytes) {
|
|
666
|
+
let raw = "";
|
|
667
|
+
for (let offset = 0; offset < bytes.length; offset += 8192) raw += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
|
|
668
|
+
return { encoding: "base64", data: btoa(raw) };
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
// src/taskset-draft-manifests.ts
|
|
672
|
+
function configuredTasksetDraftFiles(draft) {
|
|
673
|
+
const configured = [];
|
|
674
|
+
if (draft.output.renderer) {
|
|
675
|
+
configured.push({
|
|
676
|
+
relativePath: draft.output.renderer.module,
|
|
677
|
+
source: `export async function ${draft.output.renderer.exportName}(output: unknown) {
|
|
678
|
+
// Return artifact paths or bytes derived from the model's structured output.
|
|
679
|
+
return output;
|
|
680
|
+
}
|
|
681
|
+
`
|
|
682
|
+
});
|
|
683
|
+
}
|
|
684
|
+
for (const grader of draft.graders) {
|
|
685
|
+
if (grader.kind !== "custom_verifier") continue;
|
|
686
|
+
configured.push({
|
|
687
|
+
relativePath: grader.module,
|
|
688
|
+
source: `export async function ${grader.exportName}(input: { output: unknown; task: unknown }) {
|
|
689
|
+
return { passed: false, score: 0, feedback: "Implement this verifier." };
|
|
690
|
+
}
|
|
691
|
+
`
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
if (draft.metrics.customAggregator) {
|
|
695
|
+
configured.push({
|
|
696
|
+
relativePath: draft.metrics.customAggregator.module,
|
|
697
|
+
source: `export function ${draft.metrics.customAggregator.exportName}(scores: number[]) {
|
|
698
|
+
return scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : 0;
|
|
699
|
+
}
|
|
700
|
+
`
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
for (const file of configured) TasksetDraftFilePathSchema.parse(file.relativePath);
|
|
704
|
+
return configured;
|
|
705
|
+
}
|
|
706
|
+
function renderTasksetDraftManifests(input) {
|
|
707
|
+
const draft = TasksetDraftSchema.parse(input);
|
|
708
|
+
return /* @__PURE__ */ new Map([
|
|
709
|
+
["taskset.json", canonicalJson(draft)],
|
|
710
|
+
["tasks/tasks.jsonl", jsonLines(draft.tasks)],
|
|
711
|
+
["graders/graders.json", canonicalJson(draft.graders)],
|
|
712
|
+
["fixtures/grader-fixtures.json", canonicalJson(draft.graderFixtures)],
|
|
713
|
+
["metrics/policy.json", canonicalJson(draft.metrics)],
|
|
714
|
+
["assets/manifest.json", canonicalJson(draft.tasks.flatMap((task) => task.assets ?? []))],
|
|
715
|
+
["environment/contract.json", canonicalJson(draft.environment)],
|
|
716
|
+
["rubrics/preference-review.md", rubricMarkdown(draft)],
|
|
717
|
+
["comparisons/policy.json", canonicalJson(draft.review)]
|
|
718
|
+
]);
|
|
719
|
+
}
|
|
720
|
+
function rubricMarkdown(draft) {
|
|
721
|
+
const criteria = draft.review.criteria.length ? `
|
|
722
|
+
|
|
723
|
+
## Criteria
|
|
724
|
+
|
|
725
|
+
${draft.review.criteria.map(
|
|
726
|
+
(criterion) => `- **${criterion.label}** (${criterion.weight}): ${criterion.description}`
|
|
727
|
+
).join("\n")}` : "";
|
|
728
|
+
return `# Preference review rubric
|
|
729
|
+
|
|
730
|
+
${draft.review.rubric}${criteria}
|
|
731
|
+
`;
|
|
732
|
+
}
|
|
733
|
+
function jsonLines(values) {
|
|
734
|
+
return values.length ? `${values.map((value) => JSON.stringify(value)).join("\n")}
|
|
735
|
+
` : "";
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
// src/taskset-draft-workspace.ts
|
|
739
|
+
import { z as z3 } from "zod";
|
|
740
|
+
import { assertBoundedTaskJson } from "@openpond/evals/task-schema";
|
|
741
|
+
var MAX_TASKSET_DRAFT_WORKSPACE_BYTES = 64 * 1024 * 1024;
|
|
742
|
+
var HashSchema3 = z3.string().regex(/^[a-f0-9]{64}$/);
|
|
743
|
+
var TasksetDraftWorkspaceFileSchema = z3.object({
|
|
744
|
+
path: TasksetDraftFilePathSchema,
|
|
745
|
+
contentHash: HashSchema3,
|
|
746
|
+
sizeBytes: z3.number().int().nonnegative().max(MAX_TASKSET_DRAFT_WORKSPACE_BYTES),
|
|
747
|
+
base64: z3.string().max(MAX_TASKSET_DRAFT_WORKSPACE_BYTES)
|
|
748
|
+
}).strict();
|
|
749
|
+
var WorkspaceContentSchema = z3.object({
|
|
750
|
+
schemaVersion: z3.literal("openpond.tasksetDraftWorkspace.v1"),
|
|
751
|
+
draft: TasksetDraftSchema,
|
|
752
|
+
files: z3.array(TasksetDraftWorkspaceFileSchema).max(1e4)
|
|
753
|
+
}).strict();
|
|
754
|
+
var TasksetDraftWorkspaceSchema = WorkspaceContentSchema.extend({ contentHash: HashSchema3 }).strict();
|
|
755
|
+
function createTasksetDraftWorkspace(input) {
|
|
756
|
+
assertBoundedTaskJson(input, MAX_TASKSET_DRAFT_WORKSPACE_BYTES);
|
|
757
|
+
const content = WorkspaceContentSchema.parse(input);
|
|
758
|
+
content.files.sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0);
|
|
759
|
+
return validateTasksetDraftWorkspace({ ...content, contentHash: contentHash(content) });
|
|
760
|
+
}
|
|
761
|
+
function validateTasksetDraftWorkspace(input) {
|
|
762
|
+
assertBoundedTaskJson(input, MAX_TASKSET_DRAFT_WORKSPACE_BYTES);
|
|
763
|
+
const workspace = TasksetDraftWorkspaceSchema.parse(input);
|
|
764
|
+
const { contentHash: expectedHash, ...content } = workspace;
|
|
765
|
+
if (contentHash(content) !== expectedHash) throw new Error("Taskset draft workspace differs from its retained bytes.");
|
|
766
|
+
let previousPath = null;
|
|
767
|
+
const paths = /* @__PURE__ */ new Set();
|
|
768
|
+
for (const file of workspace.files) {
|
|
769
|
+
if (isManagedTasksetDraftFilePath(file.path)) throw new Error("Managed Taskset files are represented by the draft document.");
|
|
770
|
+
if (previousPath !== null && previousPath >= file.path) throw new Error("Taskset draft file paths must be unique and sorted.");
|
|
771
|
+
const segments = file.path.split("/");
|
|
772
|
+
for (let end = 1; end < segments.length; end++) if (paths.has(segments.slice(0, end).join("/"))) throw new Error("A Taskset draft file occupies another file's directory.");
|
|
773
|
+
decodeTasksetDraftWorkspaceFile(file);
|
|
774
|
+
paths.add(file.path);
|
|
775
|
+
previousPath = file.path;
|
|
776
|
+
}
|
|
777
|
+
return workspace;
|
|
778
|
+
}
|
|
779
|
+
function readTasksetDraftWorkspaceFile(input, path) {
|
|
780
|
+
const workspace = validateTasksetDraftWorkspace(input);
|
|
781
|
+
TasksetDraftFilePathSchema.parse(path);
|
|
782
|
+
const file = workspace.files.find((candidate) => candidate.path === path);
|
|
783
|
+
const manifest = isManagedTasksetDraftFilePath(path) ? renderTasksetDraftManifests(workspace.draft).get(path) : void 0;
|
|
784
|
+
if (!file && manifest === void 0) throw new Error("Taskset draft file does not exist.");
|
|
785
|
+
const result = createTasksetDraftFile(path, file ? decodeTasksetDraftWorkspaceFile(file) : new TextEncoder().encode(manifest));
|
|
786
|
+
return { draftRevision: workspace.draft.revision, file: { ...result, writable: result.writable && workspace.draft.status !== "published" } };
|
|
787
|
+
}
|
|
788
|
+
function listTasksetDraftWorkspaceFiles(input) {
|
|
789
|
+
const workspace = validateTasksetDraftWorkspace(input);
|
|
790
|
+
const files = workspace.files.map((file) => ({
|
|
791
|
+
path: file.path,
|
|
792
|
+
sizeBytes: file.sizeBytes,
|
|
793
|
+
writable: workspace.draft.status !== "published" && isWritableTasksetDraftFilePath(file.path)
|
|
794
|
+
}));
|
|
795
|
+
for (const [path, content] of renderTasksetDraftManifests(workspace.draft)) files.push({ path, sizeBytes: new TextEncoder().encode(content).byteLength, writable: false });
|
|
796
|
+
return { draftRevision: workspace.draft.revision, files: files.sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0) };
|
|
797
|
+
}
|
|
798
|
+
function saveTasksetDraftWorkspaceDocument(input) {
|
|
799
|
+
const workspace = editableWorkspace(input.workspace, input.expectedDraftRevision);
|
|
800
|
+
const draft = TasksetDraftSchema.parse(input.draft);
|
|
801
|
+
const current = workspace.draft;
|
|
802
|
+
if (draft.id !== current.id || draft.profileId !== current.profileId || contentHash(draft.modelScope) !== contentHash(current.modelScope)) throw new Error("Taskset draft Model ownership cannot change.");
|
|
803
|
+
if (draft.revision !== current.revision || draft.status !== current.status || draft.createdAt !== current.createdAt || contentHash(draft.publishedTasksetRef) !== contentHash(current.publishedTasksetRef)) throw new Error("Taskset draft lifecycle fields cannot change through form edits.");
|
|
804
|
+
const files = [...workspace.files];
|
|
805
|
+
for (const configured of configuredTasksetDraftFiles(draft)) if (!files.some((file) => file.path === configured.relativePath)) {
|
|
806
|
+
const bytes = new TextEncoder().encode(configured.source);
|
|
807
|
+
files.push({ path: configured.relativePath, contentHash: sha256(bytes), sizeBytes: bytes.byteLength, base64: base64(bytes) });
|
|
808
|
+
}
|
|
809
|
+
const aggregator = draft.metrics.customAggregator;
|
|
810
|
+
const module = aggregator ? files.find((file) => file.path === aggregator.module) : void 0;
|
|
811
|
+
return createTasksetDraftWorkspace({
|
|
812
|
+
schemaVersion: workspace.schemaVersion,
|
|
813
|
+
files,
|
|
814
|
+
draft: {
|
|
815
|
+
...draft,
|
|
816
|
+
revision: current.revision + 1,
|
|
817
|
+
status: "draft",
|
|
818
|
+
updatedAt: input.now,
|
|
819
|
+
...aggregator && module ? { metrics: { ...draft.metrics, customAggregator: { ...aggregator, contentHash: module.contentHash } } } : {}
|
|
820
|
+
}
|
|
821
|
+
});
|
|
822
|
+
}
|
|
823
|
+
function saveTasksetDraftWorkspaceFile(input) {
|
|
824
|
+
const mutation = TasksetDraftFileMutationSchema.parse(input.mutation);
|
|
825
|
+
const workspace = editableWorkspace(input.workspace, mutation.expectedDraftRevision);
|
|
826
|
+
if (workspace.draft.id !== mutation.draftId) throw new Error("Taskset draft file belongs to another draft.");
|
|
827
|
+
if (!isWritableTasksetDraftFilePath(mutation.path)) throw new Error("This file is managed by the Taskset forms or retained source history.");
|
|
828
|
+
const previous = workspace.files.find((file) => file.path === mutation.path);
|
|
829
|
+
if ((previous?.contentHash ?? null) !== mutation.expectedFileHash) throw new Error("Taskset draft file changed. Reload it before saving.");
|
|
830
|
+
if (!previous && mutation.content === null) throw new Error("Taskset draft file does not exist.");
|
|
831
|
+
const files = workspace.files.filter((file) => file.path !== mutation.path);
|
|
832
|
+
if (mutation.content !== null) {
|
|
833
|
+
const bytes = decodeTasksetDraftFileContent(mutation.content);
|
|
834
|
+
files.push({ path: mutation.path, contentHash: sha256(bytes), sizeBytes: bytes.byteLength, base64: base64(bytes) });
|
|
835
|
+
}
|
|
836
|
+
return createTasksetDraftWorkspace({
|
|
837
|
+
schemaVersion: workspace.schemaVersion,
|
|
838
|
+
files,
|
|
839
|
+
draft: { ...workspace.draft, revision: workspace.draft.revision + 1, status: "draft", updatedAt: input.now }
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
function editableWorkspace(input, expectedRevision) {
|
|
843
|
+
const workspace = validateTasksetDraftWorkspace(input);
|
|
844
|
+
if (workspace.draft.status === "published") throw new Error("Published Taskset drafts are immutable.");
|
|
845
|
+
if (workspace.draft.revision !== expectedRevision) throw new Error("Taskset draft changed. Reload before saving.");
|
|
846
|
+
return workspace;
|
|
847
|
+
}
|
|
848
|
+
function decodeTasksetDraftWorkspaceFile(input) {
|
|
849
|
+
const file = TasksetDraftWorkspaceFileSchema.parse(input);
|
|
850
|
+
let raw;
|
|
851
|
+
try {
|
|
852
|
+
raw = atob(file.base64);
|
|
853
|
+
} catch {
|
|
854
|
+
throw new Error("Taskset draft file contains invalid base64.");
|
|
855
|
+
}
|
|
856
|
+
if (btoa(raw) !== file.base64) throw new Error("Taskset draft file contains noncanonical base64.");
|
|
857
|
+
const bytes = Uint8Array.from(raw, (character) => character.charCodeAt(0));
|
|
858
|
+
if (bytes.byteLength !== file.sizeBytes || sha256(bytes) !== file.contentHash) throw new Error("Taskset draft file differs from its retained bytes.");
|
|
859
|
+
return bytes;
|
|
860
|
+
}
|
|
861
|
+
function base64(bytes) {
|
|
862
|
+
let raw = "";
|
|
863
|
+
for (let offset = 0; offset < bytes.length; offset += 8192) raw += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
|
|
864
|
+
return btoa(raw);
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
export {
|
|
868
|
+
TasksetPackageModelConfigurationSchema,
|
|
869
|
+
TasksetPackagePublicationSchema,
|
|
870
|
+
TasksetPackageReceiptSchema,
|
|
871
|
+
TasksetPackageReadbackSchema,
|
|
872
|
+
OpenPondTasksetPackageError,
|
|
873
|
+
OpenPondTasksetPackageClient,
|
|
874
|
+
TaskDataDraftSchema,
|
|
875
|
+
TasksetDraftStatusSchema,
|
|
876
|
+
TasksetDraftReviewPolicySchema,
|
|
877
|
+
TasksetDraftOutputContractSchema,
|
|
878
|
+
TasksetDraftSchema,
|
|
879
|
+
TasksetDraftPublishError,
|
|
880
|
+
createTasksetDraft,
|
|
881
|
+
draftPublishIssues,
|
|
882
|
+
tasksetDraftFromTaskset,
|
|
883
|
+
publishTasksetDraft,
|
|
884
|
+
MAX_TASKSET_DRAFT_EDIT_FILE_BYTES,
|
|
885
|
+
isWritableTasksetDraftFilePath,
|
|
886
|
+
isManagedTasksetDraftFilePath,
|
|
887
|
+
createTasksetDraftFile,
|
|
888
|
+
decodeTasksetDraftFileContent,
|
|
889
|
+
configuredTasksetDraftFiles,
|
|
890
|
+
renderTasksetDraftManifests,
|
|
891
|
+
MAX_TASKSET_DRAFT_WORKSPACE_BYTES,
|
|
892
|
+
TasksetDraftWorkspaceFileSchema,
|
|
893
|
+
TasksetDraftWorkspaceSchema,
|
|
894
|
+
createTasksetDraftWorkspace,
|
|
895
|
+
validateTasksetDraftWorkspace,
|
|
896
|
+
readTasksetDraftWorkspaceFile,
|
|
897
|
+
listTasksetDraftWorkspaceFiles,
|
|
898
|
+
saveTasksetDraftWorkspaceDocument,
|
|
899
|
+
saveTasksetDraftWorkspaceFile,
|
|
900
|
+
decodeTasksetDraftWorkspaceFile
|
|
901
|
+
};
|
|
902
|
+
//# sourceMappingURL=chunk-UJZ3XEE3.js.map
|