openpond-sdk 0.6.7 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MODEL_STARTERS.md +25 -0
- package/TASKSETS.md +8 -0
- package/dist/{chunk-EJYSXE4V.js → chunk-32H2EJRA.js} +292 -456
- package/dist/chunk-32H2EJRA.js.map +7 -0
- package/dist/chunk-3HV3G6DG.js +95 -0
- package/dist/chunk-3HV3G6DG.js.map +7 -0
- package/dist/{chunk-FPOW5PU7.js → chunk-CGIMLZE5.js} +7 -7
- package/dist/chunk-DNBAJUVR.js +380 -0
- package/dist/chunk-DNBAJUVR.js.map +7 -0
- package/dist/{chunk-YGCHXNPI.js → chunk-DT7FRNIZ.js} +3 -94
- package/dist/{chunk-YGCHXNPI.js.map → chunk-DT7FRNIZ.js.map} +4 -4
- package/dist/{chunk-ECTMEJ5N.js → chunk-IKTZ2CD4.js} +63 -42
- package/dist/chunk-IKTZ2CD4.js.map +7 -0
- package/dist/{chunk-DNEPLQXN.js → chunk-KYCQ7LZ4.js} +5 -5
- package/dist/{chunk-TXQHWOFM.js → chunk-LGEF4S5I.js} +2 -2
- package/dist/chunk-QABCKPW6.js +171 -0
- package/dist/chunk-QABCKPW6.js.map +7 -0
- package/dist/{chunk-42YOB36J.js → chunk-WOPM32SE.js} +7 -7
- package/dist/{chunk-EAEYAU4R.js → chunk-Y4NGWGTA.js} +23 -2
- package/dist/{chunk-EAEYAU4R.js.map → chunk-Y4NGWGTA.js.map} +4 -4
- package/dist/dataset-workspaces.js +122 -0
- package/dist/dataset-workspaces.js.map +7 -0
- package/dist/index.js +20 -15
- package/dist/index.js.map +2 -2
- package/dist/learning.js +4 -4
- package/dist/model-starter-attempts.js +5 -1
- package/dist/model-starter-catalog.js +5 -5
- package/dist/model-starters.js +5 -5
- package/dist/model-taskset-runs.js +14 -11
- package/dist/model-taskset-runs.js.map +2 -2
- package/dist/taskset-drafts.js +10 -8
- package/dist/taskset-drafts.js.map +1 -1
- package/dist/taskset-packages.js +42 -364
- package/dist/taskset-packages.js.map +4 -4
- package/dist/training-bundle.js +10 -8
- package/dist/training-bundle.js.map +1 -1
- package/dist/training-projects.js +134 -0
- package/dist/training-projects.js.map +7 -0
- package/dist/training.js +3 -3
- package/dist/types/packages/cloud/src/hosted-chat.d.ts +1 -0
- package/dist/types/packages/cloud/src/hosted-chat.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-workspace-client.d.ts +1631 -0
- package/dist/types/packages/sdk/src/dataset-workspace-client.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/dataset-workspace-contracts.d.ts +1625 -0
- package/dist/types/packages/sdk/src/dataset-workspace-contracts.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/dataset-workspaces.d.ts +3 -0
- package/dist/types/packages/sdk/src/dataset-workspaces.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/evaluation-project-context.d.ts +19 -0
- package/dist/types/packages/sdk/src/evaluation-project-context.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/managed-training-preparation.d.ts +2 -2
- package/dist/types/packages/sdk/src/model-batch-review-preparation.d.ts +3 -3
- package/dist/types/packages/sdk/src/model-batch-review.d.ts +7 -7
- package/dist/types/packages/sdk/src/model-learning-intent.d.ts +1 -1
- package/dist/types/packages/sdk/src/model-learning-policy.d.ts +1 -1
- package/dist/types/packages/sdk/src/model-projects.d.ts +122 -122
- package/dist/types/packages/sdk/src/model-starter-attempts.d.ts +144 -29
- package/dist/types/packages/sdk/src/model-starter-attempts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-starter-catalog.d.ts +23 -23
- package/dist/types/packages/sdk/src/model-starter-execution.d.ts +3 -3
- package/dist/types/packages/sdk/src/model-starters.d.ts +22 -22
- package/dist/types/packages/sdk/src/model-taskset-runs-contracts.d.ts +91 -16
- package/dist/types/packages/sdk/src/model-taskset-runs-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-runs.d.ts +85 -15
- package/dist/types/packages/sdk/src/model-taskset-runs.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/task-inventory.d.ts +3 -3
- package/dist/types/packages/sdk/src/taskset-authored-contracts.d.ts +51 -51
- package/dist/types/packages/sdk/src/taskset-authored-files.d.ts +30 -30
- package/dist/types/packages/sdk/src/taskset-draft-api-contracts.d.ts +86 -86
- package/dist/types/packages/sdk/src/taskset-draft-client.d.ts +183 -183
- package/dist/types/packages/sdk/src/taskset-draft-core.d.ts +42 -42
- package/dist/types/packages/sdk/src/taskset-draft-dataset-artifacts.d.ts +36 -36
- package/dist/types/packages/sdk/src/taskset-draft-dataset-sources.d.ts +34 -34
- package/dist/types/packages/sdk/src/taskset-draft-document.d.ts +43 -43
- package/dist/types/packages/sdk/src/taskset-draft-package-compiler.d.ts +1094 -48
- package/dist/types/packages/sdk/src/taskset-draft-package-compiler.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-source.d.ts +71 -71
- package/dist/types/packages/sdk/src/taskset-draft-workspace.d.ts +84 -84
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts +671 -671
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts +371 -371
- package/dist/types/packages/sdk/src/taskset-package-grader-projection.d.ts +1 -1
- package/dist/types/packages/sdk/src/taskset-package-learning.d.ts +5 -5
- package/dist/types/packages/sdk/src/taskset-package-projection.d.ts +87 -87
- package/dist/types/packages/sdk/src/training-evaluation-results.d.ts +1 -1
- package/dist/types/packages/sdk/src/training-learning-batch.d.ts +64 -64
- package/dist/types/packages/sdk/src/training-project-client.d.ts +401 -0
- package/dist/types/packages/sdk/src/training-project-client.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/training-project-contracts.d.ts +617 -0
- package/dist/types/packages/sdk/src/training-project-contracts.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/training-projects.d.ts +3 -0
- package/dist/types/packages/sdk/src/training-projects.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/training.d.ts +24 -24
- package/package.json +10 -1
- package/dist/chunk-ECTMEJ5N.js.map +0 -7
- package/dist/chunk-EJYSXE4V.js.map +0 -7
- /package/dist/{chunk-FPOW5PU7.js.map → chunk-CGIMLZE5.js.map} +0 -0
- /package/dist/{chunk-DNEPLQXN.js.map → chunk-KYCQ7LZ4.js.map} +0 -0
- /package/dist/{chunk-TXQHWOFM.js.map → chunk-LGEF4S5I.js.map} +0 -0
- /package/dist/{chunk-42YOB36J.js.map → chunk-WOPM32SE.js.map} +0 -0
|
@@ -14,185 +14,27 @@ import {
|
|
|
14
14
|
TasksetSourceRefSchema,
|
|
15
15
|
computeTasksetHash,
|
|
16
16
|
validateTaskset
|
|
17
|
-
} from "./chunk-
|
|
18
|
-
import {
|
|
19
|
-
MAX_TASKSET_PACKAGE_BYTES,
|
|
20
|
-
TasksetPackageSchema,
|
|
21
|
-
tasksetPackageRewardBinding,
|
|
22
|
-
validateTasksetPackage
|
|
23
|
-
} from "./chunk-42YOB36J.js";
|
|
17
|
+
} from "./chunk-LGEF4S5I.js";
|
|
24
18
|
import {
|
|
25
19
|
ModelTasksetDraftPreparationSchema,
|
|
26
20
|
TasksetDraftFileContentSchema,
|
|
27
21
|
TasksetDraftFileMutationSchema,
|
|
28
22
|
TasksetDraftFilePathSchema
|
|
29
23
|
} from "./chunk-2U54EOXJ.js";
|
|
30
|
-
import {
|
|
31
|
-
TasksetCatalogReleaseRefSchema
|
|
32
|
-
} from "./chunk-A4ZDOZKP.js";
|
|
33
24
|
import {
|
|
34
25
|
canonicalJson,
|
|
35
26
|
contentHash,
|
|
36
27
|
sha256
|
|
37
28
|
} from "./chunk-QUOW6URQ.js";
|
|
38
|
-
import {
|
|
39
|
-
HostedModelProjectSummarySchema,
|
|
40
|
-
HostedModelProjectSyncSchema,
|
|
41
|
-
HostedModelProjectTrainingSetupSchema
|
|
42
|
-
} from "./chunk-TCHAWJMB.js";
|
|
43
|
-
|
|
44
|
-
// src/taskset-package-client.ts
|
|
45
|
-
import { z } from "zod";
|
|
46
|
-
import { learningRef } from "@openpond/evals/learning";
|
|
47
|
-
var IdSchema = z.string().trim().min(1).max(500);
|
|
48
|
-
var HashSchema = z.string().regex(/^[a-f0-9]{64}$/);
|
|
49
|
-
var TasksetPackageModelConfigurationSchema = HostedModelProjectSyncSchema.pick({
|
|
50
|
-
portableProjectId: true,
|
|
51
|
-
name: true,
|
|
52
|
-
objective: true,
|
|
53
|
-
defaultBaseModel: true,
|
|
54
|
-
defaultDestinationId: true,
|
|
55
|
-
sourceRevision: true,
|
|
56
|
-
sourceUpdatedAt: true
|
|
57
|
-
}).extend({ trainingSetup: HostedModelProjectTrainingSetupSchema.omit({ tasksetRef: true, rewardBindingRef: true, tasksetRelease: true }) }).strict();
|
|
58
|
-
var TasksetPackagePublicationSchema = z.object({
|
|
59
|
-
schemaVersion: z.literal("openpond.tasksetPackagePublication.v1"),
|
|
60
|
-
operationId: IdSchema,
|
|
61
|
-
modelProjectId: IdSchema,
|
|
62
|
-
expectedProjectEtag: HashSchema.nullable(),
|
|
63
|
-
name: z.string().trim().min(1).max(200),
|
|
64
|
-
description: z.string().max(5e3),
|
|
65
|
-
buildIntent: z.enum(["demonstrations", "preferences", "verifiable_reward", "rubric", "discovery"]),
|
|
66
|
-
methodHint: z.enum(["sft", "dpo", "grpo", "ppo"]).nullable(),
|
|
67
|
-
package: TasksetPackageSchema,
|
|
68
|
-
modelConfiguration: TasksetPackageModelConfigurationSchema.optional(),
|
|
69
|
-
evaluationPackage: TasksetPackageSchema.optional(),
|
|
70
|
-
selection: z.enum(["select", "attach"]).optional()
|
|
71
|
-
}).strict().refine((value) => value.selection !== "attach" || value.modelConfiguration === void 0, "Attachment-only publication cannot replace Model configuration.").refine(
|
|
72
|
-
(value) => !value.evaluationPackage || value.modelConfiguration !== void 0 && value.modelConfiguration.trainingSetup.evaluationTasksetRef != null && contentHash(value.modelConfiguration.trainingSetup.evaluationTasksetRef) === contentHash(learningRef(value.evaluationPackage.taskset)),
|
|
73
|
-
"Evaluation package must match the selected immutable evaluation in Model configuration."
|
|
74
|
-
);
|
|
75
|
-
var TasksetPackageReceiptSchema = z.object({
|
|
76
|
-
schemaVersion: z.literal("openpond.tasksetPackageReceipt.v1"),
|
|
77
|
-
teamId: IdSchema,
|
|
78
|
-
modelProjectId: IdSchema,
|
|
79
|
-
operationId: IdSchema,
|
|
80
|
-
taskset: TasksetCatalogReleaseRefSchema,
|
|
81
|
-
packageHash: HashSchema,
|
|
82
|
-
hostedTasksetId: IdSchema,
|
|
83
|
-
projectEtag: HashSchema,
|
|
84
|
-
project: HostedModelProjectSummarySchema.optional(),
|
|
85
|
-
selection: z.enum(["select", "attach"]).optional(),
|
|
86
|
-
evaluation: z.object({ taskset: TasksetCatalogReleaseRefSchema, packageHash: HashSchema, hostedTasksetId: IdSchema }).strict().optional()
|
|
87
|
-
}).strict();
|
|
88
|
-
var TasksetPackageReadbackSchema = z.object({
|
|
89
|
-
schemaVersion: z.literal("openpond.tasksetPackageReadback.v1"),
|
|
90
|
-
teamId: IdSchema,
|
|
91
|
-
modelProjectId: IdSchema,
|
|
92
|
-
package: TasksetPackageSchema
|
|
93
|
-
}).strict();
|
|
94
|
-
var OpenPondTasksetPackageError = class extends Error {
|
|
95
|
-
constructor(status, code, message) {
|
|
96
|
-
super(message);
|
|
97
|
-
this.status = status;
|
|
98
|
-
this.code = code;
|
|
99
|
-
this.name = "OpenPondTasksetPackageError";
|
|
100
|
-
}
|
|
101
|
-
status;
|
|
102
|
-
code;
|
|
103
|
-
};
|
|
104
|
-
var OpenPondTasksetPackageClient = class {
|
|
105
|
-
#options;
|
|
106
|
-
constructor(options) {
|
|
107
|
-
const url = new URL(options.baseUrl);
|
|
108
|
-
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash || !options.apiKey.trim() || !options.teamId.trim()) throw new Error("Taskset package client requires a clean HTTP(S) endpoint and workspace credentials.");
|
|
109
|
-
this.#options = { ...options, baseUrl: url.toString().replace(/\/+$/, "") };
|
|
110
|
-
}
|
|
111
|
-
async publish(input, options = {}) {
|
|
112
|
-
const request = TasksetPackagePublicationSchema.parse(input);
|
|
113
|
-
validateTasksetPackage(request.package);
|
|
114
|
-
if (request.evaluationPackage) validateTasksetPackage(request.evaluationPackage);
|
|
115
|
-
const receipt = TasksetPackageReceiptSchema.parse(await this.#request("", "POST", request, options.signal));
|
|
116
|
-
const release = request.package.taskset;
|
|
117
|
-
if (receipt.teamId !== this.#options.teamId || receipt.modelProjectId !== request.modelProjectId || receipt.operationId !== request.operationId || receipt.packageHash !== request.package.contentHash || receipt.taskset.id !== release.id || receipt.taskset.revision !== release.revision || receipt.taskset.contentHash !== release.contentHash) {
|
|
118
|
-
throw new OpenPondTasksetPackageError(502, "package_receipt_mismatch", "Taskset publication receipt did not match the requested package and owner.");
|
|
119
|
-
}
|
|
120
|
-
if (request.selection !== void 0 && receipt.selection !== request.selection) throw new OpenPondTasksetPackageError(502, "package_receipt_mismatch", "Taskset publication did not retain its selection mode.");
|
|
121
|
-
if (request.evaluationPackage && (!receipt.evaluation || receipt.evaluation.packageHash !== request.evaluationPackage.contentHash || contentHash(receipt.evaluation.taskset) !== contentHash(learningRef(request.evaluationPackage.taskset)))) {
|
|
122
|
-
throw new OpenPondTasksetPackageError(502, "package_receipt_mismatch", "Taskset publication did not retain the complete selected evaluation package.");
|
|
123
|
-
}
|
|
124
|
-
if (request.selection === "attach" && receipt.projectEtag !== request.expectedProjectEtag) throw new OpenPondTasksetPackageError(502, "package_receipt_mismatch", "Attaching a historical Taskset changed the Model configuration.");
|
|
125
|
-
if (request.modelConfiguration) {
|
|
126
|
-
const expected = request.modelConfiguration;
|
|
127
|
-
const project = receipt.project;
|
|
128
|
-
const binding = tasksetPackageRewardBinding(request.package);
|
|
129
|
-
const setup = HostedModelProjectTrainingSetupSchema.parse({
|
|
130
|
-
...expected.trainingSetup,
|
|
131
|
-
tasksetRef: learningRef(release),
|
|
132
|
-
rewardBindingRef: binding ? learningRef(binding) : null,
|
|
133
|
-
tasksetRelease: null
|
|
134
|
-
});
|
|
135
|
-
if (!project || project.teamId !== this.#options.teamId || project.etag !== receipt.projectEtag || ![project.id, project.portableProjectId].includes(request.modelProjectId) || project.portableProjectId !== expected.portableProjectId || project.name !== expected.name || project.objective !== expected.objective || project.sourceRevision !== expected.sourceRevision || Date.parse(project.sourceUpdatedAt) !== Date.parse(expected.sourceUpdatedAt) || contentHash(project.defaultBaseModel) !== contentHash(expected.defaultBaseModel) || project.defaultDestinationId !== expected.defaultDestinationId || contentHash(project.trainingSetup) !== contentHash(setup)) {
|
|
136
|
-
throw new OpenPondTasksetPackageError(502, "package_receipt_mismatch", "Taskset publication did not retain the requested Model configuration.");
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
return receipt;
|
|
140
|
-
}
|
|
141
|
-
async get(modelProjectId, reference, options = {}) {
|
|
142
|
-
const project = IdSchema.parse(modelProjectId);
|
|
143
|
-
const ref = TasksetCatalogReleaseRefSchema.parse(reference);
|
|
144
|
-
const expectedPackageHash = options.expectedPackageHash === void 0 ? void 0 : HashSchema.parse(options.expectedPackageHash);
|
|
145
|
-
const response = TasksetPackageReadbackSchema.parse(await this.#request(`/${encodeURIComponent(project)}/${encodeURIComponent(ref.id)}/${ref.revision}/${ref.contentHash}`, "GET", void 0, options.signal));
|
|
146
|
-
const taskset = response.package.taskset;
|
|
147
|
-
if (response.teamId !== this.#options.teamId || response.modelProjectId !== project || taskset.id !== ref.id || taskset.revision !== ref.revision || taskset.contentHash !== ref.contentHash) throw new OpenPondTasksetPackageError(502, "package_readback_mismatch", "Taskset package did not match the requested release and owner.");
|
|
148
|
-
if (expectedPackageHash !== void 0 && response.package.contentHash !== expectedPackageHash) throw new OpenPondTasksetPackageError(502, "package_readback_mismatch", "Taskset package did not match the saved publication receipt.");
|
|
149
|
-
return validateTasksetPackage(response.package);
|
|
150
|
-
}
|
|
151
|
-
async #request(path, method, body, signal) {
|
|
152
|
-
const serialized = body === void 0 ? void 0 : JSON.stringify(body);
|
|
153
|
-
if (serialized && new TextEncoder().encode(serialized).byteLength > MAX_TASKSET_PACKAGE_BYTES) throw new OpenPondTasksetPackageError(413, "package_too_large", "Taskset publication exceeds the 64 MiB envelope.");
|
|
154
|
-
const response = await (this.#options.fetch ?? globalThis.fetch)(`${this.#options.baseUrl}/v1/taskset-packages${path}`, {
|
|
155
|
-
method,
|
|
156
|
-
headers: { Authorization: `Bearer ${this.#options.apiKey}`, "X-OpenPond-Team-Id": this.#options.teamId, Accept: "application/json", ...serialized ? { "Content-Type": "application/json" } : {} },
|
|
157
|
-
body: serialized,
|
|
158
|
-
signal,
|
|
159
|
-
redirect: "error"
|
|
160
|
-
});
|
|
161
|
-
const reader = response.body?.getReader();
|
|
162
|
-
if (!reader) throw new OpenPondTasksetPackageError(response.status, "empty_response", "Taskset package response was empty.");
|
|
163
|
-
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
164
|
-
let bytes = 0;
|
|
165
|
-
let text = "";
|
|
166
|
-
try {
|
|
167
|
-
while (true) {
|
|
168
|
-
const chunk = await reader.read();
|
|
169
|
-
if (chunk.done) break;
|
|
170
|
-
bytes += chunk.value.byteLength;
|
|
171
|
-
if (bytes > MAX_TASKSET_PACKAGE_BYTES) throw new OpenPondTasksetPackageError(413, "package_too_large", "Taskset response exceeds the 64 MiB envelope.");
|
|
172
|
-
text += decoder.decode(chunk.value, { stream: true });
|
|
173
|
-
}
|
|
174
|
-
text += decoder.decode();
|
|
175
|
-
} finally {
|
|
176
|
-
await reader.cancel();
|
|
177
|
-
reader.releaseLock();
|
|
178
|
-
}
|
|
179
|
-
const payload = JSON.parse(text);
|
|
180
|
-
if (!response.ok) {
|
|
181
|
-
const error = z.object({ code: z.string(), message: z.string() }).safeParse(payload);
|
|
182
|
-
throw new OpenPondTasksetPackageError(response.status, error.success ? error.data.code : "package_request_failed", error.success ? error.data.message : `Taskset package request failed (${response.status}).`);
|
|
183
|
-
}
|
|
184
|
-
return payload;
|
|
185
|
-
}
|
|
186
|
-
};
|
|
187
29
|
|
|
188
30
|
// src/taskset-draft-document.ts
|
|
189
|
-
import { z
|
|
31
|
+
import { z } from "zod";
|
|
190
32
|
import { TasksetMetricPolicySchema } from "@openpond/evals/metrics";
|
|
191
|
-
var
|
|
192
|
-
var TimestampSchema =
|
|
193
|
-
var
|
|
194
|
-
var CodeIdentifierSchema =
|
|
195
|
-
var MetadataSchema =
|
|
33
|
+
var IdSchema = z.string().trim().min(1).max(240);
|
|
34
|
+
var TimestampSchema = z.string().trim().min(1);
|
|
35
|
+
var HashSchema = z.string().trim().min(8).max(256);
|
|
36
|
+
var CodeIdentifierSchema = z.string().trim().regex(/^[A-Za-z_$][A-Za-z0-9_$]*$/);
|
|
37
|
+
var MetadataSchema = z.record(z.string(), z.unknown()).default({});
|
|
196
38
|
function safeRelativeFilePath(value) {
|
|
197
39
|
const normalized = value.trim().replaceAll("\\", "/");
|
|
198
40
|
if (!normalized || normalized.startsWith("/") || normalized === "." || normalized === "..") {
|
|
@@ -203,35 +45,35 @@ function safeRelativeFilePath(value) {
|
|
|
203
45
|
);
|
|
204
46
|
}
|
|
205
47
|
var TaskDataDraftSchema = TaskDataRecordSchema.extend({
|
|
206
|
-
sourceRefs:
|
|
48
|
+
sourceRefs: z.array(IdSchema).max(100).default([])
|
|
207
49
|
});
|
|
208
|
-
var TasksetDraftStatusSchema =
|
|
50
|
+
var TasksetDraftStatusSchema = z.enum([
|
|
209
51
|
"draft",
|
|
210
52
|
"validating",
|
|
211
53
|
"needs_review",
|
|
212
54
|
"published"
|
|
213
55
|
]);
|
|
214
|
-
var TasksetDraftReviewPolicySchema =
|
|
215
|
-
enabled:
|
|
216
|
-
candidateCount:
|
|
217
|
-
minimumSamples:
|
|
218
|
-
allowTies:
|
|
219
|
-
allowRejectAll:
|
|
220
|
-
rubric:
|
|
221
|
-
criteria:
|
|
222
|
-
id:
|
|
223
|
-
label:
|
|
224
|
-
description:
|
|
225
|
-
weight:
|
|
56
|
+
var TasksetDraftReviewPolicySchema = z.object({
|
|
57
|
+
enabled: z.boolean(),
|
|
58
|
+
candidateCount: z.number().int().min(2).max(4),
|
|
59
|
+
minimumSamples: z.number().int().min(1).max(1e4).default(100),
|
|
60
|
+
allowTies: z.boolean(),
|
|
61
|
+
allowRejectAll: z.boolean(),
|
|
62
|
+
rubric: z.string().trim().max(5e4),
|
|
63
|
+
criteria: z.array(z.object({
|
|
64
|
+
id: IdSchema,
|
|
65
|
+
label: z.string().trim().min(1).max(500),
|
|
66
|
+
description: z.string().trim().max(1e4),
|
|
67
|
+
weight: z.number().nonnegative().max(1e3)
|
|
226
68
|
})).max(100)
|
|
227
69
|
});
|
|
228
|
-
var TasksetDraftOutputContractSchema =
|
|
229
|
-
mode:
|
|
230
|
-
jsonSchema:
|
|
231
|
-
renderer:
|
|
232
|
-
module:
|
|
70
|
+
var TasksetDraftOutputContractSchema = z.object({
|
|
71
|
+
mode: z.enum(["text", "structured_json", "artifacts"]),
|
|
72
|
+
jsonSchema: z.record(z.string(), z.unknown()).nullable(),
|
|
73
|
+
renderer: z.object({
|
|
74
|
+
module: z.string().trim().min(1).max(1e3).refine(safeRelativeFilePath, "Renderer modules must use a safe relative path."),
|
|
233
75
|
exportName: CodeIdentifierSchema,
|
|
234
|
-
configRef:
|
|
76
|
+
configRef: z.string().trim().min(1).max(1e3).refine(safeRelativeFilePath, "Renderer configuration must use a safe relative path.").nullable().default(null)
|
|
235
77
|
}).nullable()
|
|
236
78
|
}).superRefine((output, context) => {
|
|
237
79
|
if (output.mode === "structured_json" && !output.jsonSchema) {
|
|
@@ -249,23 +91,23 @@ var TasksetDraftOutputContractSchema = z2.object({
|
|
|
249
91
|
});
|
|
250
92
|
}
|
|
251
93
|
});
|
|
252
|
-
var TasksetDraftSchema =
|
|
253
|
-
schemaVersion:
|
|
254
|
-
id:
|
|
255
|
-
revision:
|
|
256
|
-
profileId:
|
|
257
|
-
modelScope:
|
|
258
|
-
modelId:
|
|
259
|
-
expectedModelRevision:
|
|
94
|
+
var TasksetDraftSchema = z.object({
|
|
95
|
+
schemaVersion: z.literal("openpond.tasksetDraft.v1"),
|
|
96
|
+
id: IdSchema,
|
|
97
|
+
revision: z.number().int().positive(),
|
|
98
|
+
profileId: IdSchema,
|
|
99
|
+
modelScope: z.object({
|
|
100
|
+
modelId: IdSchema,
|
|
101
|
+
expectedModelRevision: z.number().int().positive(),
|
|
260
102
|
source: ModelTasksetDraftPreparationSchema.optional()
|
|
261
103
|
}).strict().nullable().default(null),
|
|
262
|
-
name:
|
|
263
|
-
objective:
|
|
104
|
+
name: z.string().trim().max(500),
|
|
105
|
+
objective: z.string().trim().max(2e4),
|
|
264
106
|
purpose: TasksetPurposeSchema.default("general"),
|
|
265
107
|
benchmark: TasksetBenchmarkBindingSchema.nullable().default(null),
|
|
266
108
|
preferenceComparison: TasksetPreferenceComparisonBindingSchema.nullable().default(null),
|
|
267
109
|
status: TasksetDraftStatusSchema,
|
|
268
|
-
sourceRefs:
|
|
110
|
+
sourceRefs: z.array(TasksetSourceRefSchema).max(1e5).default([]),
|
|
269
111
|
datasetArtifact: DatasetArtifactManifestSchema.nullable().optional(),
|
|
270
112
|
policy: TaskPolicyBoundarySchema,
|
|
271
113
|
environment: TasksetEnvironmentContractSchema,
|
|
@@ -277,14 +119,14 @@ var TasksetDraftSchema = z2.object({
|
|
|
277
119
|
capabilities: TasksetCapabilityManifestSchema,
|
|
278
120
|
metrics: TasksetMetricPolicySchema,
|
|
279
121
|
review: TasksetDraftReviewPolicySchema,
|
|
280
|
-
tasks:
|
|
281
|
-
graders:
|
|
282
|
-
graderFixtures:
|
|
122
|
+
tasks: z.array(TaskDataDraftSchema).max(1e6).default([]),
|
|
123
|
+
graders: z.array(GraderSpecSchema).max(1e3).default([]),
|
|
124
|
+
graderFixtures: z.array(GraderFixtureSchema).max(1e5).default([]),
|
|
283
125
|
learningSignals: LearningSignalInventorySchema,
|
|
284
|
-
publishedTasksetRef:
|
|
285
|
-
id:
|
|
286
|
-
revision:
|
|
287
|
-
contentHash:
|
|
126
|
+
publishedTasksetRef: z.object({
|
|
127
|
+
id: IdSchema,
|
|
128
|
+
revision: z.number().int().positive(),
|
|
129
|
+
contentHash: HashSchema
|
|
288
130
|
}).nullable(),
|
|
289
131
|
createdAt: TimestampSchema,
|
|
290
132
|
updatedAt: TimestampSchema,
|
|
@@ -320,6 +162,246 @@ var TasksetDraftSchema = z2.object({
|
|
|
320
162
|
}
|
|
321
163
|
});
|
|
322
164
|
|
|
165
|
+
// src/taskset-draft-files.ts
|
|
166
|
+
var MAX_TASKSET_DRAFT_EDIT_FILE_BYTES = 6e6;
|
|
167
|
+
var MANAGED_FILES = /* @__PURE__ */ new Set(["taskset.json", "capabilities.json", "data/tasks.jsonl", "tasks/tasks.jsonl", "graders/graders.json", "fixtures/grader-fixtures.json", "metrics/policy.json", "assets/manifest.json", "environment/contract.json", "environment/taskset.ts", "rubrics/preference-review.md", "comparisons/policy.json"]);
|
|
168
|
+
function isWritableTasksetDraftFilePath(value) {
|
|
169
|
+
const path = TasksetDraftFilePathSchema.parse(value);
|
|
170
|
+
return !MANAGED_FILES.has(path) && !path.startsWith("source-artifacts/");
|
|
171
|
+
}
|
|
172
|
+
function isManagedTasksetDraftFilePath(value) {
|
|
173
|
+
return MANAGED_FILES.has(TasksetDraftFilePathSchema.parse(value));
|
|
174
|
+
}
|
|
175
|
+
function createTasksetDraftFile(path, bytes) {
|
|
176
|
+
const writable = isWritableTasksetDraftFilePath(path);
|
|
177
|
+
if (bytes.byteLength > MAX_TASKSET_DRAFT_EDIT_FILE_BYTES) throw new Error("Taskset draft file content is invalid or exceeds 6 MB.");
|
|
178
|
+
let content;
|
|
179
|
+
try {
|
|
180
|
+
const text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
|
|
181
|
+
content = /[\u0000-\u0008\u000e-\u001f]/.test(text) ? base64Content(bytes) : { encoding: "utf8", data: text };
|
|
182
|
+
} catch {
|
|
183
|
+
content = base64Content(bytes);
|
|
184
|
+
}
|
|
185
|
+
return { path, writable, content, sizeBytes: bytes.byteLength, contentHash: sha256(bytes) };
|
|
186
|
+
}
|
|
187
|
+
function decodeTasksetDraftFileContent(input) {
|
|
188
|
+
const content = TasksetDraftFileContentSchema.parse(input);
|
|
189
|
+
let bytes;
|
|
190
|
+
try {
|
|
191
|
+
if (content.encoding === "utf8") bytes = new TextEncoder().encode(content.data);
|
|
192
|
+
else {
|
|
193
|
+
const raw = atob(content.data);
|
|
194
|
+
if (btoa(raw) !== content.data) throw new Error("Noncanonical base64");
|
|
195
|
+
bytes = Uint8Array.from(raw, (character) => character.charCodeAt(0));
|
|
196
|
+
}
|
|
197
|
+
} catch {
|
|
198
|
+
throw new Error("Taskset draft file content is invalid or exceeds 6 MB.");
|
|
199
|
+
}
|
|
200
|
+
if (bytes.byteLength > MAX_TASKSET_DRAFT_EDIT_FILE_BYTES) throw new Error("Taskset draft file content is invalid or exceeds 6 MB.");
|
|
201
|
+
return bytes;
|
|
202
|
+
}
|
|
203
|
+
function base64Content(bytes) {
|
|
204
|
+
let raw = "";
|
|
205
|
+
for (let offset = 0; offset < bytes.length; offset += 8192) raw += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
|
|
206
|
+
return { encoding: "base64", data: btoa(raw) };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// src/taskset-draft-manifests.ts
|
|
210
|
+
function configuredTasksetDraftFiles(draft) {
|
|
211
|
+
const configured = [];
|
|
212
|
+
if (draft.output.renderer) {
|
|
213
|
+
configured.push({
|
|
214
|
+
relativePath: draft.output.renderer.module,
|
|
215
|
+
source: `export async function ${draft.output.renderer.exportName}(output: unknown) {
|
|
216
|
+
// Return artifact paths or bytes derived from the model's structured output.
|
|
217
|
+
return output;
|
|
218
|
+
}
|
|
219
|
+
`
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
for (const grader of draft.graders) {
|
|
223
|
+
if (grader.kind !== "custom_verifier") continue;
|
|
224
|
+
configured.push({
|
|
225
|
+
relativePath: grader.module,
|
|
226
|
+
source: `export async function ${grader.exportName}(input: { output: unknown; task: unknown }) {
|
|
227
|
+
return { passed: false, score: 0, feedback: "Implement this verifier." };
|
|
228
|
+
}
|
|
229
|
+
`
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
if (draft.metrics.customAggregator) {
|
|
233
|
+
configured.push({
|
|
234
|
+
relativePath: draft.metrics.customAggregator.module,
|
|
235
|
+
source: `export function ${draft.metrics.customAggregator.exportName}(scores: number[]) {
|
|
236
|
+
return scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : 0;
|
|
237
|
+
}
|
|
238
|
+
`
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
for (const file of configured) TasksetDraftFilePathSchema.parse(file.relativePath);
|
|
242
|
+
return configured;
|
|
243
|
+
}
|
|
244
|
+
function renderTasksetDraftManifests(input) {
|
|
245
|
+
const draft = TasksetDraftSchema.parse(input);
|
|
246
|
+
return /* @__PURE__ */ new Map([
|
|
247
|
+
["taskset.json", canonicalJson(draft)],
|
|
248
|
+
["tasks/tasks.jsonl", jsonLines(draft.tasks)],
|
|
249
|
+
["graders/graders.json", canonicalJson(draft.graders)],
|
|
250
|
+
["fixtures/grader-fixtures.json", canonicalJson(draft.graderFixtures)],
|
|
251
|
+
["metrics/policy.json", canonicalJson(draft.metrics)],
|
|
252
|
+
["assets/manifest.json", canonicalJson(draft.tasks.flatMap((task) => task.assets ?? []))],
|
|
253
|
+
["environment/contract.json", canonicalJson(draft.environment)],
|
|
254
|
+
["rubrics/preference-review.md", rubricMarkdown(draft)],
|
|
255
|
+
["comparisons/policy.json", canonicalJson(draft.review)]
|
|
256
|
+
]);
|
|
257
|
+
}
|
|
258
|
+
function rubricMarkdown(draft) {
|
|
259
|
+
const criteria = draft.review.criteria.length ? `
|
|
260
|
+
|
|
261
|
+
## Criteria
|
|
262
|
+
|
|
263
|
+
${draft.review.criteria.map(
|
|
264
|
+
(criterion) => `- **${criterion.label}** (${criterion.weight}): ${criterion.description}`
|
|
265
|
+
).join("\n")}` : "";
|
|
266
|
+
return `# Preference review rubric
|
|
267
|
+
|
|
268
|
+
${draft.review.rubric}${criteria}
|
|
269
|
+
`;
|
|
270
|
+
}
|
|
271
|
+
function jsonLines(values) {
|
|
272
|
+
return values.length ? `${values.map((value) => JSON.stringify(value)).join("\n")}
|
|
273
|
+
` : "";
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// src/taskset-draft-workspace.ts
|
|
277
|
+
import { z as z2 } from "zod";
|
|
278
|
+
import { assertBoundedTaskJson } from "@openpond/evals/task-schema";
|
|
279
|
+
var MAX_TASKSET_DRAFT_WORKSPACE_BYTES = 64 * 1024 * 1024;
|
|
280
|
+
var HashSchema2 = z2.string().regex(/^[a-f0-9]{64}$/);
|
|
281
|
+
var TasksetDraftWorkspaceFileSchema = z2.object({
|
|
282
|
+
path: TasksetDraftFilePathSchema,
|
|
283
|
+
contentHash: HashSchema2,
|
|
284
|
+
sizeBytes: z2.number().int().nonnegative().max(MAX_TASKSET_DRAFT_WORKSPACE_BYTES),
|
|
285
|
+
base64: z2.string().max(MAX_TASKSET_DRAFT_WORKSPACE_BYTES)
|
|
286
|
+
}).strict();
|
|
287
|
+
var WorkspaceContentSchema = z2.object({
|
|
288
|
+
schemaVersion: z2.literal("openpond.tasksetDraftWorkspace.v1"),
|
|
289
|
+
draft: TasksetDraftSchema,
|
|
290
|
+
files: z2.array(TasksetDraftWorkspaceFileSchema).max(1e4)
|
|
291
|
+
}).strict();
|
|
292
|
+
var TasksetDraftWorkspaceSchema = WorkspaceContentSchema.extend({ contentHash: HashSchema2 }).strict();
|
|
293
|
+
function createTasksetDraftWorkspace(input) {
|
|
294
|
+
assertBoundedTaskJson(input, MAX_TASKSET_DRAFT_WORKSPACE_BYTES);
|
|
295
|
+
const content = WorkspaceContentSchema.parse(input);
|
|
296
|
+
content.files.sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0);
|
|
297
|
+
return validateTasksetDraftWorkspace({ ...content, contentHash: contentHash(content) });
|
|
298
|
+
}
|
|
299
|
+
function validateTasksetDraftWorkspace(input) {
|
|
300
|
+
assertBoundedTaskJson(input, MAX_TASKSET_DRAFT_WORKSPACE_BYTES);
|
|
301
|
+
const workspace = TasksetDraftWorkspaceSchema.parse(input);
|
|
302
|
+
const { contentHash: expectedHash, ...content } = workspace;
|
|
303
|
+
if (contentHash(content) !== expectedHash) throw new Error("Taskset draft workspace differs from its retained bytes.");
|
|
304
|
+
let previousPath = null;
|
|
305
|
+
const paths = /* @__PURE__ */ new Set();
|
|
306
|
+
for (const file of workspace.files) {
|
|
307
|
+
if (isManagedTasksetDraftFilePath(file.path)) throw new Error("Managed Taskset files are represented by the draft document.");
|
|
308
|
+
if (previousPath !== null && previousPath >= file.path) throw new Error("Taskset draft file paths must be unique and sorted.");
|
|
309
|
+
const segments = file.path.split("/");
|
|
310
|
+
for (let end = 1; end < segments.length; end++) if (paths.has(segments.slice(0, end).join("/"))) throw new Error("A Taskset draft file occupies another file's directory.");
|
|
311
|
+
decodeTasksetDraftWorkspaceFile(file);
|
|
312
|
+
paths.add(file.path);
|
|
313
|
+
previousPath = file.path;
|
|
314
|
+
}
|
|
315
|
+
return workspace;
|
|
316
|
+
}
|
|
317
|
+
function readTasksetDraftWorkspaceFile(input, path) {
|
|
318
|
+
const workspace = validateTasksetDraftWorkspace(input);
|
|
319
|
+
TasksetDraftFilePathSchema.parse(path);
|
|
320
|
+
const file = workspace.files.find((candidate) => candidate.path === path);
|
|
321
|
+
const manifest = isManagedTasksetDraftFilePath(path) ? renderTasksetDraftManifests(workspace.draft).get(path) : void 0;
|
|
322
|
+
if (!file && manifest === void 0) throw new Error("Taskset draft file does not exist.");
|
|
323
|
+
const result = createTasksetDraftFile(path, file ? decodeTasksetDraftWorkspaceFile(file) : new TextEncoder().encode(manifest));
|
|
324
|
+
return { draftRevision: workspace.draft.revision, file: { ...result, writable: result.writable && workspace.draft.status !== "published" } };
|
|
325
|
+
}
|
|
326
|
+
function listTasksetDraftWorkspaceFiles(input) {
|
|
327
|
+
const workspace = validateTasksetDraftWorkspace(input);
|
|
328
|
+
const files = workspace.files.map((file) => ({
|
|
329
|
+
path: file.path,
|
|
330
|
+
sizeBytes: file.sizeBytes,
|
|
331
|
+
writable: workspace.draft.status !== "published" && isWritableTasksetDraftFilePath(file.path)
|
|
332
|
+
}));
|
|
333
|
+
for (const [path, content] of renderTasksetDraftManifests(workspace.draft)) files.push({ path, sizeBytes: new TextEncoder().encode(content).byteLength, writable: false });
|
|
334
|
+
return { draftRevision: workspace.draft.revision, files: files.sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0) };
|
|
335
|
+
}
|
|
336
|
+
function saveTasksetDraftWorkspaceDocument(input) {
|
|
337
|
+
const workspace = editableWorkspace(input.workspace, input.expectedDraftRevision);
|
|
338
|
+
const draft = TasksetDraftSchema.parse(input.draft);
|
|
339
|
+
const current = workspace.draft;
|
|
340
|
+
if (draft.id !== current.id || draft.profileId !== current.profileId || contentHash(draft.modelScope) !== contentHash(current.modelScope)) throw new Error("Taskset draft Model ownership cannot change.");
|
|
341
|
+
if (draft.revision !== current.revision || draft.status !== current.status || draft.createdAt !== current.createdAt || contentHash(draft.publishedTasksetRef) !== contentHash(current.publishedTasksetRef)) throw new Error("Taskset draft lifecycle fields cannot change through form edits.");
|
|
342
|
+
const files = [...workspace.files];
|
|
343
|
+
for (const configured of configuredTasksetDraftFiles(draft)) if (!files.some((file) => file.path === configured.relativePath)) {
|
|
344
|
+
const bytes = new TextEncoder().encode(configured.source);
|
|
345
|
+
files.push({ path: configured.relativePath, contentHash: sha256(bytes), sizeBytes: bytes.byteLength, base64: base64(bytes) });
|
|
346
|
+
}
|
|
347
|
+
const aggregator = draft.metrics.customAggregator;
|
|
348
|
+
const module = aggregator ? files.find((file) => file.path === aggregator.module) : void 0;
|
|
349
|
+
return createTasksetDraftWorkspace({
|
|
350
|
+
schemaVersion: workspace.schemaVersion,
|
|
351
|
+
files,
|
|
352
|
+
draft: {
|
|
353
|
+
...draft,
|
|
354
|
+
revision: current.revision + 1,
|
|
355
|
+
status: "draft",
|
|
356
|
+
updatedAt: input.now,
|
|
357
|
+
...aggregator && module ? { metrics: { ...draft.metrics, customAggregator: { ...aggregator, contentHash: module.contentHash } } } : {}
|
|
358
|
+
}
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
function saveTasksetDraftWorkspaceFile(input) {
|
|
362
|
+
const mutation = TasksetDraftFileMutationSchema.parse(input.mutation);
|
|
363
|
+
const workspace = editableWorkspace(input.workspace, mutation.expectedDraftRevision);
|
|
364
|
+
if (workspace.draft.id !== mutation.draftId) throw new Error("Taskset draft file belongs to another draft.");
|
|
365
|
+
if (!isWritableTasksetDraftFilePath(mutation.path)) throw new Error("This file is managed by the Taskset forms or retained source history.");
|
|
366
|
+
const previous = workspace.files.find((file) => file.path === mutation.path);
|
|
367
|
+
if ((previous?.contentHash ?? null) !== mutation.expectedFileHash) throw new Error("Taskset draft file changed. Reload it before saving.");
|
|
368
|
+
if (!previous && mutation.content === null) throw new Error("Taskset draft file does not exist.");
|
|
369
|
+
const files = workspace.files.filter((file) => file.path !== mutation.path);
|
|
370
|
+
if (mutation.content !== null) {
|
|
371
|
+
const bytes = decodeTasksetDraftFileContent(mutation.content);
|
|
372
|
+
files.push({ path: mutation.path, contentHash: sha256(bytes), sizeBytes: bytes.byteLength, base64: base64(bytes) });
|
|
373
|
+
}
|
|
374
|
+
return createTasksetDraftWorkspace({
|
|
375
|
+
schemaVersion: workspace.schemaVersion,
|
|
376
|
+
files,
|
|
377
|
+
draft: { ...workspace.draft, revision: workspace.draft.revision + 1, status: "draft", updatedAt: input.now }
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
function editableWorkspace(input, expectedRevision) {
|
|
381
|
+
const workspace = validateTasksetDraftWorkspace(input);
|
|
382
|
+
if (workspace.draft.status === "published") throw new Error("Published Taskset drafts are immutable.");
|
|
383
|
+
if (workspace.draft.revision !== expectedRevision) throw new Error("Taskset draft changed. Reload before saving.");
|
|
384
|
+
return workspace;
|
|
385
|
+
}
|
|
386
|
+
function decodeTasksetDraftWorkspaceFile(input) {
|
|
387
|
+
const file = TasksetDraftWorkspaceFileSchema.parse(input);
|
|
388
|
+
let raw;
|
|
389
|
+
try {
|
|
390
|
+
raw = atob(file.base64);
|
|
391
|
+
} catch {
|
|
392
|
+
throw new Error("Taskset draft file contains invalid base64.");
|
|
393
|
+
}
|
|
394
|
+
if (btoa(raw) !== file.base64) throw new Error("Taskset draft file contains noncanonical base64.");
|
|
395
|
+
const bytes = Uint8Array.from(raw, (character) => character.charCodeAt(0));
|
|
396
|
+
if (bytes.byteLength !== file.sizeBytes || sha256(bytes) !== file.contentHash) throw new Error("Taskset draft file differs from its retained bytes.");
|
|
397
|
+
return bytes;
|
|
398
|
+
}
|
|
399
|
+
function base64(bytes) {
|
|
400
|
+
let raw = "";
|
|
401
|
+
for (let offset = 0; offset < bytes.length; offset += 8192) raw += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
|
|
402
|
+
return btoa(raw);
|
|
403
|
+
}
|
|
404
|
+
|
|
323
405
|
// src/taskset-draft-authoring.ts
|
|
324
406
|
var TasksetDraftPublishError = class extends Error {
|
|
325
407
|
issues;
|
|
@@ -642,263 +724,12 @@ function tasksetOutputContract(metadata) {
|
|
|
642
724
|
return { mode: "text", jsonSchema: null, renderer: null };
|
|
643
725
|
}
|
|
644
726
|
|
|
645
|
-
// src/taskset-draft-files.ts
|
|
646
|
-
var MAX_TASKSET_DRAFT_EDIT_FILE_BYTES = 6e6;
|
|
647
|
-
var MANAGED_FILES = /* @__PURE__ */ new Set(["taskset.json", "capabilities.json", "data/tasks.jsonl", "tasks/tasks.jsonl", "graders/graders.json", "fixtures/grader-fixtures.json", "metrics/policy.json", "assets/manifest.json", "environment/contract.json", "environment/taskset.ts", "rubrics/preference-review.md", "comparisons/policy.json"]);
|
|
648
|
-
function isWritableTasksetDraftFilePath(value) {
|
|
649
|
-
const path = TasksetDraftFilePathSchema.parse(value);
|
|
650
|
-
return !MANAGED_FILES.has(path) && !path.startsWith("source-artifacts/");
|
|
651
|
-
}
|
|
652
|
-
function isManagedTasksetDraftFilePath(value) {
|
|
653
|
-
return MANAGED_FILES.has(TasksetDraftFilePathSchema.parse(value));
|
|
654
|
-
}
|
|
655
|
-
function createTasksetDraftFile(path, bytes) {
|
|
656
|
-
const writable = isWritableTasksetDraftFilePath(path);
|
|
657
|
-
if (bytes.byteLength > MAX_TASKSET_DRAFT_EDIT_FILE_BYTES) throw new Error("Taskset draft file content is invalid or exceeds 6 MB.");
|
|
658
|
-
let content;
|
|
659
|
-
try {
|
|
660
|
-
const text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
|
|
661
|
-
content = /[\u0000-\u0008\u000e-\u001f]/.test(text) ? base64Content(bytes) : { encoding: "utf8", data: text };
|
|
662
|
-
} catch {
|
|
663
|
-
content = base64Content(bytes);
|
|
664
|
-
}
|
|
665
|
-
return { path, writable, content, sizeBytes: bytes.byteLength, contentHash: sha256(bytes) };
|
|
666
|
-
}
|
|
667
|
-
function decodeTasksetDraftFileContent(input) {
|
|
668
|
-
const content = TasksetDraftFileContentSchema.parse(input);
|
|
669
|
-
let bytes;
|
|
670
|
-
try {
|
|
671
|
-
if (content.encoding === "utf8") bytes = new TextEncoder().encode(content.data);
|
|
672
|
-
else {
|
|
673
|
-
const raw = atob(content.data);
|
|
674
|
-
if (btoa(raw) !== content.data) throw new Error("Noncanonical base64");
|
|
675
|
-
bytes = Uint8Array.from(raw, (character) => character.charCodeAt(0));
|
|
676
|
-
}
|
|
677
|
-
} catch {
|
|
678
|
-
throw new Error("Taskset draft file content is invalid or exceeds 6 MB.");
|
|
679
|
-
}
|
|
680
|
-
if (bytes.byteLength > MAX_TASKSET_DRAFT_EDIT_FILE_BYTES) throw new Error("Taskset draft file content is invalid or exceeds 6 MB.");
|
|
681
|
-
return bytes;
|
|
682
|
-
}
|
|
683
|
-
function base64Content(bytes) {
|
|
684
|
-
let raw = "";
|
|
685
|
-
for (let offset = 0; offset < bytes.length; offset += 8192) raw += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
|
|
686
|
-
return { encoding: "base64", data: btoa(raw) };
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
// src/taskset-draft-manifests.ts
|
|
690
|
-
function configuredTasksetDraftFiles(draft) {
|
|
691
|
-
const configured = [];
|
|
692
|
-
if (draft.output.renderer) {
|
|
693
|
-
configured.push({
|
|
694
|
-
relativePath: draft.output.renderer.module,
|
|
695
|
-
source: `export async function ${draft.output.renderer.exportName}(output: unknown) {
|
|
696
|
-
// Return artifact paths or bytes derived from the model's structured output.
|
|
697
|
-
return output;
|
|
698
|
-
}
|
|
699
|
-
`
|
|
700
|
-
});
|
|
701
|
-
}
|
|
702
|
-
for (const grader of draft.graders) {
|
|
703
|
-
if (grader.kind !== "custom_verifier") continue;
|
|
704
|
-
configured.push({
|
|
705
|
-
relativePath: grader.module,
|
|
706
|
-
source: `export async function ${grader.exportName}(input: { output: unknown; task: unknown }) {
|
|
707
|
-
return { passed: false, score: 0, feedback: "Implement this verifier." };
|
|
708
|
-
}
|
|
709
|
-
`
|
|
710
|
-
});
|
|
711
|
-
}
|
|
712
|
-
if (draft.metrics.customAggregator) {
|
|
713
|
-
configured.push({
|
|
714
|
-
relativePath: draft.metrics.customAggregator.module,
|
|
715
|
-
source: `export function ${draft.metrics.customAggregator.exportName}(scores: number[]) {
|
|
716
|
-
return scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : 0;
|
|
717
|
-
}
|
|
718
|
-
`
|
|
719
|
-
});
|
|
720
|
-
}
|
|
721
|
-
for (const file of configured) TasksetDraftFilePathSchema.parse(file.relativePath);
|
|
722
|
-
return configured;
|
|
723
|
-
}
|
|
724
|
-
function renderTasksetDraftManifests(input) {
|
|
725
|
-
const draft = TasksetDraftSchema.parse(input);
|
|
726
|
-
return /* @__PURE__ */ new Map([
|
|
727
|
-
["taskset.json", canonicalJson(draft)],
|
|
728
|
-
["tasks/tasks.jsonl", jsonLines(draft.tasks)],
|
|
729
|
-
["graders/graders.json", canonicalJson(draft.graders)],
|
|
730
|
-
["fixtures/grader-fixtures.json", canonicalJson(draft.graderFixtures)],
|
|
731
|
-
["metrics/policy.json", canonicalJson(draft.metrics)],
|
|
732
|
-
["assets/manifest.json", canonicalJson(draft.tasks.flatMap((task) => task.assets ?? []))],
|
|
733
|
-
["environment/contract.json", canonicalJson(draft.environment)],
|
|
734
|
-
["rubrics/preference-review.md", rubricMarkdown(draft)],
|
|
735
|
-
["comparisons/policy.json", canonicalJson(draft.review)]
|
|
736
|
-
]);
|
|
737
|
-
}
|
|
738
|
-
function rubricMarkdown(draft) {
|
|
739
|
-
const criteria = draft.review.criteria.length ? `
|
|
740
|
-
|
|
741
|
-
## Criteria
|
|
742
|
-
|
|
743
|
-
${draft.review.criteria.map(
|
|
744
|
-
(criterion) => `- **${criterion.label}** (${criterion.weight}): ${criterion.description}`
|
|
745
|
-
).join("\n")}` : "";
|
|
746
|
-
return `# Preference review rubric
|
|
747
|
-
|
|
748
|
-
${draft.review.rubric}${criteria}
|
|
749
|
-
`;
|
|
750
|
-
}
|
|
751
|
-
function jsonLines(values) {
|
|
752
|
-
return values.length ? `${values.map((value) => JSON.stringify(value)).join("\n")}
|
|
753
|
-
` : "";
|
|
754
|
-
}
|
|
755
|
-
|
|
756
|
-
// src/taskset-draft-workspace.ts
|
|
757
|
-
import { z as z3 } from "zod";
|
|
758
|
-
import { assertBoundedTaskJson } from "@openpond/evals/task-schema";
|
|
759
|
-
var MAX_TASKSET_DRAFT_WORKSPACE_BYTES = 64 * 1024 * 1024;
|
|
760
|
-
var HashSchema3 = z3.string().regex(/^[a-f0-9]{64}$/);
|
|
761
|
-
var TasksetDraftWorkspaceFileSchema = z3.object({
|
|
762
|
-
path: TasksetDraftFilePathSchema,
|
|
763
|
-
contentHash: HashSchema3,
|
|
764
|
-
sizeBytes: z3.number().int().nonnegative().max(MAX_TASKSET_DRAFT_WORKSPACE_BYTES),
|
|
765
|
-
base64: z3.string().max(MAX_TASKSET_DRAFT_WORKSPACE_BYTES)
|
|
766
|
-
}).strict();
|
|
767
|
-
var WorkspaceContentSchema = z3.object({
|
|
768
|
-
schemaVersion: z3.literal("openpond.tasksetDraftWorkspace.v1"),
|
|
769
|
-
draft: TasksetDraftSchema,
|
|
770
|
-
files: z3.array(TasksetDraftWorkspaceFileSchema).max(1e4)
|
|
771
|
-
}).strict();
|
|
772
|
-
var TasksetDraftWorkspaceSchema = WorkspaceContentSchema.extend({ contentHash: HashSchema3 }).strict();
|
|
773
|
-
function createTasksetDraftWorkspace(input) {
|
|
774
|
-
assertBoundedTaskJson(input, MAX_TASKSET_DRAFT_WORKSPACE_BYTES);
|
|
775
|
-
const content = WorkspaceContentSchema.parse(input);
|
|
776
|
-
content.files.sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0);
|
|
777
|
-
return validateTasksetDraftWorkspace({ ...content, contentHash: contentHash(content) });
|
|
778
|
-
}
|
|
779
|
-
function validateTasksetDraftWorkspace(input) {
|
|
780
|
-
assertBoundedTaskJson(input, MAX_TASKSET_DRAFT_WORKSPACE_BYTES);
|
|
781
|
-
const workspace = TasksetDraftWorkspaceSchema.parse(input);
|
|
782
|
-
const { contentHash: expectedHash, ...content } = workspace;
|
|
783
|
-
if (contentHash(content) !== expectedHash) throw new Error("Taskset draft workspace differs from its retained bytes.");
|
|
784
|
-
let previousPath = null;
|
|
785
|
-
const paths = /* @__PURE__ */ new Set();
|
|
786
|
-
for (const file of workspace.files) {
|
|
787
|
-
if (isManagedTasksetDraftFilePath(file.path)) throw new Error("Managed Taskset files are represented by the draft document.");
|
|
788
|
-
if (previousPath !== null && previousPath >= file.path) throw new Error("Taskset draft file paths must be unique and sorted.");
|
|
789
|
-
const segments = file.path.split("/");
|
|
790
|
-
for (let end = 1; end < segments.length; end++) if (paths.has(segments.slice(0, end).join("/"))) throw new Error("A Taskset draft file occupies another file's directory.");
|
|
791
|
-
decodeTasksetDraftWorkspaceFile(file);
|
|
792
|
-
paths.add(file.path);
|
|
793
|
-
previousPath = file.path;
|
|
794
|
-
}
|
|
795
|
-
return workspace;
|
|
796
|
-
}
|
|
797
|
-
function readTasksetDraftWorkspaceFile(input, path) {
|
|
798
|
-
const workspace = validateTasksetDraftWorkspace(input);
|
|
799
|
-
TasksetDraftFilePathSchema.parse(path);
|
|
800
|
-
const file = workspace.files.find((candidate) => candidate.path === path);
|
|
801
|
-
const manifest = isManagedTasksetDraftFilePath(path) ? renderTasksetDraftManifests(workspace.draft).get(path) : void 0;
|
|
802
|
-
if (!file && manifest === void 0) throw new Error("Taskset draft file does not exist.");
|
|
803
|
-
const result = createTasksetDraftFile(path, file ? decodeTasksetDraftWorkspaceFile(file) : new TextEncoder().encode(manifest));
|
|
804
|
-
return { draftRevision: workspace.draft.revision, file: { ...result, writable: result.writable && workspace.draft.status !== "published" } };
|
|
805
|
-
}
|
|
806
|
-
function listTasksetDraftWorkspaceFiles(input) {
|
|
807
|
-
const workspace = validateTasksetDraftWorkspace(input);
|
|
808
|
-
const files = workspace.files.map((file) => ({
|
|
809
|
-
path: file.path,
|
|
810
|
-
sizeBytes: file.sizeBytes,
|
|
811
|
-
writable: workspace.draft.status !== "published" && isWritableTasksetDraftFilePath(file.path)
|
|
812
|
-
}));
|
|
813
|
-
for (const [path, content] of renderTasksetDraftManifests(workspace.draft)) files.push({ path, sizeBytes: new TextEncoder().encode(content).byteLength, writable: false });
|
|
814
|
-
return { draftRevision: workspace.draft.revision, files: files.sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0) };
|
|
815
|
-
}
|
|
816
|
-
function saveTasksetDraftWorkspaceDocument(input) {
|
|
817
|
-
const workspace = editableWorkspace(input.workspace, input.expectedDraftRevision);
|
|
818
|
-
const draft = TasksetDraftSchema.parse(input.draft);
|
|
819
|
-
const current = workspace.draft;
|
|
820
|
-
if (draft.id !== current.id || draft.profileId !== current.profileId || contentHash(draft.modelScope) !== contentHash(current.modelScope)) throw new Error("Taskset draft Model ownership cannot change.");
|
|
821
|
-
if (draft.revision !== current.revision || draft.status !== current.status || draft.createdAt !== current.createdAt || contentHash(draft.publishedTasksetRef) !== contentHash(current.publishedTasksetRef)) throw new Error("Taskset draft lifecycle fields cannot change through form edits.");
|
|
822
|
-
const files = [...workspace.files];
|
|
823
|
-
for (const configured of configuredTasksetDraftFiles(draft)) if (!files.some((file) => file.path === configured.relativePath)) {
|
|
824
|
-
const bytes = new TextEncoder().encode(configured.source);
|
|
825
|
-
files.push({ path: configured.relativePath, contentHash: sha256(bytes), sizeBytes: bytes.byteLength, base64: base64(bytes) });
|
|
826
|
-
}
|
|
827
|
-
const aggregator = draft.metrics.customAggregator;
|
|
828
|
-
const module = aggregator ? files.find((file) => file.path === aggregator.module) : void 0;
|
|
829
|
-
return createTasksetDraftWorkspace({
|
|
830
|
-
schemaVersion: workspace.schemaVersion,
|
|
831
|
-
files,
|
|
832
|
-
draft: {
|
|
833
|
-
...draft,
|
|
834
|
-
revision: current.revision + 1,
|
|
835
|
-
status: "draft",
|
|
836
|
-
updatedAt: input.now,
|
|
837
|
-
...aggregator && module ? { metrics: { ...draft.metrics, customAggregator: { ...aggregator, contentHash: module.contentHash } } } : {}
|
|
838
|
-
}
|
|
839
|
-
});
|
|
840
|
-
}
|
|
841
|
-
function saveTasksetDraftWorkspaceFile(input) {
|
|
842
|
-
const mutation = TasksetDraftFileMutationSchema.parse(input.mutation);
|
|
843
|
-
const workspace = editableWorkspace(input.workspace, mutation.expectedDraftRevision);
|
|
844
|
-
if (workspace.draft.id !== mutation.draftId) throw new Error("Taskset draft file belongs to another draft.");
|
|
845
|
-
if (!isWritableTasksetDraftFilePath(mutation.path)) throw new Error("This file is managed by the Taskset forms or retained source history.");
|
|
846
|
-
const previous = workspace.files.find((file) => file.path === mutation.path);
|
|
847
|
-
if ((previous?.contentHash ?? null) !== mutation.expectedFileHash) throw new Error("Taskset draft file changed. Reload it before saving.");
|
|
848
|
-
if (!previous && mutation.content === null) throw new Error("Taskset draft file does not exist.");
|
|
849
|
-
const files = workspace.files.filter((file) => file.path !== mutation.path);
|
|
850
|
-
if (mutation.content !== null) {
|
|
851
|
-
const bytes = decodeTasksetDraftFileContent(mutation.content);
|
|
852
|
-
files.push({ path: mutation.path, contentHash: sha256(bytes), sizeBytes: bytes.byteLength, base64: base64(bytes) });
|
|
853
|
-
}
|
|
854
|
-
return createTasksetDraftWorkspace({
|
|
855
|
-
schemaVersion: workspace.schemaVersion,
|
|
856
|
-
files,
|
|
857
|
-
draft: { ...workspace.draft, revision: workspace.draft.revision + 1, status: "draft", updatedAt: input.now }
|
|
858
|
-
});
|
|
859
|
-
}
|
|
860
|
-
function editableWorkspace(input, expectedRevision) {
|
|
861
|
-
const workspace = validateTasksetDraftWorkspace(input);
|
|
862
|
-
if (workspace.draft.status === "published") throw new Error("Published Taskset drafts are immutable.");
|
|
863
|
-
if (workspace.draft.revision !== expectedRevision) throw new Error("Taskset draft changed. Reload before saving.");
|
|
864
|
-
return workspace;
|
|
865
|
-
}
|
|
866
|
-
function decodeTasksetDraftWorkspaceFile(input) {
|
|
867
|
-
const file = TasksetDraftWorkspaceFileSchema.parse(input);
|
|
868
|
-
let raw;
|
|
869
|
-
try {
|
|
870
|
-
raw = atob(file.base64);
|
|
871
|
-
} catch {
|
|
872
|
-
throw new Error("Taskset draft file contains invalid base64.");
|
|
873
|
-
}
|
|
874
|
-
if (btoa(raw) !== file.base64) throw new Error("Taskset draft file contains noncanonical base64.");
|
|
875
|
-
const bytes = Uint8Array.from(raw, (character) => character.charCodeAt(0));
|
|
876
|
-
if (bytes.byteLength !== file.sizeBytes || sha256(bytes) !== file.contentHash) throw new Error("Taskset draft file differs from its retained bytes.");
|
|
877
|
-
return bytes;
|
|
878
|
-
}
|
|
879
|
-
function base64(bytes) {
|
|
880
|
-
let raw = "";
|
|
881
|
-
for (let offset = 0; offset < bytes.length; offset += 8192) raw += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
|
|
882
|
-
return btoa(raw);
|
|
883
|
-
}
|
|
884
|
-
|
|
885
727
|
export {
|
|
886
|
-
TasksetPackageModelConfigurationSchema,
|
|
887
|
-
TasksetPackagePublicationSchema,
|
|
888
|
-
TasksetPackageReceiptSchema,
|
|
889
|
-
TasksetPackageReadbackSchema,
|
|
890
|
-
OpenPondTasksetPackageError,
|
|
891
|
-
OpenPondTasksetPackageClient,
|
|
892
728
|
TaskDataDraftSchema,
|
|
893
729
|
TasksetDraftStatusSchema,
|
|
894
730
|
TasksetDraftReviewPolicySchema,
|
|
895
731
|
TasksetDraftOutputContractSchema,
|
|
896
732
|
TasksetDraftSchema,
|
|
897
|
-
TasksetDraftPublishError,
|
|
898
|
-
createTasksetDraft,
|
|
899
|
-
draftPublishIssues,
|
|
900
|
-
tasksetDraftFromTaskset,
|
|
901
|
-
publishTasksetDraft,
|
|
902
733
|
MAX_TASKSET_DRAFT_EDIT_FILE_BYTES,
|
|
903
734
|
isWritableTasksetDraftFilePath,
|
|
904
735
|
isManagedTasksetDraftFilePath,
|
|
@@ -915,6 +746,11 @@ export {
|
|
|
915
746
|
listTasksetDraftWorkspaceFiles,
|
|
916
747
|
saveTasksetDraftWorkspaceDocument,
|
|
917
748
|
saveTasksetDraftWorkspaceFile,
|
|
918
|
-
decodeTasksetDraftWorkspaceFile
|
|
749
|
+
decodeTasksetDraftWorkspaceFile,
|
|
750
|
+
TasksetDraftPublishError,
|
|
751
|
+
createTasksetDraft,
|
|
752
|
+
draftPublishIssues,
|
|
753
|
+
tasksetDraftFromTaskset,
|
|
754
|
+
publishTasksetDraft
|
|
919
755
|
};
|
|
920
|
-
//# sourceMappingURL=chunk-
|
|
756
|
+
//# sourceMappingURL=chunk-32H2EJRA.js.map
|