openpond-sdk 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/TASKSETS.md +38 -0
- package/dist/model-batch-review.js +48 -0
- package/dist/model-batch-review.js.map +7 -0
- package/dist/taskset-packages.js +790 -71
- package/dist/taskset-packages.js.map +4 -4
- package/dist/types/packages/sdk/src/model-batch-review-contracts.d.ts +568 -0
- package/dist/types/packages/sdk/src/model-batch-review-contracts.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-batch-review-preparation.d.ts +351 -0
- package/dist/types/packages/sdk/src/model-batch-review-preparation.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-batch-review.d.ts +543 -0
- package/dist/types/packages/sdk/src/model-batch-review.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-projects.d.ts +2 -2
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts +684 -0
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts +725 -0
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-learning.d.ts +610 -0
- package/dist/types/packages/sdk/src/taskset-package-learning.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/taskset-packages.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-packages.d.ts.map +1 -1
- package/package.json +6 -2
package/dist/taskset-packages.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/taskset-package-contracts.ts
|
|
2
|
-
import { z as
|
|
2
|
+
import { z as z17 } from "zod";
|
|
3
3
|
|
|
4
4
|
// ../harness/dist/common.js
|
|
5
5
|
import { z } from "zod";
|
|
@@ -1859,6 +1859,63 @@ import { validateTaskValue } from "@openpond/evals/task-schema";
|
|
|
1859
1859
|
// src/model-projects.ts
|
|
1860
1860
|
import { z as z11 } from "zod";
|
|
1861
1861
|
import { assertBoundedTaskJson } from "@openpond/evals/task-schema";
|
|
1862
|
+
|
|
1863
|
+
// src/protocol.ts
|
|
1864
|
+
var OpenPondProtocolError = class extends Error {
|
|
1865
|
+
code;
|
|
1866
|
+
constructor(code, message) {
|
|
1867
|
+
super(message);
|
|
1868
|
+
this.name = "OpenPondProtocolError";
|
|
1869
|
+
this.code = code;
|
|
1870
|
+
}
|
|
1871
|
+
};
|
|
1872
|
+
function canonicalJson2(value) {
|
|
1873
|
+
return JSON.stringify(normalizeCanonicalJson(value));
|
|
1874
|
+
}
|
|
1875
|
+
function normalizeCanonicalJson(value) {
|
|
1876
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") {
|
|
1877
|
+
return value;
|
|
1878
|
+
}
|
|
1879
|
+
if (typeof value === "number") {
|
|
1880
|
+
if (!Number.isFinite(value)) {
|
|
1881
|
+
throw new OpenPondProtocolError(
|
|
1882
|
+
"non_json_value",
|
|
1883
|
+
"Canonical JSON cannot contain a non-finite number."
|
|
1884
|
+
);
|
|
1885
|
+
}
|
|
1886
|
+
return Object.is(value, -0) ? 0 : value;
|
|
1887
|
+
}
|
|
1888
|
+
if (Array.isArray(value)) {
|
|
1889
|
+
return value.map((entry) => normalizeCanonicalJson(entry));
|
|
1890
|
+
}
|
|
1891
|
+
if (typeof value === "object") {
|
|
1892
|
+
const prototype = Object.getPrototypeOf(value);
|
|
1893
|
+
if (prototype !== Object.prototype && prototype !== null) {
|
|
1894
|
+
throw new OpenPondProtocolError(
|
|
1895
|
+
"non_json_value",
|
|
1896
|
+
"Canonical JSON accepts only plain objects."
|
|
1897
|
+
);
|
|
1898
|
+
}
|
|
1899
|
+
const result = {};
|
|
1900
|
+
for (const key of Object.keys(value).sort()) {
|
|
1901
|
+
const entry = value[key];
|
|
1902
|
+
if (entry === void 0 || typeof entry === "function" || typeof entry === "symbol") {
|
|
1903
|
+
throw new OpenPondProtocolError(
|
|
1904
|
+
"non_json_value",
|
|
1905
|
+
`Canonical JSON cannot contain ${typeof entry} at ${key}.`
|
|
1906
|
+
);
|
|
1907
|
+
}
|
|
1908
|
+
result[key] = normalizeCanonicalJson(entry);
|
|
1909
|
+
}
|
|
1910
|
+
return result;
|
|
1911
|
+
}
|
|
1912
|
+
throw new OpenPondProtocolError(
|
|
1913
|
+
"non_json_value",
|
|
1914
|
+
`Canonical JSON cannot contain ${typeof value}.`
|
|
1915
|
+
);
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
// src/model-projects.ts
|
|
1862
1919
|
var IdSchema = z11.string().trim().min(1).max(500);
|
|
1863
1920
|
var HashSchema = z11.string().regex(/^[a-f0-9]{64}$/);
|
|
1864
1921
|
var TimestampSchema = z11.string().datetime({ offset: true });
|
|
@@ -2189,22 +2246,381 @@ function validateModelTasksetPackage(value) {
|
|
|
2189
2246
|
return result;
|
|
2190
2247
|
}
|
|
2191
2248
|
|
|
2249
|
+
// src/taskset-package-contracts.ts
|
|
2250
|
+
import { taskBatchPackageMetadata as taskBatchPackageMetadata2 } from "@openpond/evals/learning";
|
|
2251
|
+
|
|
2252
|
+
// src/taskset-package-learning.ts
|
|
2253
|
+
import { z as z16 } from "zod";
|
|
2254
|
+
import {
|
|
2255
|
+
TaskBatchSchema,
|
|
2256
|
+
TaskEvidenceSchema,
|
|
2257
|
+
TaskAdmissionDecisionSchema,
|
|
2258
|
+
LearningSourceSchema,
|
|
2259
|
+
LearningTextAssetSchema as LearningTextAssetSchema2,
|
|
2260
|
+
assertLearningContentHash as assertLearningContentHash2,
|
|
2261
|
+
compileTaskBatch,
|
|
2262
|
+
learningRef as learningRef3,
|
|
2263
|
+
sameLearningRef as sameLearningRef2,
|
|
2264
|
+
taskBatchPackageMetadata,
|
|
2265
|
+
verifyLearningTextAsset as verifyLearningTextAsset4
|
|
2266
|
+
} from "@openpond/evals/learning";
|
|
2267
|
+
|
|
2268
|
+
// src/training.ts
|
|
2269
|
+
import { z as z15 } from "zod";
|
|
2270
|
+
var IdSchema2 = z15.string().trim().min(1).max(500);
|
|
2271
|
+
var HashSchema3 = z15.string().regex(/^[a-f0-9]{64}$/);
|
|
2272
|
+
var TimestampSchema2 = z15.string().datetime({ offset: true });
|
|
2273
|
+
var TrainingInputArtifactUploadSchema = z15.object({
|
|
2274
|
+
schemaVersion: z15.literal("openpond.trainingInputArtifactUpload.v2"),
|
|
2275
|
+
kind: z15.enum(["portable_training_bundle", "reward_model_dataset"]),
|
|
2276
|
+
idempotencyKey: z15.string().trim().min(1).max(500),
|
|
2277
|
+
sourceManifest: ModelProjectImmutableRefSchema,
|
|
2278
|
+
payload: z15.unknown(),
|
|
2279
|
+
contentHash: HashSchema3
|
|
2280
|
+
}).strict();
|
|
2281
|
+
var TrainingInputArtifactSchema = z15.object({
|
|
2282
|
+
schemaVersion: z15.literal("openpond.trainingInputArtifact.v2"),
|
|
2283
|
+
kind: z15.enum(["portable_training_bundle", "reward_model_dataset"]),
|
|
2284
|
+
sourceManifest: ModelProjectImmutableRefSchema,
|
|
2285
|
+
artifactRef: z15.string().trim().min(1).max(2e3),
|
|
2286
|
+
contentHash: HashSchema3,
|
|
2287
|
+
sizeBytes: z15.number().int().positive(),
|
|
2288
|
+
createdAt: TimestampSchema2
|
|
2289
|
+
}).strict();
|
|
2290
|
+
var TrainingJobKindSchema = z15.enum([
|
|
2291
|
+
"reward_model_train",
|
|
2292
|
+
"policy_optimize"
|
|
2293
|
+
]);
|
|
2294
|
+
var TrainingJobStateSchema = z15.enum([
|
|
2295
|
+
"queued",
|
|
2296
|
+
"admitting",
|
|
2297
|
+
"provisioning",
|
|
2298
|
+
"running",
|
|
2299
|
+
"stopping",
|
|
2300
|
+
"cancelling",
|
|
2301
|
+
"succeeded",
|
|
2302
|
+
"failed",
|
|
2303
|
+
"cancelled"
|
|
2304
|
+
]);
|
|
2305
|
+
var TrainingCapabilityRequirementSchema = z15.object({
|
|
2306
|
+
id: IdSchema2,
|
|
2307
|
+
version: z15.string().trim().min(1).max(200).nullable().default(null),
|
|
2308
|
+
required: z15.boolean().default(true)
|
|
2309
|
+
}).strict();
|
|
2310
|
+
var TrainingJobSourceSchema = z15.object({
|
|
2311
|
+
modelProject: z15.object({
|
|
2312
|
+
id: IdSchema2,
|
|
2313
|
+
portableProjectId: IdSchema2,
|
|
2314
|
+
revision: z15.number().int().positive(),
|
|
2315
|
+
contentHash: HashSchema3
|
|
2316
|
+
}).strict(),
|
|
2317
|
+
harnessRunManifest: ModelProjectImmutableRefSchema,
|
|
2318
|
+
harnessRelease: ModelProjectImmutableRefSchema,
|
|
2319
|
+
taskset: ModelProjectVersionedRefSchema,
|
|
2320
|
+
tasksetRelease: ModelProjectImmutableRefSchema,
|
|
2321
|
+
dataset: ModelProjectImmutableRefSchema,
|
|
2322
|
+
evidenceSets: z15.array(ModelProjectImmutableRefSchema).max(1e4)
|
|
2323
|
+
}).strict();
|
|
2324
|
+
var RewardModelTrainingRequestSchema = z15.object({
|
|
2325
|
+
kind: z15.literal("reward_model_train"),
|
|
2326
|
+
baseModel: ModelProjectBaseModelSchema,
|
|
2327
|
+
preferenceDatasetRelease: ModelProjectImmutableRefSchema,
|
|
2328
|
+
processorRelease: ModelProjectImmutableRefSchema,
|
|
2329
|
+
recipe: z15.object({
|
|
2330
|
+
schemaVersion: z15.literal("openpond.rewardModelRecipe.v1"),
|
|
2331
|
+
method: z15.literal("reward_model"),
|
|
2332
|
+
parameterization: z15.literal("lora_with_scalar_head")
|
|
2333
|
+
}).catchall(z15.unknown())
|
|
2334
|
+
}).strict();
|
|
2335
|
+
var DeterministicRewardSourceSchema = z15.object({
|
|
2336
|
+
kind: z15.literal("deterministic"),
|
|
2337
|
+
grader: ModelProjectImmutableRefSchema,
|
|
2338
|
+
composer: ModelProjectImmutableRefSchema.nullable().default(null)
|
|
2339
|
+
}).strict();
|
|
2340
|
+
var LearnedRewardSourceSchema = z15.object({
|
|
2341
|
+
kind: z15.literal("learned_reward"),
|
|
2342
|
+
rewardModelVersion: ModelProjectImmutableRefSchema,
|
|
2343
|
+
qualificationReport: ModelProjectImmutableRefSchema.nullable().default(null),
|
|
2344
|
+
evaluationReferences: z15.array(ModelProjectImmutableRefSchema).max(1e3).default([]),
|
|
2345
|
+
scorerArtifact: z15.object({
|
|
2346
|
+
artifactRef: z15.string().trim().min(1).max(2e3),
|
|
2347
|
+
contentHash: HashSchema3,
|
|
2348
|
+
executionReceipt: ModelProjectImmutableRefSchema
|
|
2349
|
+
}).strict(),
|
|
2350
|
+
processorRelease: ModelProjectImmutableRefSchema,
|
|
2351
|
+
rewardComposerRelease: ModelProjectImmutableRefSchema
|
|
2352
|
+
}).strict();
|
|
2353
|
+
var PolicyOptimizationRequestSchema = z15.object({
|
|
2354
|
+
kind: z15.literal("policy_optimize"),
|
|
2355
|
+
baseModel: ModelProjectBaseModelSchema,
|
|
2356
|
+
recipe: ModelProjectRecipeDocumentSchema,
|
|
2357
|
+
rewardSource: z15.discriminatedUnion("kind", [
|
|
2358
|
+
DeterministicRewardSourceSchema,
|
|
2359
|
+
LearnedRewardSourceSchema
|
|
2360
|
+
]),
|
|
2361
|
+
resumeFrom: ModelProjectImmutableRefSchema.nullable().default(null)
|
|
2362
|
+
}).strict();
|
|
2363
|
+
var TrainingJobRequestSchema = z15.discriminatedUnion("kind", [
|
|
2364
|
+
RewardModelTrainingRequestSchema,
|
|
2365
|
+
PolicyOptimizationRequestSchema
|
|
2366
|
+
]);
|
|
2367
|
+
var TrainingJobApprovalSchema = z15.object({
|
|
2368
|
+
approvalHash: HashSchema3,
|
|
2369
|
+
approvedAt: TimestampSchema2,
|
|
2370
|
+
exportApproved: z15.boolean(),
|
|
2371
|
+
maximumSpendUsd: z15.number().nonnegative(),
|
|
2372
|
+
retentionDays: z15.number().int().nonnegative().nullable(),
|
|
2373
|
+
region: z15.string().trim().min(1).max(200).nullable().default(null)
|
|
2374
|
+
}).strict();
|
|
2375
|
+
var TrainingJobSubmissionSchema = z15.object({
|
|
2376
|
+
schemaVersion: z15.literal("openpond.trainingJobSubmission.v2"),
|
|
2377
|
+
idempotencyKey: z15.string().trim().min(1).max(500),
|
|
2378
|
+
name: z15.string().trim().min(1).max(200),
|
|
2379
|
+
source: TrainingJobSourceSchema,
|
|
2380
|
+
job: TrainingJobRequestSchema,
|
|
2381
|
+
requestedCapabilities: z15.array(TrainingCapabilityRequirementSchema).max(1e3),
|
|
2382
|
+
placementObjective: z15.enum(["fast", "balanced", "economical"]),
|
|
2383
|
+
budget: z15.object({
|
|
2384
|
+
maximumSpendUsd: z15.number().nonnegative(),
|
|
2385
|
+
maximumWallSeconds: z15.number().int().positive()
|
|
2386
|
+
}).strict(),
|
|
2387
|
+
approval: TrainingJobApprovalSchema,
|
|
2388
|
+
contentHash: HashSchema3
|
|
2389
|
+
}).strict().superRefine((submission, context) => {
|
|
2390
|
+
if (submission.approval.maximumSpendUsd !== submission.budget.maximumSpendUsd) {
|
|
2391
|
+
context.addIssue({
|
|
2392
|
+
code: "custom",
|
|
2393
|
+
path: ["approval", "maximumSpendUsd"],
|
|
2394
|
+
message: "Approved spend must equal the immutable Job budget."
|
|
2395
|
+
});
|
|
2396
|
+
}
|
|
2397
|
+
});
|
|
2398
|
+
var TrainingJobSchema = z15.object({
|
|
2399
|
+
schemaVersion: z15.literal("openpond.trainingJob.v2"),
|
|
2400
|
+
id: IdSchema2,
|
|
2401
|
+
teamId: IdSchema2,
|
|
2402
|
+
kind: TrainingJobKindSchema,
|
|
2403
|
+
modelProjectId: IdSchema2,
|
|
2404
|
+
portableProjectId: IdSchema2,
|
|
2405
|
+
sourceProjectRevision: z15.number().int().positive(),
|
|
2406
|
+
submissionHash: HashSchema3,
|
|
2407
|
+
state: TrainingJobStateSchema,
|
|
2408
|
+
phase: z15.string().trim().min(1).max(200),
|
|
2409
|
+
version: z15.number().int().nonnegative(),
|
|
2410
|
+
progress: z15.number().min(0).max(1),
|
|
2411
|
+
rolloutProgress: z15.object({
|
|
2412
|
+
groupsCompleted: z15.number().int().nonnegative(),
|
|
2413
|
+
groupsTarget: z15.number().int().nonnegative(),
|
|
2414
|
+
optimizerUpdatesApplied: z15.number().int().nonnegative(),
|
|
2415
|
+
optimizerUpdatesSkipped: z15.number().int().nonnegative()
|
|
2416
|
+
}).strict(),
|
|
2417
|
+
accruedSpendUsd: z15.number().nonnegative(),
|
|
2418
|
+
terminalReason: z15.string().trim().min(1).max(5e3).nullable(),
|
|
2419
|
+
createdAt: TimestampSchema2,
|
|
2420
|
+
updatedAt: TimestampSchema2,
|
|
2421
|
+
completedAt: TimestampSchema2.nullable()
|
|
2422
|
+
}).strict();
|
|
2423
|
+
var TrainingJobPageSchema = z15.object({
|
|
2424
|
+
schemaVersion: z15.literal("openpond.trainingJobPage.v2"),
|
|
2425
|
+
jobs: z15.array(TrainingJobSchema).max(1e3),
|
|
2426
|
+
nextCursor: z15.string().trim().min(1).max(2e3).nullable()
|
|
2427
|
+
}).strict();
|
|
2428
|
+
var TrainingJobEventSchema = z15.object({
|
|
2429
|
+
schemaVersion: z15.literal("openpond.trainingJobEvent.v2"),
|
|
2430
|
+
id: IdSchema2,
|
|
2431
|
+
jobId: IdSchema2,
|
|
2432
|
+
sequence: z15.number().int().nonnegative(),
|
|
2433
|
+
type: IdSchema2,
|
|
2434
|
+
phase: z15.string().trim().min(1).max(200),
|
|
2435
|
+
message: z15.string().trim().min(1).max(5e3).nullable(),
|
|
2436
|
+
data: z15.record(z15.string(), z15.unknown()).default({}),
|
|
2437
|
+
createdAt: TimestampSchema2
|
|
2438
|
+
}).strict();
|
|
2439
|
+
var TrainingJobLogSchema = z15.object({
|
|
2440
|
+
schemaVersion: z15.literal("openpond.trainingJobLog.v2"),
|
|
2441
|
+
jobId: IdSchema2,
|
|
2442
|
+
sequence: z15.number().int().nonnegative(),
|
|
2443
|
+
level: z15.enum(["debug", "info", "warning", "error"]),
|
|
2444
|
+
message: z15.string().max(2e4),
|
|
2445
|
+
createdAt: TimestampSchema2
|
|
2446
|
+
}).strict();
|
|
2447
|
+
var TrainingJobControlRequestSchema = z15.object({ expectedVersion: z15.number().int().nonnegative() }).strict();
|
|
2448
|
+
var TrainingJobOutputSchema = z15.object({
|
|
2449
|
+
schemaVersion: z15.literal("openpond.trainingJobOutput.v2"),
|
|
2450
|
+
id: IdSchema2,
|
|
2451
|
+
jobId: IdSchema2,
|
|
2452
|
+
kind: z15.enum([
|
|
2453
|
+
"checkpoint",
|
|
2454
|
+
"adapter",
|
|
2455
|
+
"scorer",
|
|
2456
|
+
"metrics",
|
|
2457
|
+
"evaluation",
|
|
2458
|
+
"trace",
|
|
2459
|
+
"receipt"
|
|
2460
|
+
]),
|
|
2461
|
+
artifactRef: z15.string().trim().min(1).max(2e3),
|
|
2462
|
+
contentHash: HashSchema3,
|
|
2463
|
+
sizeBytes: z15.number().int().nonnegative(),
|
|
2464
|
+
metadata: z15.record(z15.string(), z15.unknown()).default({}),
|
|
2465
|
+
createdAt: TimestampSchema2
|
|
2466
|
+
}).strict();
|
|
2467
|
+
var TrainingExecutionReceiptSchema = z15.object({
|
|
2468
|
+
schemaVersion: z15.literal("openpond.trainingExecutionReceipt.v2"),
|
|
2469
|
+
id: IdSchema2,
|
|
2470
|
+
teamId: IdSchema2,
|
|
2471
|
+
jobId: IdSchema2,
|
|
2472
|
+
submissionHash: HashSchema3,
|
|
2473
|
+
manifestHash: HashSchema3,
|
|
2474
|
+
recipeHash: HashSchema3,
|
|
2475
|
+
capabilityHash: HashSchema3,
|
|
2476
|
+
runtimeRelease: ModelProjectImmutableRefSchema,
|
|
2477
|
+
inputs: z15.array(ModelProjectImmutableRefSchema).max(1e4),
|
|
2478
|
+
outputs: z15.array(ModelProjectImmutableRefSchema).max(1e4),
|
|
2479
|
+
spendUsd: z15.number().nonnegative(),
|
|
2480
|
+
durationSeconds: z15.number().nonnegative(),
|
|
2481
|
+
cleanupComplete: z15.boolean(),
|
|
2482
|
+
issuer: IdSchema2,
|
|
2483
|
+
issuedAt: TimestampSchema2,
|
|
2484
|
+
signature: z15.string().trim().min(1).max(1e4).nullable()
|
|
2485
|
+
}).strict();
|
|
2486
|
+
var TrainingJobOutputsSchema = z15.object({
|
|
2487
|
+
schemaVersion: z15.literal("openpond.trainingJobOutputs.v2"),
|
|
2488
|
+
outputs: z15.array(TrainingJobOutputSchema).max(1e4),
|
|
2489
|
+
receipt: TrainingExecutionReceiptSchema.nullable()
|
|
2490
|
+
}).strict();
|
|
2491
|
+
var TrainingApiErrorSchema = z15.object({
|
|
2492
|
+
schemaVersion: z15.literal("openpond.trainingApiError.v2"),
|
|
2493
|
+
code: z15.string().trim().min(1).max(200),
|
|
2494
|
+
message: z15.string().trim().min(1).max(5e3),
|
|
2495
|
+
retryable: z15.boolean().default(false),
|
|
2496
|
+
requestId: z15.string().trim().min(1).max(500).nullable().default(null),
|
|
2497
|
+
details: z15.record(z15.string(), z15.unknown()).default({})
|
|
2498
|
+
}).strict();
|
|
2499
|
+
var TrainingCapabilitiesSchema = z15.object({
|
|
2500
|
+
schemaVersion: z15.literal("openpond.trainingCapabilities.v2"),
|
|
2501
|
+
capabilityHash: HashSchema3,
|
|
2502
|
+
jobKinds: z15.array(TrainingJobKindSchema),
|
|
2503
|
+
methods: z15.array(z15.string().trim().min(1).max(200)),
|
|
2504
|
+
placements: z15.array(
|
|
2505
|
+
z15.enum(["local", "remote", "colocated", "provider_native"])
|
|
2506
|
+
),
|
|
2507
|
+
controls: z15.object({
|
|
2508
|
+
cancel: z15.boolean(),
|
|
2509
|
+
stopAfterGroup: z15.boolean(),
|
|
2510
|
+
resumeFromCheckpoint: z15.boolean()
|
|
2511
|
+
}).strict(),
|
|
2512
|
+
limits: z15.object({
|
|
2513
|
+
maximumSpendUsd: z15.number().nonnegative(),
|
|
2514
|
+
maximumWallSeconds: z15.number().int().positive(),
|
|
2515
|
+
maximumArtifactBytes: z15.number().int().positive()
|
|
2516
|
+
}).strict(),
|
|
2517
|
+
checkedAt: TimestampSchema2,
|
|
2518
|
+
expiresAt: TimestampSchema2
|
|
2519
|
+
}).strict();
|
|
2520
|
+
|
|
2521
|
+
// src/taskset-package-learning.ts
|
|
2522
|
+
var TasksetPackageLearningResourcesSchema = z16.object({
|
|
2523
|
+
batch: TaskBatchSchema,
|
|
2524
|
+
evidence: z16.array(TaskEvidenceSchema).min(1).max(1e4),
|
|
2525
|
+
decisions: z16.array(TaskAdmissionDecisionSchema).min(1).max(1e4),
|
|
2526
|
+
sources: z16.array(LearningSourceSchema).min(1).max(1e4),
|
|
2527
|
+
assets: z16.array(LearningTextAssetSchema2).max(1e3)
|
|
2528
|
+
}).strict();
|
|
2529
|
+
function learningPackageContextFiles(resources) {
|
|
2530
|
+
return resources.evidence.filter((evidence) => evidence.submission.evaluatorContext !== null).map((evidence) => {
|
|
2531
|
+
const text = canonicalJson2(evidence.submission.evaluatorContext);
|
|
2532
|
+
const bytes = new TextEncoder().encode(text);
|
|
2533
|
+
return {
|
|
2534
|
+
asset: {
|
|
2535
|
+
id: `task-evidence:${evidence.contentHash}`,
|
|
2536
|
+
path: `learning/context/${evidence.contentHash}.json`,
|
|
2537
|
+
contentHash: sha256(bytes),
|
|
2538
|
+
sizeBytes: bytes.byteLength,
|
|
2539
|
+
mediaType: "application/json",
|
|
2540
|
+
visibility: "host_private"
|
|
2541
|
+
},
|
|
2542
|
+
base64: btoa(Array.from(bytes, (byte) => String.fromCharCode(byte)).join(""))
|
|
2543
|
+
};
|
|
2544
|
+
});
|
|
2545
|
+
}
|
|
2546
|
+
function validateTasksetLearningResources(taskset, input) {
|
|
2547
|
+
const resources = TasksetPackageLearningResourcesSchema.parse(input);
|
|
2548
|
+
const metadata = taskBatchPackageMetadata(taskset);
|
|
2549
|
+
if (!sameLearningRef2(learningRef3(resources.batch), metadata.batch)) throw new Error("Taskset learning batch differs from its admission metadata.");
|
|
2550
|
+
for (const collection of [resources.evidence, resources.decisions, resources.sources, resources.assets]) {
|
|
2551
|
+
const identities = /* @__PURE__ */ new Set();
|
|
2552
|
+
for (const resource of collection) {
|
|
2553
|
+
assertLearningContentHash2(resource);
|
|
2554
|
+
const key = `${resource.id}:${resource.revision}`;
|
|
2555
|
+
if (identities.has(key)) throw new Error("Taskset learning resource identities must be unique.");
|
|
2556
|
+
identities.add(key);
|
|
2557
|
+
}
|
|
2558
|
+
}
|
|
2559
|
+
if (resources.evidence.length !== resources.batch.examples.length || resources.decisions.length !== resources.batch.examples.length) throw new Error("Taskset learning snapshots must match the admitted batch inventory.");
|
|
2560
|
+
for (const evidence of resources.evidence) {
|
|
2561
|
+
const source = resources.sources.find((source2) => sameLearningRef2(learningRef3(source2), evidence.source));
|
|
2562
|
+
if (!source || !sameLearningRef2(source.taskDefinition, learningRef3(metadata.definition))) throw new Error("Taskset evidence source differs from its task definition.");
|
|
2563
|
+
}
|
|
2564
|
+
if (resources.sources.some((source) => !resources.evidence.some((evidence) => sameLearningRef2(evidence.source, learningRef3(source))))) throw new Error("Taskset package includes an unrelated learning source.");
|
|
2565
|
+
const rewardAssets = metadata.rewards.flatMap((reward) => [
|
|
2566
|
+
...reward.assets,
|
|
2567
|
+
...reward.implementation.kind === "custom_verifier" ? [reward.implementation.verifierRef] : reward.implementation.kind === "model_judge" || reward.implementation.kind === "human" ? [reward.implementation.rubricRef] : [],
|
|
2568
|
+
..."inputContract" in reward.implementation ? [reward.implementation.inputContract] : []
|
|
2569
|
+
]);
|
|
2570
|
+
for (const ref of rewardAssets) {
|
|
2571
|
+
const asset = resources.assets.find((asset2) => asset2.id === ref.id);
|
|
2572
|
+
if (!asset) throw new Error("Taskset learning Reward asset is missing.");
|
|
2573
|
+
verifyLearningTextAsset4(asset, ref);
|
|
2574
|
+
}
|
|
2575
|
+
if (resources.assets.some((asset) => !rewardAssets.some((ref) => ref.id === asset.id))) throw new Error("Taskset package includes an unrelated learning asset.");
|
|
2576
|
+
const compiled = compileTaskBatch({
|
|
2577
|
+
batch: resources.batch,
|
|
2578
|
+
definition: metadata.definition,
|
|
2579
|
+
binding: metadata.binding,
|
|
2580
|
+
rewards: metadata.rewards,
|
|
2581
|
+
evidence: resources.evidence,
|
|
2582
|
+
decisions: resources.decisions
|
|
2583
|
+
});
|
|
2584
|
+
const compiledMetadata = taskBatchPackageMetadata(compiled);
|
|
2585
|
+
if (contentHash(compiledMetadata) !== contentHash(metadata)) throw new Error("Taskset admission metadata differs from its reviewed snapshots.");
|
|
2586
|
+
const execution = (release) => ({ policy: release.policy, environment: release.environment, tools: release.tools });
|
|
2587
|
+
if (contentHash(execution(compiled)) !== contentHash(execution(taskset))) throw new Error("Taskset execution differs from its reviewed task definition.");
|
|
2588
|
+
const taskContent = (task) => ({
|
|
2589
|
+
id: task.id,
|
|
2590
|
+
clusterKey: task.clusterKey,
|
|
2591
|
+
split: task.split,
|
|
2592
|
+
input: task.input,
|
|
2593
|
+
expectedOutput: task.expectedOutput,
|
|
2594
|
+
policyVisibleContext: task.policyVisibleContext,
|
|
2595
|
+
privilegedContextRef: task.privilegedContextRef,
|
|
2596
|
+
artifactRefs: task.artifactRefs,
|
|
2597
|
+
requiredOutputs: task.requiredOutputs ?? [],
|
|
2598
|
+
tags: task.tags
|
|
2599
|
+
});
|
|
2600
|
+
if (contentHash(compiled.tasks.map(taskContent)) !== contentHash(taskset.tasks.map(taskContent))) throw new Error("Taskset rows differ from their reviewed evidence.");
|
|
2601
|
+
return resources;
|
|
2602
|
+
}
|
|
2603
|
+
|
|
2192
2604
|
// src/taskset-package-contracts.ts
|
|
2193
2605
|
var MAX_TASKSET_PACKAGE_BYTES = 64 * 1024 * 1024;
|
|
2194
|
-
var TasksetPackageFileSchema =
|
|
2606
|
+
var TasksetPackageFileSchema = z17.object({
|
|
2195
2607
|
asset: ImmutableAssetRefSchema,
|
|
2196
|
-
base64:
|
|
2608
|
+
base64: z17.string().max(Math.ceil(MAX_TASKSET_PACKAGE_BYTES / 3) * 4)
|
|
2197
2609
|
}).strict();
|
|
2198
2610
|
var BoundModelResourcesSchema = ModelTasksetPackageSchema.omit({ taskset: true, executionResources: true });
|
|
2199
|
-
var TasksetPackageContentSchema =
|
|
2200
|
-
schemaVersion:
|
|
2611
|
+
var TasksetPackageContentSchema = z17.object({
|
|
2612
|
+
schemaVersion: z17.literal("openpond.tasksetPackage.v1"),
|
|
2201
2613
|
taskset: TasksetReleaseSchema2,
|
|
2202
2614
|
environment: EnvironmentReleaseSchema3,
|
|
2203
2615
|
verifierSet: VerifierSetReleaseSchema3,
|
|
2204
|
-
files:
|
|
2205
|
-
modelResources: BoundModelResourcesSchema.optional()
|
|
2616
|
+
files: z17.array(TasksetPackageFileSchema).max(1e4),
|
|
2617
|
+
modelResources: BoundModelResourcesSchema.optional(),
|
|
2618
|
+
learningResources: TasksetPackageLearningResourcesSchema.optional()
|
|
2206
2619
|
}).strict();
|
|
2207
|
-
var TasksetPackageSchema = TasksetPackageContentSchema.extend({ contentHash:
|
|
2620
|
+
var TasksetPackageSchema = TasksetPackageContentSchema.extend({ contentHash: z17.string().regex(/^[a-f0-9]{64}$/) }).strict();
|
|
2621
|
+
function tasksetPackageRewardBinding(value) {
|
|
2622
|
+
return value.learningResources ? taskBatchPackageMetadata2(value.taskset).binding : value.modelResources?.rewardBinding ?? null;
|
|
2623
|
+
}
|
|
2208
2624
|
function decodeTasksetPackageFile(value) {
|
|
2209
2625
|
const file = TasksetPackageFileSchema.parse(value);
|
|
2210
2626
|
const raw = atob(file.base64);
|
|
@@ -2250,11 +2666,18 @@ function validateTasksetPackage(value) {
|
|
|
2250
2666
|
}
|
|
2251
2667
|
for (const declarations of [taskset.metadata.environmentResources, environment.metadata.resources]) {
|
|
2252
2668
|
if (declarations === void 0) continue;
|
|
2253
|
-
for (const resource of
|
|
2669
|
+
for (const resource of z17.array(z17.object({ path: z17.string().min(1) }).passthrough()).parse(declarations)) {
|
|
2254
2670
|
if (!result.files.some((file) => file.asset.path === resource.path)) throw new Error(`Taskset environment resource is missing: ${resource.path}.`);
|
|
2255
2671
|
}
|
|
2256
2672
|
}
|
|
2257
|
-
if (result.
|
|
2673
|
+
if (result.learningResources) {
|
|
2674
|
+
if (result.modelResources) throw new Error("Taskset package cannot declare competing authoring graphs.");
|
|
2675
|
+
const learning = validateTasksetLearningResources(taskset, result.learningResources);
|
|
2676
|
+
for (const expected of learningPackageContextFiles(learning)) {
|
|
2677
|
+
if (!same2(files.get(expected.asset.id), expected)) throw new Error("Taskset private context differs from its reviewed evidence.");
|
|
2678
|
+
}
|
|
2679
|
+
for (const asset of learning.assets) requireAsset(asset.asset);
|
|
2680
|
+
} else if (result.modelResources) {
|
|
2258
2681
|
const model = validateModelTasksetPackage({ ...result.modelResources, taskset, executionResources: { environment, verifierSet } });
|
|
2259
2682
|
for (const asset of model.assets) requireAsset(asset.asset);
|
|
2260
2683
|
} else if (["starter", "rewardBinding", "rewardExecution", "modelTasksetDerivation", "learning"].some((key) => taskset.metadata[key] !== void 0)) {
|
|
@@ -2272,45 +2695,45 @@ function same2(left, right) {
|
|
|
2272
2695
|
}
|
|
2273
2696
|
|
|
2274
2697
|
// src/taskset-package-client.ts
|
|
2275
|
-
import { z as
|
|
2698
|
+
import { z as z19 } from "zod";
|
|
2276
2699
|
|
|
2277
2700
|
// src/taskset-catalog.ts
|
|
2278
|
-
import { z as
|
|
2279
|
-
var
|
|
2280
|
-
var
|
|
2281
|
-
var TasksetCatalogReleaseRefSchema =
|
|
2282
|
-
var HostedTasksetSummarySchema =
|
|
2283
|
-
schemaVersion:
|
|
2284
|
-
id:
|
|
2285
|
-
teamId:
|
|
2701
|
+
import { z as z18 } from "zod";
|
|
2702
|
+
var IdSchema3 = z18.string().trim().min(1).max(500);
|
|
2703
|
+
var HashSchema4 = z18.string().regex(/^[a-f0-9]{64}$/);
|
|
2704
|
+
var TasksetCatalogReleaseRefSchema = z18.object({ id: IdSchema3, revision: z18.number().int().positive(), contentHash: HashSchema4 }).strict();
|
|
2705
|
+
var HostedTasksetSummarySchema = z18.object({
|
|
2706
|
+
schemaVersion: z18.literal("openpond.hostedTasksetSummary.v1"),
|
|
2707
|
+
id: IdSchema3,
|
|
2708
|
+
teamId: IdSchema3,
|
|
2286
2709
|
release: TasksetCatalogReleaseRefSchema,
|
|
2287
|
-
name:
|
|
2288
|
-
description:
|
|
2289
|
-
taskCount:
|
|
2290
|
-
buildIntent:
|
|
2291
|
-
methodHint:
|
|
2292
|
-
packageBytes:
|
|
2293
|
-
storedBytes:
|
|
2294
|
-
createdAt:
|
|
2295
|
-
}).strict();
|
|
2296
|
-
var TasksetCatalogQuerySchema =
|
|
2297
|
-
afterId:
|
|
2298
|
-
limit:
|
|
2299
|
-
modelProjectId:
|
|
2300
|
-
}).strict();
|
|
2301
|
-
var TasksetCatalogPageSchema =
|
|
2302
|
-
items:
|
|
2303
|
-
nextCursor:
|
|
2304
|
-
}).strict();
|
|
2305
|
-
var TasksetCatalogErrorSchema =
|
|
2306
|
-
code:
|
|
2307
|
-
message:
|
|
2710
|
+
name: z18.string().min(1).max(500),
|
|
2711
|
+
description: z18.string().max(2e4),
|
|
2712
|
+
taskCount: z18.number().int().nonnegative(),
|
|
2713
|
+
buildIntent: z18.string().min(1).max(100),
|
|
2714
|
+
methodHint: z18.string().min(1).max(100).nullable(),
|
|
2715
|
+
packageBytes: z18.number().int().nonnegative().nullable(),
|
|
2716
|
+
storedBytes: z18.number().int().nonnegative().nullable(),
|
|
2717
|
+
createdAt: z18.string().datetime({ offset: true })
|
|
2718
|
+
}).strict();
|
|
2719
|
+
var TasksetCatalogQuerySchema = z18.object({
|
|
2720
|
+
afterId: IdSchema3.optional(),
|
|
2721
|
+
limit: z18.number().int().min(1).max(100).default(30),
|
|
2722
|
+
modelProjectId: IdSchema3.optional()
|
|
2723
|
+
}).strict();
|
|
2724
|
+
var TasksetCatalogPageSchema = z18.object({
|
|
2725
|
+
items: z18.array(HostedTasksetSummarySchema).max(100),
|
|
2726
|
+
nextCursor: IdSchema3.nullable()
|
|
2727
|
+
}).strict();
|
|
2728
|
+
var TasksetCatalogErrorSchema = z18.object({
|
|
2729
|
+
code: z18.string().min(1),
|
|
2730
|
+
message: z18.string().min(1)
|
|
2308
2731
|
}).strict();
|
|
2309
2732
|
|
|
2310
2733
|
// src/taskset-package-client.ts
|
|
2311
|
-
import { learningRef as
|
|
2312
|
-
var
|
|
2313
|
-
var
|
|
2734
|
+
import { learningRef as learningRef4 } from "@openpond/evals/learning";
|
|
2735
|
+
var IdSchema4 = z19.string().trim().min(1).max(500);
|
|
2736
|
+
var HashSchema5 = z19.string().regex(/^[a-f0-9]{64}$/);
|
|
2314
2737
|
var TasksetPackageModelConfigurationSchema = HostedModelProjectSyncSchema.pick({
|
|
2315
2738
|
portableProjectId: true,
|
|
2316
2739
|
name: true,
|
|
@@ -2320,35 +2743,35 @@ var TasksetPackageModelConfigurationSchema = HostedModelProjectSyncSchema.pick({
|
|
|
2320
2743
|
sourceRevision: true,
|
|
2321
2744
|
sourceUpdatedAt: true
|
|
2322
2745
|
}).extend({ trainingSetup: HostedModelProjectTrainingSetupSchema.omit({ tasksetRef: true, rewardBindingRef: true, tasksetRelease: true, recipe: true }) }).strict();
|
|
2323
|
-
var TasksetPackagePublicationSchema =
|
|
2324
|
-
schemaVersion:
|
|
2325
|
-
operationId:
|
|
2326
|
-
modelProjectId:
|
|
2327
|
-
expectedProjectEtag:
|
|
2328
|
-
name:
|
|
2329
|
-
description:
|
|
2330
|
-
buildIntent:
|
|
2331
|
-
methodHint:
|
|
2746
|
+
var TasksetPackagePublicationSchema = z19.object({
|
|
2747
|
+
schemaVersion: z19.literal("openpond.tasksetPackagePublication.v1"),
|
|
2748
|
+
operationId: IdSchema4,
|
|
2749
|
+
modelProjectId: IdSchema4,
|
|
2750
|
+
expectedProjectEtag: HashSchema5.nullable(),
|
|
2751
|
+
name: z19.string().trim().min(1).max(200),
|
|
2752
|
+
description: z19.string().max(5e3),
|
|
2753
|
+
buildIntent: z19.enum(["demonstrations", "preferences", "verifiable_reward", "rubric", "discovery"]),
|
|
2754
|
+
methodHint: z19.enum(["sft", "dpo", "grpo", "ppo"]).nullable(),
|
|
2332
2755
|
package: TasksetPackageSchema,
|
|
2333
2756
|
modelConfiguration: TasksetPackageModelConfigurationSchema.optional(),
|
|
2334
|
-
selection:
|
|
2757
|
+
selection: z19.enum(["select", "attach"]).optional()
|
|
2335
2758
|
}).strict().refine((value) => value.selection !== "attach" || value.modelConfiguration === void 0, "Attachment-only publication cannot replace Model configuration.");
|
|
2336
|
-
var TasksetPackageReceiptSchema =
|
|
2337
|
-
schemaVersion:
|
|
2338
|
-
teamId:
|
|
2339
|
-
modelProjectId:
|
|
2340
|
-
operationId:
|
|
2759
|
+
var TasksetPackageReceiptSchema = z19.object({
|
|
2760
|
+
schemaVersion: z19.literal("openpond.tasksetPackageReceipt.v1"),
|
|
2761
|
+
teamId: IdSchema4,
|
|
2762
|
+
modelProjectId: IdSchema4,
|
|
2763
|
+
operationId: IdSchema4,
|
|
2341
2764
|
taskset: TasksetCatalogReleaseRefSchema,
|
|
2342
|
-
packageHash:
|
|
2343
|
-
hostedTasksetId:
|
|
2344
|
-
projectEtag:
|
|
2765
|
+
packageHash: HashSchema5,
|
|
2766
|
+
hostedTasksetId: IdSchema4,
|
|
2767
|
+
projectEtag: HashSchema5,
|
|
2345
2768
|
project: HostedModelProjectSummarySchema.optional(),
|
|
2346
|
-
selection:
|
|
2769
|
+
selection: z19.enum(["select", "attach"]).optional()
|
|
2347
2770
|
}).strict();
|
|
2348
|
-
var TasksetPackageReadbackSchema =
|
|
2349
|
-
schemaVersion:
|
|
2350
|
-
teamId:
|
|
2351
|
-
modelProjectId:
|
|
2771
|
+
var TasksetPackageReadbackSchema = z19.object({
|
|
2772
|
+
schemaVersion: z19.literal("openpond.tasksetPackageReadback.v1"),
|
|
2773
|
+
teamId: IdSchema4,
|
|
2774
|
+
modelProjectId: IdSchema4,
|
|
2352
2775
|
package: TasksetPackageSchema
|
|
2353
2776
|
}).strict();
|
|
2354
2777
|
var OpenPondTasksetPackageError = class extends Error {
|
|
@@ -2381,10 +2804,11 @@ var OpenPondTasksetPackageClient = class {
|
|
|
2381
2804
|
if (request.modelConfiguration) {
|
|
2382
2805
|
const expected = request.modelConfiguration;
|
|
2383
2806
|
const project = receipt.project;
|
|
2807
|
+
const binding = tasksetPackageRewardBinding(request.package);
|
|
2384
2808
|
const setup = HostedModelProjectTrainingSetupSchema.parse({
|
|
2385
2809
|
...expected.trainingSetup,
|
|
2386
|
-
tasksetRef:
|
|
2387
|
-
rewardBindingRef:
|
|
2810
|
+
tasksetRef: learningRef4(release),
|
|
2811
|
+
rewardBindingRef: binding ? learningRef4(binding) : null,
|
|
2388
2812
|
tasksetRelease: null,
|
|
2389
2813
|
recipe: null
|
|
2390
2814
|
});
|
|
@@ -2395,9 +2819,9 @@ var OpenPondTasksetPackageClient = class {
|
|
|
2395
2819
|
return receipt;
|
|
2396
2820
|
}
|
|
2397
2821
|
async get(modelProjectId, reference, options = {}) {
|
|
2398
|
-
const project =
|
|
2822
|
+
const project = IdSchema4.parse(modelProjectId);
|
|
2399
2823
|
const ref = TasksetCatalogReleaseRefSchema.parse(reference);
|
|
2400
|
-
const expectedPackageHash = options.expectedPackageHash === void 0 ? void 0 :
|
|
2824
|
+
const expectedPackageHash = options.expectedPackageHash === void 0 ? void 0 : HashSchema5.parse(options.expectedPackageHash);
|
|
2401
2825
|
const response = TasksetPackageReadbackSchema.parse(await this.#request(`/${encodeURIComponent(project)}/${encodeURIComponent(ref.id)}/${ref.revision}/${ref.contentHash}`, "GET", void 0, options.signal));
|
|
2402
2826
|
const taskset = response.package.taskset;
|
|
2403
2827
|
if (response.teamId !== this.#options.teamId || response.modelProjectId !== project || taskset.id !== ref.id || taskset.revision !== ref.revision || taskset.contentHash !== ref.contentHash) throw new OpenPondTasksetPackageError(502, "package_readback_mismatch", "Taskset package did not match the requested release and owner.");
|
|
@@ -2434,25 +2858,320 @@ var OpenPondTasksetPackageClient = class {
|
|
|
2434
2858
|
}
|
|
2435
2859
|
const payload = JSON.parse(text);
|
|
2436
2860
|
if (!response.ok) {
|
|
2437
|
-
const error =
|
|
2861
|
+
const error = z19.object({ code: z19.string(), message: z19.string() }).safeParse(payload);
|
|
2438
2862
|
throw new OpenPondTasksetPackageError(response.status, error.success ? error.data.code : "package_request_failed", error.success ? error.data.message : `Taskset package request failed (${response.status}).`);
|
|
2439
2863
|
}
|
|
2440
2864
|
return payload;
|
|
2441
2865
|
}
|
|
2442
2866
|
};
|
|
2867
|
+
|
|
2868
|
+
// src/model-batch-review.ts
|
|
2869
|
+
import {
|
|
2870
|
+
LearningDomainError as LearningDomainError2,
|
|
2871
|
+
learningRef as learningRef6,
|
|
2872
|
+
sameLearningRef as sameLearningRef4,
|
|
2873
|
+
requireLearningRelease,
|
|
2874
|
+
requireLearningResource,
|
|
2875
|
+
taskBatchPackageMetadata as taskBatchPackageMetadata4
|
|
2876
|
+
} from "@openpond/evals/learning";
|
|
2877
|
+
|
|
2878
|
+
// src/model-batch-review-contracts.ts
|
|
2879
|
+
import { z as z20 } from "zod";
|
|
2880
|
+
import { LearningJsonObjectSchema, LearningRevisionRefSchema, LearningSourceSchema as LearningSourceSchema2, TaskEvidenceSchema as TaskEvidenceSchema2, TaskFeedbackSchema, TaskDefinitionSchema as TaskDefinitionSchema2, TaskAdmissionDecisionSchema as TaskAdmissionDecisionSchema2 } from "@openpond/evals/learning";
|
|
2881
|
+
import { RewardBindingSchema as RewardBindingSchema2 } from "@openpond/evals/rewards";
|
|
2882
|
+
var ReleaseIdSchema2 = LearningRevisionRefSchema.shape.id;
|
|
2883
|
+
var ModelBatchReviewInspectionSchema = z20.object({
|
|
2884
|
+
schemaVersion: z20.literal("openpond.modelBatchReviewInspection.v1"),
|
|
2885
|
+
packageHash: z20.string().regex(/^[a-f0-9]{64}$/),
|
|
2886
|
+
definition: TaskDefinitionSchema2,
|
|
2887
|
+
binding: RewardBindingSchema2,
|
|
2888
|
+
evidence: z20.array(TaskEvidenceSchema2).min(1).max(1e4),
|
|
2889
|
+
decisions: z20.array(TaskAdmissionDecisionSchema2).min(1).max(1e4)
|
|
2890
|
+
}).strict();
|
|
2891
|
+
var ModelBatchReviewRequestSchema = z20.object({
|
|
2892
|
+
schemaVersion: z20.literal("openpond.modelBatchReviewRequest.v1"),
|
|
2893
|
+
operationId: ReleaseIdSchema2,
|
|
2894
|
+
modelId: ReleaseIdSchema2,
|
|
2895
|
+
expectedModelRevision: z20.number().int().positive(),
|
|
2896
|
+
tasksetRef: LearningRevisionRefSchema,
|
|
2897
|
+
rewardBindingRef: LearningRevisionRefSchema.nullable(),
|
|
2898
|
+
definition: z20.object({
|
|
2899
|
+
name: z20.string().trim().min(1).max(500).optional(),
|
|
2900
|
+
instructions: z20.string().trim().min(1).max(2e4).optional(),
|
|
2901
|
+
inputSchema: LearningJsonObjectSchema.optional(),
|
|
2902
|
+
outputSchema: LearningJsonObjectSchema.optional()
|
|
2903
|
+
}).strict(),
|
|
2904
|
+
examples: z20.array(z20.object({
|
|
2905
|
+
evidence: LearningRevisionRefSchema,
|
|
2906
|
+
input: LearningJsonObjectSchema.optional(),
|
|
2907
|
+
expected: LearningJsonObjectSchema.nullable().optional(),
|
|
2908
|
+
evaluatorContext: LearningJsonObjectSchema.nullable().optional(),
|
|
2909
|
+
proposedTarget: LearningJsonObjectSchema.nullable().optional()
|
|
2910
|
+
}).strict()).max(1e4)
|
|
2911
|
+
}).strict();
|
|
2912
|
+
var ModelBatchReviewReceiptSchema = z20.object({
|
|
2913
|
+
schemaVersion: z20.literal("openpond.modelBatchReviewReceipt.v1"),
|
|
2914
|
+
operationId: ReleaseIdSchema2,
|
|
2915
|
+
modelId: ReleaseIdSchema2,
|
|
2916
|
+
source: LearningSourceSchema2,
|
|
2917
|
+
evidence: z20.array(TaskEvidenceSchema2).min(1).max(1e4),
|
|
2918
|
+
proposals: z20.array(TaskFeedbackSchema).max(1e4)
|
|
2919
|
+
}).strict();
|
|
2920
|
+
|
|
2921
|
+
// src/model-batch-review-preparation.ts
|
|
2922
|
+
import {
|
|
2923
|
+
LearningDomainError,
|
|
2924
|
+
LearningSourceSchema as LearningSourceSchema3,
|
|
2925
|
+
TaskDefinitionSchema as TaskDefinitionSchema3,
|
|
2926
|
+
TaskEvidenceSchema as TaskEvidenceSchema3,
|
|
2927
|
+
TaskFeedbackSchema as TaskFeedbackSchema2,
|
|
2928
|
+
assertLearningContentHash as assertLearningContentHash3,
|
|
2929
|
+
learningEvidenceId,
|
|
2930
|
+
learningRef as learningRef5,
|
|
2931
|
+
sameLearningRef as sameLearningRef3,
|
|
2932
|
+
sealLearningContent as sealLearningContent4,
|
|
2933
|
+
taskBatchPackageMetadata as taskBatchPackageMetadata3,
|
|
2934
|
+
verifyLearningTextAsset as verifyLearningTextAsset5
|
|
2935
|
+
} from "@openpond/evals/learning";
|
|
2936
|
+
import { compileBoundGraders as compileBoundGraders2, createRewardBinding, createRewardRelease, resolveBoundRewards } from "@openpond/evals/rewards";
|
|
2937
|
+
function prepareModelBatchReview(input) {
|
|
2938
|
+
const { request, package: value } = input;
|
|
2939
|
+
const learning = value.learningResources;
|
|
2940
|
+
if (!learning) throw new LearningDomainError("model_review_batch_required", 422);
|
|
2941
|
+
const metadata = taskBatchPackageMetadata3(value.taskset);
|
|
2942
|
+
const id = `model-review-${contentHash([input.scope, request.modelId, request.operationId])}`;
|
|
2943
|
+
const edits = new Map(request.examples.map((edit) => [contentHash(edit.evidence), edit]));
|
|
2944
|
+
const parentRefs = new Set(learning.evidence.map((evidence2) => contentHash(learningRef5(evidence2))));
|
|
2945
|
+
const decisions = new Map(learning.decisions.map((decision) => [contentHash(decision.evidence), decision]));
|
|
2946
|
+
if (edits.size !== request.examples.length || request.examples.some((edit) => !parentRefs.has(contentHash(edit.evidence)))) {
|
|
2947
|
+
throw new LearningDomainError("model_review_evidence_mismatch", 409);
|
|
2948
|
+
}
|
|
2949
|
+
for (const resource of [input.binding, ...input.rewards, ...input.assets]) assertLearningContentHash3(resource);
|
|
2950
|
+
resolveBoundRewards(input.binding, input.rewards);
|
|
2951
|
+
const rewardByHash = /* @__PURE__ */ new Map();
|
|
2952
|
+
for (const source2 of input.binding.sources) {
|
|
2953
|
+
const original = input.rewards.find((reward) => sameLearningRef3(learningRef5(reward), source2.reward));
|
|
2954
|
+
const { contentHash: _hash, ...content } = original;
|
|
2955
|
+
rewardByHash.set(original.contentHash, createRewardRelease({ ...content, id: `${id}-reward-${original.contentHash}`, revision: 1 }));
|
|
2956
|
+
}
|
|
2957
|
+
const rewards = [...rewardByHash.values()];
|
|
2958
|
+
const { contentHash: _bindingHash, recipeRef: _recipe, ...bindingContent } = input.binding;
|
|
2959
|
+
const binding = createRewardBinding({
|
|
2960
|
+
...bindingContent,
|
|
2961
|
+
id: `${id}-binding`,
|
|
2962
|
+
revision: 1,
|
|
2963
|
+
sources: input.binding.sources.map((source2) => ({ ...source2, reward: learningRef5(rewardByHash.get(source2.reward.contentHash)) }))
|
|
2964
|
+
}, rewards);
|
|
2965
|
+
const assetRefs = rewards.flatMap((reward) => [
|
|
2966
|
+
...reward.assets,
|
|
2967
|
+
..."verifierRef" in reward.implementation ? [reward.implementation.verifierRef] : [],
|
|
2968
|
+
..."rubricRef" in reward.implementation ? [reward.implementation.rubricRef] : [],
|
|
2969
|
+
..."inputContract" in reward.implementation ? [reward.implementation.inputContract] : []
|
|
2970
|
+
]);
|
|
2971
|
+
const assets = [...new Map(assetRefs.map((ref) => {
|
|
2972
|
+
const asset = input.assets.find((asset2) => asset2.id === ref.id);
|
|
2973
|
+
if (!asset) throw new LearningDomainError("model_review_reward_asset_missing", 422);
|
|
2974
|
+
verifyLearningTextAsset5(asset, ref);
|
|
2975
|
+
if (ref.visibility === "policy") throw new LearningDomainError("reward_asset_visibility_invalid", 422);
|
|
2976
|
+
return [asset.id, asset];
|
|
2977
|
+
})).values()];
|
|
2978
|
+
const { environmentRelease: _environment, verifierSetRelease: _verifiers, ...execution } = metadata.definition.execution;
|
|
2979
|
+
const { contentHash: _definitionHash, ...definitionContent } = metadata.definition;
|
|
2980
|
+
const definition = TaskDefinitionSchema3.parse(sealLearningContent4({
|
|
2981
|
+
...definitionContent,
|
|
2982
|
+
...request.definition,
|
|
2983
|
+
id: `${id}-definition`,
|
|
2984
|
+
revision: 1,
|
|
2985
|
+
rewardBinding: learningRef5(binding),
|
|
2986
|
+
execution: {
|
|
2987
|
+
...execution,
|
|
2988
|
+
policy: { ...execution.policy, hiddenGraderRefs: compileBoundGraders2(binding, rewards).filter((grader) => grader.privileged).map((grader) => grader.id) }
|
|
2989
|
+
}
|
|
2990
|
+
}));
|
|
2991
|
+
const source = LearningSourceSchema3.parse(sealLearningContent4({
|
|
2992
|
+
schemaVersion: "openpond.learningSource.v1",
|
|
2993
|
+
id,
|
|
2994
|
+
revision: 1,
|
|
2995
|
+
name: `${definition.name} review`,
|
|
2996
|
+
kind: "direct",
|
|
2997
|
+
taskDefinition: learningRef5(definition),
|
|
2998
|
+
enabled: true,
|
|
2999
|
+
allowedSplits: [...new Set(learning.evidence.map((item) => item.submission.split))],
|
|
3000
|
+
mapping: null,
|
|
3001
|
+
adapterVersion: null,
|
|
3002
|
+
reviewOrigin: {
|
|
3003
|
+
modelId: request.modelId,
|
|
3004
|
+
packageHash: value.contentHash,
|
|
3005
|
+
taskset: learningRef5(value.taskset),
|
|
3006
|
+
batch: learningRef5(learning.batch),
|
|
3007
|
+
rewardBinding: learningRef5(input.binding)
|
|
3008
|
+
}
|
|
3009
|
+
}));
|
|
3010
|
+
const evidence = learning.evidence.map((parent) => {
|
|
3011
|
+
const edit = edits.get(contentHash(learningRef5(parent)));
|
|
3012
|
+
const attemptId = `review-${parent.contentHash}`;
|
|
3013
|
+
const policyChanged = contentHash({
|
|
3014
|
+
instructions: definition.instructions,
|
|
3015
|
+
inputSchema: definition.inputSchema,
|
|
3016
|
+
outputSchema: definition.outputSchema,
|
|
3017
|
+
input: edit?.input ?? parent.submission.input
|
|
3018
|
+
}) !== contentHash({
|
|
3019
|
+
instructions: metadata.definition.instructions,
|
|
3020
|
+
inputSchema: metadata.definition.inputSchema,
|
|
3021
|
+
outputSchema: metadata.definition.outputSchema,
|
|
3022
|
+
input: parent.submission.input
|
|
3023
|
+
});
|
|
3024
|
+
return TaskEvidenceSchema3.parse(sealLearningContent4({
|
|
3025
|
+
schemaVersion: "openpond.taskEvidence.v1",
|
|
3026
|
+
id: learningEvidenceId(source.id, parent.submission.exampleId, attemptId),
|
|
3027
|
+
revision: 1,
|
|
3028
|
+
source: learningRef5(source),
|
|
3029
|
+
submission: {
|
|
3030
|
+
...parent.submission,
|
|
3031
|
+
sourceId: source.id,
|
|
3032
|
+
taskDefinition: learningRef5(definition),
|
|
3033
|
+
attemptId,
|
|
3034
|
+
// The old response was observed under the old policy-visible task. Its
|
|
3035
|
+
// parent snapshot remains available, but it is not a run of new inputs.
|
|
3036
|
+
observedOutput: policyChanged ? null : parent.submission.observedOutput,
|
|
3037
|
+
idempotencyKey: `${id}-${parent.contentHash}`,
|
|
3038
|
+
...edit?.input === void 0 ? {} : { input: edit.input },
|
|
3039
|
+
...edit?.expected === void 0 ? {} : { expected: edit.expected },
|
|
3040
|
+
...edit?.evaluatorContext === void 0 ? {} : { evaluatorContext: edit.evaluatorContext }
|
|
3041
|
+
},
|
|
3042
|
+
supersedes: learningRef5(parent),
|
|
3043
|
+
correctionFeedbackId: null,
|
|
3044
|
+
receivedAt: input.now
|
|
3045
|
+
}));
|
|
3046
|
+
});
|
|
3047
|
+
const proposals = evidence.flatMap((item) => {
|
|
3048
|
+
const edit = edits.get(contentHash(item.supersedes));
|
|
3049
|
+
const decision = decisions.get(contentHash(item.supersedes));
|
|
3050
|
+
const target = edit?.proposedTarget === void 0 ? decision.approvedTarget : edit.proposedTarget;
|
|
3051
|
+
return target === null ? [] : [TaskFeedbackSchema2.parse({
|
|
3052
|
+
schemaVersion: "openpond.taskFeedbackRecord.v1",
|
|
3053
|
+
id: `feedback-${contentHash([id, item.id])}`,
|
|
3054
|
+
revision: 1,
|
|
3055
|
+
submission: {
|
|
3056
|
+
schemaVersion: "openpond.taskFeedback.v1",
|
|
3057
|
+
sourceId: source.id,
|
|
3058
|
+
idempotencyKey: `${id}-${item.id}`,
|
|
3059
|
+
exampleId: item.submission.exampleId,
|
|
3060
|
+
attemptId: item.submission.attemptId,
|
|
3061
|
+
expectedEvidenceHash: item.contentHash,
|
|
3062
|
+
occurredAt: input.now,
|
|
3063
|
+
kind: "target_correction",
|
|
3064
|
+
value: target,
|
|
3065
|
+
note: "Proposed target for the revised task. Grade and review before admission."
|
|
3066
|
+
},
|
|
3067
|
+
status: "pending_review",
|
|
3068
|
+
evidence: learningRef5(item),
|
|
3069
|
+
createdAt: input.now,
|
|
3070
|
+
review: null
|
|
3071
|
+
})];
|
|
3072
|
+
});
|
|
3073
|
+
return { source, definition, binding, rewards, assets, evidence, proposals };
|
|
3074
|
+
}
|
|
3075
|
+
|
|
3076
|
+
// src/model-batch-review.ts
|
|
3077
|
+
import { LearningDomainError as LearningDomainError3 } from "@openpond/evals/learning";
|
|
3078
|
+
var operation = (request) => `model-batch-review-${contentHash([request.modelId, request.operationId])}`;
|
|
3079
|
+
function inspectModelBatchPackage(raw) {
|
|
3080
|
+
const value = validateTasksetPackage(raw);
|
|
3081
|
+
if (!value.learningResources) throw new LearningDomainError2("model_review_batch_required", 422);
|
|
3082
|
+
const metadata = taskBatchPackageMetadata4(value.taskset);
|
|
3083
|
+
return ModelBatchReviewInspectionSchema.parse({
|
|
3084
|
+
schemaVersion: "openpond.modelBatchReviewInspection.v1",
|
|
3085
|
+
packageHash: value.contentHash,
|
|
3086
|
+
definition: metadata.definition,
|
|
3087
|
+
binding: metadata.binding,
|
|
3088
|
+
evidence: value.learningResources.evidence,
|
|
3089
|
+
decisions: value.learningResources.decisions
|
|
3090
|
+
});
|
|
3091
|
+
}
|
|
3092
|
+
async function findModelBatchReview(transaction, raw) {
|
|
3093
|
+
const request = ModelBatchReviewRequestSchema.parse(raw);
|
|
3094
|
+
const prior = await transaction.operation(operation(request));
|
|
3095
|
+
if (!prior) return null;
|
|
3096
|
+
if (prior.requestHash !== contentHash(request)) throw new LearningDomainError2("learning_idempotency_conflict", 409);
|
|
3097
|
+
return readReceipt(transaction, request, prior.resources);
|
|
3098
|
+
}
|
|
3099
|
+
async function beginModelBatchReview(transaction, input) {
|
|
3100
|
+
const request = ModelBatchReviewRequestSchema.parse(input.request);
|
|
3101
|
+
const prior = await findModelBatchReview(transaction, request);
|
|
3102
|
+
if (prior) return prior;
|
|
3103
|
+
const value = validateTasksetPackage(input.package);
|
|
3104
|
+
if (!value.learningResources) throw new LearningDomainError2("model_review_batch_required", 422);
|
|
3105
|
+
const metadata = taskBatchPackageMetadata4(value.taskset);
|
|
3106
|
+
const selected = request.rewardBindingRef;
|
|
3107
|
+
const retained = !selected || sameLearningRef4(selected, learningRef6(metadata.binding));
|
|
3108
|
+
const binding = retained ? metadata.binding : await requireLearningRelease(transaction, "binding", selected);
|
|
3109
|
+
const rewards = retained ? metadata.rewards : await Promise.all(binding.sources.map((source) => requireLearningRelease(transaction, "reward", source.reward)));
|
|
3110
|
+
const assetIds = new Set(rewards.flatMap((reward) => [
|
|
3111
|
+
...reward.assets,
|
|
3112
|
+
..."verifierRef" in reward.implementation ? [reward.implementation.verifierRef] : [],
|
|
3113
|
+
..."rubricRef" in reward.implementation ? [reward.implementation.rubricRef] : [],
|
|
3114
|
+
..."inputContract" in reward.implementation ? [reward.implementation.inputContract] : []
|
|
3115
|
+
]).map((asset) => asset.id));
|
|
3116
|
+
const assets = retained ? value.learningResources.assets : await Promise.all([...assetIds].map((id) => requireLearningResource(transaction, "asset", id, 1)));
|
|
3117
|
+
const prepared = prepareModelBatchReview({ ...input, request, package: value, binding, rewards, assets });
|
|
3118
|
+
const pointers = [];
|
|
3119
|
+
for (const asset of prepared.assets) {
|
|
3120
|
+
const existing = await transaction.get("asset", asset.id, 1);
|
|
3121
|
+
if (existing && existing.contentHash !== asset.contentHash) throw new LearningDomainError2("model_review_asset_conflict", 409);
|
|
3122
|
+
if (!existing) await transaction.put("asset", asset, 0);
|
|
3123
|
+
}
|
|
3124
|
+
for (const reward of prepared.rewards) await transaction.put("reward", reward, 0);
|
|
3125
|
+
await transaction.put("binding", prepared.binding, 0);
|
|
3126
|
+
await transaction.put("definition", prepared.definition, 0);
|
|
3127
|
+
await transaction.put("source", prepared.source, 0, { parentId: request.modelId });
|
|
3128
|
+
pointers.push({ kind: "source", id: prepared.source.id, revision: 1 });
|
|
3129
|
+
for (const evidence of prepared.evidence) {
|
|
3130
|
+
await transaction.put("evidence", evidence, 0, { parentId: prepared.source.id });
|
|
3131
|
+
pointers.push({ kind: "evidence", id: evidence.id, revision: 1 });
|
|
3132
|
+
}
|
|
3133
|
+
for (const proposal of prepared.proposals) {
|
|
3134
|
+
await transaction.put("feedback", proposal, 0, { parentId: proposal.evidence.id, status: proposal.status });
|
|
3135
|
+
pointers.push({ kind: "feedback", id: proposal.id, revision: 1 });
|
|
3136
|
+
}
|
|
3137
|
+
await transaction.saveOperation(operation(request), { requestHash: contentHash(request), resources: pointers });
|
|
3138
|
+
return readReceipt(transaction, request, pointers);
|
|
3139
|
+
}
|
|
3140
|
+
async function readReceipt(transaction, request, pointers) {
|
|
3141
|
+
const resources = await Promise.all(pointers.map((pointer) => requireLearningResource(transaction, pointer.kind, pointer.id, pointer.revision)));
|
|
3142
|
+
return ModelBatchReviewReceiptSchema.parse({
|
|
3143
|
+
schemaVersion: "openpond.modelBatchReviewReceipt.v1",
|
|
3144
|
+
operationId: request.operationId,
|
|
3145
|
+
modelId: request.modelId,
|
|
3146
|
+
source: resources.find((resource) => resource.schemaVersion === "openpond.learningSource.v1"),
|
|
3147
|
+
evidence: resources.filter((resource) => resource.schemaVersion === "openpond.taskEvidence.v1"),
|
|
3148
|
+
proposals: resources.filter((resource) => resource.schemaVersion === "openpond.taskFeedbackRecord.v1")
|
|
3149
|
+
});
|
|
3150
|
+
}
|
|
2443
3151
|
export {
|
|
2444
3152
|
MAX_TASKSET_PACKAGE_BYTES,
|
|
3153
|
+
LearningDomainError3 as ModelBatchReviewError,
|
|
3154
|
+
ModelBatchReviewInspectionSchema,
|
|
3155
|
+
ModelBatchReviewReceiptSchema,
|
|
3156
|
+
ModelBatchReviewRequestSchema,
|
|
2445
3157
|
OpenPondTasksetPackageClient,
|
|
2446
3158
|
OpenPondTasksetPackageError,
|
|
2447
3159
|
TasksetPackageContentSchema,
|
|
2448
3160
|
TasksetPackageFileSchema,
|
|
3161
|
+
TasksetPackageLearningResourcesSchema,
|
|
2449
3162
|
TasksetPackageModelConfigurationSchema,
|
|
2450
3163
|
TasksetPackagePublicationSchema,
|
|
2451
3164
|
TasksetPackageReadbackSchema,
|
|
2452
3165
|
TasksetPackageReceiptSchema,
|
|
2453
3166
|
TasksetPackageSchema,
|
|
3167
|
+
beginModelBatchReview,
|
|
2454
3168
|
createTasksetPackage,
|
|
2455
3169
|
decodeTasksetPackageFile,
|
|
3170
|
+
findModelBatchReview,
|
|
3171
|
+
inspectModelBatchPackage,
|
|
3172
|
+
learningPackageContextFiles,
|
|
3173
|
+
tasksetPackageRewardBinding,
|
|
3174
|
+
validateTasksetLearningResources,
|
|
2456
3175
|
validateTasksetPackage
|
|
2457
3176
|
};
|
|
2458
3177
|
//# sourceMappingURL=taskset-packages.js.map
|