openpond-sdk 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/TASKSETS.md +96 -0
- package/dist/model-batch-review.js +48 -0
- package/dist/model-batch-review.js.map +7 -0
- package/dist/model-taskset-authoring.js +56 -0
- package/dist/model-taskset-authoring.js.map +7 -0
- package/dist/taskset-packages.js +566 -75
- package/dist/taskset-packages.js.map +4 -4
- package/dist/types/packages/sdk/src/model-batch-review-contracts.d.ts +568 -0
- package/dist/types/packages/sdk/src/model-batch-review-contracts.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-batch-review-preparation.d.ts +351 -0
- package/dist/types/packages/sdk/src/model-batch-review-preparation.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-batch-review.d.ts +543 -0
- package/dist/types/packages/sdk/src/model-batch-review.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-projects.d.ts +2 -2
- package/dist/types/packages/sdk/src/model-starter-catalog.d.ts +16 -2
- package/dist/types/packages/sdk/src/model-starter-catalog.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-starters.d.ts +41 -5
- package/dist/types/packages/sdk/src/model-starters.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-authoring-contracts.d.ts +107 -0
- package/dist/types/packages/sdk/src/model-taskset-authoring-contracts.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-taskset-authoring-lineage.d.ts +27 -0
- package/dist/types/packages/sdk/src/model-taskset-authoring-lineage.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-taskset-authoring.d.ts +20 -0
- package/dist/types/packages/sdk/src/model-taskset-authoring.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/model-taskset-derivation.d.ts +25 -3
- package/dist/types/packages/sdk/src/model-taskset-derivation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts +88 -6
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts +90 -24
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-execution.d.ts +19 -0
- package/dist/types/packages/sdk/src/taskset-package-execution.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/taskset-package-files.d.ts +21 -0
- package/dist/types/packages/sdk/src/taskset-package-files.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/taskset-package-learning.d.ts +38 -0
- package/dist/types/packages/sdk/src/taskset-package-learning.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-packages.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-packages.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training.d.ts +3 -3
- package/package.json +10 -2
package/dist/taskset-packages.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/taskset-package-contracts.ts
|
|
2
|
-
import { z as
|
|
2
|
+
import { z as z19 } from "zod";
|
|
3
3
|
|
|
4
4
|
// ../harness/dist/common.js
|
|
5
5
|
import { z } from "zod";
|
|
@@ -2136,6 +2136,18 @@ var ModelStarterExecutionSchema = z12.object({
|
|
|
2136
2136
|
var ModelStarterToolFixtureScriptSchema = z12.object({
|
|
2137
2137
|
actions: z12.array(z12.object({ name: z12.string().min(1).max(64), arguments: z12.record(z12.string(), z12.unknown()) }).strict()).max(1e3)
|
|
2138
2138
|
}).strict();
|
|
2139
|
+
function modelStarterExecutionAssetId(context) {
|
|
2140
|
+
if (!context.environmentRelease || !context.verifierSetRelease) throw new Error("Starter execution release references are missing.");
|
|
2141
|
+
return `starter-execution-${sealLearningContent({ environment: context.environmentRelease, verifiers: context.verifierSetRelease }).contentHash}`;
|
|
2142
|
+
}
|
|
2143
|
+
function createModelStarterExecutionAsset(value) {
|
|
2144
|
+
const execution = ModelStarterExecutionSchema.parse(value);
|
|
2145
|
+
assertExecutionIntegrity(execution);
|
|
2146
|
+
const id = modelStarterExecutionAssetId({ environmentRelease: { id: execution.environment.id, contentHash: execution.environment.contentHash }, verifierSetRelease: { id: execution.verifierSet.id, contentHash: execution.verifierSet.contentHash } });
|
|
2147
|
+
const generated = createLearningTextAsset({ text: canonicalJson2(execution), path: "environment/execution.json", mediaType: "application/json", visibility: "host_private" });
|
|
2148
|
+
const { contentHash: _hash, ...content } = generated;
|
|
2149
|
+
return sealLearningContent({ ...content, id, asset: { ...content.asset, id } });
|
|
2150
|
+
}
|
|
2139
2151
|
function equal(left, right) {
|
|
2140
2152
|
return sealLearningContent({ value: left }).contentHash === sealLearningContent({ value: right }).contentHash;
|
|
2141
2153
|
}
|
|
@@ -2601,34 +2613,138 @@ function validateTasksetLearningResources(taskset, input) {
|
|
|
2601
2613
|
return resources;
|
|
2602
2614
|
}
|
|
2603
2615
|
|
|
2604
|
-
// src/taskset-
|
|
2616
|
+
// src/model-taskset-authoring-lineage.ts
|
|
2617
|
+
import { sameLearningRef as sameLearningRef3 } from "@openpond/evals/learning";
|
|
2618
|
+
|
|
2619
|
+
// src/model-taskset-authoring-contracts.ts
|
|
2620
|
+
import { z as z17 } from "zod";
|
|
2621
|
+
import { LearningRevisionRefSchema } from "@openpond/evals/learning";
|
|
2622
|
+
var IdSchema3 = LearningRevisionRefSchema.shape.id;
|
|
2623
|
+
var HashSchema4 = z17.string().regex(/^[a-f0-9]{64}$/);
|
|
2624
|
+
var ModelTasksetAuthoringOwnerSchema = z17.object({ scopeId: IdSchema3, modelId: IdSchema3 }).strict();
|
|
2625
|
+
var ModelTasksetAuthoringSchema = z17.object({
|
|
2626
|
+
schemaVersion: z17.literal("openpond.modelTasksetAuthoring.v1"),
|
|
2627
|
+
owner: ModelTasksetAuthoringOwnerSchema,
|
|
2628
|
+
root: LearningRevisionRefSchema,
|
|
2629
|
+
parent: LearningRevisionRefSchema
|
|
2630
|
+
}).strict();
|
|
2631
|
+
var ModelTasksetDraftRequestSchema = z17.object({
|
|
2632
|
+
schemaVersion: z17.literal("openpond.modelTasksetDraftRequest.v1"),
|
|
2633
|
+
operationId: IdSchema3,
|
|
2634
|
+
modelId: IdSchema3,
|
|
2635
|
+
expectedModelRevision: z17.number().int().positive(),
|
|
2636
|
+
sourcePackageHash: HashSchema4
|
|
2637
|
+
}).strict();
|
|
2638
|
+
var ModelTasksetDraftPreparationSchema = z17.object({
|
|
2639
|
+
schemaVersion: z17.literal("openpond.modelTasksetDraftPreparation.v1"),
|
|
2640
|
+
draftId: IdSchema3,
|
|
2641
|
+
requestHash: HashSchema4,
|
|
2642
|
+
sourcePackageHash: HashSchema4,
|
|
2643
|
+
sourceTasksetRef: LearningRevisionRefSchema,
|
|
2644
|
+
tasksetId: IdSchema3,
|
|
2645
|
+
tasksetRevision: z17.number().int().positive(),
|
|
2646
|
+
lineage: ModelTasksetAuthoringSchema
|
|
2647
|
+
}).strict();
|
|
2648
|
+
var TasksetDraftFilePathSchema = z17.string().min(1).max(1024).refine((value) => !value.includes("\\") && !value.includes(":") && !value.includes("\0") && !value.startsWith("/") && !value.split("/").some((part) => !part || part === "." || part === ".." || part.startsWith(".env")), "Use a relative Taskset file path without environment files.");
|
|
2649
|
+
var TasksetDraftFileInfoSchema = z17.object({
|
|
2650
|
+
path: TasksetDraftFilePathSchema,
|
|
2651
|
+
sizeBytes: z17.number().int().nonnegative(),
|
|
2652
|
+
writable: z17.boolean()
|
|
2653
|
+
}).strict();
|
|
2654
|
+
var TasksetDraftFileContentSchema = z17.object({ encoding: z17.enum(["utf8", "base64"]), data: z17.string().max(8e6) }).strict();
|
|
2655
|
+
var TasksetDraftFileSchema = TasksetDraftFileInfoSchema.extend({ contentHash: HashSchema4, content: TasksetDraftFileContentSchema }).strict();
|
|
2656
|
+
var TasksetDraftFileMutationSchema = z17.object({
|
|
2657
|
+
draftId: IdSchema3,
|
|
2658
|
+
expectedDraftRevision: z17.number().int().positive(),
|
|
2659
|
+
path: TasksetDraftFilePathSchema,
|
|
2660
|
+
expectedFileHash: HashSchema4.nullable(),
|
|
2661
|
+
content: TasksetDraftFileContentSchema.nullable()
|
|
2662
|
+
}).strict();
|
|
2663
|
+
|
|
2664
|
+
// src/model-taskset-authoring-lineage.ts
|
|
2665
|
+
function modelAuthoredTasksetId(lineage) {
|
|
2666
|
+
return `model-authored-taskset-${contentHash({ owner: lineage.owner, root: lineage.root })}`;
|
|
2667
|
+
}
|
|
2668
|
+
function assertModelTasksetAuthoring(taskset) {
|
|
2669
|
+
if (taskset.metadata.modelTasksetAuthoring === void 0) return null;
|
|
2670
|
+
const lineage = ModelTasksetAuthoringSchema.parse(taskset.metadata.modelTasksetAuthoring);
|
|
2671
|
+
if (taskset.id !== modelAuthoredTasksetId(lineage) || (taskset.revision === 1 ? !sameLearningRef3(lineage.parent, lineage.root) : lineage.parent.id !== taskset.id || lineage.parent.revision !== taskset.revision - 1)) {
|
|
2672
|
+
throw new Error("Authored Taskset revision differs from its ownership lineage.");
|
|
2673
|
+
}
|
|
2674
|
+
return lineage;
|
|
2675
|
+
}
|
|
2676
|
+
|
|
2677
|
+
// src/taskset-package-execution.ts
|
|
2678
|
+
import { LearningTextAssetSchema as LearningTextAssetSchema3, sealLearningContent as sealLearningContent4 } from "@openpond/evals/learning";
|
|
2679
|
+
|
|
2680
|
+
// src/taskset-package-files.ts
|
|
2681
|
+
import { z as z18 } from "zod";
|
|
2605
2682
|
var MAX_TASKSET_PACKAGE_BYTES = 64 * 1024 * 1024;
|
|
2606
|
-
var TasksetPackageFileSchema =
|
|
2683
|
+
var TasksetPackageFileSchema = z18.object({
|
|
2607
2684
|
asset: ImmutableAssetRefSchema,
|
|
2608
|
-
base64:
|
|
2685
|
+
base64: z18.string().max(Math.ceil(MAX_TASKSET_PACKAGE_BYTES / 3) * 4)
|
|
2609
2686
|
}).strict();
|
|
2687
|
+
function decodeTasksetPackageFile(value) {
|
|
2688
|
+
const file = TasksetPackageFileSchema.parse(value);
|
|
2689
|
+
const raw = atob(file.base64);
|
|
2690
|
+
if (btoa(raw) !== file.base64) throw new Error(`Taskset asset ${file.asset.id} has noncanonical base64.`);
|
|
2691
|
+
const bytes = Uint8Array.from(raw, (character) => character.charCodeAt(0));
|
|
2692
|
+
if (bytes.byteLength !== file.asset.sizeBytes || sha256(bytes) !== file.asset.contentHash) throw new Error(`Taskset asset ${file.asset.id} differs from its immutable bytes.`);
|
|
2693
|
+
return bytes;
|
|
2694
|
+
}
|
|
2695
|
+
|
|
2696
|
+
// src/taskset-package-execution.ts
|
|
2697
|
+
function resolveTasksetPackageExecution(input) {
|
|
2698
|
+
if (input.taskset.environment.entrypoint !== "openpond.javascript-environment.v1") return null;
|
|
2699
|
+
const byId = new Map(input.files.map((file) => [file.asset.id, file]));
|
|
2700
|
+
const read = (id) => {
|
|
2701
|
+
const file = byId.get(id);
|
|
2702
|
+
if (!file) throw new Error(`Taskset environment asset is missing: ${id}.`);
|
|
2703
|
+
if (file.asset.sizeBytes > 524288) throw new Error("Taskset environment source exceeds the authored asset limit.");
|
|
2704
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(decodeTasksetPackageFile(file));
|
|
2705
|
+
return LearningTextAssetSchema3.parse(sealLearningContent4({ schemaVersion: "openpond.learningTextAsset.v1", id, revision: 1, asset: file.asset, text }));
|
|
2706
|
+
};
|
|
2707
|
+
let execution;
|
|
2708
|
+
if (input.modelResources) {
|
|
2709
|
+
if (!input.modelResources.execution) throw new Error("Bound Taskset JavaScript execution resources are missing.");
|
|
2710
|
+
execution = ModelStarterExecutionSchema.parse(input.modelResources.execution);
|
|
2711
|
+
} else {
|
|
2712
|
+
const declaration = read(modelStarterExecutionAssetId(input.taskset));
|
|
2713
|
+
if (declaration.asset.visibility !== "host_private" || declaration.asset.mediaType !== "application/json" || declaration.asset.path !== "environment/execution.json") throw new Error("Taskset execution declaration must be private JSON at environment/execution.json.");
|
|
2714
|
+
execution = ModelStarterExecutionSchema.parse(JSON.parse(declaration.text));
|
|
2715
|
+
}
|
|
2716
|
+
const ids = /* @__PURE__ */ new Set([
|
|
2717
|
+
execution.javascript.module.id,
|
|
2718
|
+
...[execution.environment.actionSchemaRef, execution.environment.observationSchemaRef, execution.environment.stateSchemaRef].flatMap((ref) => ref ? [ref.id] : []),
|
|
2719
|
+
...input.taskset.tasks.flatMap((task) => task.privilegedContextRef ? [task.privilegedContextRef] : [])
|
|
2720
|
+
]);
|
|
2721
|
+
const assets = [...ids].map(read);
|
|
2722
|
+
validateModelStarterExecution(execution, input.taskset, assets);
|
|
2723
|
+
return { execution, assets };
|
|
2724
|
+
}
|
|
2725
|
+
function createTasksetPackageExecutionFile(execution) {
|
|
2726
|
+
const asset = createModelStarterExecutionAsset(execution);
|
|
2727
|
+
const bytes = new TextEncoder().encode(asset.text);
|
|
2728
|
+
let raw = "";
|
|
2729
|
+
for (const byte of bytes) raw += String.fromCharCode(byte);
|
|
2730
|
+
return { asset: asset.asset, base64: btoa(raw) };
|
|
2731
|
+
}
|
|
2732
|
+
|
|
2733
|
+
// src/taskset-package-contracts.ts
|
|
2610
2734
|
var BoundModelResourcesSchema = ModelTasksetPackageSchema.omit({ taskset: true, executionResources: true });
|
|
2611
|
-
var TasksetPackageContentSchema =
|
|
2612
|
-
schemaVersion:
|
|
2735
|
+
var TasksetPackageContentSchema = z19.object({
|
|
2736
|
+
schemaVersion: z19.literal("openpond.tasksetPackage.v1"),
|
|
2613
2737
|
taskset: TasksetReleaseSchema2,
|
|
2614
2738
|
environment: EnvironmentReleaseSchema3,
|
|
2615
2739
|
verifierSet: VerifierSetReleaseSchema3,
|
|
2616
|
-
files:
|
|
2740
|
+
files: z19.array(TasksetPackageFileSchema).max(1e4),
|
|
2617
2741
|
modelResources: BoundModelResourcesSchema.optional(),
|
|
2618
2742
|
learningResources: TasksetPackageLearningResourcesSchema.optional()
|
|
2619
2743
|
}).strict();
|
|
2620
|
-
var TasksetPackageSchema = TasksetPackageContentSchema.extend({ contentHash:
|
|
2744
|
+
var TasksetPackageSchema = TasksetPackageContentSchema.extend({ contentHash: z19.string().regex(/^[a-f0-9]{64}$/) }).strict();
|
|
2621
2745
|
function tasksetPackageRewardBinding(value) {
|
|
2622
2746
|
return value.learningResources ? taskBatchPackageMetadata2(value.taskset).binding : value.modelResources?.rewardBinding ?? null;
|
|
2623
2747
|
}
|
|
2624
|
-
function decodeTasksetPackageFile(value) {
|
|
2625
|
-
const file = TasksetPackageFileSchema.parse(value);
|
|
2626
|
-
const raw = atob(file.base64);
|
|
2627
|
-
if (btoa(raw) !== file.base64) throw new Error(`Taskset asset ${file.asset.id} has noncanonical base64.`);
|
|
2628
|
-
const bytes = Uint8Array.from(raw, (character) => character.charCodeAt(0));
|
|
2629
|
-
if (bytes.byteLength !== file.asset.sizeBytes || sha256(bytes) !== file.asset.contentHash) throw new Error(`Taskset asset ${file.asset.id} differs from its immutable bytes.`);
|
|
2630
|
-
return bytes;
|
|
2631
|
-
}
|
|
2632
2748
|
function validateTasksetPackage(value) {
|
|
2633
2749
|
assertBoundedTaskJson3(value, MAX_TASKSET_PACKAGE_BYTES);
|
|
2634
2750
|
const result = TasksetPackageSchema.parse(value);
|
|
@@ -2636,6 +2752,8 @@ function validateTasksetPackage(value) {
|
|
|
2636
2752
|
if (contentHash(content) !== hash) throw new Error("Taskset package content hash differs from its bytes.");
|
|
2637
2753
|
const { taskset, environment, verifierSet } = result;
|
|
2638
2754
|
assertTasksetRelease2(taskset);
|
|
2755
|
+
const authoring = assertModelTasksetAuthoring(taskset);
|
|
2756
|
+
if (authoring && (result.modelResources || result.learningResources)) throw new Error("Taskset package cannot declare competing authoring graphs.");
|
|
2639
2757
|
if (!verifyEnvironmentRelease3(environment) || !verifyVerifierSetRelease3(verifierSet) || !same2(taskset.environmentRelease, { id: environment.id, contentHash: environment.contentHash }) || !same2(taskset.verifierSetRelease, { id: verifierSet.id, contentHash: verifierSet.contentHash }) || !same2(taskset.environment, environment.contract) || !same2(taskset.graders, verifierSet.graders)) {
|
|
2640
2758
|
throw new Error("Taskset package execution releases do not match its Taskset.");
|
|
2641
2759
|
}
|
|
@@ -2664,9 +2782,16 @@ function validateTasksetPackage(value) {
|
|
|
2664
2782
|
if (ref.visibility === "policy") throw new Error(`Taskset grader asset must be private: ${ref.id}.`);
|
|
2665
2783
|
}
|
|
2666
2784
|
}
|
|
2785
|
+
if (taskset.metrics?.customAggregator) {
|
|
2786
|
+
const aggregator = taskset.metrics.customAggregator;
|
|
2787
|
+
const modules = result.files.filter((file) => file.asset.path === aggregator.module);
|
|
2788
|
+
if (modules.length !== 1 || modules[0].asset.contentHash !== aggregator.contentHash || modules[0].asset.sizeBytes > 524288 || modules[0].asset.visibility === "policy") {
|
|
2789
|
+
throw new Error("Taskset metric module is missing or mismatched, public, or exceeds the source limit.");
|
|
2790
|
+
}
|
|
2791
|
+
}
|
|
2667
2792
|
for (const declarations of [taskset.metadata.environmentResources, environment.metadata.resources]) {
|
|
2668
2793
|
if (declarations === void 0) continue;
|
|
2669
|
-
for (const resource of
|
|
2794
|
+
for (const resource of z19.array(z19.object({ path: z19.string().min(1) }).passthrough()).parse(declarations)) {
|
|
2670
2795
|
if (!result.files.some((file) => file.asset.path === resource.path)) throw new Error(`Taskset environment resource is missing: ${resource.path}.`);
|
|
2671
2796
|
}
|
|
2672
2797
|
}
|
|
@@ -2683,6 +2808,7 @@ function validateTasksetPackage(value) {
|
|
|
2683
2808
|
} else if (["starter", "rewardBinding", "rewardExecution", "modelTasksetDerivation", "learning"].some((key) => taskset.metadata[key] !== void 0)) {
|
|
2684
2809
|
throw new Error("Bound Taskset publication requires its complete model resources.");
|
|
2685
2810
|
}
|
|
2811
|
+
resolveTasksetPackageExecution(result);
|
|
2686
2812
|
return result;
|
|
2687
2813
|
}
|
|
2688
2814
|
function createTasksetPackage(value) {
|
|
@@ -2695,45 +2821,45 @@ function same2(left, right) {
|
|
|
2695
2821
|
}
|
|
2696
2822
|
|
|
2697
2823
|
// src/taskset-package-client.ts
|
|
2698
|
-
import { z as
|
|
2824
|
+
import { z as z21 } from "zod";
|
|
2699
2825
|
|
|
2700
2826
|
// src/taskset-catalog.ts
|
|
2701
|
-
import { z as
|
|
2702
|
-
var
|
|
2703
|
-
var
|
|
2704
|
-
var TasksetCatalogReleaseRefSchema =
|
|
2705
|
-
var HostedTasksetSummarySchema =
|
|
2706
|
-
schemaVersion:
|
|
2707
|
-
id:
|
|
2708
|
-
teamId:
|
|
2827
|
+
import { z as z20 } from "zod";
|
|
2828
|
+
var IdSchema4 = z20.string().trim().min(1).max(500);
|
|
2829
|
+
var HashSchema5 = z20.string().regex(/^[a-f0-9]{64}$/);
|
|
2830
|
+
var TasksetCatalogReleaseRefSchema = z20.object({ id: IdSchema4, revision: z20.number().int().positive(), contentHash: HashSchema5 }).strict();
|
|
2831
|
+
var HostedTasksetSummarySchema = z20.object({
|
|
2832
|
+
schemaVersion: z20.literal("openpond.hostedTasksetSummary.v1"),
|
|
2833
|
+
id: IdSchema4,
|
|
2834
|
+
teamId: IdSchema4,
|
|
2709
2835
|
release: TasksetCatalogReleaseRefSchema,
|
|
2710
|
-
name:
|
|
2711
|
-
description:
|
|
2712
|
-
taskCount:
|
|
2713
|
-
buildIntent:
|
|
2714
|
-
methodHint:
|
|
2715
|
-
packageBytes:
|
|
2716
|
-
storedBytes:
|
|
2717
|
-
createdAt:
|
|
2718
|
-
}).strict();
|
|
2719
|
-
var TasksetCatalogQuerySchema =
|
|
2720
|
-
afterId:
|
|
2721
|
-
limit:
|
|
2722
|
-
modelProjectId:
|
|
2723
|
-
}).strict();
|
|
2724
|
-
var TasksetCatalogPageSchema =
|
|
2725
|
-
items:
|
|
2726
|
-
nextCursor:
|
|
2727
|
-
}).strict();
|
|
2728
|
-
var TasksetCatalogErrorSchema =
|
|
2729
|
-
code:
|
|
2730
|
-
message:
|
|
2836
|
+
name: z20.string().min(1).max(500),
|
|
2837
|
+
description: z20.string().max(2e4),
|
|
2838
|
+
taskCount: z20.number().int().nonnegative(),
|
|
2839
|
+
buildIntent: z20.string().min(1).max(100),
|
|
2840
|
+
methodHint: z20.string().min(1).max(100).nullable(),
|
|
2841
|
+
packageBytes: z20.number().int().nonnegative().nullable(),
|
|
2842
|
+
storedBytes: z20.number().int().nonnegative().nullable(),
|
|
2843
|
+
createdAt: z20.string().datetime({ offset: true })
|
|
2844
|
+
}).strict();
|
|
2845
|
+
var TasksetCatalogQuerySchema = z20.object({
|
|
2846
|
+
afterId: IdSchema4.optional(),
|
|
2847
|
+
limit: z20.number().int().min(1).max(100).default(30),
|
|
2848
|
+
modelProjectId: IdSchema4.optional()
|
|
2849
|
+
}).strict();
|
|
2850
|
+
var TasksetCatalogPageSchema = z20.object({
|
|
2851
|
+
items: z20.array(HostedTasksetSummarySchema).max(100),
|
|
2852
|
+
nextCursor: IdSchema4.nullable()
|
|
2853
|
+
}).strict();
|
|
2854
|
+
var TasksetCatalogErrorSchema = z20.object({
|
|
2855
|
+
code: z20.string().min(1),
|
|
2856
|
+
message: z20.string().min(1)
|
|
2731
2857
|
}).strict();
|
|
2732
2858
|
|
|
2733
2859
|
// src/taskset-package-client.ts
|
|
2734
2860
|
import { learningRef as learningRef4 } from "@openpond/evals/learning";
|
|
2735
|
-
var
|
|
2736
|
-
var
|
|
2861
|
+
var IdSchema5 = z21.string().trim().min(1).max(500);
|
|
2862
|
+
var HashSchema6 = z21.string().regex(/^[a-f0-9]{64}$/);
|
|
2737
2863
|
var TasksetPackageModelConfigurationSchema = HostedModelProjectSyncSchema.pick({
|
|
2738
2864
|
portableProjectId: true,
|
|
2739
2865
|
name: true,
|
|
@@ -2743,35 +2869,35 @@ var TasksetPackageModelConfigurationSchema = HostedModelProjectSyncSchema.pick({
|
|
|
2743
2869
|
sourceRevision: true,
|
|
2744
2870
|
sourceUpdatedAt: true
|
|
2745
2871
|
}).extend({ trainingSetup: HostedModelProjectTrainingSetupSchema.omit({ tasksetRef: true, rewardBindingRef: true, tasksetRelease: true, recipe: true }) }).strict();
|
|
2746
|
-
var TasksetPackagePublicationSchema =
|
|
2747
|
-
schemaVersion:
|
|
2748
|
-
operationId:
|
|
2749
|
-
modelProjectId:
|
|
2750
|
-
expectedProjectEtag:
|
|
2751
|
-
name:
|
|
2752
|
-
description:
|
|
2753
|
-
buildIntent:
|
|
2754
|
-
methodHint:
|
|
2872
|
+
var TasksetPackagePublicationSchema = z21.object({
|
|
2873
|
+
schemaVersion: z21.literal("openpond.tasksetPackagePublication.v1"),
|
|
2874
|
+
operationId: IdSchema5,
|
|
2875
|
+
modelProjectId: IdSchema5,
|
|
2876
|
+
expectedProjectEtag: HashSchema6.nullable(),
|
|
2877
|
+
name: z21.string().trim().min(1).max(200),
|
|
2878
|
+
description: z21.string().max(5e3),
|
|
2879
|
+
buildIntent: z21.enum(["demonstrations", "preferences", "verifiable_reward", "rubric", "discovery"]),
|
|
2880
|
+
methodHint: z21.enum(["sft", "dpo", "grpo", "ppo"]).nullable(),
|
|
2755
2881
|
package: TasksetPackageSchema,
|
|
2756
2882
|
modelConfiguration: TasksetPackageModelConfigurationSchema.optional(),
|
|
2757
|
-
selection:
|
|
2883
|
+
selection: z21.enum(["select", "attach"]).optional()
|
|
2758
2884
|
}).strict().refine((value) => value.selection !== "attach" || value.modelConfiguration === void 0, "Attachment-only publication cannot replace Model configuration.");
|
|
2759
|
-
var TasksetPackageReceiptSchema =
|
|
2760
|
-
schemaVersion:
|
|
2761
|
-
teamId:
|
|
2762
|
-
modelProjectId:
|
|
2763
|
-
operationId:
|
|
2885
|
+
var TasksetPackageReceiptSchema = z21.object({
|
|
2886
|
+
schemaVersion: z21.literal("openpond.tasksetPackageReceipt.v1"),
|
|
2887
|
+
teamId: IdSchema5,
|
|
2888
|
+
modelProjectId: IdSchema5,
|
|
2889
|
+
operationId: IdSchema5,
|
|
2764
2890
|
taskset: TasksetCatalogReleaseRefSchema,
|
|
2765
|
-
packageHash:
|
|
2766
|
-
hostedTasksetId:
|
|
2767
|
-
projectEtag:
|
|
2891
|
+
packageHash: HashSchema6,
|
|
2892
|
+
hostedTasksetId: IdSchema5,
|
|
2893
|
+
projectEtag: HashSchema6,
|
|
2768
2894
|
project: HostedModelProjectSummarySchema.optional(),
|
|
2769
|
-
selection:
|
|
2895
|
+
selection: z21.enum(["select", "attach"]).optional()
|
|
2770
2896
|
}).strict();
|
|
2771
|
-
var TasksetPackageReadbackSchema =
|
|
2772
|
-
schemaVersion:
|
|
2773
|
-
teamId:
|
|
2774
|
-
modelProjectId:
|
|
2897
|
+
var TasksetPackageReadbackSchema = z21.object({
|
|
2898
|
+
schemaVersion: z21.literal("openpond.tasksetPackageReadback.v1"),
|
|
2899
|
+
teamId: IdSchema5,
|
|
2900
|
+
modelProjectId: IdSchema5,
|
|
2775
2901
|
package: TasksetPackageSchema
|
|
2776
2902
|
}).strict();
|
|
2777
2903
|
var OpenPondTasksetPackageError = class extends Error {
|
|
@@ -2819,9 +2945,9 @@ var OpenPondTasksetPackageClient = class {
|
|
|
2819
2945
|
return receipt;
|
|
2820
2946
|
}
|
|
2821
2947
|
async get(modelProjectId, reference, options = {}) {
|
|
2822
|
-
const project =
|
|
2948
|
+
const project = IdSchema5.parse(modelProjectId);
|
|
2823
2949
|
const ref = TasksetCatalogReleaseRefSchema.parse(reference);
|
|
2824
|
-
const expectedPackageHash = options.expectedPackageHash === void 0 ? void 0 :
|
|
2950
|
+
const expectedPackageHash = options.expectedPackageHash === void 0 ? void 0 : HashSchema6.parse(options.expectedPackageHash);
|
|
2825
2951
|
const response = TasksetPackageReadbackSchema.parse(await this.#request(`/${encodeURIComponent(project)}/${encodeURIComponent(ref.id)}/${ref.revision}/${ref.contentHash}`, "GET", void 0, options.signal));
|
|
2826
2952
|
const taskset = response.package.taskset;
|
|
2827
2953
|
if (response.teamId !== this.#options.teamId || response.modelProjectId !== project || taskset.id !== ref.id || taskset.revision !== ref.revision || taskset.contentHash !== ref.contentHash) throw new OpenPondTasksetPackageError(502, "package_readback_mismatch", "Taskset package did not match the requested release and owner.");
|
|
@@ -2858,16 +2984,374 @@ var OpenPondTasksetPackageClient = class {
|
|
|
2858
2984
|
}
|
|
2859
2985
|
const payload = JSON.parse(text);
|
|
2860
2986
|
if (!response.ok) {
|
|
2861
|
-
const error =
|
|
2987
|
+
const error = z21.object({ code: z21.string(), message: z21.string() }).safeParse(payload);
|
|
2862
2988
|
throw new OpenPondTasksetPackageError(response.status, error.success ? error.data.code : "package_request_failed", error.success ? error.data.message : `Taskset package request failed (${response.status}).`);
|
|
2863
2989
|
}
|
|
2864
2990
|
return payload;
|
|
2865
2991
|
}
|
|
2866
2992
|
};
|
|
2993
|
+
|
|
2994
|
+
// src/model-batch-review.ts
|
|
2995
|
+
import {
|
|
2996
|
+
LearningDomainError as LearningDomainError2,
|
|
2997
|
+
learningRef as learningRef6,
|
|
2998
|
+
sameLearningRef as sameLearningRef5,
|
|
2999
|
+
requireLearningRelease,
|
|
3000
|
+
requireLearningResource,
|
|
3001
|
+
taskBatchPackageMetadata as taskBatchPackageMetadata4
|
|
3002
|
+
} from "@openpond/evals/learning";
|
|
3003
|
+
|
|
3004
|
+
// src/model-batch-review-contracts.ts
|
|
3005
|
+
import { z as z22 } from "zod";
|
|
3006
|
+
import { LearningJsonObjectSchema, LearningRevisionRefSchema as LearningRevisionRefSchema2, LearningSourceSchema as LearningSourceSchema2, TaskEvidenceSchema as TaskEvidenceSchema2, TaskFeedbackSchema, TaskDefinitionSchema as TaskDefinitionSchema2, TaskAdmissionDecisionSchema as TaskAdmissionDecisionSchema2 } from "@openpond/evals/learning";
|
|
3007
|
+
import { RewardBindingSchema as RewardBindingSchema2 } from "@openpond/evals/rewards";
|
|
3008
|
+
var ReleaseIdSchema2 = LearningRevisionRefSchema2.shape.id;
|
|
3009
|
+
var ModelBatchReviewInspectionSchema = z22.object({
|
|
3010
|
+
schemaVersion: z22.literal("openpond.modelBatchReviewInspection.v1"),
|
|
3011
|
+
packageHash: z22.string().regex(/^[a-f0-9]{64}$/),
|
|
3012
|
+
definition: TaskDefinitionSchema2,
|
|
3013
|
+
binding: RewardBindingSchema2,
|
|
3014
|
+
evidence: z22.array(TaskEvidenceSchema2).min(1).max(1e4),
|
|
3015
|
+
decisions: z22.array(TaskAdmissionDecisionSchema2).min(1).max(1e4)
|
|
3016
|
+
}).strict();
|
|
3017
|
+
var ModelBatchReviewRequestSchema = z22.object({
|
|
3018
|
+
schemaVersion: z22.literal("openpond.modelBatchReviewRequest.v1"),
|
|
3019
|
+
operationId: ReleaseIdSchema2,
|
|
3020
|
+
modelId: ReleaseIdSchema2,
|
|
3021
|
+
expectedModelRevision: z22.number().int().positive(),
|
|
3022
|
+
tasksetRef: LearningRevisionRefSchema2,
|
|
3023
|
+
rewardBindingRef: LearningRevisionRefSchema2.nullable(),
|
|
3024
|
+
definition: z22.object({
|
|
3025
|
+
name: z22.string().trim().min(1).max(500).optional(),
|
|
3026
|
+
instructions: z22.string().trim().min(1).max(2e4).optional(),
|
|
3027
|
+
inputSchema: LearningJsonObjectSchema.optional(),
|
|
3028
|
+
outputSchema: LearningJsonObjectSchema.optional()
|
|
3029
|
+
}).strict(),
|
|
3030
|
+
examples: z22.array(z22.object({
|
|
3031
|
+
evidence: LearningRevisionRefSchema2,
|
|
3032
|
+
input: LearningJsonObjectSchema.optional(),
|
|
3033
|
+
expected: LearningJsonObjectSchema.nullable().optional(),
|
|
3034
|
+
evaluatorContext: LearningJsonObjectSchema.nullable().optional(),
|
|
3035
|
+
proposedTarget: LearningJsonObjectSchema.nullable().optional()
|
|
3036
|
+
}).strict()).max(1e4)
|
|
3037
|
+
}).strict();
|
|
3038
|
+
var ModelBatchReviewReceiptSchema = z22.object({
|
|
3039
|
+
schemaVersion: z22.literal("openpond.modelBatchReviewReceipt.v1"),
|
|
3040
|
+
operationId: ReleaseIdSchema2,
|
|
3041
|
+
modelId: ReleaseIdSchema2,
|
|
3042
|
+
source: LearningSourceSchema2,
|
|
3043
|
+
evidence: z22.array(TaskEvidenceSchema2).min(1).max(1e4),
|
|
3044
|
+
proposals: z22.array(TaskFeedbackSchema).max(1e4)
|
|
3045
|
+
}).strict();
|
|
3046
|
+
|
|
3047
|
+
// src/model-batch-review-preparation.ts
|
|
3048
|
+
import {
|
|
3049
|
+
LearningDomainError,
|
|
3050
|
+
LearningSourceSchema as LearningSourceSchema3,
|
|
3051
|
+
TaskDefinitionSchema as TaskDefinitionSchema3,
|
|
3052
|
+
TaskEvidenceSchema as TaskEvidenceSchema3,
|
|
3053
|
+
TaskFeedbackSchema as TaskFeedbackSchema2,
|
|
3054
|
+
assertLearningContentHash as assertLearningContentHash3,
|
|
3055
|
+
learningEvidenceId,
|
|
3056
|
+
learningRef as learningRef5,
|
|
3057
|
+
sameLearningRef as sameLearningRef4,
|
|
3058
|
+
sealLearningContent as sealLearningContent5,
|
|
3059
|
+
taskBatchPackageMetadata as taskBatchPackageMetadata3,
|
|
3060
|
+
verifyLearningTextAsset as verifyLearningTextAsset5
|
|
3061
|
+
} from "@openpond/evals/learning";
|
|
3062
|
+
import { compileBoundGraders as compileBoundGraders2, createRewardBinding, createRewardRelease, resolveBoundRewards } from "@openpond/evals/rewards";
|
|
3063
|
+
function prepareModelBatchReview(input) {
|
|
3064
|
+
const { request, package: value } = input;
|
|
3065
|
+
const learning = value.learningResources;
|
|
3066
|
+
if (!learning) throw new LearningDomainError("model_review_batch_required", 422);
|
|
3067
|
+
const metadata = taskBatchPackageMetadata3(value.taskset);
|
|
3068
|
+
const id = `model-review-${contentHash([input.scope, request.modelId, request.operationId])}`;
|
|
3069
|
+
const edits = new Map(request.examples.map((edit) => [contentHash(edit.evidence), edit]));
|
|
3070
|
+
const parentRefs = new Set(learning.evidence.map((evidence2) => contentHash(learningRef5(evidence2))));
|
|
3071
|
+
const decisions = new Map(learning.decisions.map((decision) => [contentHash(decision.evidence), decision]));
|
|
3072
|
+
if (edits.size !== request.examples.length || request.examples.some((edit) => !parentRefs.has(contentHash(edit.evidence)))) {
|
|
3073
|
+
throw new LearningDomainError("model_review_evidence_mismatch", 409);
|
|
3074
|
+
}
|
|
3075
|
+
for (const resource of [input.binding, ...input.rewards, ...input.assets]) assertLearningContentHash3(resource);
|
|
3076
|
+
resolveBoundRewards(input.binding, input.rewards);
|
|
3077
|
+
const rewardByHash = /* @__PURE__ */ new Map();
|
|
3078
|
+
for (const source2 of input.binding.sources) {
|
|
3079
|
+
const original = input.rewards.find((reward) => sameLearningRef4(learningRef5(reward), source2.reward));
|
|
3080
|
+
const { contentHash: _hash, ...content } = original;
|
|
3081
|
+
rewardByHash.set(original.contentHash, createRewardRelease({ ...content, id: `${id}-reward-${original.contentHash}`, revision: 1 }));
|
|
3082
|
+
}
|
|
3083
|
+
const rewards = [...rewardByHash.values()];
|
|
3084
|
+
const { contentHash: _bindingHash, recipeRef: _recipe, ...bindingContent } = input.binding;
|
|
3085
|
+
const binding = createRewardBinding({
|
|
3086
|
+
...bindingContent,
|
|
3087
|
+
id: `${id}-binding`,
|
|
3088
|
+
revision: 1,
|
|
3089
|
+
sources: input.binding.sources.map((source2) => ({ ...source2, reward: learningRef5(rewardByHash.get(source2.reward.contentHash)) }))
|
|
3090
|
+
}, rewards);
|
|
3091
|
+
const assetRefs = rewards.flatMap((reward) => [
|
|
3092
|
+
...reward.assets,
|
|
3093
|
+
..."verifierRef" in reward.implementation ? [reward.implementation.verifierRef] : [],
|
|
3094
|
+
..."rubricRef" in reward.implementation ? [reward.implementation.rubricRef] : [],
|
|
3095
|
+
..."inputContract" in reward.implementation ? [reward.implementation.inputContract] : []
|
|
3096
|
+
]);
|
|
3097
|
+
const assets = [...new Map(assetRefs.map((ref) => {
|
|
3098
|
+
const asset = input.assets.find((asset2) => asset2.id === ref.id);
|
|
3099
|
+
if (!asset) throw new LearningDomainError("model_review_reward_asset_missing", 422);
|
|
3100
|
+
verifyLearningTextAsset5(asset, ref);
|
|
3101
|
+
if (ref.visibility === "policy") throw new LearningDomainError("reward_asset_visibility_invalid", 422);
|
|
3102
|
+
return [asset.id, asset];
|
|
3103
|
+
})).values()];
|
|
3104
|
+
const { environmentRelease: _environment, verifierSetRelease: _verifiers, ...execution } = metadata.definition.execution;
|
|
3105
|
+
const { contentHash: _definitionHash, ...definitionContent } = metadata.definition;
|
|
3106
|
+
const definition = TaskDefinitionSchema3.parse(sealLearningContent5({
|
|
3107
|
+
...definitionContent,
|
|
3108
|
+
...request.definition,
|
|
3109
|
+
id: `${id}-definition`,
|
|
3110
|
+
revision: 1,
|
|
3111
|
+
rewardBinding: learningRef5(binding),
|
|
3112
|
+
execution: {
|
|
3113
|
+
...execution,
|
|
3114
|
+
policy: { ...execution.policy, hiddenGraderRefs: compileBoundGraders2(binding, rewards).filter((grader) => grader.privileged).map((grader) => grader.id) }
|
|
3115
|
+
}
|
|
3116
|
+
}));
|
|
3117
|
+
const source = LearningSourceSchema3.parse(sealLearningContent5({
|
|
3118
|
+
schemaVersion: "openpond.learningSource.v1",
|
|
3119
|
+
id,
|
|
3120
|
+
revision: 1,
|
|
3121
|
+
name: `${definition.name} review`,
|
|
3122
|
+
kind: "direct",
|
|
3123
|
+
taskDefinition: learningRef5(definition),
|
|
3124
|
+
enabled: true,
|
|
3125
|
+
allowedSplits: [...new Set(learning.evidence.map((item) => item.submission.split))],
|
|
3126
|
+
mapping: null,
|
|
3127
|
+
adapterVersion: null,
|
|
3128
|
+
reviewOrigin: {
|
|
3129
|
+
modelId: request.modelId,
|
|
3130
|
+
packageHash: value.contentHash,
|
|
3131
|
+
taskset: learningRef5(value.taskset),
|
|
3132
|
+
batch: learningRef5(learning.batch),
|
|
3133
|
+
rewardBinding: learningRef5(input.binding)
|
|
3134
|
+
}
|
|
3135
|
+
}));
|
|
3136
|
+
const evidence = learning.evidence.map((parent) => {
|
|
3137
|
+
const edit = edits.get(contentHash(learningRef5(parent)));
|
|
3138
|
+
const attemptId = `review-${parent.contentHash}`;
|
|
3139
|
+
const policyChanged = contentHash({
|
|
3140
|
+
instructions: definition.instructions,
|
|
3141
|
+
inputSchema: definition.inputSchema,
|
|
3142
|
+
outputSchema: definition.outputSchema,
|
|
3143
|
+
input: edit?.input ?? parent.submission.input
|
|
3144
|
+
}) !== contentHash({
|
|
3145
|
+
instructions: metadata.definition.instructions,
|
|
3146
|
+
inputSchema: metadata.definition.inputSchema,
|
|
3147
|
+
outputSchema: metadata.definition.outputSchema,
|
|
3148
|
+
input: parent.submission.input
|
|
3149
|
+
});
|
|
3150
|
+
return TaskEvidenceSchema3.parse(sealLearningContent5({
|
|
3151
|
+
schemaVersion: "openpond.taskEvidence.v1",
|
|
3152
|
+
id: learningEvidenceId(source.id, parent.submission.exampleId, attemptId),
|
|
3153
|
+
revision: 1,
|
|
3154
|
+
source: learningRef5(source),
|
|
3155
|
+
submission: {
|
|
3156
|
+
...parent.submission,
|
|
3157
|
+
sourceId: source.id,
|
|
3158
|
+
taskDefinition: learningRef5(definition),
|
|
3159
|
+
attemptId,
|
|
3160
|
+
// The old response was observed under the old policy-visible task. Its
|
|
3161
|
+
// parent snapshot remains available, but it is not a run of new inputs.
|
|
3162
|
+
observedOutput: policyChanged ? null : parent.submission.observedOutput,
|
|
3163
|
+
idempotencyKey: `${id}-${parent.contentHash}`,
|
|
3164
|
+
...edit?.input === void 0 ? {} : { input: edit.input },
|
|
3165
|
+
...edit?.expected === void 0 ? {} : { expected: edit.expected },
|
|
3166
|
+
...edit?.evaluatorContext === void 0 ? {} : { evaluatorContext: edit.evaluatorContext }
|
|
3167
|
+
},
|
|
3168
|
+
supersedes: learningRef5(parent),
|
|
3169
|
+
correctionFeedbackId: null,
|
|
3170
|
+
receivedAt: input.now
|
|
3171
|
+
}));
|
|
3172
|
+
});
|
|
3173
|
+
const proposals = evidence.flatMap((item) => {
|
|
3174
|
+
const edit = edits.get(contentHash(item.supersedes));
|
|
3175
|
+
const decision = decisions.get(contentHash(item.supersedes));
|
|
3176
|
+
const target = edit?.proposedTarget === void 0 ? decision.approvedTarget : edit.proposedTarget;
|
|
3177
|
+
return target === null ? [] : [TaskFeedbackSchema2.parse({
|
|
3178
|
+
schemaVersion: "openpond.taskFeedbackRecord.v1",
|
|
3179
|
+
id: `feedback-${contentHash([id, item.id])}`,
|
|
3180
|
+
revision: 1,
|
|
3181
|
+
submission: {
|
|
3182
|
+
schemaVersion: "openpond.taskFeedback.v1",
|
|
3183
|
+
sourceId: source.id,
|
|
3184
|
+
idempotencyKey: `${id}-${item.id}`,
|
|
3185
|
+
exampleId: item.submission.exampleId,
|
|
3186
|
+
attemptId: item.submission.attemptId,
|
|
3187
|
+
expectedEvidenceHash: item.contentHash,
|
|
3188
|
+
occurredAt: input.now,
|
|
3189
|
+
kind: "target_correction",
|
|
3190
|
+
value: target,
|
|
3191
|
+
note: "Proposed target for the revised task. Grade and review before admission."
|
|
3192
|
+
},
|
|
3193
|
+
status: "pending_review",
|
|
3194
|
+
evidence: learningRef5(item),
|
|
3195
|
+
createdAt: input.now,
|
|
3196
|
+
review: null
|
|
3197
|
+
})];
|
|
3198
|
+
});
|
|
3199
|
+
return { source, definition, binding, rewards, assets, evidence, proposals };
|
|
3200
|
+
}
|
|
3201
|
+
|
|
3202
|
+
// src/model-batch-review.ts
|
|
3203
|
+
import { LearningDomainError as LearningDomainError3 } from "@openpond/evals/learning";
|
|
3204
|
+
var operation = (request) => `model-batch-review-${contentHash([request.modelId, request.operationId])}`;
|
|
3205
|
+
function inspectModelBatchPackage(raw) {
|
|
3206
|
+
const value = validateTasksetPackage(raw);
|
|
3207
|
+
if (!value.learningResources) throw new LearningDomainError2("model_review_batch_required", 422);
|
|
3208
|
+
const metadata = taskBatchPackageMetadata4(value.taskset);
|
|
3209
|
+
return ModelBatchReviewInspectionSchema.parse({
|
|
3210
|
+
schemaVersion: "openpond.modelBatchReviewInspection.v1",
|
|
3211
|
+
packageHash: value.contentHash,
|
|
3212
|
+
definition: metadata.definition,
|
|
3213
|
+
binding: metadata.binding,
|
|
3214
|
+
evidence: value.learningResources.evidence,
|
|
3215
|
+
decisions: value.learningResources.decisions
|
|
3216
|
+
});
|
|
3217
|
+
}
|
|
3218
|
+
async function findModelBatchReview(transaction, raw) {
|
|
3219
|
+
const request = ModelBatchReviewRequestSchema.parse(raw);
|
|
3220
|
+
const prior = await transaction.operation(operation(request));
|
|
3221
|
+
if (!prior) return null;
|
|
3222
|
+
if (prior.requestHash !== contentHash(request)) throw new LearningDomainError2("learning_idempotency_conflict", 409);
|
|
3223
|
+
return readReceipt(transaction, request, prior.resources);
|
|
3224
|
+
}
|
|
3225
|
+
async function beginModelBatchReview(transaction, input) {
|
|
3226
|
+
const request = ModelBatchReviewRequestSchema.parse(input.request);
|
|
3227
|
+
const prior = await findModelBatchReview(transaction, request);
|
|
3228
|
+
if (prior) return prior;
|
|
3229
|
+
const value = validateTasksetPackage(input.package);
|
|
3230
|
+
if (!value.learningResources) throw new LearningDomainError2("model_review_batch_required", 422);
|
|
3231
|
+
const metadata = taskBatchPackageMetadata4(value.taskset);
|
|
3232
|
+
const selected = request.rewardBindingRef;
|
|
3233
|
+
const retained = !selected || sameLearningRef5(selected, learningRef6(metadata.binding));
|
|
3234
|
+
const binding = retained ? metadata.binding : await requireLearningRelease(transaction, "binding", selected);
|
|
3235
|
+
const rewards = retained ? metadata.rewards : await Promise.all(binding.sources.map((source) => requireLearningRelease(transaction, "reward", source.reward)));
|
|
3236
|
+
const assetIds = new Set(rewards.flatMap((reward) => [
|
|
3237
|
+
...reward.assets,
|
|
3238
|
+
..."verifierRef" in reward.implementation ? [reward.implementation.verifierRef] : [],
|
|
3239
|
+
..."rubricRef" in reward.implementation ? [reward.implementation.rubricRef] : [],
|
|
3240
|
+
..."inputContract" in reward.implementation ? [reward.implementation.inputContract] : []
|
|
3241
|
+
]).map((asset) => asset.id));
|
|
3242
|
+
const assets = retained ? value.learningResources.assets : await Promise.all([...assetIds].map((id) => requireLearningResource(transaction, "asset", id, 1)));
|
|
3243
|
+
const prepared = prepareModelBatchReview({ ...input, request, package: value, binding, rewards, assets });
|
|
3244
|
+
const pointers = [];
|
|
3245
|
+
for (const asset of prepared.assets) {
|
|
3246
|
+
const existing = await transaction.get("asset", asset.id, 1);
|
|
3247
|
+
if (existing && existing.contentHash !== asset.contentHash) throw new LearningDomainError2("model_review_asset_conflict", 409);
|
|
3248
|
+
if (!existing) await transaction.put("asset", asset, 0);
|
|
3249
|
+
}
|
|
3250
|
+
for (const reward of prepared.rewards) await transaction.put("reward", reward, 0);
|
|
3251
|
+
await transaction.put("binding", prepared.binding, 0);
|
|
3252
|
+
await transaction.put("definition", prepared.definition, 0);
|
|
3253
|
+
await transaction.put("source", prepared.source, 0, { parentId: request.modelId });
|
|
3254
|
+
pointers.push({ kind: "source", id: prepared.source.id, revision: 1 });
|
|
3255
|
+
for (const evidence of prepared.evidence) {
|
|
3256
|
+
await transaction.put("evidence", evidence, 0, { parentId: prepared.source.id });
|
|
3257
|
+
pointers.push({ kind: "evidence", id: evidence.id, revision: 1 });
|
|
3258
|
+
}
|
|
3259
|
+
for (const proposal of prepared.proposals) {
|
|
3260
|
+
await transaction.put("feedback", proposal, 0, { parentId: proposal.evidence.id, status: proposal.status });
|
|
3261
|
+
pointers.push({ kind: "feedback", id: proposal.id, revision: 1 });
|
|
3262
|
+
}
|
|
3263
|
+
await transaction.saveOperation(operation(request), { requestHash: contentHash(request), resources: pointers });
|
|
3264
|
+
return readReceipt(transaction, request, pointers);
|
|
3265
|
+
}
|
|
3266
|
+
async function readReceipt(transaction, request, pointers) {
|
|
3267
|
+
const resources = await Promise.all(pointers.map((pointer) => requireLearningResource(transaction, pointer.kind, pointer.id, pointer.revision)));
|
|
3268
|
+
return ModelBatchReviewReceiptSchema.parse({
|
|
3269
|
+
schemaVersion: "openpond.modelBatchReviewReceipt.v1",
|
|
3270
|
+
operationId: request.operationId,
|
|
3271
|
+
modelId: request.modelId,
|
|
3272
|
+
source: resources.find((resource) => resource.schemaVersion === "openpond.learningSource.v1"),
|
|
3273
|
+
evidence: resources.filter((resource) => resource.schemaVersion === "openpond.taskEvidence.v1"),
|
|
3274
|
+
proposals: resources.filter((resource) => resource.schemaVersion === "openpond.taskFeedbackRecord.v1")
|
|
3275
|
+
});
|
|
3276
|
+
}
|
|
3277
|
+
|
|
3278
|
+
// src/model-taskset-authoring.ts
|
|
3279
|
+
import { createVerifierSetRelease as createVerifierSetRelease2 } from "@openpond/evals";
|
|
3280
|
+
import { learningRef as learningRef7, sameLearningRef as sameLearningRef6, sealLearningContent as sealLearningContent6 } from "@openpond/evals/learning";
|
|
3281
|
+
import { TasksetReleaseSchema as TasksetReleaseSchema3 } from "@openpond/evals/tasksets";
|
|
3282
|
+
function prepareModelTasksetDraft(input) {
|
|
3283
|
+
const request = ModelTasksetDraftRequestSchema.parse(input.request);
|
|
3284
|
+
const owner = ModelTasksetAuthoringOwnerSchema.parse(input.owner);
|
|
3285
|
+
const source = requireOrdinaryPackage(input.source);
|
|
3286
|
+
if (source.contentHash !== request.sourcePackageHash) throw new Error("Taskset draft source package changed.");
|
|
3287
|
+
const previous = assertModelTasksetAuthoring(source.taskset);
|
|
3288
|
+
const owned = previous && contentHash(previous.owner) === contentHash(owner);
|
|
3289
|
+
const root = owned ? previous.root : learningRef7(source.taskset);
|
|
3290
|
+
const lineage = { schemaVersion: "openpond.modelTasksetAuthoring.v1", owner, root, parent: learningRef7(source.taskset) };
|
|
3291
|
+
return ModelTasksetDraftPreparationSchema.parse({
|
|
3292
|
+
schemaVersion: "openpond.modelTasksetDraftPreparation.v1",
|
|
3293
|
+
draftId: `model-taskset-draft-${contentHash({ owner, operationId: request.operationId })}`,
|
|
3294
|
+
requestHash: contentHash(request),
|
|
3295
|
+
sourcePackageHash: source.contentHash,
|
|
3296
|
+
sourceTasksetRef: learningRef7(source.taskset),
|
|
3297
|
+
tasksetId: modelAuthoredTasksetId(lineage),
|
|
3298
|
+
tasksetRevision: owned ? source.taskset.revision + 1 : 1,
|
|
3299
|
+
lineage
|
|
3300
|
+
});
|
|
3301
|
+
}
|
|
3302
|
+
function publishModelTasksetDraftPackage(input) {
|
|
3303
|
+
const prepared = ModelTasksetDraftPreparationSchema.parse(input.preparation);
|
|
3304
|
+
if (!sameLearningRef6(prepared.sourceTasksetRef, prepared.lineage.parent)) throw new Error("Taskset draft preparation differs from its source lineage.");
|
|
3305
|
+
const edited = requireOrdinaryPackage(input.edited);
|
|
3306
|
+
const { contentHash: _verifierHash, ...verifierContent } = edited.verifierSet;
|
|
3307
|
+
const verifierSet = createVerifierSetRelease2({
|
|
3308
|
+
...verifierContent,
|
|
3309
|
+
id: `${prepared.tasksetId}-verifiers`,
|
|
3310
|
+
revision: prepared.tasksetRevision,
|
|
3311
|
+
calibrationReceiptRefs: [],
|
|
3312
|
+
metadata: { sourceVerifierSet: learningRef7(edited.verifierSet) }
|
|
3313
|
+
});
|
|
3314
|
+
const { contentHash: _hash, ...content } = edited.taskset;
|
|
3315
|
+
const taskset = TasksetReleaseSchema3.parse(sealLearningContent6({
|
|
3316
|
+
...content,
|
|
3317
|
+
id: prepared.tasksetId,
|
|
3318
|
+
revision: prepared.tasksetRevision,
|
|
3319
|
+
verifierSetRelease: { id: verifierSet.id, contentHash: verifierSet.contentHash },
|
|
3320
|
+
metadata: {
|
|
3321
|
+
modelTasksetAuthoring: prepared.lineage,
|
|
3322
|
+
...edited.taskset.metadata.environmentResources === void 0 ? {} : { environmentResources: edited.taskset.metadata.environmentResources },
|
|
3323
|
+
...edited.taskset.metadata.ordinaryAuthoring === void 0 ? {} : { ordinaryAuthoring: edited.taskset.metadata.ordinaryAuthoring }
|
|
3324
|
+
}
|
|
3325
|
+
}));
|
|
3326
|
+
const resolved = resolveTasksetPackageExecution(edited);
|
|
3327
|
+
const files = resolved ? [
|
|
3328
|
+
...edited.files.filter((file) => file.asset.id !== modelStarterExecutionAssetId(edited.taskset)),
|
|
3329
|
+
createTasksetPackageExecutionFile({ ...resolved.execution, verifierSet })
|
|
3330
|
+
] : edited.files;
|
|
3331
|
+
return createTasksetPackage({ schemaVersion: "openpond.tasksetPackage.v1", taskset, environment: edited.environment, verifierSet, files });
|
|
3332
|
+
}
|
|
3333
|
+
function requireOrdinaryPackage(value) {
|
|
3334
|
+
const result = validateTasksetPackage(value);
|
|
3335
|
+
if (result.modelResources || result.learningResources) throw new Error("This Taskset requires its bound or reviewed authoring workflow.");
|
|
3336
|
+
return result;
|
|
3337
|
+
}
|
|
2867
3338
|
export {
|
|
2868
3339
|
MAX_TASKSET_PACKAGE_BYTES,
|
|
3340
|
+
LearningDomainError3 as ModelBatchReviewError,
|
|
3341
|
+
ModelBatchReviewInspectionSchema,
|
|
3342
|
+
ModelBatchReviewReceiptSchema,
|
|
3343
|
+
ModelBatchReviewRequestSchema,
|
|
3344
|
+
ModelTasksetAuthoringOwnerSchema,
|
|
3345
|
+
ModelTasksetAuthoringSchema,
|
|
3346
|
+
ModelTasksetDraftPreparationSchema,
|
|
3347
|
+
ModelTasksetDraftRequestSchema,
|
|
2869
3348
|
OpenPondTasksetPackageClient,
|
|
2870
3349
|
OpenPondTasksetPackageError,
|
|
3350
|
+
TasksetDraftFileContentSchema,
|
|
3351
|
+
TasksetDraftFileInfoSchema,
|
|
3352
|
+
TasksetDraftFileMutationSchema,
|
|
3353
|
+
TasksetDraftFilePathSchema,
|
|
3354
|
+
TasksetDraftFileSchema,
|
|
2871
3355
|
TasksetPackageContentSchema,
|
|
2872
3356
|
TasksetPackageFileSchema,
|
|
2873
3357
|
TasksetPackageLearningResourcesSchema,
|
|
@@ -2876,9 +3360,16 @@ export {
|
|
|
2876
3360
|
TasksetPackageReadbackSchema,
|
|
2877
3361
|
TasksetPackageReceiptSchema,
|
|
2878
3362
|
TasksetPackageSchema,
|
|
3363
|
+
beginModelBatchReview,
|
|
2879
3364
|
createTasksetPackage,
|
|
3365
|
+
createTasksetPackageExecutionFile,
|
|
2880
3366
|
decodeTasksetPackageFile,
|
|
3367
|
+
findModelBatchReview,
|
|
3368
|
+
inspectModelBatchPackage,
|
|
2881
3369
|
learningPackageContextFiles,
|
|
3370
|
+
prepareModelTasksetDraft,
|
|
3371
|
+
publishModelTasksetDraftPackage,
|
|
3372
|
+
resolveTasksetPackageExecution,
|
|
2882
3373
|
tasksetPackageRewardBinding,
|
|
2883
3374
|
validateTasksetLearningResources,
|
|
2884
3375
|
validateTasksetPackage
|