openpond-sdk 0.12.2 → 0.12.3

package/dist/training-bundle.js

@@ -1,32 +1,37 @@
 import {
+  prepareImportedTasksetPackage,
   projectLearningBatchGraders
-} from "./chunk-PGBFI5HC.js";
+} from "./chunk-FY7NHCEH.js";
 import {
+  projectPortableTaskRecord,
   resolvePortableTasksetRewardExecution
 } from "./chunk-ROBAX5TN.js";
 import {
   TasksetSchema,
   computeTasksetHash
-} from "./chunk-7AZDNZP6.js";
+} from "./chunk-CWXM57QR.js";
 import {
   LearningBatchDatasetSourceRefSchema,
-  MAX_TASKSET_PACKAGE_BYTES
+  MAX_TASKSET_PACKAGE_BYTES,
+  decodeTasksetPackageFile,
+  validateTasksetPackage
 } from "./chunk-FHCSDXIA.js";
 import "./chunk-2U54EOXJ.js";
 import {
   TRAINING_EVALUATION_SOURCE_PATH,
   TrainingEvaluationSourceSchema,
   assertTrainingEvaluationIsolation,
+  deterministicTrainingRewardSource,
   parseAndVerifyTrainingInputArtifactUpload,
   parseAndVerifyTrainingJobSubmission,
   trainingEvaluationSourceRef,
   trainingInputArtifactUploadHash,
   trainingJobSubmissionHash
-} from "./chunk-YVFERCUC.js";
+} from "./chunk-TMUAKJWX.js";
 import {
   TrainingHandoffOriginSchema
-} from "./chunk-OEPYLZXK.js";
-import "./chunk-YSKUXQ2N.js";
+} from "./chunk-MLPPSCDQ.js";
+import "./chunk-G53MA5XX.js";
 import {
   HarnessSourceSelectionSchema,
   createAgentSnapshot,
@@ -35,8 +40,8 @@ import {
   validateHarnessSourcePackage
 } from "./chunk-DUKW3R7O.js";
 import "./chunk-KAE7AMYE.js";
-import "./chunk-L5PQPAU6.js";
-import "./chunk-7WUJBFCH.js";
+import "./chunk-QTJQIHDZ.js";
+import "./chunk-ZJ3H3VS7.js";
 import "./chunk-CK3ONRRC.js";
 import "./chunk-TCHAWJMB.js";
 import "./chunk-3KXDH6OB.js";
@@ -51,7 +56,7 @@ import {
 } from "./chunk-COPVBJPL.js";
 
 // src/training-bundle.ts
-import { compileBoundGraders } from "@openpond/evals/rewards";
+import { compileBoundGraders as compileBoundGraders2 } from "@openpond/evals/rewards";
 import { verifyLearningTextAsset as verifyLearningTextAsset3 } from "@openpond/evals/learning";
 
 // src/training-bundle-contracts.ts
@@ -106,6 +111,7 @@ var TrainingEngineBindingSchema = z.object({
 }).strict();
 var HarnessRunManifestContentSchema = z.object({
   schemaVersion: z.literal("openpond.harnessRunManifest.v1"),
+  executionMode: z.literal("qualification").optional(),
   id: ReleaseIdSchema,
   harnessRelease: ImmutableReleaseRefSchema,
   datasetRelease: ImmutableReleaseRefSchema,
@@ -139,6 +145,7 @@ var HarnessRunManifestSchema = HarnessRunManifestContentSchema.extend({
 }).strict();
 var ResolvedTrainingBundleContentSchema = z.object({
   schemaVersion: z.literal("openpond.resolvedTrainingBundle.v1"),
+  executionMode: z.literal("qualification").optional(),
   projection: z.literal("trainer"),
   harnessRelease: ImmutableReleaseRefSchema,
   datasetRelease: ImmutableReleaseRefSchema,
@@ -354,6 +361,13 @@ async function prepareManagedTrainingSubmission(raw) {
   const { project, recipe, taskset, approval } = input;
   const manifest = HarnessRunManifestSchema.parse(input.manifest);
   const bundle = ResolvedTrainingBundleManifestSchema.parse(input.bundleManifest);
+  const qualification = input.executionMode === "qualification";
+  if (manifest.executionMode !== input.executionMode || bundle.executionMode !== input.executionMode) {
+    throw new Error("Managed submission execution mode differs from its immutable bundle.");
+  }
+  if (qualification && (input.evaluationSource || input.resumeFrom || input.rewardSource.kind !== "deterministic")) {
+    throw new Error("Qualification requires the original learner, deterministic rewards, and no evaluation source.");
+  }
   const { contentHash: manifestHash, ...manifestContent } = manifest;
   const { contentHash: bundleHash, ...bundleContent } = bundle;
   if (contentHash(manifestContent) !== manifestHash || contentHash(bundleContent) !== bundleHash || manifest.resolvedBundleHash !== bundleHash || contentHash(manifest.harnessRelease) !== contentHash(bundle.harnessRelease) || contentHash(manifest.datasetRelease) !== contentHash(bundle.datasetRelease) || contentHash(manifest.evidenceSets) !== contentHash(bundle.evidenceSetRelease ? [bundle.evidenceSetRelease] : [])) {
@@ -372,14 +386,21 @@ async function prepareManagedTrainingSubmission(raw) {
   if (input.idempotencyKey !== void 0)
     z2.string().trim().min(16).max(191).parse(input.idempotencyKey);
   const files = verifyFiles(bundle, input.files);
-  const evaluationSource = TrainingEvaluationSourceSchema.parse(input.evaluationSource);
+  const evaluationSource = qualification ? null : TrainingEvaluationSourceSchema.parse(input.evaluationSource);
   const evaluationFile = files.find((file) => file.path === TRAINING_EVALUATION_SOURCE_PATH);
-  if (!evaluationFile || contentHash(JSON.parse(Buffer.from(evaluationFile.content, "base64").toString("utf8"))) !== contentHash(evaluationSource)) {
+  if (evaluationSource && (!evaluationFile || contentHash(JSON.parse(Buffer.from(evaluationFile.content, "base64").toString("utf8"))) !== contentHash(evaluationSource))) {
     throw new Error("The evaluation source differs from the immutable training bundle.");
   }
-  const evaluation = await trainingEvaluationSourceRef(evaluationSource);
+  if (qualification && (evaluationFile || files.some((file) => file.path === "dataset/train.json") || !files.some((file) => file.path === "dataset/qualification.json") || !files.some((file) => file.path === "qualification-source.json"))) {
+    throw new Error("Qualification requires its dedicated dataset and no training/evaluation datasets.");
+  }
+  if (!qualification && files.some((file) => file.path === "dataset/qualification.json" || file.path === "qualification-source.json")) {
+    throw new Error("Qualification datasets cannot be prepared as training.");
+  }
+  const evaluation = evaluationSource ? await trainingEvaluationSourceRef(evaluationSource) : null;
   const payload = {
     schemaVersion: "openpond.managedRlPortableSubmission.v1",
+    ...qualification ? { executionMode: "qualification" } : {},
     ...input.origin ? { origin: TrainingHandoffOriginSchema.parse(input.origin) } : {},
     sourceRunRef: `openpond:model-run:${manifest.id}`,
     name: `OpenPond Managed \xB7 ${project.id}`.slice(0, 191),
@@ -393,8 +414,8 @@ async function prepareManagedTrainingSubmission(raw) {
     modelImprovementQualification: input.modelImprovementQualification ?? null,
     recipe,
     resolvedBundle: { manifest: bundle, files },
-    validationTasks: evaluationSource.tasks,
-    validationAssets: evaluationSource.assets
+    validationTasks: evaluationSource?.tasks ?? [],
+    validationAssets: evaluationSource?.assets ?? []
   };
   const artifactContent = {
     schemaVersion: "openpond.trainingInputArtifactUpload.v2",
@@ -424,18 +445,19 @@ async function prepareManagedTrainingSubmission(raw) {
       taskset,
       tasksetRelease: { id: taskset.id, contentHash: taskset.contentHash },
       dataset: bundle.datasetRelease,
-      evaluation,
+      ...evaluation ? { evaluation } : {},
       evidenceSets: bundle.evidenceSetRelease ? [bundle.evidenceSetRelease] : []
     },
     job: {
       kind: "policy_optimize",
+      ...qualification ? { executionMode: "qualification" } : {},
       baseModel,
       recipe,
       rewardSource: input.rewardSource,
       resumeFrom: input.resumeFrom
     },
     requestedCapabilities: [
-      { id: "managed_rl.policy.grpo", version: "1", required: true },
+      { id: qualification ? "managed_rl.policy.qualification" : "managed_rl.policy.grpo", version: "1", required: true },
       {
         id: `managed_rl.rollouts.${manifest.runtimeTarget.placement}`,
         version: "1",
@@ -734,10 +756,114 @@ function asset(input) {
   };
 }
 
+// src/training-qualification-preparation.ts
+import { createRewardBinding, createRewardRelease, compileBoundGraders } from "@openpond/evals/rewards";
+import { LearningTextAssetSchema, sealLearningContent } from "@openpond/evals/learning";
+async function prepareOrdinaryTasksetQualification(input) {
+  const source = validateTasksetPackage(input.package);
+  if (source.modelResources || source.learningResources || !source.taskset.tasks.length) {
+    throw new Error("Qualification requires a nonempty ordinary Taskset package.");
+  }
+  if (source.taskset.tasks.some((task) => task.privilegedContextRef !== null || task.artifactRefs.some((asset2) => asset2.visibility !== "policy"))) {
+    throw new Error("Qualification runtime requires private targets in expectedOutput and policy-only task assets.");
+  }
+  const rewards = source.taskset.graders.map((grader) => {
+    const { id, version, weight: _weight, hardGate: _gate, rewardEligible: _eligible, privileged: _private, taskIds: _tasks, ...implementation } = grader;
+    const revision = Number(version);
+    if (!Number.isSafeInteger(revision) || revision < 1 || String(revision) !== version || implementation.kind === "model_judge" || implementation.kind === "human") {
+      throw new Error("Qualification requires deterministic graders with canonical integer versions.");
+    }
+    return createRewardRelease({
+      schemaVersion: "openpond.rewardRelease.v1",
+      id: `qualification-${contentHash({ source: source.contentHash, id }).slice(0, 40)}`,
+      revision,
+      name: id,
+      description: "Exact ordinary package grader for no-update qualification.",
+      implementation,
+      rawScore: { minimum: 0, maximum: 1 },
+      assets: implementation.kind === "custom_verifier" ? [implementation.verifierRef] : []
+    });
+  });
+  const binding = createRewardBinding({
+    schemaVersion: "openpond.rewardBinding.v1",
+    id: `qualification-${source.contentHash.slice(0, 40)}`,
+    revision: 1,
+    sources: source.taskset.graders.map((grader, index) => ({
+      graderId: grader.id,
+      ...grader.taskIds ? { taskIds: grader.taskIds } : {},
+      reward: { id: rewards[index].id, revision: rewards[index].revision, contentHash: rewards[index].contentHash },
+      role: grader.rewardEligible ? "training" : "evaluation",
+      normalization: { kind: "identity" },
+      weight: grader.weight,
+      required: true,
+      hardGate: grader.hardGate,
+      privileged: grader.privileged,
+      fixtureRefs: []
+    })),
+    aggregation: "weighted_mean",
+    unscorable: "exclude_optional_require_all_required"
+  }, rewards);
+  if (!binding.sources.some((item) => item.role === "training" && item.weight > 0) || contentHash(compileBoundGraders(binding, rewards)) !== contentHash(source.taskset.graders)) {
+    throw new Error("Qualification must preserve an executable positively weighted source grader set.");
+  }
+  const verifierIds = new Set(rewards.flatMap((reward) => reward.assets.map((asset2) => asset2.id)));
+  const verifierAssets = source.files.filter((file) => verifierIds.has(file.asset.id)).map((file) => LearningTextAssetSchema.parse(sealLearningContent({
+    schemaVersion: "openpond.learningTextAsset.v1",
+    id: file.asset.id,
+    revision: 1,
+    asset: file.asset,
+    text: new TextDecoder("utf-8", { fatal: true }).decode(decodeTasksetPackageFile(file))
+  })));
+  const { taskset } = prepareImportedTasksetPackage({
+    package: source,
+    profileId: input.modelProject.id,
+    name: source.taskset.id,
+    createdAt: input.modelProject.updatedAt
+  });
+  const taskAssets = new Set(taskset.tasks.flatMap((task) => (task.assets ?? []).map((asset2) => asset2.artifactRef)));
+  const tasksetAssetBytes = new Map(source.files.filter((file) => taskAssets.has(file.asset.path)).map((file) => [file.asset.path, decodeTasksetPackageFile(file)]));
+  const tasksetRef = { id: source.taskset.id, revision: source.taskset.revision, contentHash: source.taskset.contentHash };
+  const rewardExecution = { binding, rewards };
+  const bundle = buildTasksetTrainingBundle({
+    ...input,
+    taskset,
+    executionMode: "qualification",
+    qualificationSource: source,
+    tasksetRelease: { id: tasksetRef.id, contentHash: tasksetRef.contentHash },
+    rewardExecution,
+    verifierAssets,
+    tasksetAssetBytes,
+    approval: { approvalHash: input.approval.approvalHash, approvedAt: input.approval.approvedAt, maximumSpendUsd: input.approval.maximumSpendUsd }
+  });
+  const recipe = input.modelProject.trainingSetup.recipe;
+  if (!recipe || recipe.method !== "grpo") throw new Error("Qualification requires a pinned GRPO rollout recipe.");
+  const prepared = await prepareManagedTrainingSubmission({
+    executionMode: "qualification",
+    project: input.modelProject,
+    manifest: bundle.manifest,
+    bundleManifest: bundle.resolvedBundleManifest,
+    taskset: tasksetRef,
+    recipe,
+    files: [...bundle.assets].map(([path, bytes2]) => ({ ...bundle.resolvedBundleManifest.files.find((file) => file.path === path), encoding: "base64", content: Buffer.from(bytes2).toString("base64") })),
+    rewardSource: await deterministicTrainingRewardSource({ graders: source.taskset.graders, rewardExecution }),
+    resumeFrom: null,
+    approval: { approvalHash: input.approval.approvalHash, maximumSpendUsd: input.approval.maximumSpendUsd, retentionDays: input.approval.retentionDays, region: input.approval.region },
+    ...input.idempotencyKey ? { idempotencyKey: input.idempotencyKey } : {}
+  });
+  return { ...prepared, bundle };
+}
+
 // src/training-bundle.ts
 function buildTasksetTrainingBundle(input) {
   const { taskset, modelProject, harnessRelease, tasksetRelease } = input;
   const setup = modelProject.trainingSetup;
+  const qualification = input.executionMode === "qualification";
+  if (qualification && input.evaluationSource) throw new Error("Qualification does not consume an independent evaluation source.");
+  const qualificationSource = qualification ? validateTasksetPackage(input.qualificationSource) : null;
+  if (!qualification && input.qualificationSource) throw new Error("Qualification sources cannot be used for training.");
+  if (qualificationSource && (qualificationSource.modelResources || qualificationSource.learningResources || contentHash(taskset.tasks.map(projectPortableTaskRecord)) !== contentHash(qualificationSource.taskset.tasks) || releasedRefHash(input.tasksetRelease) !== releasedRefHash(qualificationSource.taskset))) {
+    throw new Error("Qualification must preserve its ordinary source release and complete task population.");
+  }
   const releasedHarness = ImmutableReleaseRefSchema.parse(harnessRelease);
   const releasedTaskset = ImmutableReleaseRefSchema.parse(tasksetRelease);
   void input.openpondRelease;
@@ -748,7 +874,7 @@ function buildTasksetTrainingBundle(input) {
       "Taskset authoring state changed after its release was selected."
     );
   }
-  if (!setup.baseModel?.revision || !setup.baseModel.tokenizerRevision || !setup.baseModel.chatTemplateHash || !setup.recipe || !setup.tasksetRef || setup.tasksetRef.id !== taskset.id || setup.tasksetRef.contentHash !== taskset.contentHash) {
+  if (!setup.baseModel?.revision || !setup.baseModel.tokenizerRevision || !setup.baseModel.chatTemplateHash || !setup.recipe || !setup.tasksetRef || setup.tasksetRef.id !== (qualification ? tasksetRelease.id : taskset.id) || setup.tasksetRef.contentHash !== (qualification ? tasksetRelease.contentHash : taskset.contentHash)) {
     throw new Error(
       "Model Run must bind an exact Taskset, Model revision, tokenizer, chat template, and Recipe."
     );
@@ -763,6 +889,7 @@ function buildTasksetTrainingBundle(input) {
     })
   };
   const assets = /* @__PURE__ */ new Map();
+  if (qualificationSource) addJsonAsset(assets, "qualification-source.json", qualificationSource);
   if (input.evaluationSource) {
     const evaluation = TrainingEvaluationSourceSchema.parse(input.evaluationSource);
     if (contentHash(evaluation.taskset) !== contentHash(setup.evaluationTasksetRef)) {
@@ -770,7 +897,7 @@ function buildTasksetTrainingBundle(input) {
     }
     assertTrainingEvaluationIsolation(taskset.tasks.filter((task) => task.split === "train"), evaluation.tasks);
     assets.set(TRAINING_EVALUATION_SOURCE_PATH, new TextEncoder().encode(canonicalJson(evaluation)));
-  } else if (setup.evaluationTasksetRef) {
+  } else if (!qualification && setup.evaluationTasksetRef) {
     throw new Error("The prepared Model requires its pinned evaluation source bytes.");
   }
   const harnessSource = input.harnessSource ? input.harnessSource.schemaVersion === "openpond.harnessPolicySourcePackage.v1" ? validateHarnessPolicySourcePackage(input.harnessSource, releasedHarness) : validateHarnessSourcePackage(input.harnessSource, releasedHarness) : null;
@@ -794,11 +921,14 @@ function buildTasksetTrainingBundle(input) {
     sourcePackageHash: harnessSource?.contentHash ?? null
   }));
   const rewardExecution = resolvePortableTasksetRewardExecution(taskset, input.rewardExecution);
+  if (qualificationSource && (!rewardExecution || contentHash(compileBoundGraders2(rewardExecution.binding, rewardExecution.rewards)) !== contentHash(qualificationSource.taskset.graders) || qualificationSource.taskset.graders.some((grader) => grader.kind === "model_judge" || grader.kind === "human"))) {
+    throw new Error("Qualification requires the source package's exact deterministic graders.");
+  }
   if (rewardExecution) {
     const verifierAssets = input.verifierAssets ?? [];
     const references = [
       ...rewardExecution.rewards.flatMap((reward) => reward.assets),
-      ...compileBoundGraders(rewardExecution.binding, rewardExecution.rewards).flatMap((grader) => grader.kind === "custom_verifier" ? [grader.verifierRef] : grader.kind === "model_judge" ? [grader.rubricRef] : [])
+      ...compileBoundGraders2(rewardExecution.binding, rewardExecution.rewards).flatMap((grader) => grader.kind === "custom_verifier" ? [grader.verifierRef] : grader.kind === "model_judge" ? [grader.rubricRef] : [])
     ];
     const expected = new Set(references.map((reference) => reference.id));
     if (new Set(verifierAssets.map((asset2) => asset2.id)).size !== verifierAssets.length || new Set(verifierAssets.map((asset2) => asset2.asset.path)).size !== verifierAssets.length || verifierAssets.some((asset2) => !expected.has(asset2.id))) throw new Error("Unexpected or duplicate private verifier asset.");
@@ -813,7 +943,7 @@ function buildTasksetTrainingBundle(input) {
   }
   addJsonAsset(assets, "environment.json", {
     schemaVersion: "openpond.harnessEnvironment.v1",
-    environment: taskset.environment,
+    environment: qualification ? { ...taskset.environment, metadata: { ...taskset.environment.metadata, policySystemPrompt: taskset.objective } } : taskset.environment,
     capabilities: taskset.capabilities,
     policy: taskset.policy,
     outputContract: taskset.metadata.tasksetOutputContract?.mode === "text" ? null : taskset.metadata.tasksetOutputContract ?? null
@@ -830,11 +960,11 @@ function buildTasksetTrainingBundle(input) {
     capabilities: taskset.capabilities,
     connectedAppScopes: taskset.policy.connectedAppScopes
   });
-  const trainTasks = taskset.tasks.filter((task) => task.split === "train");
+  const trainTasks = qualification ? taskset.tasks : taskset.tasks.filter((task) => task.split === "train");
   if (trainTasks.length > 0) {
-    addJsonAsset(assets, "dataset/train.json", {
-      schemaVersion: "openpond.datasetSplit.v1",
-      split: "train",
+    addJsonAsset(assets, qualification ? "dataset/qualification.json" : "dataset/train.json", {
+      schemaVersion: qualification ? "openpond.qualificationDataset.v1" : "openpond.datasetSplit.v1",
+      ...qualification ? {} : { split: "train" },
       tasks: trainTasks
     });
     addTasksetAssets({
@@ -858,7 +988,7 @@ function buildTasksetTrainingBundle(input) {
       taskset: releasedTaskset,
       sourceRefs: taskset.sourceRefs,
       datasetArtifact: taskset.datasetArtifact ?? null,
-      trainTasks
+      ...qualification ? { executionMode: "qualification", tasks: trainTasks } : { trainTasks }
     })
   };
   const evidenceSetRelease = approvedSignals.length > 0 ? {
@@ -873,6 +1003,7 @@ function buildTasksetTrainingBundle(input) {
   } : null;
   const bundleContent = ResolvedTrainingBundleContentSchema.parse({
     schemaVersion: "openpond.resolvedTrainingBundle.v1",
+    ...qualification ? { executionMode: "qualification" } : {},
     projection: "trainer",
     harnessRelease: releasedHarness,
     datasetRelease,
@@ -889,6 +1020,7 @@ function buildTasksetTrainingBundle(input) {
   });
   const manifestContent = HarnessRunManifestContentSchema.parse({
     schemaVersion: "openpond.harnessRunManifest.v1",
+    ...qualification ? { executionMode: "qualification" } : {},
     id: `manifest_${input.modelRunId}_${input.approval.approvalHash.slice(0, 16)}`,
     harnessRelease,
     datasetRelease,
@@ -961,6 +1093,9 @@ function addJsonAsset(assets, assetPath, value) {
 function bytes(value) {
   return Buffer.from(canonicalJson(value), "utf8");
 }
+function releasedRefHash(value) {
+  return contentHash({ id: value.id, contentHash: value.contentHash });
+}
 export {
   AdamwOptimizerConfigSchema,
   ComputeTargetBindingSchema,
@@ -976,6 +1111,7 @@ export {
   createPolicyHarnessContext,
   materializeLearningBatchTaskset,
   prepareManagedTrainingSubmission,
+  prepareOrdinaryTasksetQualification,
   prepareReviewedLearningBatch,
   scanAndRedactEvidence,
   withAuthoritativeRecipeHashes