openpond-sdk 0.12.5 → 0.12.7

package/dist/types/packages/sdk/src/training.d.ts

@@ -1,5 +1,11 @@
 export { TrainingAcceptanceGroupReferenceSchema, TrainingAcceptanceGroupSummarySchema, TrainingAcceptanceGroupPageSchema, TrainingAcceptanceGroupQuerySchema, type TrainingAcceptanceGroupReference, type TrainingAcceptanceGroupSummary, type TrainingAcceptanceGroupPage, type TrainingAcceptanceGroupQuery, type TrainingAcceptanceTarget, } from "./training-acceptance.js";
 import { z } from "zod";
+import { TrainingCheckpointRecoveryRequestSchema } from "./training-recovery.js";
+export { TrainingCheckpointRecoveryRequestSchema, TrainingCheckpointRecoveryResourceApprovalSchema, createTrainingCheckpointRecoveryResourceApproval } from "./training-recovery.js";
+export { TrainingPhysicalPlacementSchema, type TrainingPhysicalPlacement } from "./training-physical-placement.js";
+export { TrainingGpuCatalogSchema, type TrainingGpuCatalog } from "./training-compute-selection.js";
+export { TrainingQueueStatusSchema, type TrainingQueueStatus } from "./training-queue.js";
+export { TrainingGpuSelectionSchema, TrainingEstimateSourceSchema, TrainingEstimateRequestSchema, TrainingEstimateComponentSchema, TrainingEstimateReviewSchema, TrainingEstimateSchema, trainingEstimateSourceIdentity, type TrainingGpuSelection, type TrainingEstimateRequest, type TrainingEstimate, type TrainingEstimateReview, } from "./training-estimates.js";
 export { TrainingPreparationRequestSchema, TrainingPreparationPlanContentSchema, TrainingPreparationPlanSchema, TrainingPreparationReceiptSchema, TrainingPreparationControlSchema, parseAndVerifyTrainingPreparationReceipt, type TrainingPreparationRequest, type TrainingPreparationPlan, type TrainingPreparationReceipt, type TrainingPreparationControl, } from "./training-preparations.js";
 export { TrainingPreparationListQuerySchema, TrainingPreparationPageSchema, type TrainingPreparationListQuery, type TrainingPreparationPage, } from "./training-preparations.js";
 export { TrainingHarnessSourcePublicationSchema } from "./training-harness-sources.js";
@@ -907,6 +913,30 @@ export declare const TrainingJobSubmissionSchema: z.ZodObject<{
         economical: "economical";
         fast: "fast";
     }>;
+    gpuSelection: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        mode: z.ZodLiteral<"automatic">;
+        objective: z.ZodEnum<{
+            balanced: "balanced";
+            economical: "economical";
+            fast: "fast";
+        }>;
+    }, z.core.$strict>, z.ZodObject<{
+        mode: z.ZodLiteral<"manual">;
+        availabilityId: z.ZodString;
+        gpuType: z.ZodString;
+    }, z.core.$strict>], "mode">>;
+    physicalPlacement: z.ZodOptional<z.ZodObject<{
+        mode: z.ZodLiteral<"reuse_only">;
+        physicalWorkerId: z.ZodString;
+        providerPodId: z.ZodString;
+        workerImageDigest: z.ZodString;
+        baseProfileSha256: z.ZodString;
+        gpuType: z.ZodString;
+        sourceJobId: z.ZodString;
+        sourceInputManifestSha256: z.ZodString;
+        ownerTeamId: z.ZodString;
+        deadlineAt: z.ZodString;
+    }, z.core.$strict>>;
     budget: z.ZodObject<{
         maximumSpendUsd: z.ZodNumber;
         maximumWallSeconds: z.ZodNumber;
@@ -923,6 +953,32 @@ export declare const TrainingJobSubmissionSchema: z.ZodObject<{
 }, z.core.$strict>;
 export declare function trainingJobSubmissionHash(submission: Omit<TrainingJobSubmission, "contentHash"> | TrainingJobSubmission): Promise<string>;
 export declare function parseAndVerifyTrainingJobSubmission(value: unknown): Promise<TrainingJobSubmission>;
+export declare const TrainingJobRecoverySchema: z.ZodObject<{
+    resourceApproval: z.ZodOptional<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.checkpointRecoveryResourceApproval.v1">;
+        ownerUserId: z.ZodString;
+        teamId: z.ZodString;
+        sourceJobId: z.ZodString;
+        sourceJobVersion: z.ZodNumber;
+        sourceInputManifestSha256: z.ZodString;
+        checkpointId: z.ZodString;
+        adapterSha256: z.ZodString;
+        resumeSha256: z.ZodString;
+        idempotencyKey: z.ZodString;
+        approvedAt: z.ZodISODateTime;
+        maximumSpendUsd: z.ZodNumber;
+        maximumGpuSeconds: z.ZodNumber;
+        maximumWallSeconds: z.ZodNumber;
+        approvalHash: z.ZodString;
+    }, z.core.$strict>>;
+    mode: z.ZodOptional<z.ZodLiteral<"evaluation_only">>;
+    trainingWorkloadComplete: z.ZodOptional<z.ZodBoolean>;
+    sourceJobId: z.ZodString;
+    checkpointId: z.ZodString;
+    policyVersion: z.ZodNumber;
+    completedGroups: z.ZodNumber;
+    optimizerMode: z.ZodLiteral<"continue">;
+}, z.core.$strict>;
 export declare const TrainingJobSchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.trainingJob.v2">;
     id: z.ZodString;
@@ -932,6 +988,32 @@ export declare const TrainingJobSchema: z.ZodObject<{
         reward_model_train: "reward_model_train";
     }>;
     executionMode: z.ZodOptional<z.ZodLiteral<"qualification">>;
+    recovery: z.ZodOptional<z.ZodObject<{
+        resourceApproval: z.ZodOptional<z.ZodObject<{
+            schemaVersion: z.ZodLiteral<"openpond.checkpointRecoveryResourceApproval.v1">;
+            ownerUserId: z.ZodString;
+            teamId: z.ZodString;
+            sourceJobId: z.ZodString;
+            sourceJobVersion: z.ZodNumber;
+            sourceInputManifestSha256: z.ZodString;
+            checkpointId: z.ZodString;
+            adapterSha256: z.ZodString;
+            resumeSha256: z.ZodString;
+            idempotencyKey: z.ZodString;
+            approvedAt: z.ZodISODateTime;
+            maximumSpendUsd: z.ZodNumber;
+            maximumGpuSeconds: z.ZodNumber;
+            maximumWallSeconds: z.ZodNumber;
+            approvalHash: z.ZodString;
+        }, z.core.$strict>>;
+        mode: z.ZodOptional<z.ZodLiteral<"evaluation_only">>;
+        trainingWorkloadComplete: z.ZodOptional<z.ZodBoolean>;
+        sourceJobId: z.ZodString;
+        checkpointId: z.ZodString;
+        policyVersion: z.ZodNumber;
+        completedGroups: z.ZodNumber;
+        optimizerMode: z.ZodLiteral<"continue">;
+    }, z.core.$strict>>;
     modelProjectId: z.ZodString;
     portableProjectId: z.ZodString;
     sourceProjectRevision: z.ZodNumber;
@@ -947,6 +1029,25 @@ export declare const TrainingJobSchema: z.ZodObject<{
         stopping: "stopping";
         succeeded: "succeeded";
     }>;
+    queue: z.ZodOptional<z.ZodObject<{
+        ticketId: z.ZodString;
+        ticketNumber: z.ZodNumber;
+        label: z.ZodLiteral<"Waiting for resources">;
+        position: z.ZodNullable<z.ZodNumber>;
+        positionScope: z.ZodLiteral<"compatible_fair_queue">;
+        enqueuedAt: z.ZodString;
+        asOf: z.ZodString;
+        estimate: z.ZodNullable<z.ZodObject<{
+            revision: z.ZodNumber;
+            estimatedStartAt: z.ZodString;
+            expiresAt: z.ZodString;
+            evidence: z.ZodEnum<{
+                measured_bootstrap: "measured_bootstrap";
+                observed_capacity: "observed_capacity";
+                scheduled_release: "scheduled_release";
+            }>;
+        }, z.core.$strict>>;
+    }, z.core.$strict>>;
     phase: z.ZodString;
     version: z.ZodNumber;
     progress: z.ZodNumber;
@@ -973,6 +1074,32 @@ export declare const TrainingJobPageSchema: z.ZodObject<{
             reward_model_train: "reward_model_train";
         }>;
         executionMode: z.ZodOptional<z.ZodLiteral<"qualification">>;
+        recovery: z.ZodOptional<z.ZodObject<{
+            resourceApproval: z.ZodOptional<z.ZodObject<{
+                schemaVersion: z.ZodLiteral<"openpond.checkpointRecoveryResourceApproval.v1">;
+                ownerUserId: z.ZodString;
+                teamId: z.ZodString;
+                sourceJobId: z.ZodString;
+                sourceJobVersion: z.ZodNumber;
+                sourceInputManifestSha256: z.ZodString;
+                checkpointId: z.ZodString;
+                adapterSha256: z.ZodString;
+                resumeSha256: z.ZodString;
+                idempotencyKey: z.ZodString;
+                approvedAt: z.ZodISODateTime;
+                maximumSpendUsd: z.ZodNumber;
+                maximumGpuSeconds: z.ZodNumber;
+                maximumWallSeconds: z.ZodNumber;
+                approvalHash: z.ZodString;
+            }, z.core.$strict>>;
+            mode: z.ZodOptional<z.ZodLiteral<"evaluation_only">>;
+            trainingWorkloadComplete: z.ZodOptional<z.ZodBoolean>;
+            sourceJobId: z.ZodString;
+            checkpointId: z.ZodString;
+            policyVersion: z.ZodNumber;
+            completedGroups: z.ZodNumber;
+            optimizerMode: z.ZodLiteral<"continue">;
+        }, z.core.$strict>>;
         modelProjectId: z.ZodString;
         portableProjectId: z.ZodString;
         sourceProjectRevision: z.ZodNumber;
@@ -988,6 +1115,25 @@ export declare const TrainingJobPageSchema: z.ZodObject<{
             stopping: "stopping";
             succeeded: "succeeded";
         }>;
+        queue: z.ZodOptional<z.ZodObject<{
+            ticketId: z.ZodString;
+            ticketNumber: z.ZodNumber;
+            label: z.ZodLiteral<"Waiting for resources">;
+            position: z.ZodNullable<z.ZodNumber>;
+            positionScope: z.ZodLiteral<"compatible_fair_queue">;
+            enqueuedAt: z.ZodString;
+            asOf: z.ZodString;
+            estimate: z.ZodNullable<z.ZodObject<{
+                revision: z.ZodNumber;
+                estimatedStartAt: z.ZodString;
+                expiresAt: z.ZodString;
+                evidence: z.ZodEnum<{
+                    measured_bootstrap: "measured_bootstrap";
+                    observed_capacity: "observed_capacity";
+                    scheduled_release: "scheduled_release";
+                }>;
+            }, z.core.$strict>>;
+        }, z.core.$strict>>;
         phase: z.ZodString;
         version: z.ZodNumber;
         progress: z.ZodNumber;
@@ -1254,6 +1400,27 @@ export declare class OpenPondTrainingApiError extends Error {
     readonly details: Record<string, unknown>;
     constructor(status: number, error: z.infer<typeof TrainingApiErrorSchema>);
 }
+export declare const TrainingPolicyLoraCapabilitiesSchema: z.ZodObject<{
+    workerImageDigest: z.ZodString;
+    envelopeRank: z.ZodNumber;
+    supportedRanks: z.ZodArray<z.ZodNumber>;
+    qualification: z.ZodNullable<z.ZodObject<{
+        evidenceRef: z.ZodString;
+        evidenceSha256: z.ZodString;
+    }, z.core.$strict>>;
+    continuation: z.ZodObject<{
+        sharedRequiresSameRank: z.ZodLiteral<true>;
+        mergedRequiresSameRank: z.ZodLiteral<true>;
+        residualAppendRequiresFreeCapacity: z.ZodLiteral<true>;
+        stageOptimizer: z.ZodLiteral<"reset">;
+        interruptedRecoveryOptimizer: z.ZodLiteral<"restore">;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export type TrainingPolicyLoraCapabilities = z.infer<typeof TrainingPolicyLoraCapabilitiesSchema>;
+export declare function supportsTrainingPolicyLoraRank(capabilities: TrainingPolicyLoraCapabilities, rank: number, continuation?: {
+    mode: "shared" | "sequential_merge" | "residual_append";
+    parentEnabledRank: number;
+}): boolean;
 export declare const TrainingCapabilitiesSchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.trainingCapabilities.v2">;
     capabilityHash: z.ZodString;
@@ -1266,6 +1433,22 @@ export declare const TrainingCapabilitiesSchema: z.ZodObject<{
         qualification: "qualification";
         training: "training";
     }>>>;
+    policyLora: z.ZodOptional<z.ZodObject<{
+        workerImageDigest: z.ZodString;
+        envelopeRank: z.ZodNumber;
+        supportedRanks: z.ZodArray<z.ZodNumber>;
+        qualification: z.ZodNullable<z.ZodObject<{
+            evidenceRef: z.ZodString;
+            evidenceSha256: z.ZodString;
+        }, z.core.$strict>>;
+        continuation: z.ZodObject<{
+            sharedRequiresSameRank: z.ZodLiteral<true>;
+            mergedRequiresSameRank: z.ZodLiteral<true>;
+            residualAppendRequiresFreeCapacity: z.ZodLiteral<true>;
+            stageOptimizer: z.ZodLiteral<"reset">;
+            interruptedRecoveryOptimizer: z.ZodLiteral<"restore">;
+        }, z.core.$strict>;
+    }, z.core.$strict>>;
     structuredOutputs: z.ZodOptional<z.ZodObject<{
         capability: z.ZodLiteral<"work.structured-output.v1">;
         harnessRuntimeHash: z.ZodString;
@@ -1286,6 +1469,9 @@ export declare const TrainingCapabilitiesSchema: z.ZodObject<{
         stopAfterGroup: z.ZodBoolean;
         resumeFromCheckpoint: z.ZodBoolean;
     }, z.core.$strict>;
+    estimates: z.ZodOptional<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.trainingEstimate.v1">;
+    }, z.core.$strict>>;
     limits: z.ZodObject<{
         maximumSpendUsd: z.ZodNumber;
         maximumWallSeconds: z.ZodNumber;
@@ -1661,6 +1847,26 @@ export declare function createTrainingClient(input: {
                     };
                     name: string;
                     maximumSpendUsd: number;
+                    gpuSelection?: {
+                        mode: "automatic";
+                        objective: "balanced" | "economical" | "fast";
+                    } | {
+                        mode: "manual";
+                        availabilityId: string;
+                        gpuType: string;
+                    } | undefined;
+                    physicalPlacement?: {
+                        mode: "reuse_only";
+                        physicalWorkerId: string;
+                        providerPodId: string;
+                        workerImageDigest: string;
+                        baseProfileSha256: string;
+                        gpuType: string;
+                        sourceJobId: string;
+                        sourceInputManifestSha256: string;
+                        ownerTeamId: string;
+                        deadlineAt: string;
+                    } | undefined;
                 };
                 requestHash: string;
                 source: {
@@ -2097,6 +2303,26 @@ export declare function createTrainingClient(input: {
                 };
                 name: string;
                 maximumSpendUsd: number;
+                gpuSelection?: {
+                    mode: "automatic";
+                    objective: "balanced" | "economical" | "fast";
+                } | {
+                    mode: "manual";
+                    availabilityId: string;
+                    gpuType: string;
+                } | undefined;
+                physicalPlacement?: {
+                    mode: "reuse_only";
+                    physicalWorkerId: string;
+                    providerPodId: string;
+                    workerImageDigest: string;
+                    baseProfileSha256: string;
+                    gpuType: string;
+                    sourceJobId: string;
+                    sourceInputManifestSha256: string;
+                    ownerTeamId: string;
+                    deadlineAt: string;
+                } | undefined;
             };
             requestHash: string;
             source: {
@@ -2532,6 +2758,26 @@ export declare function createTrainingClient(input: {
                 };
                 name: string;
                 maximumSpendUsd: number;
+                gpuSelection?: {
+                    mode: "automatic";
+                    objective: "balanced" | "economical" | "fast";
+                } | {
+                    mode: "manual";
+                    availabilityId: string;
+                    gpuType: string;
+                } | undefined;
+                physicalPlacement?: {
+                    mode: "reuse_only";
+                    physicalWorkerId: string;
+                    providerPodId: string;
+                    workerImageDigest: string;
+                    baseProfileSha256: string;
+                    gpuType: string;
+                    sourceJobId: string;
+                    sourceInputManifestSha256: string;
+                    ownerTeamId: string;
+                    deadlineAt: string;
+                } | undefined;
             };
             requestHash: string;
             source: {
@@ -2967,6 +3213,26 @@ export declare function createTrainingClient(input: {
                 };
                 name: string;
                 maximumSpendUsd: number;
+                gpuSelection?: {
+                    mode: "automatic";
+                    objective: "balanced" | "economical" | "fast";
+                } | {
+                    mode: "manual";
+                    availabilityId: string;
+                    gpuType: string;
+                } | undefined;
+                physicalPlacement?: {
+                    mode: "reuse_only";
+                    physicalWorkerId: string;
+                    providerPodId: string;
+                    workerImageDigest: string;
+                    baseProfileSha256: string;
+                    gpuType: string;
+                    sourceJobId: string;
+                    sourceInputManifestSha256: string;
+                    ownerTeamId: string;
+                    deadlineAt: string;
+                } | undefined;
             };
             requestHash: string;
             source: {
@@ -3402,6 +3668,26 @@ export declare function createTrainingClient(input: {
                 };
                 name: string;
                 maximumSpendUsd: number;
+                gpuSelection?: {
+                    mode: "automatic";
+                    objective: "balanced" | "economical" | "fast";
+                } | {
+                    mode: "manual";
+                    availabilityId: string;
+                    gpuType: string;
+                } | undefined;
+                physicalPlacement?: {
+                    mode: "reuse_only";
+                    physicalWorkerId: string;
+                    providerPodId: string;
+                    workerImageDigest: string;
+                    baseProfileSha256: string;
+                    gpuType: string;
+                    sourceJobId: string;
+                    sourceInputManifestSha256: string;
+                    ownerTeamId: string;
+                    deadlineAt: string;
+                } | undefined;
             };
             requestHash: string;
             source: {
@@ -3635,6 +3921,7 @@ export declare function createTrainingClient(input: {
         state: "cancelled" | "prepared" | "submitted";
         jobId: string | null;
     }>;
+    estimate(input: import("./training-estimates.js").TrainingEstimateRequest): Promise<import("./training-estimates.js").TrainingEstimate>;
     publishHarnessSource(value: import("@openpond/harness").HarnessSourcePackage | import("@openpond/harness").HarnessPolicySourcePackage): Promise<{
         schemaVersion: "openpond.trainingHarnessSourcePublication.v1";
         harnessRelease: {
@@ -4353,6 +4640,22 @@ export declare function createTrainingClient(input: {
         jobKinds: ("policy_optimize" | "reward_model_train")[];
         methods: string[];
         executionModes?: ("qualification" | "training")[] | undefined;
+        policyLora?: {
+            workerImageDigest: string;
+            envelopeRank: number;
+            supportedRanks: number[];
+            qualification: {
+                evidenceRef: string;
+                evidenceSha256: string;
+            } | null;
+            continuation: {
+                sharedRequiresSameRank: true;
+                mergedRequiresSameRank: true;
+                residualAppendRequiresFreeCapacity: true;
+                stageOptimizer: "reset";
+                interruptedRecoveryOptimizer: "restore";
+            };
+        } | undefined;
         structuredOutputs?: {
             capability: "work.structured-output.v1";
             harnessRuntimeHash: string;
@@ -4368,6 +4671,9 @@ export declare function createTrainingClient(input: {
             stopAfterGroup: boolean;
             resumeFromCheckpoint: boolean;
         };
+        estimates?: {
+            schemaVersion: "openpond.trainingEstimate.v1";
+        } | undefined;
         limits: {
             maximumSpendUsd: number;
             maximumWallSeconds: number;
@@ -4376,6 +4682,27 @@ export declare function createTrainingClient(input: {
         checkedAt: string;
         expiresAt: string;
     }>;
+    gpuCatalog(): Promise<{
+        schemaVersion: "openpond.trainingGpuCatalog.v1";
+        observedAt: string;
+        expiresAt: string;
+        workerImageDigest: string;
+        supportedLoraRanks: number[];
+        pricingBasis: "accepted_compute_and_disk_hourly_before_credits";
+        offerings: {
+            availabilityId: string;
+            gpuType: string;
+            provider: string;
+            region: string;
+            dataCenterId: string | null;
+            gpuCount: number;
+            gpuMemoryGb: number;
+            stockStatus: "High" | "Low" | "Medium";
+            computeHourlyUsd: number;
+            diskHourlyUsd: number;
+            totalHourlyUsd: number;
+        }[];
+    }>;
     stageArtifact(upload: TrainingInputArtifactUpload): Promise<{
         schemaVersion: "openpond.trainingInputArtifact.v2";
         kind: "portable_training_bundle" | "reward_model_dataset";
@@ -4394,11 +4721,52 @@ export declare function createTrainingClient(input: {
         teamId: string;
         kind: "policy_optimize" | "reward_model_train";
         executionMode?: "qualification" | undefined;
+        recovery?: {
+            resourceApproval?: {
+                schemaVersion: "openpond.checkpointRecoveryResourceApproval.v1";
+                ownerUserId: string;
+                teamId: string;
+                sourceJobId: string;
+                sourceJobVersion: number;
+                sourceInputManifestSha256: string;
+                checkpointId: string;
+                adapterSha256: string;
+                resumeSha256: string;
+                idempotencyKey: string;
+                approvedAt: string;
+                maximumSpendUsd: number;
+                maximumGpuSeconds: number;
+                maximumWallSeconds: number;
+                approvalHash: string;
+            } | undefined;
+            mode?: "evaluation_only" | undefined;
+            trainingWorkloadComplete?: boolean | undefined;
+            sourceJobId: string;
+            checkpointId: string;
+            policyVersion: number;
+            completedGroups: number;
+            optimizerMode: "continue";
+        } | undefined;
         modelProjectId: string;
         portableProjectId: string;
         sourceProjectRevision: number;
         submissionHash: string;
         state: "admitting" | "cancelled" | "cancelling" | "failed" | "provisioning" | "queued" | "running" | "stopping" | "succeeded";
+        queue?: {
+            ticketId: string;
+            ticketNumber: number;
+            label: "Waiting for resources";
+            position: number | null;
+            positionScope: "compatible_fair_queue";
+            enqueuedAt: string;
+            asOf: string;
+            estimate: {
+                revision: number;
+                estimatedStartAt: string;
+                expiresAt: string;
+                evidence: "measured_bootstrap" | "observed_capacity" | "scheduled_release";
+            } | null;
+        } | undefined;
         phase: string;
         version: number;
         progress: number;
@@ -4426,11 +4794,52 @@ export declare function createTrainingClient(input: {
             teamId: string;
             kind: "policy_optimize" | "reward_model_train";
             executionMode?: "qualification" | undefined;
+            recovery?: {
+                resourceApproval?: {
+                    schemaVersion: "openpond.checkpointRecoveryResourceApproval.v1";
+                    ownerUserId: string;
+                    teamId: string;
+                    sourceJobId: string;
+                    sourceJobVersion: number;
+                    sourceInputManifestSha256: string;
+                    checkpointId: string;
+                    adapterSha256: string;
+                    resumeSha256: string;
+                    idempotencyKey: string;
+                    approvedAt: string;
+                    maximumSpendUsd: number;
+                    maximumGpuSeconds: number;
+                    maximumWallSeconds: number;
+                    approvalHash: string;
+                } | undefined;
+                mode?: "evaluation_only" | undefined;
+                trainingWorkloadComplete?: boolean | undefined;
+                sourceJobId: string;
+                checkpointId: string;
+                policyVersion: number;
+                completedGroups: number;
+                optimizerMode: "continue";
+            } | undefined;
             modelProjectId: string;
             portableProjectId: string;
             sourceProjectRevision: number;
             submissionHash: string;
             state: "admitting" | "cancelled" | "cancelling" | "failed" | "provisioning" | "queued" | "running" | "stopping" | "succeeded";
+            queue?: {
+                ticketId: string;
+                ticketNumber: number;
+                label: "Waiting for resources";
+                position: number | null;
+                positionScope: "compatible_fair_queue";
+                enqueuedAt: string;
+                asOf: string;
+                estimate: {
+                    revision: number;
+                    estimatedStartAt: string;
+                    expiresAt: string;
+                    evidence: "measured_bootstrap" | "observed_capacity" | "scheduled_release";
+                } | null;
+            } | undefined;
             phase: string;
             version: number;
             progress: number;
@@ -4454,11 +4863,52 @@ export declare function createTrainingClient(input: {
         teamId: string;
         kind: "policy_optimize" | "reward_model_train";
         executionMode?: "qualification" | undefined;
+        recovery?: {
+            resourceApproval?: {
+                schemaVersion: "openpond.checkpointRecoveryResourceApproval.v1";
+                ownerUserId: string;
+                teamId: string;
+                sourceJobId: string;
+                sourceJobVersion: number;
+                sourceInputManifestSha256: string;
+                checkpointId: string;
+                adapterSha256: string;
+                resumeSha256: string;
+                idempotencyKey: string;
+                approvedAt: string;
+                maximumSpendUsd: number;
+                maximumGpuSeconds: number;
+                maximumWallSeconds: number;
+                approvalHash: string;
+            } | undefined;
+            mode?: "evaluation_only" | undefined;
+            trainingWorkloadComplete?: boolean | undefined;
+            sourceJobId: string;
+            checkpointId: string;
+            policyVersion: number;
+            completedGroups: number;
+            optimizerMode: "continue";
+        } | undefined;
         modelProjectId: string;
         portableProjectId: string;
         sourceProjectRevision: number;
         submissionHash: string;
         state: "admitting" | "cancelled" | "cancelling" | "failed" | "provisioning" | "queued" | "running" | "stopping" | "succeeded";
+        queue?: {
+            ticketId: string;
+            ticketNumber: number;
+            label: "Waiting for resources";
+            position: number | null;
+            positionScope: "compatible_fair_queue";
+            enqueuedAt: string;
+            asOf: string;
+            estimate: {
+                revision: number;
+                estimatedStartAt: string;
+                expiresAt: string;
+                evidence: "measured_bootstrap" | "observed_capacity" | "scheduled_release";
+            } | null;
+        } | undefined;
         phase: string;
         version: number;
         progress: number;
@@ -4480,11 +4930,52 @@ export declare function createTrainingClient(input: {
         teamId: string;
         kind: "policy_optimize" | "reward_model_train";
         executionMode?: "qualification" | undefined;
+        recovery?: {
+            resourceApproval?: {
+                schemaVersion: "openpond.checkpointRecoveryResourceApproval.v1";
+                ownerUserId: string;
+                teamId: string;
+                sourceJobId: string;
+                sourceJobVersion: number;
+                sourceInputManifestSha256: string;
+                checkpointId: string;
+                adapterSha256: string;
+                resumeSha256: string;
+                idempotencyKey: string;
+                approvedAt: string;
+                maximumSpendUsd: number;
+                maximumGpuSeconds: number;
+                maximumWallSeconds: number;
+                approvalHash: string;
+            } | undefined;
+            mode?: "evaluation_only" | undefined;
+            trainingWorkloadComplete?: boolean | undefined;
+            sourceJobId: string;
+            checkpointId: string;
+            policyVersion: number;
+            completedGroups: number;
+            optimizerMode: "continue";
+        } | undefined;
         modelProjectId: string;
         portableProjectId: string;
         sourceProjectRevision: number;
         submissionHash: string;
         state: "admitting" | "cancelled" | "cancelling" | "failed" | "provisioning" | "queued" | "running" | "stopping" | "succeeded";
+        queue?: {
+            ticketId: string;
+            ticketNumber: number;
+            label: "Waiting for resources";
+            position: number | null;
+            positionScope: "compatible_fair_queue";
+            enqueuedAt: string;
+            asOf: string;
+            estimate: {
+                revision: number;
+                estimatedStartAt: string;
+                expiresAt: string;
+                evidence: "measured_bootstrap" | "observed_capacity" | "scheduled_release";
+            } | null;
+        } | undefined;
         phase: string;
         version: number;
         progress: number;
@@ -4506,11 +4997,119 @@ export declare function createTrainingClient(input: {
         teamId: string;
         kind: "policy_optimize" | "reward_model_train";
         executionMode?: "qualification" | undefined;
+        recovery?: {
+            resourceApproval?: {
+                schemaVersion: "openpond.checkpointRecoveryResourceApproval.v1";
+                ownerUserId: string;
+                teamId: string;
+                sourceJobId: string;
+                sourceJobVersion: number;
+                sourceInputManifestSha256: string;
+                checkpointId: string;
+                adapterSha256: string;
+                resumeSha256: string;
+                idempotencyKey: string;
+                approvedAt: string;
+                maximumSpendUsd: number;
+                maximumGpuSeconds: number;
+                maximumWallSeconds: number;
+                approvalHash: string;
+            } | undefined;
+            mode?: "evaluation_only" | undefined;
+            trainingWorkloadComplete?: boolean | undefined;
+            sourceJobId: string;
+            checkpointId: string;
+            policyVersion: number;
+            completedGroups: number;
+            optimizerMode: "continue";
+        } | undefined;
         modelProjectId: string;
         portableProjectId: string;
         sourceProjectRevision: number;
         submissionHash: string;
         state: "admitting" | "cancelled" | "cancelling" | "failed" | "provisioning" | "queued" | "running" | "stopping" | "succeeded";
+        queue?: {
+            ticketId: string;
+            ticketNumber: number;
+            label: "Waiting for resources";
+            position: number | null;
+            positionScope: "compatible_fair_queue";
+            enqueuedAt: string;
+            asOf: string;
+            estimate: {
+                revision: number;
+                estimatedStartAt: string;
+                expiresAt: string;
+                evidence: "measured_bootstrap" | "observed_capacity" | "scheduled_release";
+            } | null;
+        } | undefined;
+        phase: string;
+        version: number;
+        progress: number;
+        rolloutProgress: {
+            groupsCompleted: number;
+            groupsTarget: number;
+            optimizerUpdatesApplied: number;
+            optimizerUpdatesSkipped: number;
+        };
+        accruedSpendUsd: number;
+        terminalReason: string | null;
+        createdAt: string;
+        updatedAt: string;
+        completedAt: string | null;
+    }>;
+    recoverCheckpoint(jobId: string, input: z.input<typeof TrainingCheckpointRecoveryRequestSchema>): Promise<{
+        schemaVersion: "openpond.trainingJob.v2";
+        id: string;
+        teamId: string;
+        kind: "policy_optimize" | "reward_model_train";
+        executionMode?: "qualification" | undefined;
+        recovery?: {
+            resourceApproval?: {
+                schemaVersion: "openpond.checkpointRecoveryResourceApproval.v1";
+                ownerUserId: string;
+                teamId: string;
+                sourceJobId: string;
+                sourceJobVersion: number;
+                sourceInputManifestSha256: string;
+                checkpointId: string;
+                adapterSha256: string;
+                resumeSha256: string;
+                idempotencyKey: string;
+                approvedAt: string;
+                maximumSpendUsd: number;
+                maximumGpuSeconds: number;
+                maximumWallSeconds: number;
+                approvalHash: string;
+            } | undefined;
+            mode?: "evaluation_only" | undefined;
+            trainingWorkloadComplete?: boolean | undefined;
+            sourceJobId: string;
+            checkpointId: string;
+            policyVersion: number;
+            completedGroups: number;
+            optimizerMode: "continue";
+        } | undefined;
+        modelProjectId: string;
+        portableProjectId: string;
+        sourceProjectRevision: number;
+        submissionHash: string;
+        state: "admitting" | "cancelled" | "cancelling" | "failed" | "provisioning" | "queued" | "running" | "stopping" | "succeeded";
+        queue?: {
+            ticketId: string;
+            ticketNumber: number;
+            label: "Waiting for resources";
+            position: number | null;
+            positionScope: "compatible_fair_queue";
+            enqueuedAt: string;
+            asOf: string;
+            estimate: {
+                revision: number;
+                estimatedStartAt: string;
+                expiresAt: string;
+                evidence: "measured_bootstrap" | "observed_capacity" | "scheduled_release";
+            } | null;
+        } | undefined;
         phase: string;
         version: number;
         progress: number;