openai 4.96.2 → 4.98.0

package/src/resources/evals/index.ts

@@ -4,10 +4,7 @@ export {
   EvalListResponsesPage,
   Evals,
   type EvalCustomDataSourceConfig,
-  type EvalLabelModelGrader,
   type EvalStoredCompletionsDataSourceConfig,
-  type EvalStringCheckGrader,
-  type EvalTextSimilarityGrader,
   type EvalCreateResponse,
   type EvalRetrieveResponse,
   type EvalUpdateResponse,

package/src/resources/fine-tuning/alpha/alpha.ts

@@ -0,0 +1,27 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../../resource';
+import * as GradersAPI from './graders';
+import {
+  GraderRunParams,
+  GraderRunResponse,
+  GraderValidateParams,
+  GraderValidateResponse,
+  Graders,
+} from './graders';
+
+export class Alpha extends APIResource {
+  graders: GradersAPI.Graders = new GradersAPI.Graders(this._client);
+}
+
+Alpha.Graders = Graders;
+
+export declare namespace Alpha {
+  export {
+    Graders as Graders,
+    type GraderRunResponse as GraderRunResponse,
+    type GraderValidateResponse as GraderValidateResponse,
+    type GraderRunParams as GraderRunParams,
+    type GraderValidateParams as GraderValidateParams,
+  };
+}

package/src/resources/fine-tuning/alpha/graders.ts

@@ -0,0 +1,168 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../../resource';
+import * as Core from '../../../core';
+import * as GraderModelsAPI from '../../graders/grader-models';
+
+export class Graders extends APIResource {
+  /**
+   * Run a grader.
+   *
+   * @example
+   * ```ts
+   * const response = await client.fineTuning.alpha.graders.run({
+   *   grader: {
+   *     input: 'input',
+   *     name: 'name',
+   *     operation: 'eq',
+   *     reference: 'reference',
+   *     type: 'string_check',
+   *   },
+   *   model_sample: 'model_sample',
+   *   reference_answer: 'string',
+   * });
+   * ```
+   */
+  run(body: GraderRunParams, options?: Core.RequestOptions): Core.APIPromise<GraderRunResponse> {
+    return this._client.post('/fine_tuning/alpha/graders/run', { body, ...options });
+  }
+
+  /**
+   * Validate a grader.
+   *
+   * @example
+   * ```ts
+   * const response =
+   *   await client.fineTuning.alpha.graders.validate({
+   *     grader: {
+   *       input: 'input',
+   *       name: 'name',
+   *       operation: 'eq',
+   *       reference: 'reference',
+   *       type: 'string_check',
+   *     },
+   *   });
+   * ```
+   */
+  validate(
+    body: GraderValidateParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<GraderValidateResponse> {
+    return this._client.post('/fine_tuning/alpha/graders/validate', { body, ...options });
+  }
+}
+
+export interface GraderRunResponse {
+  metadata: GraderRunResponse.Metadata;
+
+  model_grader_token_usage_per_model: Record<string, unknown>;
+
+  reward: number;
+
+  sub_rewards: Record<string, unknown>;
+}
+
+export namespace GraderRunResponse {
+  export interface Metadata {
+    errors: Metadata.Errors;
+
+    execution_time: number;
+
+    name: string;
+
+    sampled_model_name: string | null;
+
+    scores: Record<string, unknown>;
+
+    token_usage: number | null;
+
+    type: string;
+  }
+
+  export namespace Metadata {
+    export interface Errors {
+      formula_parse_error: boolean;
+
+      invalid_variable_error: boolean;
+
+      model_grader_parse_error: boolean;
+
+      model_grader_refusal_error: boolean;
+
+      model_grader_server_error: boolean;
+
+      model_grader_server_error_details: string | null;
+
+      other_error: boolean;
+
+      python_grader_runtime_error: boolean;
+
+      python_grader_runtime_error_details: string | null;
+
+      python_grader_server_error: boolean;
+
+      python_grader_server_error_type: string | null;
+
+      sample_parse_error: boolean;
+
+      truncated_observation_error: boolean;
+
+      unresponsive_reward_error: boolean;
+    }
+  }
+}
+
+export interface GraderValidateResponse {
+  /**
+   * The grader used for the fine-tuning job.
+   */
+  grader?:
+    | GraderModelsAPI.StringCheckGrader
+    | GraderModelsAPI.TextSimilarityGrader
+    | GraderModelsAPI.PythonGrader
+    | GraderModelsAPI.ScoreModelGrader
+    | GraderModelsAPI.MultiGrader;
+}
+
+export interface GraderRunParams {
+  /**
+   * The grader used for the fine-tuning job.
+   */
+  grader:
+    | GraderModelsAPI.StringCheckGrader
+    | GraderModelsAPI.TextSimilarityGrader
+    | GraderModelsAPI.PythonGrader
+    | GraderModelsAPI.ScoreModelGrader
+    | GraderModelsAPI.MultiGrader;
+
+  /**
+   * The model sample to be evaluated.
+   */
+  model_sample: string;
+
+  /**
+   * The reference answer for the evaluation.
+   */
+  reference_answer: string | unknown | Array<unknown> | number;
+}
+
+export interface GraderValidateParams {
+  /**
+   * The grader used for the fine-tuning job.
+   */
+  grader:
+    | GraderModelsAPI.StringCheckGrader
+    | GraderModelsAPI.TextSimilarityGrader
+    | GraderModelsAPI.PythonGrader
+    | GraderModelsAPI.ScoreModelGrader
+    | GraderModelsAPI.MultiGrader;
+}
+
+export declare namespace Graders {
+  export {
+    type GraderRunResponse as GraderRunResponse,
+    type GraderValidateResponse as GraderValidateResponse,
+    type GraderRunParams as GraderRunParams,
+    type GraderValidateParams as GraderValidateParams,
+  };
+}

package/src/resources/fine-tuning/alpha/index.ts

@@ -0,0 +1,10 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export { Alpha } from './alpha';
+export {
+  Graders,
+  type GraderRunResponse,
+  type GraderValidateResponse,
+  type GraderRunParams,
+  type GraderValidateParams,
+} from './graders';

package/src/resources/fine-tuning/alpha.ts

@@ -0,0 +1,3 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export * from './alpha/index';

package/src/resources/fine-tuning/checkpoints/permissions.ts

@@ -11,6 +11,17 @@ export class Permissions extends APIResource {
    *
    * This enables organization owners to share fine-tuned models with other projects
    * in their organization.
+   *
+   * @example
+   * ```ts
+   * // Automatically fetches more pages as needed.
+   * for await (const permissionCreateResponse of client.fineTuning.checkpoints.permissions.create(
+   *   'ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd',
+   *   { project_ids: ['string'] },
+   * )) {
+   *   // ...
+   * }
+   * ```
    */
   create(
     fineTunedModelCheckpoint: string,
@@ -29,6 +40,14 @@ export class Permissions extends APIResource {
    *
    * Organization owners can use this endpoint to view all permissions for a
    * fine-tuned model checkpoint.
+   *
+   * @example
+   * ```ts
+   * const permission =
+   *   await client.fineTuning.checkpoints.permissions.retrieve(
+   *     'ft-AF1WoRqd3aJAHsqc9NY7iL8F',
+   *   );
+   * ```
    */
   retrieve(
     fineTunedModelCheckpoint: string,
@@ -58,6 +77,15 @@ export class Permissions extends APIResource {
    *
    * Organization owners can use this endpoint to delete a permission for a
    * fine-tuned model checkpoint.
+   *
+   * @example
+   * ```ts
+   * const permission =
+   *   await client.fineTuning.checkpoints.permissions.del(
+   *     'ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd',
+   *     'cp_zc4Q7MP6XxulcVzj4MZdwsAB',
+   *   );
+   * ```
    */
   del(
     fineTunedModelCheckpoint: string,

package/src/resources/fine-tuning/fine-tuning.ts

@@ -1,6 +1,18 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 import { APIResource } from '../../resource';
+import * as MethodsAPI from './methods';
+import {
+  DpoHyperparameters,
+  DpoMethod,
+  Methods,
+  ReinforcementHyperparameters,
+  ReinforcementMethod,
+  SupervisedHyperparameters,
+  SupervisedMethod,
+} from './methods';
+import * as AlphaAPI from './alpha/alpha';
+import { Alpha } from './alpha/alpha';
 import * as CheckpointsAPI from './checkpoints/checkpoints';
 import { Checkpoints } from './checkpoints/checkpoints';
 import * as JobsAPI from './jobs/jobs';
@@ -19,16 +31,30 @@ import {
 } from './jobs/jobs';
 
 export class FineTuning extends APIResource {
+  methods: MethodsAPI.Methods = new MethodsAPI.Methods(this._client);
   jobs: JobsAPI.Jobs = new JobsAPI.Jobs(this._client);
   checkpoints: CheckpointsAPI.Checkpoints = new CheckpointsAPI.Checkpoints(this._client);
+  alpha: AlphaAPI.Alpha = new AlphaAPI.Alpha(this._client);
 }
 
+FineTuning.Methods = Methods;
 FineTuning.Jobs = Jobs;
 FineTuning.FineTuningJobsPage = FineTuningJobsPage;
 FineTuning.FineTuningJobEventsPage = FineTuningJobEventsPage;
 FineTuning.Checkpoints = Checkpoints;
+FineTuning.Alpha = Alpha;
 
 export declare namespace FineTuning {
+  export {
+    Methods as Methods,
+    type DpoHyperparameters as DpoHyperparameters,
+    type DpoMethod as DpoMethod,
+    type ReinforcementHyperparameters as ReinforcementHyperparameters,
+    type ReinforcementMethod as ReinforcementMethod,
+    type SupervisedHyperparameters as SupervisedHyperparameters,
+    type SupervisedMethod as SupervisedMethod,
+  };
+
   export {
     Jobs as Jobs,
     type FineTuningJob as FineTuningJob,
@@ -44,4 +70,6 @@ export declare namespace FineTuning {
   };
 
   export { Checkpoints as Checkpoints };
+
+  export { Alpha as Alpha };
 }

package/src/resources/fine-tuning/index.ts

@@ -1,5 +1,6 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+export { Alpha } from './alpha/index';
 export { Checkpoints } from './checkpoints/index';
 export { FineTuning } from './fine-tuning';
 export {
@@ -15,3 +16,12 @@ export {
   type JobListParams,
   type JobListEventsParams,
 } from './jobs/index';
+export {
+  Methods,
+  type DpoHyperparameters,
+  type DpoMethod,
+  type ReinforcementHyperparameters,
+  type ReinforcementMethod,
+  type SupervisedHyperparameters,
+  type SupervisedMethod,
+} from './methods';

package/src/resources/fine-tuning/jobs/checkpoints.ts

@@ -8,6 +8,16 @@ import { CursorPage, type CursorPageParams } from '../../../pagination';
 export class Checkpoints extends APIResource {
   /**
    * List checkpoints for a fine-tuning job.
+   *
+   * @example
+   * ```ts
+   * // Automatically fetches more pages as needed.
+   * for await (const fineTuningJobCheckpoint of client.fineTuning.jobs.checkpoints.list(
+   *   'ft-AF1WoRqd3aJAHsqc9NY7iL8F',
+   * )) {
+   *   // ...
+   * }
+   * ```
    */
   list(
     fineTuningJobId: string,

package/src/resources/fine-tuning/jobs/jobs.ts

@@ -3,6 +3,7 @@
 import { APIResource } from '../../../resource';
 import { isRequestOptions } from '../../../core';
 import * as Core from '../../../core';
+import * as MethodsAPI from '../methods';
 import * as CheckpointsAPI from './checkpoints';
 import {
   CheckpointListParams,
@@ -23,6 +24,14 @@ export class Jobs extends APIResource {
    * of the fine-tuned models once complete.
    *
    * [Learn more about fine-tuning](https://platform.openai.com/docs/guides/fine-tuning)
+   *
+   * @example
+   * ```ts
+   * const fineTuningJob = await client.fineTuning.jobs.create({
+   *   model: 'gpt-4o-mini',
+   *   training_file: 'file-abc123',
+   * });
+   * ```
    */
   create(body: JobCreateParams, options?: Core.RequestOptions): Core.APIPromise<FineTuningJob> {
     return this._client.post('/fine_tuning/jobs', { body, ...options });
@@ -32,6 +41,13 @@ export class Jobs extends APIResource {
    * Get info about a fine-tuning job.
    *
    * [Learn more about fine-tuning](https://platform.openai.com/docs/guides/fine-tuning)
+   *
+   * @example
+   * ```ts
+   * const fineTuningJob = await client.fineTuning.jobs.retrieve(
+   *   'ft-AF1WoRqd3aJAHsqc9NY7iL8F',
+   * );
+   * ```
    */
   retrieve(fineTuningJobId: string, options?: Core.RequestOptions): Core.APIPromise<FineTuningJob> {
     return this._client.get(`/fine_tuning/jobs/${fineTuningJobId}`, options);
@@ -39,6 +55,14 @@ export class Jobs extends APIResource {
 
   /**
    * List your organization's fine-tuning jobs
+   *
+   * @example
+   * ```ts
+   * // Automatically fetches more pages as needed.
+   * for await (const fineTuningJob of client.fineTuning.jobs.list()) {
+   *   // ...
+   * }
+   * ```
    */
   list(
     query?: JobListParams,
@@ -57,6 +81,13 @@ export class Jobs extends APIResource {
 
   /**
    * Immediately cancel a fine-tune job.
+   *
+   * @example
+   * ```ts
+   * const fineTuningJob = await client.fineTuning.jobs.cancel(
+   *   'ft-AF1WoRqd3aJAHsqc9NY7iL8F',
+   * );
+   * ```
    */
   cancel(fineTuningJobId: string, options?: Core.RequestOptions): Core.APIPromise<FineTuningJob> {
     return this._client.post(`/fine_tuning/jobs/${fineTuningJobId}/cancel`, options);
@@ -64,6 +95,16 @@ export class Jobs extends APIResource {
 
   /**
    * Get status updates for a fine-tuning job.
+   *
+   * @example
+   * ```ts
+   * // Automatically fetches more pages as needed.
+   * for await (const fineTuningJobEvent of client.fineTuning.jobs.listEvents(
+   *   'ft-AF1WoRqd3aJAHsqc9NY7iL8F',
+   * )) {
+   *   // ...
+   * }
+   * ```
    */
   listEvents(
     fineTuningJobId: string,
@@ -87,6 +128,34 @@ export class Jobs extends APIResource {
       ...options,
     });
   }
+
+  /**
+   * Pause a fine-tune job.
+   *
+   * @example
+   * ```ts
+   * const fineTuningJob = await client.fineTuning.jobs.pause(
+   *   'ft-AF1WoRqd3aJAHsqc9NY7iL8F',
+   * );
+   * ```
+   */
+  pause(fineTuningJobId: string, options?: Core.RequestOptions): Core.APIPromise<FineTuningJob> {
+    return this._client.post(`/fine_tuning/jobs/${fineTuningJobId}/pause`, options);
+  }
+
+  /**
+   * Resume a fine-tune job.
+   *
+   * @example
+   * ```ts
+   * const fineTuningJob = await client.fineTuning.jobs.resume(
+   *   'ft-AF1WoRqd3aJAHsqc9NY7iL8F',
+   * );
+   * ```
+   */
+  resume(fineTuningJobId: string, options?: Core.RequestOptions): Core.APIPromise<FineTuningJob> {
+    return this._client.post(`/fine_tuning/jobs/${fineTuningJobId}/resume`, options);
+  }
 }
 
 export class FineTuningJobsPage extends CursorPage<FineTuningJob> {}
@@ -253,97 +322,24 @@ export namespace FineTuningJob {
    */
   export interface Method {
     /**
-     * Configuration for the DPO fine-tuning method.
-     */
-    dpo?: Method.Dpo;
-
-    /**
-     * Configuration for the supervised fine-tuning method.
+     * The type of method. Is either `supervised`, `dpo`, or `reinforcement`.
      */
-    supervised?: Method.Supervised;
+    type: 'supervised' | 'dpo' | 'reinforcement';
 
     /**
-     * The type of method. Is either `supervised` or `dpo`.
+     * Configuration for the DPO fine-tuning method.
      */
-    type?: 'supervised' | 'dpo';
-  }
+    dpo?: MethodsAPI.DpoMethod;
 
-  export namespace Method {
     /**
-     * Configuration for the DPO fine-tuning method.
+     * Configuration for the reinforcement fine-tuning method.
      */
-    export interface Dpo {
-      /**
-       * The hyperparameters used for the fine-tuning job.
-       */
-      hyperparameters?: Dpo.Hyperparameters;
-    }
-
-    export namespace Dpo {
-      /**
-       * The hyperparameters used for the fine-tuning job.
-       */
-      export interface Hyperparameters {
-        /**
-         * Number of examples in each batch. A larger batch size means that model
-         * parameters are updated less frequently, but with lower variance.
-         */
-        batch_size?: 'auto' | number;
-
-        /**
-         * The beta value for the DPO method. A higher beta value will increase the weight
-         * of the penalty between the policy and reference model.
-         */
-        beta?: 'auto' | number;
-
-        /**
-         * Scaling factor for the learning rate. A smaller learning rate may be useful to
-         * avoid overfitting.
-         */
-        learning_rate_multiplier?: 'auto' | number;
-
-        /**
-         * The number of epochs to train the model for. An epoch refers to one full cycle
-         * through the training dataset.
-         */
-        n_epochs?: 'auto' | number;
-      }
-    }
+    reinforcement?: MethodsAPI.ReinforcementMethod;
 
     /**
      * Configuration for the supervised fine-tuning method.
      */
-    export interface Supervised {
-      /**
-       * The hyperparameters used for the fine-tuning job.
-       */
-      hyperparameters?: Supervised.Hyperparameters;
-    }
-
-    export namespace Supervised {
-      /**
-       * The hyperparameters used for the fine-tuning job.
-       */
-      export interface Hyperparameters {
-        /**
-         * Number of examples in each batch. A larger batch size means that model
-         * parameters are updated less frequently, but with lower variance.
-         */
-        batch_size?: 'auto' | number;
-
-        /**
-         * Scaling factor for the learning rate. A smaller learning rate may be useful to
-         * avoid overfitting.
-         */
-        learning_rate_multiplier?: 'auto' | number;
-
-        /**
-         * The number of epochs to train the model for. An epoch refers to one full cycle
-         * through the training dataset.
-         */
-        n_epochs?: 'auto' | number;
-      }
-    }
+    supervised?: MethodsAPI.SupervisedMethod;
   }
 }
 
@@ -466,8 +462,9 @@ export interface JobCreateParams {
   training_file: string;
 
   /**
-   * The hyperparameters used for the fine-tuning job. This value is now deprecated
-   * in favor of `method`, and should be passed in under the `method` parameter.
+   * @deprecated The hyperparameters used for the fine-tuning job. This value is now
+   * deprecated in favor of `method`, and should be passed in under the `method`
+   * parameter.
    */
   hyperparameters?: JobCreateParams.Hyperparameters;
 
@@ -596,97 +593,24 @@ export namespace JobCreateParams {
    */
   export interface Method {
     /**
-     * Configuration for the DPO fine-tuning method.
+     * The type of method. Is either `supervised`, `dpo`, or `reinforcement`.
      */
-    dpo?: Method.Dpo;
+    type: 'supervised' | 'dpo' | 'reinforcement';
 
     /**
-     * Configuration for the supervised fine-tuning method.
-     */
-    supervised?: Method.Supervised;
-
-    /**
-     * The type of method. Is either `supervised` or `dpo`.
+     * Configuration for the DPO fine-tuning method.
      */
-    type?: 'supervised' | 'dpo';
-  }
+    dpo?: MethodsAPI.DpoMethod;
 
-  export namespace Method {
     /**
-     * Configuration for the DPO fine-tuning method.
+     * Configuration for the reinforcement fine-tuning method.
      */
-    export interface Dpo {
-      /**
-       * The hyperparameters used for the fine-tuning job.
-       */
-      hyperparameters?: Dpo.Hyperparameters;
-    }
-
-    export namespace Dpo {
-      /**
-       * The hyperparameters used for the fine-tuning job.
-       */
-      export interface Hyperparameters {
-        /**
-         * Number of examples in each batch. A larger batch size means that model
-         * parameters are updated less frequently, but with lower variance.
-         */
-        batch_size?: 'auto' | number;
-
-        /**
-         * The beta value for the DPO method. A higher beta value will increase the weight
-         * of the penalty between the policy and reference model.
-         */
-        beta?: 'auto' | number;
-
-        /**
-         * Scaling factor for the learning rate. A smaller learning rate may be useful to
-         * avoid overfitting.
-         */
-        learning_rate_multiplier?: 'auto' | number;
-
-        /**
-         * The number of epochs to train the model for. An epoch refers to one full cycle
-         * through the training dataset.
-         */
-        n_epochs?: 'auto' | number;
-      }
-    }
+    reinforcement?: MethodsAPI.ReinforcementMethod;
 
     /**
      * Configuration for the supervised fine-tuning method.
      */
-    export interface Supervised {
-      /**
-       * The hyperparameters used for the fine-tuning job.
-       */
-      hyperparameters?: Supervised.Hyperparameters;
-    }
-
-    export namespace Supervised {
-      /**
-       * The hyperparameters used for the fine-tuning job.
-       */
-      export interface Hyperparameters {
-        /**
-         * Number of examples in each batch. A larger batch size means that model
-         * parameters are updated less frequently, but with lower variance.
-         */
-        batch_size?: 'auto' | number;
-
-        /**
-         * Scaling factor for the learning rate. A smaller learning rate may be useful to
-         * avoid overfitting.
-         */
-        learning_rate_multiplier?: 'auto' | number;
-
-        /**
-         * The number of epochs to train the model for. An epoch refers to one full cycle
-         * through the training dataset.
-         */
-        n_epochs?: 'auto' | number;
-      }
-    }
+    supervised?: MethodsAPI.SupervisedMethod;
   }
 }