llama-stack-client 0.4.3 → 0.4.5

package/dist/src/resources/alpha/alpha.ts

@@ -0,0 +1,93 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../resource';
+import * as AdminAPI from './admin';
+import { Admin, AdminListRoutesParams } from './admin';
+import * as BenchmarksAPI from './benchmarks';
+import {
+  Benchmark,
+  BenchmarkListResponse,
+  BenchmarkRegisterParams,
+  Benchmarks,
+  ListBenchmarksResponse,
+} from './benchmarks';
+import * as InferenceAPI from './inference';
+import { Inference, InferenceRerankParams, InferenceRerankResponse } from './inference';
+import * as EvalAPI from './eval/eval';
+import {
+  BenchmarkConfig,
+  Eval,
+  EvalEvaluateRowsAlphaParams,
+  EvalEvaluateRowsParams,
+  EvalRunEvalAlphaParams,
+  EvalRunEvalParams,
+  EvaluateResponse,
+  Job,
+} from './eval/eval';
+import * as PostTrainingAPI from './post-training/post-training';
+import {
+  AlgorithmConfig,
+  ListPostTrainingJobsResponse,
+  PostTraining,
+  PostTrainingJob,
+  PostTrainingPreferenceOptimizeParams,
+  PostTrainingSupervisedFineTuneParams,
+} from './post-training/post-training';
+
+export class Alpha extends APIResource {
+  postTraining: PostTrainingAPI.PostTraining = new PostTrainingAPI.PostTraining(this._client);
+  benchmarks: BenchmarksAPI.Benchmarks = new BenchmarksAPI.Benchmarks(this._client);
+  eval: EvalAPI.Eval = new EvalAPI.Eval(this._client);
+  admin: AdminAPI.Admin = new AdminAPI.Admin(this._client);
+  inference: InferenceAPI.Inference = new InferenceAPI.Inference(this._client);
+}
+
+Alpha.PostTraining = PostTraining;
+Alpha.Benchmarks = Benchmarks;
+Alpha.Eval = Eval;
+Alpha.Admin = Admin;
+Alpha.Inference = Inference;
+
+export declare namespace Alpha {
+  export {
+    PostTraining as PostTraining,
+    type AlgorithmConfig as AlgorithmConfig,
+    type ListPostTrainingJobsResponse as ListPostTrainingJobsResponse,
+    type PostTrainingJob as PostTrainingJob,
+    type PostTrainingPreferenceOptimizeParams as PostTrainingPreferenceOptimizeParams,
+    type PostTrainingSupervisedFineTuneParams as PostTrainingSupervisedFineTuneParams,
+  };
+
+  export {
+    Benchmarks as Benchmarks,
+    type Benchmark as Benchmark,
+    type ListBenchmarksResponse as ListBenchmarksResponse,
+    type BenchmarkListResponse as BenchmarkListResponse,
+    type BenchmarkRegisterParams as BenchmarkRegisterParams,
+  };
+
+  export {
+    Eval as Eval,
+    type BenchmarkConfig as BenchmarkConfig,
+    type EvaluateResponse as EvaluateResponse,
+    type Job as Job,
+    type EvalEvaluateRowsParams as EvalEvaluateRowsParams,
+    type EvalEvaluateRowsAlphaParams as EvalEvaluateRowsAlphaParams,
+    type EvalRunEvalParams as EvalRunEvalParams,
+    type EvalRunEvalAlphaParams as EvalRunEvalAlphaParams,
+  };
+
+  export { Admin as Admin, type AdminListRoutesParams as AdminListRoutesParams };
+
+  export {
+    Inference as Inference,
+    type InferenceRerankResponse as InferenceRerankResponse,
+    type InferenceRerankParams as InferenceRerankParams,
+  };
+}

package/dist/src/resources/alpha/benchmarks.ts

@@ -0,0 +1,152 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../resource';
+import * as Core from '../../core';
+
+export class Benchmarks extends APIResource {
+  /**
+   * Get a benchmark by its ID.
+   */
+  retrieve(benchmarkId: string, options?: Core.RequestOptions): Core.APIPromise<Benchmark> {
+    return this._client.get(`/v1alpha/eval/benchmarks/${benchmarkId}`, options);
+  }
+
+  /**
+   * List all benchmarks.
+   */
+  list(options?: Core.RequestOptions): Core.APIPromise<BenchmarkListResponse> {
+    return (
+      this._client.get('/v1alpha/eval/benchmarks', options) as Core.APIPromise<{
+        data: BenchmarkListResponse;
+      }>
+    )._thenUnwrap((obj) => obj.data);
+  }
+
+  /**
+   * Register a benchmark.
+   *
+   * @deprecated
+   */
+  register(body: BenchmarkRegisterParams, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.post('/v1alpha/eval/benchmarks', {
+      body,
+      ...options,
+      headers: { Accept: '*/*', ...options?.headers },
+    });
+  }
+
+  /**
+   * Unregister a benchmark.
+   *
+   * @deprecated
+   */
+  unregister(benchmarkId: string, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.delete(`/v1alpha/eval/benchmarks/${benchmarkId}`, {
+      ...options,
+      headers: { Accept: '*/*', ...options?.headers },
+    });
+  }
+}
+
+/**
+ * A benchmark resource for evaluating model performance.
+ */
+export interface Benchmark {
+  /**
+   * Identifier of the dataset to use for the benchmark evaluation.
+   */
+  dataset_id: string;
+
+  /**
+   * Unique identifier for this resource in llama stack
+   */
+  identifier: string;
+
+  /**
+   * ID of the provider that owns this resource
+   */
+  provider_id: string;
+
+  /**
+   * List of scoring function identifiers to apply during evaluation.
+   */
+  scoring_functions: Array<string>;
+
+  /**
+   * Metadata for this evaluation task.
+   */
+  metadata?: { [key: string]: unknown };
+
+  /**
+   * Unique identifier for this resource in the provider
+   */
+  provider_resource_id?: string | null;
+
+  /**
+   * The resource type, always benchmark.
+   */
+  type?: 'benchmark';
+}
+
+/**
+ * Response containing a list of benchmark objects.
+ */
+export interface ListBenchmarksResponse {
+  /**
+   * List of benchmark objects.
+   */
+  data: BenchmarkListResponse;
+}
+
+/**
+ * List of benchmark objects.
+ */
+export type BenchmarkListResponse = Array<Benchmark>;
+
+export interface BenchmarkRegisterParams {
+  /**
+   * The ID of the benchmark to register.
+   */
+  benchmark_id: string;
+
+  /**
+   * The ID of the dataset to use for the benchmark.
+   */
+  dataset_id: string;
+
+  /**
+   * The scoring functions to use for the benchmark.
+   */
+  scoring_functions: Array<string>;
+
+  /**
+   * The metadata to use for the benchmark.
+   */
+  metadata?: { [key: string]: unknown } | null;
+
+  /**
+   * The ID of the provider benchmark to use for the benchmark.
+   */
+  provider_benchmark_id?: string | null;
+
+  /**
+   * The ID of the provider to use for the benchmark.
+   */
+  provider_id?: string | null;
+}
+
+export declare namespace Benchmarks {
+  export {
+    type Benchmark as Benchmark,
+    type ListBenchmarksResponse as ListBenchmarksResponse,
+    type BenchmarkListResponse as BenchmarkListResponse,
+    type BenchmarkRegisterParams as BenchmarkRegisterParams,
+  };
+}

package/dist/src/resources/alpha/eval/eval.ts

@@ -0,0 +1,255 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../../resource';
+import * as Core from '../../../core';
+import * as Shared from '../../shared';
+import * as JobsAPI from './jobs';
+import { Jobs } from './jobs';
+
+export class Eval extends APIResource {
+  jobs: JobsAPI.Jobs = new JobsAPI.Jobs(this._client);
+
+  /**
+   * Evaluate a list of rows on a benchmark.
+   */
+  evaluateRows(
+    benchmarkId: string,
+    body: EvalEvaluateRowsParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvaluateResponse> {
+    return this._client.post(`/v1alpha/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
+  }
+
+  /**
+   * Evaluate a list of rows on a benchmark.
+   */
+  evaluateRowsAlpha(
+    benchmarkId: string,
+    body: EvalEvaluateRowsAlphaParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvaluateResponse> {
+    return this._client.post(`/v1alpha/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
+  }
+
+  /**
+   * Run an evaluation on a benchmark.
+   */
+  runEval(benchmarkId: string, body: EvalRunEvalParams, options?: Core.RequestOptions): Core.APIPromise<Job> {
+    return this._client.post(`/v1alpha/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
+  }
+
+  /**
+   * Run an evaluation on a benchmark.
+   */
+  runEvalAlpha(
+    benchmarkId: string,
+    body: EvalRunEvalAlphaParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<Job> {
+    return this._client.post(`/v1alpha/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
+  }
+}
+
+/**
+ * A benchmark configuration for evaluation.
+ */
+export interface BenchmarkConfig {
+  /**
+   * The candidate to evaluate
+   */
+  eval_candidate: BenchmarkConfig.EvalCandidate;
+
+  /**
+   * Number of examples to evaluate (useful for testing), if not provided, all
+   * examples in the dataset will be evaluated
+   */
+  num_examples?: number | null;
+
+  /**
+   * Map between scoring function id and parameters for each scoring function you
+   * want to run
+   */
+  scoring_params?: {
+    [key: string]:
+      | BenchmarkConfig.LlmAsJudgeScoringFnParams
+      | BenchmarkConfig.RegexParserScoringFnParams
+      | BenchmarkConfig.BasicScoringFnParams;
+  };
+}
+
+export namespace BenchmarkConfig {
+  /**
+   * The candidate to evaluate
+   */
+  export interface EvalCandidate {
+    /**
+     * The model ID to evaluate
+     */
+    model: string;
+
+    /**
+     * The sampling parameters for the model
+     */
+    sampling_params: Shared.SamplingParams;
+
+    /**
+     * A system message providing instructions or context to the model.
+     */
+    system_message?: Shared.SystemMessage | null;
+
+    type?: 'model';
+  }
+
+  /**
+   * Parameters for LLM-as-judge scoring function configuration.
+   */
+  export interface LlmAsJudgeScoringFnParams {
+    judge_model: string;
+
+    /**
+     * Aggregation functions to apply to the scores of each row
+     */
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
+
+    /**
+     * Regexes to extract the answer from generated response
+     */
+    judge_score_regexes?: Array<string>;
+
+    prompt_template?: string | null;
+
+    type?: 'llm_as_judge';
+  }
+
+  /**
+   * Parameters for regex parser scoring function configuration.
+   */
+  export interface RegexParserScoringFnParams {
+    /**
+     * Aggregation functions to apply to the scores of each row
+     */
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
+
+    /**
+     * Regex to extract the answer from generated response
+     */
+    parsing_regexes?: Array<string>;
+
+    type?: 'regex_parser';
+  }
+
+  /**
+   * Parameters for basic scoring function configuration.
+   */
+  export interface BasicScoringFnParams {
+    /**
+     * Aggregation functions to apply to the scores of each row
+     */
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
+
+    type?: 'basic';
+  }
+}
+
+/**
+ * The response from an evaluation.
+ */
+export interface EvaluateResponse {
+  /**
+   * The generations from the evaluation
+   */
+  generations: Array<{ [key: string]: unknown }>;
+
+  /**
+   * The scores from the evaluation. Each key in the dict is a scoring function name
+   */
+  scores: { [key: string]: Shared.ScoringResult };
+}
+
+/**
+ * A job execution instance with status tracking.
+ */
+export interface Job {
+  job_id: string;
+
+  /**
+   * Status of a job execution.
+   */
+  status: 'completed' | 'in_progress' | 'failed' | 'scheduled' | 'cancelled';
+}
+
+export interface EvalEvaluateRowsParams {
+  /**
+   * The configuration for the benchmark
+   */
+  benchmark_config: BenchmarkConfig;
+
+  /**
+   * The rows to evaluate
+   */
+  input_rows: Array<{ [key: string]: unknown }>;
+
+  /**
+   * The scoring functions to use for the evaluation
+   */
+  scoring_functions: Array<string>;
+}
+
+export interface EvalEvaluateRowsAlphaParams {
+  /**
+   * The configuration for the benchmark
+   */
+  benchmark_config: BenchmarkConfig;
+
+  /**
+   * The rows to evaluate
+   */
+  input_rows: Array<{ [key: string]: unknown }>;
+
+  /**
+   * The scoring functions to use for the evaluation
+   */
+  scoring_functions: Array<string>;
+}
+
+export interface EvalRunEvalParams {
+  /**
+   * The configuration for the benchmark
+   */
+  benchmark_config: BenchmarkConfig;
+}
+
+export interface EvalRunEvalAlphaParams {
+  /**
+   * The configuration for the benchmark
+   */
+  benchmark_config: BenchmarkConfig;
+}
+
+Eval.Jobs = Jobs;
+
+export declare namespace Eval {
+  export {
+    type BenchmarkConfig as BenchmarkConfig,
+    type EvaluateResponse as EvaluateResponse,
+    type Job as Job,
+    type EvalEvaluateRowsParams as EvalEvaluateRowsParams,
+    type EvalEvaluateRowsAlphaParams as EvalEvaluateRowsAlphaParams,
+    type EvalRunEvalParams as EvalRunEvalParams,
+    type EvalRunEvalAlphaParams as EvalRunEvalAlphaParams,
+  };
+
+  export { Jobs as Jobs };
+}

package/dist/src/resources/alpha/eval/index.ts

@@ -0,0 +1,19 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export {
+  Eval,
+  type BenchmarkConfig,
+  type EvaluateResponse,
+  type Job,
+  type EvalEvaluateRowsParams,
+  type EvalEvaluateRowsAlphaParams,
+  type EvalRunEvalParams,
+  type EvalRunEvalAlphaParams,
+} from './eval';
+export { Jobs } from './jobs';

package/dist/src/resources/alpha/eval/jobs.ts

@@ -0,0 +1,41 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../../resource';
+import * as Core from '../../../core';
+import * as EvalAPI from './eval';
+
+export class Jobs extends APIResource {
+  /**
+   * Get the result of a job.
+   */
+  retrieve(
+    benchmarkId: string,
+    jobId: string,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvalAPI.EvaluateResponse> {
+    return this._client.get(`/v1alpha/eval/benchmarks/${benchmarkId}/jobs/${jobId}/result`, options);
+  }
+
+  /**
+   * Cancel a job.
+   */
+  cancel(benchmarkId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.delete(`/v1alpha/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, {
+      ...options,
+      headers: { Accept: '*/*', ...options?.headers },
+    });
+  }
+
+  /**
+   * Get the status of a job.
+   */
+  status(benchmarkId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<EvalAPI.Job> {
+    return this._client.get(`/v1alpha/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, options);
+  }
+}

package/dist/src/resources/alpha/eval.ts

@@ -0,0 +1,10 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export * from './eval/index';

package/dist/src/resources/alpha/index.ts

@@ -0,0 +1,36 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export { Admin, type AdminListRoutesParams } from './admin';
+export { Alpha } from './alpha';
+export {
+  Benchmarks,
+  type Benchmark,
+  type ListBenchmarksResponse,
+  type BenchmarkListResponse,
+  type BenchmarkRegisterParams,
+} from './benchmarks';
+export {
+  Eval,
+  type BenchmarkConfig,
+  type EvaluateResponse,
+  type Job,
+  type EvalEvaluateRowsParams,
+  type EvalEvaluateRowsAlphaParams,
+  type EvalRunEvalParams,
+  type EvalRunEvalAlphaParams,
+} from './eval/index';
+export { Inference, type InferenceRerankResponse, type InferenceRerankParams } from './inference';
+export {
+  PostTraining,
+  type AlgorithmConfig,
+  type ListPostTrainingJobsResponse,
+  type PostTrainingJob,
+  type PostTrainingPreferenceOptimizeParams,
+  type PostTrainingSupervisedFineTuneParams,
+} from './post-training/index';