llama-stack-client 0.3.4-rc3 → 0.4.0-rc1

package/src/lib/response-helpers.ts

@@ -0,0 +1,60 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+
+/**
+ * Helper utilities for working with response objects.
+ */
+
+import type { ResponseObject } from '../resources/responses/responses';
+
+/**
+ * Extracts aggregated text output from a ResponseObject.
+ * This concatenates all `output_text` entries from the response's output array.
+ *
+ * Useful for streaming responses where you want to get the final text from chunk.response:
+ *
+ * @example
+ * ```ts
+ * const stream = await client.responses.create({ stream: true, ... });
+ * for await (const chunk of stream) {
+ *   if (chunk.type === 'response.completed') {
+ *     const text = getResponseOutputText(chunk.response);
+ *     console.log(text);
+ *   }
+ * }
+ * ```
+ */
+export function getResponseOutputText(response: ResponseObject): string {
+  const pieces: string[] = [];
+
+  for (const output of response.output ?? []) {
+    if (!output || output.type !== 'message') {
+      continue;
+    }
+
+    const content = output.content;
+    if (typeof content === 'string') {
+      pieces.push(content);
+      continue;
+    }
+
+    if (!Array.isArray(content)) {
+      continue;
+    }
+
+    for (const item of content) {
+      if (typeof item === 'string') {
+        pieces.push(item);
+        continue;
+      }
+      if (item && item.type === 'output_text' && 'text' in item && typeof item.text === 'string') {
+        pieces.push(item.text);
+      }
+    }
+  }
+
+  return pieces.join('');
+}

package/src/pagination.ts

@@ -4,6 +4,7 @@
 // This source code is licensed under the terms described in the LICENSE file in
 // the root directory of this source tree.
 
+//
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 import { AbstractPage, Response, APIClient, FinalRequestOptions, PageInfo } from './core';
@@ -54,11 +55,7 @@ export class DatasetsIterrows<Item> extends AbstractPage<Item> implements Datase
   }
 
   nextPageInfo(): PageInfo | null {
-    const offset = this.next_index;
-    if (!offset) {
-      return null;
-    }
-
+    const offset = this.next_index ?? 0;
     const length = this.getPaginatedItems().length;
     const currentCount = offset + length;
 

package/src/resource.ts

@@ -4,6 +4,7 @@
 // This source code is licensed under the terms described in the LICENSE file in
 // the root directory of this source tree.
 
+//
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 import type { LlamaStackClient } from './index';

package/src/resources/alpha/alpha.ts

@@ -0,0 +1,85 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../resource';
+import * as BenchmarksAPI from './benchmarks';
+import {
+  Benchmark,
+  BenchmarkListResponse,
+  BenchmarkRegisterParams,
+  Benchmarks,
+  ListBenchmarksResponse,
+} from './benchmarks';
+import * as InferenceAPI from './inference';
+import { Inference, InferenceRerankParams, InferenceRerankResponse } from './inference';
+import * as EvalAPI from './eval/eval';
+import {
+  BenchmarkConfig,
+  Eval,
+  EvalEvaluateRowsAlphaParams,
+  EvalEvaluateRowsParams,
+  EvalRunEvalAlphaParams,
+  EvalRunEvalParams,
+  EvaluateResponse,
+  Job,
+} from './eval/eval';
+import * as PostTrainingAPI from './post-training/post-training';
+import {
+  AlgorithmConfig,
+  PostTraining,
+  PostTrainingJob,
+  PostTrainingPreferenceOptimizeParams,
+  PostTrainingSupervisedFineTuneParams,
+} from './post-training/post-training';
+
+export class Alpha extends APIResource {
+  inference: InferenceAPI.Inference = new InferenceAPI.Inference(this._client);
+  postTraining: PostTrainingAPI.PostTraining = new PostTrainingAPI.PostTraining(this._client);
+  benchmarks: BenchmarksAPI.Benchmarks = new BenchmarksAPI.Benchmarks(this._client);
+  eval: EvalAPI.Eval = new EvalAPI.Eval(this._client);
+}
+
+Alpha.Inference = Inference;
+Alpha.PostTraining = PostTraining;
+Alpha.Benchmarks = Benchmarks;
+Alpha.Eval = Eval;
+
+export declare namespace Alpha {
+  export {
+    Inference as Inference,
+    type InferenceRerankResponse as InferenceRerankResponse,
+    type InferenceRerankParams as InferenceRerankParams,
+  };
+
+  export {
+    PostTraining as PostTraining,
+    type AlgorithmConfig as AlgorithmConfig,
+    type PostTrainingJob as PostTrainingJob,
+    type PostTrainingPreferenceOptimizeParams as PostTrainingPreferenceOptimizeParams,
+    type PostTrainingSupervisedFineTuneParams as PostTrainingSupervisedFineTuneParams,
+  };
+
+  export {
+    Benchmarks as Benchmarks,
+    type Benchmark as Benchmark,
+    type ListBenchmarksResponse as ListBenchmarksResponse,
+    type BenchmarkListResponse as BenchmarkListResponse,
+    type BenchmarkRegisterParams as BenchmarkRegisterParams,
+  };
+
+  export {
+    Eval as Eval,
+    type BenchmarkConfig as BenchmarkConfig,
+    type EvaluateResponse as EvaluateResponse,
+    type Job as Job,
+    type EvalEvaluateRowsParams as EvalEvaluateRowsParams,
+    type EvalEvaluateRowsAlphaParams as EvalEvaluateRowsAlphaParams,
+    type EvalRunEvalParams as EvalRunEvalParams,
+    type EvalRunEvalAlphaParams as EvalRunEvalAlphaParams,
+  };
+}

package/src/resources/{benchmarks.ts → alpha/benchmarks.ts}

@@ -4,17 +4,18 @@
 // This source code is licensed under the terms described in the LICENSE file in
 // the root directory of this source tree.
 
+//
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { APIResource } from '../resource';
-import * as Core from '../core';
+import { APIResource } from '../../resource';
+import * as Core from '../../core';
 
 export class Benchmarks extends APIResource {
   /**
    * Get a benchmark by its ID.
    */
   retrieve(benchmarkId: string, options?: Core.RequestOptions): Core.APIPromise<Benchmark> {
-    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}`, options);
+    return this._client.get(`/v1alpha/eval/benchmarks/${benchmarkId}`, options);
   }
 
   /**
@@ -22,51 +23,67 @@ export class Benchmarks extends APIResource {
    */
   list(options?: Core.RequestOptions): Core.APIPromise<BenchmarkListResponse> {
     return (
-      this._client.get('/v1/eval/benchmarks', options) as Core.APIPromise<{ data: BenchmarkListResponse }>
+      this._client.get('/v1alpha/eval/benchmarks', options) as Core.APIPromise<{
+        data: BenchmarkListResponse;
+      }>
     )._thenUnwrap((obj) => obj.data);
   }
 
   /**
    * Register a benchmark.
+   *
+   * @deprecated
    */
   register(body: BenchmarkRegisterParams, options?: Core.RequestOptions): Core.APIPromise<void> {
-    return this._client.post('/v1/eval/benchmarks', {
+    return this._client.post('/v1alpha/eval/benchmarks', {
       body,
       ...options,
       headers: { Accept: '*/*', ...options?.headers },
     });
   }
+
+  /**
+   * Unregister a benchmark.
+   *
+   * @deprecated
+   */
+  unregister(benchmarkId: string, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.delete(`/v1alpha/eval/benchmarks/${benchmarkId}`, {
+      ...options,
+      headers: { Accept: '*/*', ...options?.headers },
+    });
+  }
 }
 
 /**
  * A benchmark resource for evaluating model performance.
  */
 export interface Benchmark {
-  /**
-   * Identifier of the dataset to use for the benchmark evaluation
-   */
   dataset_id: string;
 
+  /**
+   * Unique identifier for this resource in llama stack
+   */
   identifier: string;
 
   /**
-   * Metadata for this evaluation task
+   * ID of the provider that owns this resource
    */
-  metadata: { [key: string]: boolean | number | string | Array<unknown> | unknown | null };
-
   provider_id: string;
 
+  scoring_functions: Array<string>;
+
   /**
-   * List of scoring function identifiers to apply during evaluation
+   * Metadata for this evaluation task
    */
-  scoring_functions: Array<string>;
+  metadata?: { [key: string]: unknown };
 
   /**
-   * The resource type, always benchmark
+   * Unique identifier for this resource in the provider
    */
-  type: 'benchmark';
+  provider_resource_id?: string | null;
 
-  provider_resource_id?: string;
+  type?: 'benchmark';
 }
 
 export interface ListBenchmarksResponse {
@@ -76,35 +93,17 @@ export interface ListBenchmarksResponse {
 export type BenchmarkListResponse = Array<Benchmark>;
 
 export interface BenchmarkRegisterParams {
-  /**
-   * The ID of the benchmark to register.
-   */
   benchmark_id: string;
 
-  /**
-   * The ID of the dataset to use for the benchmark.
-   */
   dataset_id: string;
 
-  /**
-   * The scoring functions to use for the benchmark.
-   */
   scoring_functions: Array<string>;
 
-  /**
-   * The metadata to use for the benchmark.
-   */
-  metadata?: { [key: string]: boolean | number | string | Array<unknown> | unknown | null };
+  metadata?: { [key: string]: unknown } | null;
 
-  /**
-   * The ID of the provider benchmark to use for the benchmark.
-   */
-  provider_benchmark_id?: string;
+  provider_benchmark_id?: string | null;
 
-  /**
-   * The ID of the provider to use for the benchmark.
-   */
-  provider_id?: string;
+  provider_id?: string | null;
 }
 
 export declare namespace Benchmarks {

package/src/resources/{eval → alpha/eval}/eval.ts

@@ -3,13 +3,12 @@
 //
 // This source code is licensed under the terms described in the LICENSE file in
 // the root directory of this source tree.
-
+//
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { APIResource } from '../../resource';
-import * as Core from '../../core';
-import * as ScoringFunctionsAPI from '../scoring-functions';
-import * as Shared from '../shared';
+import { APIResource } from '../../../resource';
+import * as Core from '../../../core';
+import * as Shared from '../../shared';
 import * as JobsAPI from './jobs';
 import { Jobs } from './jobs';
 
@@ -24,7 +23,7 @@ export class Eval extends APIResource {
     body: EvalEvaluateRowsParams,
     options?: Core.RequestOptions,
   ): Core.APIPromise<EvaluateResponse> {
-    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
+    return this._client.post(`/v1alpha/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
   }
 
   /**
@@ -35,14 +34,14 @@ export class Eval extends APIResource {
     body: EvalEvaluateRowsAlphaParams,
     options?: Core.RequestOptions,
   ): Core.APIPromise<EvaluateResponse> {
-    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
+    return this._client.post(`/v1alpha/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
   }
 
   /**
    * Run an evaluation on a benchmark.
    */
   runEval(benchmarkId: string, body: EvalRunEvalParams, options?: Core.RequestOptions): Core.APIPromise<Job> {
-    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
+    return this._client.post(`/v1alpha/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
   }
 
   /**
@@ -53,7 +52,7 @@ export class Eval extends APIResource {
     body: EvalRunEvalAlphaParams,
     options?: Core.RequestOptions,
   ): Core.APIPromise<Job> {
-    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
+    return this._client.post(`/v1alpha/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
   }
 }
 
@@ -62,61 +61,102 @@ export class Eval extends APIResource {
  */
 export interface BenchmarkConfig {
   /**
-   * The candidate to evaluate.
+   * A model candidate for evaluation.
+   */
+  eval_candidate: BenchmarkConfig.EvalCandidate;
+
+  /**
+   * Number of examples to evaluate (useful for testing), if not provided, all
+   * examples in the dataset will be evaluated
    */
-  eval_candidate: EvalCandidate;
+  num_examples?: number | null;
 
   /**
    * Map between scoring function id and parameters for each scoring function you
    * want to run
    */
-  scoring_params: { [key: string]: ScoringFunctionsAPI.ScoringFnParams };
+  scoring_params?: {
+    [key: string]:
+      | BenchmarkConfig.LlmAsJudgeScoringFnParams
+      | BenchmarkConfig.RegexParserScoringFnParams
+      | BenchmarkConfig.BasicScoringFnParams;
+  };
+}
 
+export namespace BenchmarkConfig {
   /**
-   * (Optional) The number of examples to evaluate. If not provided, all examples in
-   * the dataset will be evaluated
+   * A model candidate for evaluation.
    */
-  num_examples?: number;
-}
+  export interface EvalCandidate {
+    model: string;
 
-/**
- * A model candidate for evaluation.
- */
-export type EvalCandidate = EvalCandidate.ModelCandidate | EvalCandidate.AgentCandidate;
+    /**
+     * Sampling parameters.
+     */
+    sampling_params: Shared.SamplingParams;
+
+    /**
+     * A system message providing instructions or context to the model.
+     */
+    system_message?: Shared.SystemMessage | null;
+
+    type?: 'model';
+  }
 
-export namespace EvalCandidate {
   /**
-   * A model candidate for evaluation.
+   * Parameters for LLM-as-judge scoring function configuration.
    */
-  export interface ModelCandidate {
+  export interface LlmAsJudgeScoringFnParams {
+    judge_model: string;
+
     /**
-     * The model ID to evaluate.
+     * Aggregation functions to apply to the scores of each row
      */
-    model: string;
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
 
     /**
-     * The sampling parameters for the model.
+     * Regexes to extract the answer from generated response
      */
-    sampling_params: Shared.SamplingParams;
+    judge_score_regexes?: Array<string>;
+
+    prompt_template?: string | null;
+
+    type?: 'llm_as_judge';
+  }
 
-    type: 'model';
+  /**
+   * Parameters for regex parser scoring function configuration.
+   */
+  export interface RegexParserScoringFnParams {
+    /**
+     * Aggregation functions to apply to the scores of each row
+     */
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
 
     /**
-     * (Optional) The system message providing instructions or context to the model.
+     * Regex to extract the answer from generated response
      */
-    system_message?: Shared.SystemMessage;
+    parsing_regexes?: Array<string>;
+
+    type?: 'regex_parser';
   }
 
   /**
-   * An agent candidate for evaluation.
+   * Parameters for basic scoring function configuration.
    */
-  export interface AgentCandidate {
+  export interface BasicScoringFnParams {
     /**
-     * The configuration for the agent candidate.
+     * Aggregation functions to apply to the scores of each row
      */
-    config: Shared.AgentConfig;
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
 
-    type: 'agent';
+    type?: 'basic';
   }
 }
 
@@ -124,14 +164,8 @@ export namespace EvalCandidate {
  * The response from an evaluation.
  */
 export interface EvaluateResponse {
-  /**
-   * The generations from the evaluation.
-   */
-  generations: Array<{ [key: string]: boolean | number | string | Array<unknown> | unknown | null }>;
+  generations: Array<{ [key: string]: unknown }>;
 
-  /**
-   * The scores from the evaluation.
-   */
   scores: { [key: string]: Shared.ScoringResult };
 }
 
@@ -139,61 +173,46 @@ export interface EvaluateResponse {
  * A job execution instance with status tracking.
  */
 export interface Job {
-  /**
-   * Unique identifier for the job
-   */
   job_id: string;
 
   /**
-   * Current execution status of the job
+   * Status of a job execution.
    */
   status: 'completed' | 'in_progress' | 'failed' | 'scheduled' | 'cancelled';
 }
 
 export interface EvalEvaluateRowsParams {
   /**
-   * The configuration for the benchmark.
+   * A benchmark configuration for evaluation.
    */
   benchmark_config: BenchmarkConfig;
 
-  /**
-   * The rows to evaluate.
-   */
-  input_rows: Array<{ [key: string]: boolean | number | string | Array<unknown> | unknown | null }>;
+  input_rows: Array<{ [key: string]: unknown }>;
 
-  /**
-   * The scoring functions to use for the evaluation.
-   */
   scoring_functions: Array<string>;
 }
 
 export interface EvalEvaluateRowsAlphaParams {
   /**
-   * The configuration for the benchmark.
+   * A benchmark configuration for evaluation.
    */
   benchmark_config: BenchmarkConfig;
 
-  /**
-   * The rows to evaluate.
-   */
-  input_rows: Array<{ [key: string]: boolean | number | string | Array<unknown> | unknown | null }>;
+  input_rows: Array<{ [key: string]: unknown }>;
 
-  /**
-   * The scoring functions to use for the evaluation.
-   */
   scoring_functions: Array<string>;
 }
 
 export interface EvalRunEvalParams {
   /**
-   * The configuration for the benchmark.
+   * A benchmark configuration for evaluation.
    */
   benchmark_config: BenchmarkConfig;
 }
 
 export interface EvalRunEvalAlphaParams {
   /**
-   * The configuration for the benchmark.
+   * A benchmark configuration for evaluation.
    */
   benchmark_config: BenchmarkConfig;
 }
@@ -203,7 +222,6 @@ Eval.Jobs = Jobs;
 export declare namespace Eval {
   export {
     type BenchmarkConfig as BenchmarkConfig,
-    type EvalCandidate as EvalCandidate,
     type EvaluateResponse as EvaluateResponse,
     type Job as Job,
     type EvalEvaluateRowsParams as EvalEvaluateRowsParams,

package/src/resources/{eval → alpha/eval}/index.ts

@@ -3,13 +3,12 @@
 //
 // This source code is licensed under the terms described in the LICENSE file in
 // the root directory of this source tree.
-
+//
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 export {
   Eval,
   type BenchmarkConfig,
-  type EvalCandidate,
   type EvaluateResponse,
   type Job,
   type EvalEvaluateRowsParams,

package/src/resources/{eval → alpha/eval}/jobs.ts

@@ -3,11 +3,11 @@
 //
 // This source code is licensed under the terms described in the LICENSE file in
 // the root directory of this source tree.
-
+//
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-import { APIResource } from '../../resource';
-import * as Core from '../../core';
+import { APIResource } from '../../../resource';
+import * as Core from '../../../core';
 import * as EvalAPI from './eval';
 
 export class Jobs extends APIResource {
@@ -19,14 +19,14 @@ export class Jobs extends APIResource {
     jobId: string,
     options?: Core.RequestOptions,
   ): Core.APIPromise<EvalAPI.EvaluateResponse> {
-    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}/result`, options);
+    return this._client.get(`/v1alpha/eval/benchmarks/${benchmarkId}/jobs/${jobId}/result`, options);
   }
 
   /**
    * Cancel a job.
    */
   cancel(benchmarkId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<void> {
-    return this._client.delete(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, {
+    return this._client.delete(`/v1alpha/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, {
       ...options,
       headers: { Accept: '*/*', ...options?.headers },
     });
@@ -36,6 +36,6 @@ export class Jobs extends APIResource {
    * Get the status of a job.
    */
   status(benchmarkId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<EvalAPI.Job> {
-    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, options);
+    return this._client.get(`/v1alpha/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, options);
   }
 }

package/src/resources/{eval.ts → alpha/eval.ts}

@@ -4,6 +4,7 @@
 // This source code is licensed under the terms described in the LICENSE file in
 // the root directory of this source tree.
 
+//
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 export * from './eval/index';