llama-stack-client 0.1.0 → 0.1.3

package/src/resources/batch-inference.ts

@@ -31,12 +31,23 @@ export interface BatchInferenceChatCompletionParams {
 
   logprobs?: BatchInferenceChatCompletionParams.Logprobs;
 
+  /**
+   * Configuration for JSON schema-guided response generation.
+   */
   response_format?: Shared.ResponseFormat;
 
   sampling_params?: Shared.SamplingParams;
 
+  /**
+   * Whether tool use is required or automatic. This is a hint to the model which may
+   * not be followed. It depends on the Instruction Following capabilities of the
+   * model.
+   */
   tool_choice?: 'auto' | 'required';
 
+  /**
+   * Prompt format for calling custom / zero shot tools.
+   */
   tool_prompt_format?: 'json' | 'function_tag' | 'python_list';
 
   tools?: Array<BatchInferenceChatCompletionParams.Tool>;
@@ -66,6 +77,9 @@ export interface BatchInferenceCompletionParams {
 
   logprobs?: BatchInferenceCompletionParams.Logprobs;
 
+  /**
+   * Configuration for JSON schema-guided response generation.
+   */
   response_format?: Shared.ResponseFormat;
 
   sampling_params?: Shared.SamplingParams;

package/src/resources/benchmarks.ts

@@ -0,0 +1,69 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../resource';
+import * as Core from '../core';
+
+export class Benchmarks extends APIResource {
+  retrieve(benchmarkId: string, options?: Core.RequestOptions): Core.APIPromise<Benchmark | null> {
+    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}`, options);
+  }
+
+  list(options?: Core.RequestOptions): Core.APIPromise<BenchmarkListResponse> {
+    return (
+      this._client.get('/v1/eval/benchmarks', options) as Core.APIPromise<{ data: BenchmarkListResponse }>
+    )._thenUnwrap((obj) => obj.data);
+  }
+
+  register(body: BenchmarkRegisterParams, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.post('/v1/eval/benchmarks', {
+      body,
+      ...options,
+      headers: { Accept: '*/*', ...options?.headers },
+    });
+  }
+}
+
+export interface Benchmark {
+  dataset_id: string;
+
+  identifier: string;
+
+  metadata: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
+
+  provider_id: string;
+
+  provider_resource_id: string;
+
+  scoring_functions: Array<string>;
+
+  type: 'benchmark';
+}
+
+export interface ListBenchmarksResponse {
+  data: BenchmarkListResponse;
+}
+
+export type BenchmarkListResponse = Array<Benchmark>;
+
+export interface BenchmarkRegisterParams {
+  benchmark_id: string;
+
+  dataset_id: string;
+
+  scoring_functions: Array<string>;
+
+  metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
+
+  provider_benchmark_id?: string;
+
+  provider_id?: string;
+}
+
+export declare namespace Benchmarks {
+  export {
+    type Benchmark as Benchmark,
+    type ListBenchmarksResponse as ListBenchmarksResponse,
+    type BenchmarkListResponse as BenchmarkListResponse,
+    type BenchmarkRegisterParams as BenchmarkRegisterParams,
+  };
+}

package/src/resources/datasets.ts

@@ -35,25 +35,7 @@ export class Datasets extends APIResource {
 }
 
 export interface ListDatasetsResponse {
-  data: Array<ListDatasetsResponse.Data>;
-}
-
-export namespace ListDatasetsResponse {
-  export interface Data {
-    dataset_schema: Record<string, Shared.ParamType>;
-
-    identifier: string;
-
-    metadata: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
-
-    provider_id: string;
-
-    provider_resource_id: string;
-
-    type: 'dataset';
-
-    url: Shared.URL;
-  }
+  data: DatasetListResponse;
 }
 
 export interface DatasetRetrieveResponse {
@@ -69,7 +51,13 @@ export interface DatasetRetrieveResponse {
 
   type: 'dataset';
 
-  url: Shared.URL;
+  url: DatasetRetrieveResponse.URL;
+}
+
+export namespace DatasetRetrieveResponse {
+  export interface URL {
+    uri: string;
+  }
 }
 
 export type DatasetListResponse = Array<DatasetListResponse.DatasetListResponseItem>;
@@ -88,7 +76,13 @@ export namespace DatasetListResponse {
 
     type: 'dataset';
 
-    url: Shared.URL;
+    url: DatasetListResponseItem.URL;
+  }
+
+  export namespace DatasetListResponseItem {
+    export interface URL {
+      uri: string;
+    }
   }
 }
 
@@ -97,7 +91,7 @@ export interface DatasetRegisterParams {
 
   dataset_schema: Record<string, Shared.ParamType>;
 
-  url: Shared.URL;
+  url: DatasetRegisterParams.URL;
 
   metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
 
@@ -106,6 +100,12 @@ export interface DatasetRegisterParams {
   provider_id?: string;
 }
 
+export namespace DatasetRegisterParams {
+  export interface URL {
+    uri: string;
+  }
+}
+
 export declare namespace Datasets {
   export {
     type ListDatasetsResponse as ListDatasetsResponse,

package/src/resources/eval/eval.ts

@@ -2,7 +2,6 @@
 
 import { APIResource } from '../../resource';
 import * as Core from '../../core';
-import * as EvalAPI from './eval';
 import * as ScoringFunctionsAPI from '../scoring-functions';
 import * as Shared from '../shared';
 import * as JobsAPI from './jobs';
@@ -19,9 +18,35 @@ export class Eval extends APIResource {
     return this._client.post(`/v1/eval/tasks/${taskId}/evaluations`, { body, ...options });
   }
 
+  evaluateRowsAlpha(
+    benchmarkId: string,
+    body: EvalEvaluateRowsAlphaParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvaluateResponse> {
+    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
+  }
+
   runEval(taskId: string, body: EvalRunEvalParams, options?: Core.RequestOptions): Core.APIPromise<Job> {
     return this._client.post(`/v1/eval/tasks/${taskId}/jobs`, { body, ...options });
   }
+
+  runEvalAlpha(
+    benchmarkId: string,
+    body: EvalRunEvalAlphaParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<Job> {
+    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
+  }
+}
+
+export interface BenchmarkConfig {
+  eval_candidate: EvalCandidate;
+
+  scoring_params: Record<string, ScoringFunctionsAPI.ScoringFnParams>;
+
+  type: 'benchmark';
+
+  num_examples?: number;
 }
 
 export type EvalCandidate = EvalCandidate.ModelCandidate | EvalCandidate.AgentCandidate;
@@ -34,6 +59,9 @@ export namespace EvalCandidate {
 
     type: 'model';
 
+    /**
+     * A system message providing instructions or context to the model.
+     */
     system_message?: Shared.SystemMessage;
   }
 
@@ -44,28 +72,6 @@ export namespace EvalCandidate {
   }
 }
 
-export type EvalTaskConfig = EvalTaskConfig.BenchmarkEvalTaskConfig | EvalTaskConfig.AppEvalTaskConfig;
-
-export namespace EvalTaskConfig {
-  export interface BenchmarkEvalTaskConfig {
-    eval_candidate: EvalAPI.EvalCandidate;
-
-    type: 'benchmark';
-
-    num_examples?: number;
-  }
-
-  export interface AppEvalTaskConfig {
-    eval_candidate: EvalAPI.EvalCandidate;
-
-    scoring_params: Record<string, ScoringFunctionsAPI.ScoringFnParams>;
-
-    type: 'app';
-
-    num_examples?: number;
-  }
-}
-
 export interface EvaluateResponse {
   generations: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;
 
@@ -81,23 +87,37 @@ export interface EvalEvaluateRowsParams {
 
   scoring_functions: Array<string>;
 
-  task_config: EvalTaskConfig;
+  task_config: BenchmarkConfig;
+}
+
+export interface EvalEvaluateRowsAlphaParams {
+  input_rows: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;
+
+  scoring_functions: Array<string>;
+
+  task_config: BenchmarkConfig;
 }
 
 export interface EvalRunEvalParams {
-  task_config: EvalTaskConfig;
+  task_config: BenchmarkConfig;
+}
+
+export interface EvalRunEvalAlphaParams {
+  task_config: BenchmarkConfig;
 }
 
 Eval.Jobs = Jobs;
 
 export declare namespace Eval {
   export {
+    type BenchmarkConfig as BenchmarkConfig,
     type EvalCandidate as EvalCandidate,
-    type EvalTaskConfig as EvalTaskConfig,
     type EvaluateResponse as EvaluateResponse,
     type Job as Job,
     type EvalEvaluateRowsParams as EvalEvaluateRowsParams,
+    type EvalEvaluateRowsAlphaParams as EvalEvaluateRowsAlphaParams,
     type EvalRunEvalParams as EvalRunEvalParams,
+    type EvalRunEvalAlphaParams as EvalRunEvalAlphaParams,
   };
 
   export { Jobs as Jobs, type JobStatusResponse as JobStatusResponse };

package/src/resources/eval/index.ts

@@ -2,11 +2,13 @@
 
 export {
   Eval,
+  type BenchmarkConfig,
   type EvalCandidate,
-  type EvalTaskConfig,
   type EvaluateResponse,
   type Job,
   type EvalEvaluateRowsParams,
+  type EvalEvaluateRowsAlphaParams,
   type EvalRunEvalParams,
+  type EvalRunEvalAlphaParams,
 } from './eval';
 export { Jobs, type JobStatusResponse } from './jobs';

package/src/resources/eval/jobs.ts

@@ -6,26 +6,26 @@ import * as EvalAPI from './eval';
 
 export class Jobs extends APIResource {
   retrieve(
-    taskId: string,
+    benchmarkId: string,
     jobId: string,
     options?: Core.RequestOptions,
   ): Core.APIPromise<EvalAPI.EvaluateResponse> {
-    return this._client.get(`/v1/eval/tasks/${taskId}/jobs/${jobId}/result`, options);
+    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}/result`, options);
   }
 
-  cancel(taskId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<void> {
-    return this._client.delete(`/v1/eval/tasks/${taskId}/jobs/${jobId}`, {
+  cancel(benchmarkId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.delete(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, {
       ...options,
       headers: { Accept: '*/*', ...options?.headers },
     });
   }
 
   status(
-    taskId: string,
+    benchmarkId: string,
     jobId: string,
     options?: Core.RequestOptions,
   ): Core.APIPromise<JobStatusResponse | null> {
-    return this._client.get(`/v1/eval/tasks/${taskId}/jobs/${jobId}`, options);
+    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, options);
   }
 }
 

package/src/resources/eval-tasks.ts

@@ -2,15 +2,21 @@
 
 import { APIResource } from '../resource';
 import * as Core from '../core';
+import * as BenchmarksAPI from './benchmarks';
 
 export class EvalTasks extends APIResource {
-  retrieve(evalTaskId: string, options?: Core.RequestOptions): Core.APIPromise<EvalTask | null> {
+  retrieve(
+    evalTaskId: string,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<BenchmarksAPI.Benchmark | null> {
     return this._client.get(`/v1/eval-tasks/${evalTaskId}`, options);
   }
 
-  list(options?: Core.RequestOptions): Core.APIPromise<EvalTaskListResponse> {
+  list(options?: Core.RequestOptions): Core.APIPromise<BenchmarksAPI.BenchmarkListResponse> {
     return (
-      this._client.get('/v1/eval-tasks', options) as Core.APIPromise<{ data: EvalTaskListResponse }>
+      this._client.get('/v1/eval-tasks', options) as Core.APIPromise<{
+        data: BenchmarksAPI.BenchmarkListResponse;
+      }>
     )._thenUnwrap((obj) => obj.data);
   }
 
@@ -23,27 +29,7 @@ export class EvalTasks extends APIResource {
   }
 }
 
-export interface EvalTask {
-  dataset_id: string;
-
-  identifier: string;
-
-  metadata: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
-
-  provider_id: string;
-
-  provider_resource_id: string;
-
-  scoring_functions: Array<string>;
-
-  type: 'eval_task';
-}
-
-export interface ListEvalTasksResponse {
-  data: Array<EvalTask>;
-}
-
-export type EvalTaskListResponse = Array<EvalTask>;
+export type EvalTaskListResponse = Array<BenchmarksAPI.Benchmark>;
 
 export interface EvalTaskRegisterParams {
   dataset_id: string;
@@ -54,15 +40,13 @@ export interface EvalTaskRegisterParams {
 
   metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
 
-  provider_eval_task_id?: string;
+  provider_benchmark_id?: string;
 
   provider_id?: string;
 }
 
 export declare namespace EvalTasks {
   export {
-    type EvalTask as EvalTask,
-    type ListEvalTasksResponse as ListEvalTasksResponse,
     type EvalTaskListResponse as EvalTaskListResponse,
     type EvalTaskRegisterParams as EvalTaskRegisterParams,
   };

package/src/resources/index.ts

@@ -17,6 +17,13 @@ export {
   type BatchInferenceChatCompletionParams,
   type BatchInferenceCompletionParams,
 } from './batch-inference';
+export {
+  Benchmarks,
+  type Benchmark,
+  type ListBenchmarksResponse,
+  type BenchmarkListResponse,
+  type BenchmarkRegisterParams,
+} from './benchmarks';
 export {
   Datasetio,
   type PaginatedRowsResult,
@@ -32,20 +39,16 @@ export {
 } from './datasets';
 export {
   Eval,
+  type BenchmarkConfig,
   type EvalCandidate,
-  type EvalTaskConfig,
   type EvaluateResponse,
   type Job,
   type EvalEvaluateRowsParams,
+  type EvalEvaluateRowsAlphaParams,
   type EvalRunEvalParams,
+  type EvalRunEvalAlphaParams,
 } from './eval/eval';
-export {
-  EvalTasks,
-  type EvalTask,
-  type ListEvalTasksResponse,
-  type EvalTaskListResponse,
-  type EvalTaskRegisterParams,
-} from './eval-tasks';
+export { EvalTasks, type EvalTaskListResponse, type EvalTaskRegisterParams } from './eval-tasks';
 export {
   Inference,
   type ChatCompletionResponseStreamChunk,

package/src/resources/inference.ts

@@ -71,11 +71,16 @@ export class Inference extends APIResource {
   }
 }
 
+/**
+ * A chunk of a streamed chat completion response.
+ */
 export interface ChatCompletionResponseStreamChunk {
   /**
    * The event containing the new content
    */
   event: ChatCompletionResponseStreamChunk.Event;
+
+  metrics?: Array<ChatCompletionResponseStreamChunk.Metric>;
 }
 
 export namespace ChatCompletionResponseStreamChunk {
@@ -104,8 +109,29 @@ export namespace ChatCompletionResponseStreamChunk {
      */
     stop_reason?: 'end_of_turn' | 'end_of_message' | 'out_of_tokens';
   }
+
+  export interface Metric {
+    metric: string;
+
+    span_id: string;
+
+    timestamp: string;
+
+    trace_id: string;
+
+    type: 'metric';
+
+    unit: string;
+
+    value: number;
+
+    attributes?: Record<string, string | number | boolean | null>;
+  }
 }
 
+/**
+ * Response from a completion request.
+ */
 export interface CompletionResponse {
   /**
    * The generated completion text
@@ -123,6 +149,9 @@ export interface CompletionResponse {
   logprobs?: Array<TokenLogProbs>;
 }
 
+/**
+ * Response containing generated embeddings.
+ */
 export interface EmbeddingsResponse {
   /**
    * List of embedding vectors, one per input content. Each embedding is a list of
@@ -132,6 +161,9 @@ export interface EmbeddingsResponse {
   embeddings: Array<Array<number>>;
 }
 
+/**
+ * Log probabilities for generated tokens.
+ */
 export interface TokenLogProbs {
   /**
    * Dictionary mapping tokens to their log probabilities
@@ -182,17 +214,23 @@ export interface InferenceChatCompletionParamsBase {
 
   /**
    * (Optional) Whether tool use is required or automatic. Defaults to
-   * ToolChoice.auto.
+   * ToolChoice.auto. .. deprecated:: Use tool_config instead.
    */
   tool_choice?: 'auto' | 'required';
 
+  /**
+   * (Optional) Configuration for tool use.
+   */
+  tool_config?: InferenceChatCompletionParams.ToolConfig;
+
   /**
    * (Optional) Instructs the model how to format tool calls. By default, Llama Stack
    * will attempt to use a format that is best adapted to the model. -
    * `ToolPromptFormat.json`: The tool calls are formatted as a JSON object. -
    * `ToolPromptFormat.function_tag`: The tool calls are enclosed in a
    * <function=function_name> tag. - `ToolPromptFormat.python_list`: The tool calls
-   * are output as Python syntax -- a list of function calls.
+   * are output as Python syntax -- a list of function calls. .. deprecated:: Use
+   * tool_config instead.
    */
   tool_prompt_format?: 'json' | 'function_tag' | 'python_list';
 
@@ -214,6 +252,37 @@ export namespace InferenceChatCompletionParams {
     top_k?: number;
   }
 
+  /**
+   * (Optional) Configuration for tool use.
+   */
+  export interface ToolConfig {
+    /**
+     * (Optional) Config for how to override the default system prompt. -
+     * `SystemMessageBehavior.append`: Appends the provided system message to the
+     * default system prompt. - `SystemMessageBehavior.replace`: Replaces the default
+     * system prompt with the provided system message. The system message can include
+     * the string '{{function_definitions}}' to indicate where the function definitions
+     * should be inserted.
+     */
+    system_message_behavior: 'append' | 'replace';
+
+    /**
+     * (Optional) Whether tool use is required or automatic. Defaults to
+     * ToolChoice.auto.
+     */
+    tool_choice?: 'auto' | 'required';
+
+    /**
+     * (Optional) Instructs the model how to format tool calls. By default, Llama Stack
+     * will attempt to use a format that is best adapted to the model. -
+     * `ToolPromptFormat.json`: The tool calls are formatted as a JSON object. -
+     * `ToolPromptFormat.function_tag`: The tool calls are enclosed in a
+     * <function=function_name> tag. - `ToolPromptFormat.python_list`: The tool calls
+     * are output as Python syntax -- a list of function calls.
+     */
+    tool_prompt_format?: 'json' | 'function_tag' | 'python_list';
+  }
+
   export interface Tool {
     tool_name: 'brave_search' | 'wolfram_alpha' | 'photogen' | 'code_interpreter' | (string & {});
 

package/src/resources/models.ts

@@ -27,7 +27,7 @@ export class Models extends APIResource {
 }
 
 export interface ListModelsResponse {
-  data: Array<Model>;
+  data: ModelListResponse;
 }
 
 export interface Model {

package/src/resources/post-training/job.ts

@@ -2,11 +2,17 @@
 
 import { APIResource } from '../../resource';
 import * as Core from '../../core';
+import { ListPostTrainingJobsResponse } from './post-training';
+import * as PostTrainingAPI from './post-training';
 
 export class Job extends APIResource {
-  list(options?: Core.RequestOptions): Core.APIPromise<JobListResponse> {
+  list(
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<Array<PostTrainingAPI.ListPostTrainingJobsResponse.Data>> {
     return (
-      this._client.get('/v1/post-training/jobs', options) as Core.APIPromise<{ data: JobListResponse }>
+      this._client.get('/v1/post-training/jobs', options) as Core.APIPromise<{
+        data: Array<PostTrainingAPI.ListPostTrainingJobsResponse.Data>;
+      }>
     )._thenUnwrap((obj) => obj.data);
   }
 
@@ -38,12 +44,18 @@ export namespace JobListResponse {
   }
 }
 
+/**
+ * Artifacts of a finetuning job.
+ */
 export interface JobArtifactsResponse {
   checkpoints: Array<unknown>;
 
   job_uuid: string;
 }
 
+/**
+ * Status of a finetuning job.
+ */
 export interface JobStatusResponse {
   checkpoints: Array<unknown>;
 

package/src/resources/providers.ts

@@ -13,7 +13,7 @@ export class Providers extends APIResource {
 }
 
 export interface ListProvidersResponse {
-  data: Array<InspectAPI.ProviderInfo>;
+  data: ProviderListResponse;
 }
 
 export type ProviderListResponse = Array<InspectAPI.ProviderInfo>;

package/src/resources/routes.ts

@@ -13,7 +13,7 @@ export class Routes extends APIResource {
 }
 
 export interface ListRoutesResponse {
-  data: Array<InspectAPI.RouteInfo>;
+  data: RouteListResponse;
 }
 
 export type RouteListResponse = Array<InspectAPI.RouteInfo>;

package/src/resources/scoring-functions.ts

@@ -27,7 +27,7 @@ export class ScoringFunctions extends APIResource {
 }
 
 export interface ListScoringFunctionsResponse {
-  data: Array<ScoringFn>;
+  data: ScoringFunctionListResponse;
 }
 
 export interface ScoringFn {