llama-stack-client 0.1.2 → 0.1.4

package/src/resources/datasets.ts

@@ -51,7 +51,13 @@ export interface DatasetRetrieveResponse {
 
   type: 'dataset';
 
-  url: Shared.URL;
+  url: DatasetRetrieveResponse.URL;
+}
+
+export namespace DatasetRetrieveResponse {
+  export interface URL {
+    uri: string;
+  }
 }
 
 export type DatasetListResponse = Array<DatasetListResponse.DatasetListResponseItem>;
@@ -70,7 +76,13 @@ export namespace DatasetListResponse {
 
     type: 'dataset';
 
-    url: Shared.URL;
+    url: DatasetListResponseItem.URL;
+  }
+
+  export namespace DatasetListResponseItem {
+    export interface URL {
+      uri: string;
+    }
   }
 }
 
@@ -79,7 +91,7 @@ export interface DatasetRegisterParams {
 
   dataset_schema: Record<string, Shared.ParamType>;
 
-  url: Shared.URL;
+  url: DatasetRegisterParams.URL;
 
   metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
 
@@ -88,6 +100,12 @@ export interface DatasetRegisterParams {
   provider_id?: string;
 }
 
+export namespace DatasetRegisterParams {
+  export interface URL {
+    uri: string;
+  }
+}
+
 export declare namespace Datasets {
   export {
     type ListDatasetsResponse as ListDatasetsResponse,

package/src/resources/eval/eval.ts

@@ -2,7 +2,6 @@
 
 import { APIResource } from '../../resource';
 import * as Core from '../../core';
-import * as EvalAPI from './eval';
 import * as ScoringFunctionsAPI from '../scoring-functions';
 import * as Shared from '../shared';
 import * as JobsAPI from './jobs';
@@ -12,18 +11,42 @@ export class Eval extends APIResource {
   jobs: JobsAPI.Jobs = new JobsAPI.Jobs(this._client);
 
   evaluateRows(
-    taskId: string,
+    benchmarkId: string,
     body: EvalEvaluateRowsParams,
     options?: Core.RequestOptions,
   ): Core.APIPromise<EvaluateResponse> {
-    return this._client.post(`/v1/eval/tasks/${taskId}/evaluations`, { body, ...options });
+    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
   }
 
-  runEval(taskId: string, body: EvalRunEvalParams, options?: Core.RequestOptions): Core.APIPromise<Job> {
-    return this._client.post(`/v1/eval/tasks/${taskId}/jobs`, { body, ...options });
+  evaluateRowsAlpha(
+    benchmarkId: string,
+    body: EvalEvaluateRowsAlphaParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvaluateResponse> {
+    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
+  }
+
+  runEval(benchmarkId: string, body: EvalRunEvalParams, options?: Core.RequestOptions): Core.APIPromise<Job> {
+    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
+  }
+
+  runEvalAlpha(
+    benchmarkId: string,
+    body: EvalRunEvalAlphaParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<Job> {
+    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
   }
 }
 
+export interface BenchmarkConfig {
+  eval_candidate: EvalCandidate;
+
+  scoring_params: Record<string, ScoringFunctionsAPI.ScoringFnParams>;
+
+  num_examples?: number;
+}
+
 export type EvalCandidate = EvalCandidate.ModelCandidate | EvalCandidate.AgentCandidate;
 
 export namespace EvalCandidate {
@@ -47,28 +70,6 @@ export namespace EvalCandidate {
   }
 }
 
-export type EvalTaskConfig = EvalTaskConfig.BenchmarkEvalTaskConfig | EvalTaskConfig.AppEvalTaskConfig;
-
-export namespace EvalTaskConfig {
-  export interface BenchmarkEvalTaskConfig {
-    eval_candidate: EvalAPI.EvalCandidate;
-
-    type: 'benchmark';
-
-    num_examples?: number;
-  }
-
-  export interface AppEvalTaskConfig {
-    eval_candidate: EvalAPI.EvalCandidate;
-
-    scoring_params: Record<string, ScoringFunctionsAPI.ScoringFnParams>;
-
-    type: 'app';
-
-    num_examples?: number;
-  }
-}
-
 export interface EvaluateResponse {
   generations: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;
 
@@ -84,23 +85,37 @@ export interface EvalEvaluateRowsParams {
 
   scoring_functions: Array<string>;
 
-  task_config: EvalTaskConfig;
+  task_config: BenchmarkConfig;
+}
+
+export interface EvalEvaluateRowsAlphaParams {
+  input_rows: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;
+
+  scoring_functions: Array<string>;
+
+  task_config: BenchmarkConfig;
 }
 
 export interface EvalRunEvalParams {
-  task_config: EvalTaskConfig;
+  task_config: BenchmarkConfig;
+}
+
+export interface EvalRunEvalAlphaParams {
+  task_config: BenchmarkConfig;
 }
 
 Eval.Jobs = Jobs;
 
 export declare namespace Eval {
   export {
+    type BenchmarkConfig as BenchmarkConfig,
     type EvalCandidate as EvalCandidate,
-    type EvalTaskConfig as EvalTaskConfig,
     type EvaluateResponse as EvaluateResponse,
     type Job as Job,
     type EvalEvaluateRowsParams as EvalEvaluateRowsParams,
+    type EvalEvaluateRowsAlphaParams as EvalEvaluateRowsAlphaParams,
     type EvalRunEvalParams as EvalRunEvalParams,
+    type EvalRunEvalAlphaParams as EvalRunEvalAlphaParams,
   };
 
   export { Jobs as Jobs, type JobStatusResponse as JobStatusResponse };

package/src/resources/eval/index.ts

@@ -2,11 +2,13 @@
 
 export {
   Eval,
+  type BenchmarkConfig,
   type EvalCandidate,
-  type EvalTaskConfig,
   type EvaluateResponse,
   type Job,
   type EvalEvaluateRowsParams,
+  type EvalEvaluateRowsAlphaParams,
   type EvalRunEvalParams,
+  type EvalRunEvalAlphaParams,
 } from './eval';
 export { Jobs, type JobStatusResponse } from './jobs';

package/src/resources/eval/jobs.ts

@@ -6,26 +6,26 @@ import * as EvalAPI from './eval';
 
 export class Jobs extends APIResource {
   retrieve(
-    taskId: string,
+    benchmarkId: string,
     jobId: string,
     options?: Core.RequestOptions,
   ): Core.APIPromise<EvalAPI.EvaluateResponse> {
-    return this._client.get(`/v1/eval/tasks/${taskId}/jobs/${jobId}/result`, options);
+    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}/result`, options);
   }
 
-  cancel(taskId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<void> {
-    return this._client.delete(`/v1/eval/tasks/${taskId}/jobs/${jobId}`, {
+  cancel(benchmarkId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.delete(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, {
       ...options,
       headers: { Accept: '*/*', ...options?.headers },
     });
   }
 
   status(
-    taskId: string,
+    benchmarkId: string,
     jobId: string,
     options?: Core.RequestOptions,
   ): Core.APIPromise<JobStatusResponse | null> {
-    return this._client.get(`/v1/eval/tasks/${taskId}/jobs/${jobId}`, options);
+    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, options);
   }
 }
 

package/src/resources/eval-tasks.ts

@@ -2,15 +2,21 @@
 
 import { APIResource } from '../resource';
 import * as Core from '../core';
+import * as BenchmarksAPI from './benchmarks';
 
 export class EvalTasks extends APIResource {
-  retrieve(evalTaskId: string, options?: Core.RequestOptions): Core.APIPromise<EvalTask | null> {
+  retrieve(
+    evalTaskId: string,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<BenchmarksAPI.Benchmark | null> {
     return this._client.get(`/v1/eval-tasks/${evalTaskId}`, options);
   }
 
-  list(options?: Core.RequestOptions): Core.APIPromise<EvalTaskListResponse> {
+  list(options?: Core.RequestOptions): Core.APIPromise<BenchmarksAPI.BenchmarkListResponse> {
     return (
-      this._client.get('/v1/eval-tasks', options) as Core.APIPromise<{ data: EvalTaskListResponse }>
+      this._client.get('/v1/eval-tasks', options) as Core.APIPromise<{
+        data: BenchmarksAPI.BenchmarkListResponse;
+      }>
     )._thenUnwrap((obj) => obj.data);
   }
 
@@ -23,27 +29,7 @@ export class EvalTasks extends APIResource {
   }
 }
 
-export interface EvalTask {
-  dataset_id: string;
-
-  identifier: string;
-
-  metadata: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
-
-  provider_id: string;
-
-  provider_resource_id: string;
-
-  scoring_functions: Array<string>;
-
-  type: 'eval_task';
-}
-
-export interface ListEvalTasksResponse {
-  data: EvalTaskListResponse;
-}
-
-export type EvalTaskListResponse = Array<EvalTask>;
+export type EvalTaskListResponse = Array<BenchmarksAPI.Benchmark>;
 
 export interface EvalTaskRegisterParams {
   dataset_id: string;
@@ -54,15 +40,13 @@ export interface EvalTaskRegisterParams {
 
   metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
 
-  provider_eval_task_id?: string;
+  provider_benchmark_id?: string;
 
   provider_id?: string;
 }
 
 export declare namespace EvalTasks {
   export {
-    type EvalTask as EvalTask,
-    type ListEvalTasksResponse as ListEvalTasksResponse,
     type EvalTaskListResponse as EvalTaskListResponse,
     type EvalTaskRegisterParams as EvalTaskRegisterParams,
   };

package/src/resources/index.ts

@@ -17,6 +17,13 @@ export {
   type BatchInferenceChatCompletionParams,
   type BatchInferenceCompletionParams,
 } from './batch-inference';
+export {
+  Benchmarks,
+  type Benchmark,
+  type ListBenchmarksResponse,
+  type BenchmarkListResponse,
+  type BenchmarkRegisterParams,
+} from './benchmarks';
 export {
   Datasetio,
   type PaginatedRowsResult,
@@ -32,20 +39,15 @@ export {
 } from './datasets';
 export {
   Eval,
+  type BenchmarkConfig,
   type EvalCandidate,
-  type EvalTaskConfig,
   type EvaluateResponse,
   type Job,
   type EvalEvaluateRowsParams,
+  type EvalEvaluateRowsAlphaParams,
   type EvalRunEvalParams,
+  type EvalRunEvalAlphaParams,
 } from './eval/eval';
-export {
-  EvalTasks,
-  type EvalTask,
-  type ListEvalTasksResponse,
-  type EvalTaskListResponse,
-  type EvalTaskRegisterParams,
-} from './eval-tasks';
 export {
   Inference,
   type ChatCompletionResponseStreamChunk,

package/src/resources/inference.ts

@@ -79,6 +79,8 @@ export interface ChatCompletionResponseStreamChunk {
    * The event containing the new content
    */
   event: ChatCompletionResponseStreamChunk.Event;
+
+  metrics?: Array<ChatCompletionResponseStreamChunk.Metric>;
 }
 
 export namespace ChatCompletionResponseStreamChunk {
@@ -107,6 +109,24 @@ export namespace ChatCompletionResponseStreamChunk {
      */
     stop_reason?: 'end_of_turn' | 'end_of_message' | 'out_of_tokens';
   }
+
+  export interface Metric {
+    metric: string;
+
+    span_id: string;
+
+    timestamp: string;
+
+    trace_id: string;
+
+    type: 'metric';
+
+    unit: string;
+
+    value: number;
+
+    attributes?: Record<string, string | number | boolean | null>;
+  }
 }
 
 /**
@@ -196,7 +216,7 @@ export interface InferenceChatCompletionParamsBase {
    * (Optional) Whether tool use is required or automatic. Defaults to
    * ToolChoice.auto. .. deprecated:: Use tool_config instead.
    */
-  tool_choice?: 'auto' | 'required';
+  tool_choice?: 'auto' | 'required' | 'none';
 
   /**
    * (Optional) Configuration for tool use.
@@ -244,13 +264,13 @@ export namespace InferenceChatCompletionParams {
      * the string '{{function_definitions}}' to indicate where the function definitions
      * should be inserted.
      */
-    system_message_behavior: 'append' | 'replace';
+    system_message_behavior?: 'append' | 'replace';
 
     /**
-     * (Optional) Whether tool use is required or automatic. Defaults to
-     * ToolChoice.auto.
+     * (Optional) Whether tool use is automatic, required, or none. Can also specify a
+     * tool name to use a specific tool. Defaults to ToolChoice.auto.
      */
-    tool_choice?: 'auto' | 'required';
+    tool_choice?: 'auto' | 'required' | 'none' | (string & {});
 
     /**
      * (Optional) Instructs the model how to format tool calls. By default, Llama Stack
@@ -365,17 +385,35 @@ export interface InferenceCompletionParamsStreaming extends InferenceCompletionP
 
 export interface InferenceEmbeddingsParams {
   /**
-   * List of contents to generate embeddings for. Note that content can be
-   * multimodal. The behavior depends on the model and provider. Some models may only
-   * support text.
+   * List of contents to generate embeddings for. Each content can be a string or an
+   * InterleavedContentItem (and hence can be multimodal). The behavior depends on
+   * the model and provider. Some models may only support text.
    */
-  contents: Array<Shared.InterleavedContent>;
+  contents: Array<string> | Array<Shared.InterleavedContentItem>;
 
   /**
    * The identifier of the model to use. The model must be an embedding model
    * registered with Llama Stack and available via the /models endpoint.
    */
   model_id: string;
+
+  /**
+   * (Optional) Output dimensionality for the embeddings. Only supported by
+   * Matryoshka models.
+   */
+  output_dimension?: number;
+
+  /**
+   * (Optional) How is the embedding being used? This is only supported by asymmetric
+   * embedding models.
+   */
+  task_type?: 'query' | 'document';
+
+  /**
+   * (Optional) Config for how to truncate text for embedding when text is longer
+   * than the model's max sequence length.
+   */
+  text_truncation?: 'none' | 'start' | 'end';
 }
 
 export declare namespace Inference {

package/src/resources/shared.ts

@@ -5,18 +5,18 @@ import * as InferenceAPI from './inference';
 import * as ToolRuntimeAPI from './tool-runtime/tool-runtime';
 
 export interface AgentConfig {
-  enable_session_persistence: boolean;
-
   instructions: string;
 
-  max_infer_iters: number;
-
   model: string;
 
   client_tools?: Array<ToolRuntimeAPI.ToolDef>;
 
+  enable_session_persistence?: boolean;
+
   input_shields?: Array<string>;
 
+  max_infer_iters?: number;
+
   output_shields?: Array<string>;
 
   /**
@@ -27,22 +27,58 @@ export interface AgentConfig {
   sampling_params?: SamplingParams;
 
   /**
-   * Whether tool use is required or automatic. This is a hint to the model which may
-   * not be followed. It depends on the Instruction Following capabilities of the
-   * model.
+   * @deprecated Whether tool use is required or automatic. This is a hint to the
+   * model which may not be followed. It depends on the Instruction Following
+   * capabilities of the model.
+   */
+  tool_choice?: 'auto' | 'required' | 'none';
+
+  /**
+   * Configuration for tool use.
    */
-  tool_choice?: 'auto' | 'required';
+  tool_config?: AgentConfig.ToolConfig;
 
   /**
-   * Prompt format for calling custom / zero shot tools.
+   * @deprecated Prompt format for calling custom / zero shot tools.
    */
   tool_prompt_format?: 'json' | 'function_tag' | 'python_list';
 
-  toolgroups?: Array<string | AgentConfig.UnionMember1>;
+  toolgroups?: Array<string | AgentConfig.AgentToolGroupWithArgs>;
 }
 
 export namespace AgentConfig {
-  export interface UnionMember1 {
+  /**
+   * Configuration for tool use.
+   */
+  export interface ToolConfig {
+    /**
+     * (Optional) Config for how to override the default system prompt. -
+     * `SystemMessageBehavior.append`: Appends the provided system message to the
+     * default system prompt. - `SystemMessageBehavior.replace`: Replaces the default
+     * system prompt with the provided system message. The system message can include
+     * the string '{{function_definitions}}' to indicate where the function definitions
+     * should be inserted.
+     */
+    system_message_behavior?: 'append' | 'replace';
+
+    /**
+     * (Optional) Whether tool use is automatic, required, or none. Can also specify a
+     * tool name to use a specific tool. Defaults to ToolChoice.auto.
+     */
+    tool_choice?: 'auto' | 'required' | 'none' | (string & {});
+
+    /**
+     * (Optional) Instructs the model how to format tool calls. By default, Llama Stack
+     * will attempt to use a format that is best adapted to the model. -
+     * `ToolPromptFormat.json`: The tool calls are formatted as a JSON object. -
+     * `ToolPromptFormat.function_tag`: The tool calls are enclosed in a
+     * <function=function_name> tag. - `ToolPromptFormat.python_list`: The tool calls
+     * are output as Python syntax -- a list of function calls.
+     */
+    tool_prompt_format?: 'json' | 'function_tag' | 'python_list';
+  }
+
+  export interface AgentToolGroupWithArgs {
     args: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
 
     name: string;
@@ -66,6 +102,28 @@ export interface ChatCompletionResponse {
    * Optional log probabilities for generated tokens
    */
   logprobs?: Array<InferenceAPI.TokenLogProbs>;
+
+  metrics?: Array<ChatCompletionResponse.Metric>;
+}
+
+export namespace ChatCompletionResponse {
+  export interface Metric {
+    metric: string;
+
+    span_id: string;
+
+    timestamp: string;
+
+    trace_id: string;
+
+    type: 'metric';
+
+    unit: string;
+
+    value: number;
+
+    attributes?: Record<string, string | number | boolean | null>;
+  }
 }
 
 /**
@@ -95,7 +153,7 @@ export interface CompletionMessage {
   /**
    * List of tool calls. Each tool call is a ToolCall object.
    */
-  tool_calls: Array<ToolCall>;
+  tool_calls?: Array<ToolCall>;
 }
 
 export type ContentDelta = ContentDelta.TextDelta | ContentDelta.ImageDelta | ContentDelta.ToolCallDelta;
@@ -116,7 +174,7 @@ export namespace ContentDelta {
   export interface ToolCallDelta {
     parse_status: 'started' | 'in_progress' | 'failed' | 'succeeded';
 
-    tool_call: string | Shared.ToolCall;
+    tool_call: Shared.ToolCallOrString;
 
     type: 'tool_call';
   }
@@ -131,7 +189,7 @@ export interface Document {
     | Document.ImageContentItem
     | Document.TextContentItem
     | Array<InterleavedContentItem>
-    | URL;
+    | Document.URL;
 
   document_id: string;
 
@@ -170,7 +228,17 @@ export namespace Document {
        * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
        * Note that URL could have length limits.
        */
-      url?: Shared.URL;
+      url?: Image.URL;
+    }
+
+    export namespace Image {
+      /**
+       * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
+       * Note that URL could have length limits.
+       */
+      export interface URL {
+        uri: string;
+      }
     }
   }
 
@@ -188,6 +256,10 @@ export namespace Document {
      */
     type: 'text';
   }
+
+  export interface URL {
+    uri: string;
+  }
 }
 
 /**
@@ -229,7 +301,17 @@ export namespace InterleavedContent {
        * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
        * Note that URL could have length limits.
        */
-      url?: Shared.URL;
+      url?: Image.URL;
+    }
+
+    export namespace Image {
+      /**
+       * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
+       * Note that URL could have length limits.
+       */
+      export interface URL {
+        uri: string;
+      }
     }
   }
 
@@ -286,7 +368,17 @@ export namespace InterleavedContentItem {
        * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
        * Note that URL could have length limits.
        */
-      url?: Shared.URL;
+      url?: Image.URL;
+    }
+
+    export namespace Image {
+      /**
+       * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
+       * Note that URL could have length limits.
+       */
+      export interface URL {
+        uri: string;
+      }
     }
   }
 
@@ -394,6 +486,8 @@ export namespace QueryGeneratorConfig {
 }
 
 export interface QueryResult {
+  metadata: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
+
   /**
    * A image content item
    */
@@ -530,6 +624,8 @@ export interface ToolCall {
   tool_name: 'brave_search' | 'wolfram_alpha' | 'photogen' | 'code_interpreter' | (string & {});
 }
 
+export type ToolCallOrString = string | ToolCall;
+
 export interface ToolParamDefinition {
   param_type: string;
 
@@ -565,10 +661,6 @@ export interface ToolResponseMessage {
   tool_name: 'brave_search' | 'wolfram_alpha' | 'photogen' | 'code_interpreter' | (string & {});
 }
 
-export interface URL {
-  uri: string;
-}
-
 /**
  * A message from the user in a chat conversation.
  */