llama-stack-client 0.1.2 → 0.1.3

package/src/internal/decoders/line.ts

@@ -13,52 +13,58 @@ export class LineDecoder {
   static NEWLINE_CHARS = new Set(['\n', '\r']);
   static NEWLINE_REGEXP = /\r\n|[\n\r]/g;
 
-  buffer: string[];
-  trailingCR: boolean;
+  buffer: Uint8Array;
+  #carriageReturnIndex: number | null;
   textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either "dom" or "node" types.
 
   constructor() {
-    this.buffer = [];
-    this.trailingCR = false;
+    this.buffer = new Uint8Array();
+    this.#carriageReturnIndex = null;
   }
 
   decode(chunk: Bytes): string[] {
-    let text = this.decodeText(chunk);
-
-    if (this.trailingCR) {
-      text = '\r' + text;
-      this.trailingCR = false;
-    }
-    if (text.endsWith('\r')) {
-      this.trailingCR = true;
-      text = text.slice(0, -1);
-    }
-
-    if (!text) {
+    if (chunk == null) {
       return [];
     }
 
-    const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || '');
-    let lines = text.split(LineDecoder.NEWLINE_REGEXP);
+    const binaryChunk =
+      chunk instanceof ArrayBuffer ? new Uint8Array(chunk)
+      : typeof chunk === 'string' ? new TextEncoder().encode(chunk)
+      : chunk;
+
+    let newData = new Uint8Array(this.buffer.length + binaryChunk.length);
+    newData.set(this.buffer);
+    newData.set(binaryChunk, this.buffer.length);
+    this.buffer = newData;
+
+    const lines: string[] = [];
+    let patternIndex;
+    while ((patternIndex = findNewlineIndex(this.buffer, this.#carriageReturnIndex)) != null) {
+      if (patternIndex.carriage && this.#carriageReturnIndex == null) {
+        // skip until we either get a corresponding `\n`, a new `\r` or nothing
+        this.#carriageReturnIndex = patternIndex.index;
+        continue;
+      }
 
-    // if there is a trailing new line then the last entry will be an empty
-    // string which we don't care about
-    if (trailingNewline) {
-      lines.pop();
-    }
+      // we got double \r or \rtext\n
+      if (
+        this.#carriageReturnIndex != null &&
+        (patternIndex.index !== this.#carriageReturnIndex + 1 || patternIndex.carriage)
+      ) {
+        lines.push(this.decodeText(this.buffer.slice(0, this.#carriageReturnIndex - 1)));
+        this.buffer = this.buffer.slice(this.#carriageReturnIndex);
+        this.#carriageReturnIndex = null;
+        continue;
+      }
 
-    if (lines.length === 1 && !trailingNewline) {
-      this.buffer.push(lines[0]!);
-      return [];
-    }
+      const endIndex =
+        this.#carriageReturnIndex !== null ? patternIndex.preceding - 1 : patternIndex.preceding;
 
-    if (this.buffer.length > 0) {
-      lines = [this.buffer.join('') + lines[0], ...lines.slice(1)];
-      this.buffer = [];
-    }
+      const line = this.decodeText(this.buffer.slice(0, endIndex));
+      lines.push(line);
 
-    if (!trailingNewline) {
-      this.buffer = [lines.pop() || ''];
+      this.buffer = this.buffer.slice(patternIndex.index);
+      this.#carriageReturnIndex = null;
     }
 
     return lines;
@@ -102,13 +108,38 @@ export class LineDecoder {
   }
 
   flush(): string[] {
-    if (!this.buffer.length && !this.trailingCR) {
+    if (!this.buffer.length) {
       return [];
     }
+    return this.decode('\n');
+  }
+}
 
-    const lines = [this.buffer.join('')];
-    this.buffer = [];
-    this.trailingCR = false;
-    return lines;
+/**
+ * This function searches the buffer for the end patterns, (\r or \n)
+ * and returns an object with the index preceding the matched newline and the
+ * index after the newline char. `null` is returned if no new line is found.
+ *
+ * ```ts
+ * findNewLineIndex('abc\ndef') -> { preceding: 2, index: 3 }
+ * ```
+ */
+function findNewlineIndex(
+  buffer: Uint8Array,
+  startIndex: number | null,
+): { preceding: number; index: number; carriage: boolean } | null {
+  const newline = 0x0a; // \n
+  const carriage = 0x0d; // \r
+
+  for (let i = startIndex ?? 0; i < buffer.length; i++) {
+    if (buffer[i] === newline) {
+      return { preceding: i, index: i + 1, carriage: false };
+    }
+
+    if (buffer[i] === carriage) {
+      return { preceding: i, index: i + 1, carriage: true };
+    }
   }
+
+  return null;
 }

package/src/resources/agents/turn.ts

@@ -63,8 +63,6 @@ export interface AgentTurnResponseStreamChunk {
 export interface Turn {
   input_messages: Array<Shared.UserMessage | Shared.ToolResponseMessage>;
 
-  output_attachments: Array<Turn.OutputAttachment>;
-
   /**
    * A message containing the model's (assistant) response in a chat conversation.
    */
@@ -84,6 +82,8 @@ export interface Turn {
   turn_id: string;
 
   completed_at?: string;
+
+  output_attachments?: Array<Turn.OutputAttachment>;
 }
 
 export namespace Turn {
@@ -96,7 +96,7 @@ export namespace Turn {
       | OutputAttachment.ImageContentItem
       | OutputAttachment.TextContentItem
       | Array<Shared.InterleavedContentItem>
-      | Shared.URL;
+      | OutputAttachment.URL;
 
     mime_type: string;
   }
@@ -131,7 +131,17 @@ export namespace Turn {
          * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
          * Note that URL could have length limits.
          */
-        url?: Shared.URL;
+        url?: Image.URL;
+      }
+
+      export namespace Image {
+        /**
+         * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
+         * Note that URL could have length limits.
+         */
+        export interface URL {
+          uri: string;
+        }
       }
     }
 
@@ -149,6 +159,10 @@ export namespace Turn {
        */
       type: 'text';
     }
+
+    export interface URL {
+      uri: string;
+    }
   }
 }
 
@@ -223,6 +237,11 @@ export interface TurnCreateParamsBase {
 
   stream?: boolean;
 
+  /**
+   * Configuration for tool use.
+   */
+  tool_config?: TurnCreateParams.ToolConfig;
+
   toolgroups?: Array<string | TurnCreateParams.UnionMember1>;
 }
 
@@ -236,7 +255,7 @@ export namespace TurnCreateParams {
       | Document.ImageContentItem
       | Document.TextContentItem
       | Array<Shared.InterleavedContentItem>
-      | Shared.URL;
+      | Document.URL;
 
     mime_type: string;
   }
@@ -271,7 +290,17 @@ export namespace TurnCreateParams {
          * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
          * Note that URL could have length limits.
          */
-        url?: Shared.URL;
+        url?: Image.URL;
+      }
+
+      export namespace Image {
+        /**
+         * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
+         * Note that URL could have length limits.
+         */
+        export interface URL {
+          uri: string;
+        }
       }
     }
 
@@ -289,6 +318,41 @@ export namespace TurnCreateParams {
        */
       type: 'text';
     }
+
+    export interface URL {
+      uri: string;
+    }
+  }
+
+  /**
+   * Configuration for tool use.
+   */
+  export interface ToolConfig {
+    /**
+     * (Optional) Config for how to override the default system prompt. -
+     * `SystemMessageBehavior.append`: Appends the provided system message to the
+     * default system prompt. - `SystemMessageBehavior.replace`: Replaces the default
+     * system prompt with the provided system message. The system message can include
+     * the string '{{function_definitions}}' to indicate where the function definitions
+     * should be inserted.
+     */
+    system_message_behavior: 'append' | 'replace';
+
+    /**
+     * (Optional) Whether tool use is required or automatic. Defaults to
+     * ToolChoice.auto.
+     */
+    tool_choice?: 'auto' | 'required';
+
+    /**
+     * (Optional) Instructs the model how to format tool calls. By default, Llama Stack
+     * will attempt to use a format that is best adapted to the model. -
+     * `ToolPromptFormat.json`: The tool calls are formatted as a JSON object. -
+     * `ToolPromptFormat.function_tag`: The tool calls are enclosed in a
+     * <function=function_name> tag. - `ToolPromptFormat.python_list`: The tool calls
+     * are output as Python syntax -- a list of function calls.
+     */
+    tool_prompt_format?: 'json' | 'function_tag' | 'python_list';
   }
 
   export interface UnionMember1 {

package/src/resources/benchmarks.ts

@@ -0,0 +1,69 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../resource';
+import * as Core from '../core';
+
+export class Benchmarks extends APIResource {
+  retrieve(benchmarkId: string, options?: Core.RequestOptions): Core.APIPromise<Benchmark | null> {
+    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}`, options);
+  }
+
+  list(options?: Core.RequestOptions): Core.APIPromise<BenchmarkListResponse> {
+    return (
+      this._client.get('/v1/eval/benchmarks', options) as Core.APIPromise<{ data: BenchmarkListResponse }>
+    )._thenUnwrap((obj) => obj.data);
+  }
+
+  register(body: BenchmarkRegisterParams, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.post('/v1/eval/benchmarks', {
+      body,
+      ...options,
+      headers: { Accept: '*/*', ...options?.headers },
+    });
+  }
+}
+
+export interface Benchmark {
+  dataset_id: string;
+
+  identifier: string;
+
+  metadata: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
+
+  provider_id: string;
+
+  provider_resource_id: string;
+
+  scoring_functions: Array<string>;
+
+  type: 'benchmark';
+}
+
+export interface ListBenchmarksResponse {
+  data: BenchmarkListResponse;
+}
+
+export type BenchmarkListResponse = Array<Benchmark>;
+
+export interface BenchmarkRegisterParams {
+  benchmark_id: string;
+
+  dataset_id: string;
+
+  scoring_functions: Array<string>;
+
+  metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
+
+  provider_benchmark_id?: string;
+
+  provider_id?: string;
+}
+
+export declare namespace Benchmarks {
+  export {
+    type Benchmark as Benchmark,
+    type ListBenchmarksResponse as ListBenchmarksResponse,
+    type BenchmarkListResponse as BenchmarkListResponse,
+    type BenchmarkRegisterParams as BenchmarkRegisterParams,
+  };
+}

package/src/resources/datasets.ts

@@ -51,7 +51,13 @@ export interface DatasetRetrieveResponse {
 
   type: 'dataset';
 
-  url: Shared.URL;
+  url: DatasetRetrieveResponse.URL;
+}
+
+export namespace DatasetRetrieveResponse {
+  export interface URL {
+    uri: string;
+  }
 }
 
 export type DatasetListResponse = Array<DatasetListResponse.DatasetListResponseItem>;
@@ -70,7 +76,13 @@ export namespace DatasetListResponse {
 
     type: 'dataset';
 
-    url: Shared.URL;
+    url: DatasetListResponseItem.URL;
+  }
+
+  export namespace DatasetListResponseItem {
+    export interface URL {
+      uri: string;
+    }
   }
 }
 
@@ -79,7 +91,7 @@ export interface DatasetRegisterParams {
 
   dataset_schema: Record<string, Shared.ParamType>;
 
-  url: Shared.URL;
+  url: DatasetRegisterParams.URL;
 
   metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
 
@@ -88,6 +100,12 @@ export interface DatasetRegisterParams {
   provider_id?: string;
 }
 
+export namespace DatasetRegisterParams {
+  export interface URL {
+    uri: string;
+  }
+}
+
 export declare namespace Datasets {
   export {
     type ListDatasetsResponse as ListDatasetsResponse,

package/src/resources/eval/eval.ts

@@ -2,7 +2,6 @@
 
 import { APIResource } from '../../resource';
 import * as Core from '../../core';
-import * as EvalAPI from './eval';
 import * as ScoringFunctionsAPI from '../scoring-functions';
 import * as Shared from '../shared';
 import * as JobsAPI from './jobs';
@@ -19,9 +18,35 @@ export class Eval extends APIResource {
     return this._client.post(`/v1/eval/tasks/${taskId}/evaluations`, { body, ...options });
   }
 
+  evaluateRowsAlpha(
+    benchmarkId: string,
+    body: EvalEvaluateRowsAlphaParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvaluateResponse> {
+    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
+  }
+
   runEval(taskId: string, body: EvalRunEvalParams, options?: Core.RequestOptions): Core.APIPromise<Job> {
     return this._client.post(`/v1/eval/tasks/${taskId}/jobs`, { body, ...options });
   }
+
+  runEvalAlpha(
+    benchmarkId: string,
+    body: EvalRunEvalAlphaParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<Job> {
+    return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
+  }
+}
+
+export interface BenchmarkConfig {
+  eval_candidate: EvalCandidate;
+
+  scoring_params: Record<string, ScoringFunctionsAPI.ScoringFnParams>;
+
+  type: 'benchmark';
+
+  num_examples?: number;
 }
 
 export type EvalCandidate = EvalCandidate.ModelCandidate | EvalCandidate.AgentCandidate;
@@ -47,28 +72,6 @@ export namespace EvalCandidate {
   }
 }
 
-export type EvalTaskConfig = EvalTaskConfig.BenchmarkEvalTaskConfig | EvalTaskConfig.AppEvalTaskConfig;
-
-export namespace EvalTaskConfig {
-  export interface BenchmarkEvalTaskConfig {
-    eval_candidate: EvalAPI.EvalCandidate;
-
-    type: 'benchmark';
-
-    num_examples?: number;
-  }
-
-  export interface AppEvalTaskConfig {
-    eval_candidate: EvalAPI.EvalCandidate;
-
-    scoring_params: Record<string, ScoringFunctionsAPI.ScoringFnParams>;
-
-    type: 'app';
-
-    num_examples?: number;
-  }
-}
-
 export interface EvaluateResponse {
   generations: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;
 
@@ -84,23 +87,37 @@ export interface EvalEvaluateRowsParams {
 
   scoring_functions: Array<string>;
 
-  task_config: EvalTaskConfig;
+  task_config: BenchmarkConfig;
+}
+
+export interface EvalEvaluateRowsAlphaParams {
+  input_rows: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;
+
+  scoring_functions: Array<string>;
+
+  task_config: BenchmarkConfig;
 }
 
 export interface EvalRunEvalParams {
-  task_config: EvalTaskConfig;
+  task_config: BenchmarkConfig;
+}
+
+export interface EvalRunEvalAlphaParams {
+  task_config: BenchmarkConfig;
 }
 
 Eval.Jobs = Jobs;
 
 export declare namespace Eval {
   export {
+    type BenchmarkConfig as BenchmarkConfig,
     type EvalCandidate as EvalCandidate,
-    type EvalTaskConfig as EvalTaskConfig,
     type EvaluateResponse as EvaluateResponse,
     type Job as Job,
     type EvalEvaluateRowsParams as EvalEvaluateRowsParams,
+    type EvalEvaluateRowsAlphaParams as EvalEvaluateRowsAlphaParams,
     type EvalRunEvalParams as EvalRunEvalParams,
+    type EvalRunEvalAlphaParams as EvalRunEvalAlphaParams,
   };
 
   export { Jobs as Jobs, type JobStatusResponse as JobStatusResponse };

package/src/resources/eval/index.ts

@@ -2,11 +2,13 @@
 
 export {
   Eval,
+  type BenchmarkConfig,
   type EvalCandidate,
-  type EvalTaskConfig,
   type EvaluateResponse,
   type Job,
   type EvalEvaluateRowsParams,
+  type EvalEvaluateRowsAlphaParams,
   type EvalRunEvalParams,
+  type EvalRunEvalAlphaParams,
 } from './eval';
 export { Jobs, type JobStatusResponse } from './jobs';

package/src/resources/eval/jobs.ts

@@ -6,26 +6,26 @@ import * as EvalAPI from './eval';
 
 export class Jobs extends APIResource {
   retrieve(
-    taskId: string,
+    benchmarkId: string,
     jobId: string,
     options?: Core.RequestOptions,
   ): Core.APIPromise<EvalAPI.EvaluateResponse> {
-    return this._client.get(`/v1/eval/tasks/${taskId}/jobs/${jobId}/result`, options);
+    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}/result`, options);
   }
 
-  cancel(taskId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<void> {
-    return this._client.delete(`/v1/eval/tasks/${taskId}/jobs/${jobId}`, {
+  cancel(benchmarkId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.delete(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, {
       ...options,
       headers: { Accept: '*/*', ...options?.headers },
     });
   }
 
   status(
-    taskId: string,
+    benchmarkId: string,
     jobId: string,
     options?: Core.RequestOptions,
   ): Core.APIPromise<JobStatusResponse | null> {
-    return this._client.get(`/v1/eval/tasks/${taskId}/jobs/${jobId}`, options);
+    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, options);
   }
 }
 

package/src/resources/eval-tasks.ts

@@ -2,15 +2,21 @@
 
 import { APIResource } from '../resource';
 import * as Core from '../core';
+import * as BenchmarksAPI from './benchmarks';
 
 export class EvalTasks extends APIResource {
-  retrieve(evalTaskId: string, options?: Core.RequestOptions): Core.APIPromise<EvalTask | null> {
+  retrieve(
+    evalTaskId: string,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<BenchmarksAPI.Benchmark | null> {
     return this._client.get(`/v1/eval-tasks/${evalTaskId}`, options);
   }
 
-  list(options?: Core.RequestOptions): Core.APIPromise<EvalTaskListResponse> {
+  list(options?: Core.RequestOptions): Core.APIPromise<BenchmarksAPI.BenchmarkListResponse> {
     return (
-      this._client.get('/v1/eval-tasks', options) as Core.APIPromise<{ data: EvalTaskListResponse }>
+      this._client.get('/v1/eval-tasks', options) as Core.APIPromise<{
+        data: BenchmarksAPI.BenchmarkListResponse;
+      }>
     )._thenUnwrap((obj) => obj.data);
   }
 
@@ -23,27 +29,7 @@ export class EvalTasks extends APIResource {
   }
 }
 
-export interface EvalTask {
-  dataset_id: string;
-
-  identifier: string;
-
-  metadata: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
-
-  provider_id: string;
-
-  provider_resource_id: string;
-
-  scoring_functions: Array<string>;
-
-  type: 'eval_task';
-}
-
-export interface ListEvalTasksResponse {
-  data: EvalTaskListResponse;
-}
-
-export type EvalTaskListResponse = Array<EvalTask>;
+export type EvalTaskListResponse = Array<BenchmarksAPI.Benchmark>;
 
 export interface EvalTaskRegisterParams {
   dataset_id: string;
@@ -54,15 +40,13 @@ export interface EvalTaskRegisterParams {
 
   metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
 
-  provider_eval_task_id?: string;
+  provider_benchmark_id?: string;
 
   provider_id?: string;
 }
 
 export declare namespace EvalTasks {
   export {
-    type EvalTask as EvalTask,
-    type ListEvalTasksResponse as ListEvalTasksResponse,
     type EvalTaskListResponse as EvalTaskListResponse,
     type EvalTaskRegisterParams as EvalTaskRegisterParams,
   };

package/src/resources/index.ts

@@ -17,6 +17,13 @@ export {
   type BatchInferenceChatCompletionParams,
   type BatchInferenceCompletionParams,
 } from './batch-inference';
+export {
+  Benchmarks,
+  type Benchmark,
+  type ListBenchmarksResponse,
+  type BenchmarkListResponse,
+  type BenchmarkRegisterParams,
+} from './benchmarks';
 export {
   Datasetio,
   type PaginatedRowsResult,
@@ -32,20 +39,16 @@ export {
 } from './datasets';
 export {
   Eval,
+  type BenchmarkConfig,
   type EvalCandidate,
-  type EvalTaskConfig,
   type EvaluateResponse,
   type Job,
   type EvalEvaluateRowsParams,
+  type EvalEvaluateRowsAlphaParams,
   type EvalRunEvalParams,
+  type EvalRunEvalAlphaParams,
 } from './eval/eval';
-export {
-  EvalTasks,
-  type EvalTask,
-  type ListEvalTasksResponse,
-  type EvalTaskListResponse,
-  type EvalTaskRegisterParams,
-} from './eval-tasks';
+export { EvalTasks, type EvalTaskListResponse, type EvalTaskRegisterParams } from './eval-tasks';
 export {
   Inference,
   type ChatCompletionResponseStreamChunk,

package/src/resources/inference.ts

@@ -79,6 +79,8 @@ export interface ChatCompletionResponseStreamChunk {
    * The event containing the new content
    */
   event: ChatCompletionResponseStreamChunk.Event;
+
+  metrics?: Array<ChatCompletionResponseStreamChunk.Metric>;
 }
 
 export namespace ChatCompletionResponseStreamChunk {
@@ -107,6 +109,24 @@ export namespace ChatCompletionResponseStreamChunk {
      */
     stop_reason?: 'end_of_turn' | 'end_of_message' | 'out_of_tokens';
   }
+
+  export interface Metric {
+    metric: string;
+
+    span_id: string;
+
+    timestamp: string;
+
+    trace_id: string;
+
+    type: 'metric';
+
+    unit: string;
+
+    value: number;
+
+    attributes?: Record<string, string | number | boolean | null>;
+  }
 }
 
 /**