llama-stack-client 0.1.2 → 0.1.4

package/src/index.ts

@@ -12,6 +12,13 @@ import {
   BatchInferenceChatCompletionResponse,
   BatchInferenceCompletionParams,
 } from './resources/batch-inference';
+import {
+  Benchmark,
+  BenchmarkListResponse,
+  BenchmarkRegisterParams,
+  Benchmarks,
+  ListBenchmarksResponse,
+} from './resources/benchmarks';
 import {
   Datasetio,
   DatasetioAppendRowsParams,
@@ -25,13 +32,6 @@ import {
   Datasets,
   ListDatasetsResponse,
 } from './resources/datasets';
-import {
-  EvalTask,
-  EvalTaskListResponse,
-  EvalTaskRegisterParams,
-  EvalTasks,
-  ListEvalTasksResponse,
-} from './resources/eval-tasks';
 import {
   ChatCompletionResponseStreamChunk,
   CompletionResponse,
@@ -134,11 +134,13 @@ import {
   ToolResponse,
 } from './resources/agents/agents';
 import {
+  BenchmarkConfig,
   Eval,
   EvalCandidate,
+  EvalEvaluateRowsAlphaParams,
   EvalEvaluateRowsParams,
+  EvalRunEvalAlphaParams,
   EvalRunEvalParams,
-  EvalTaskConfig,
   EvaluateResponse,
   Job,
 } from './resources/eval/eval';
@@ -160,14 +162,14 @@ import {
 
 export interface ClientOptions {
   /**
-   * Defaults to process.env['LLAMA_STACK_CLIENT_API_KEY'].
+   * Defaults to process.env['LLAMA_STACK_API_KEY'].
    */
   apiKey?: string | null | undefined;
 
   /**
    * Override the default base URL for the API, e.g., "https://api.example.com/v2/"
    *
-   * Defaults to process.env['LLAMA_STACK_CLIENT_BASE_URL'].
+   * Defaults to process.env['LLAMA_STACK_BASE_URL'].
    */
   baseURL?: string | null | undefined;
 
@@ -232,8 +234,8 @@ export class LlamaStackClient extends Core.APIClient {
   /**
    * API Client for interfacing with the Llama Stack Client API.
    *
-   * @param {string | null | undefined} [opts.apiKey=process.env['LLAMA_STACK_CLIENT_API_KEY'] ?? null]
-   * @param {string} [opts.baseURL=process.env['LLAMA_STACK_CLIENT_BASE_URL'] ?? http://any-hosted-llama-stack.com] - Override the default base URL for the API.
+   * @param {string | null | undefined} [opts.apiKey=process.env['LLAMA_STACK_API_KEY'] ?? null]
+   * @param {string} [opts.baseURL=process.env['LLAMA_STACK_BASE_URL'] ?? http://any-hosted-llama-stack.com] - Override the default base URL for the API.
    * @param {number} [opts.timeout=1 minute] - The maximum amount of time (in milliseconds) the client will wait for a response before timing out.
    * @param {number} [opts.httpAgent] - An HTTP agent used to manage HTTP(s) connections.
    * @param {Core.Fetch} [opts.fetch] - Specify a custom `fetch` function implementation.
@@ -242,8 +244,8 @@ export class LlamaStackClient extends Core.APIClient {
    * @param {Core.DefaultQuery} opts.defaultQuery - Default query parameters to include with every request to the API.
    */
   constructor({
-    baseURL = Core.readEnv('LLAMA_STACK_CLIENT_BASE_URL'),
-    apiKey = Core.readEnv('LLAMA_STACK_CLIENT_API_KEY') ?? null,
+    baseURL = Core.readEnv('LLAMA_STACK_BASE_URL'),
+    apiKey = Core.readEnv('LLAMA_STACK_API_KEY') ?? null,
     ...opts
   }: ClientOptions = {}) {
     const options: ClientOptions = {
@@ -287,7 +289,7 @@ export class LlamaStackClient extends Core.APIClient {
   datasetio: API.Datasetio = new API.Datasetio(this);
   scoring: API.Scoring = new API.Scoring(this);
   scoringFunctions: API.ScoringFunctions = new API.ScoringFunctions(this);
-  evalTasks: API.EvalTasks = new API.EvalTasks(this);
+  benchmarks: API.Benchmarks = new API.Benchmarks(this);
 
   protected override defaultQuery(): Core.DefaultQuery | undefined {
     return this._options.defaultQuery;
@@ -354,7 +356,7 @@ LlamaStackClient.Telemetry = Telemetry;
 LlamaStackClient.Datasetio = Datasetio;
 LlamaStackClient.Scoring = Scoring;
 LlamaStackClient.ScoringFunctions = ScoringFunctions;
-LlamaStackClient.EvalTasks = EvalTasks;
+LlamaStackClient.Benchmarks = Benchmarks;
 export declare namespace LlamaStackClient {
   export type RequestOptions = Core.RequestOptions;
 
@@ -410,12 +412,14 @@ export declare namespace LlamaStackClient {
 
   export {
     Eval as Eval,
+    type BenchmarkConfig as BenchmarkConfig,
     type EvalCandidate as EvalCandidate,
-    type EvalTaskConfig as EvalTaskConfig,
     type EvaluateResponse as EvaluateResponse,
     type Job as Job,
     type EvalEvaluateRowsParams as EvalEvaluateRowsParams,
+    type EvalEvaluateRowsAlphaParams as EvalEvaluateRowsAlphaParams,
     type EvalRunEvalParams as EvalRunEvalParams,
+    type EvalRunEvalAlphaParams as EvalRunEvalAlphaParams,
   };
 
   export {
@@ -549,11 +553,11 @@ export declare namespace LlamaStackClient {
   };
 
   export {
-    EvalTasks as EvalTasks,
-    type EvalTask as EvalTask,
-    type ListEvalTasksResponse as ListEvalTasksResponse,
-    type EvalTaskListResponse as EvalTaskListResponse,
-    type EvalTaskRegisterParams as EvalTaskRegisterParams,
+    Benchmarks as Benchmarks,
+    type Benchmark as Benchmark,
+    type ListBenchmarksResponse as ListBenchmarksResponse,
+    type BenchmarkListResponse as BenchmarkListResponse,
+    type BenchmarkRegisterParams as BenchmarkRegisterParams,
   };
 
   export type AgentConfig = API.AgentConfig;
@@ -576,9 +580,9 @@ export declare namespace LlamaStackClient {
   export type ScoringResult = API.ScoringResult;
   export type SystemMessage = API.SystemMessage;
   export type ToolCall = API.ToolCall;
+  export type ToolCallOrString = API.ToolCallOrString;
   export type ToolParamDefinition = API.ToolParamDefinition;
   export type ToolResponseMessage = API.ToolResponseMessage;
-  export type URL = API.URL;
   export type UserMessage = API.UserMessage;
 }
 

package/src/internal/decoders/line.ts

@@ -13,52 +13,58 @@ export class LineDecoder {
   static NEWLINE_CHARS = new Set(['\n', '\r']);
   static NEWLINE_REGEXP = /\r\n|[\n\r]/g;
 
-  buffer: string[];
-  trailingCR: boolean;
+  buffer: Uint8Array;
+  #carriageReturnIndex: number | null;
   textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either "dom" or "node" types.
 
   constructor() {
-    this.buffer = [];
-    this.trailingCR = false;
+    this.buffer = new Uint8Array();
+    this.#carriageReturnIndex = null;
   }
 
   decode(chunk: Bytes): string[] {
-    let text = this.decodeText(chunk);
-
-    if (this.trailingCR) {
-      text = '\r' + text;
-      this.trailingCR = false;
-    }
-    if (text.endsWith('\r')) {
-      this.trailingCR = true;
-      text = text.slice(0, -1);
-    }
-
-    if (!text) {
+    if (chunk == null) {
       return [];
     }
 
-    const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || '');
-    let lines = text.split(LineDecoder.NEWLINE_REGEXP);
+    const binaryChunk =
+      chunk instanceof ArrayBuffer ? new Uint8Array(chunk)
+      : typeof chunk === 'string' ? new TextEncoder().encode(chunk)
+      : chunk;
+
+    let newData = new Uint8Array(this.buffer.length + binaryChunk.length);
+    newData.set(this.buffer);
+    newData.set(binaryChunk, this.buffer.length);
+    this.buffer = newData;
+
+    const lines: string[] = [];
+    let patternIndex;
+    while ((patternIndex = findNewlineIndex(this.buffer, this.#carriageReturnIndex)) != null) {
+      if (patternIndex.carriage && this.#carriageReturnIndex == null) {
+        // skip until we either get a corresponding `\n`, a new `\r` or nothing
+        this.#carriageReturnIndex = patternIndex.index;
+        continue;
+      }
 
-    // if there is a trailing new line then the last entry will be an empty
-    // string which we don't care about
-    if (trailingNewline) {
-      lines.pop();
-    }
+      // we got double \r or \rtext\n
+      if (
+        this.#carriageReturnIndex != null &&
+        (patternIndex.index !== this.#carriageReturnIndex + 1 || patternIndex.carriage)
+      ) {
+        lines.push(this.decodeText(this.buffer.slice(0, this.#carriageReturnIndex - 1)));
+        this.buffer = this.buffer.slice(this.#carriageReturnIndex);
+        this.#carriageReturnIndex = null;
+        continue;
+      }
 
-    if (lines.length === 1 && !trailingNewline) {
-      this.buffer.push(lines[0]!);
-      return [];
-    }
+      const endIndex =
+        this.#carriageReturnIndex !== null ? patternIndex.preceding - 1 : patternIndex.preceding;
 
-    if (this.buffer.length > 0) {
-      lines = [this.buffer.join('') + lines[0], ...lines.slice(1)];
-      this.buffer = [];
-    }
+      const line = this.decodeText(this.buffer.slice(0, endIndex));
+      lines.push(line);
 
-    if (!trailingNewline) {
-      this.buffer = [lines.pop() || ''];
+      this.buffer = this.buffer.slice(patternIndex.index);
+      this.#carriageReturnIndex = null;
     }
 
     return lines;
@@ -102,13 +108,69 @@ export class LineDecoder {
   }
 
   flush(): string[] {
-    if (!this.buffer.length && !this.trailingCR) {
+    if (!this.buffer.length) {
       return [];
     }
+    return this.decode('\n');
+  }
+}
 
-    const lines = [this.buffer.join('')];
-    this.buffer = [];
-    this.trailingCR = false;
-    return lines;
+/**
+ * This function searches the buffer for the end patterns, (\r or \n)
+ * and returns an object with the index preceding the matched newline and the
+ * index after the newline char. `null` is returned if no new line is found.
+ *
+ * ```ts
+ * findNewLineIndex('abc\ndef') -> { preceding: 2, index: 3 }
+ * ```
+ */
+function findNewlineIndex(
+  buffer: Uint8Array,
+  startIndex: number | null,
+): { preceding: number; index: number; carriage: boolean } | null {
+  const newline = 0x0a; // \n
+  const carriage = 0x0d; // \r
+
+  for (let i = startIndex ?? 0; i < buffer.length; i++) {
+    if (buffer[i] === newline) {
+      return { preceding: i, index: i + 1, carriage: false };
+    }
+
+    if (buffer[i] === carriage) {
+      return { preceding: i, index: i + 1, carriage: true };
+    }
   }
+
+  return null;
+}
+
+export function findDoubleNewlineIndex(buffer: Uint8Array): number {
+  // This function searches the buffer for the end patterns (\r\r, \n\n, \r\n\r\n)
+  // and returns the index right after the first occurrence of any pattern,
+  // or -1 if none of the patterns are found.
+  const newline = 0x0a; // \n
+  const carriage = 0x0d; // \r
+
+  for (let i = 0; i < buffer.length - 1; i++) {
+    if (buffer[i] === newline && buffer[i + 1] === newline) {
+      // \n\n
+      return i + 2;
+    }
+    if (buffer[i] === carriage && buffer[i + 1] === carriage) {
+      // \r\r
+      return i + 2;
+    }
+    if (
+      buffer[i] === carriage &&
+      buffer[i + 1] === newline &&
+      i + 3 < buffer.length &&
+      buffer[i + 2] === carriage &&
+      buffer[i + 3] === newline
+    ) {
+      // \r\n\r\n
+      return i + 4;
+    }
+  }
+
+  return -1;
 }

package/src/resources/agents/agents.ts

@@ -23,6 +23,9 @@ import {
   TurnResource,
   TurnResponseEvent,
   TurnResponseEventPayload,
+  TurnResumeParams,
+  TurnResumeParamsNonStreaming,
+  TurnResumeParamsStreaming,
 } from './turn';
 
 export class Agents extends APIResource {
@@ -117,6 +120,8 @@ export interface ToolResponse {
   content: Shared.InterleavedContent;
 
   tool_name: 'brave_search' | 'wolfram_alpha' | 'photogen' | 'code_interpreter' | (string & {});
+
+  metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
 }
 
 export interface AgentCreateResponse {
@@ -161,5 +166,8 @@ export declare namespace Agents {
     type TurnCreateParams as TurnCreateParams,
     type TurnCreateParamsNonStreaming as TurnCreateParamsNonStreaming,
     type TurnCreateParamsStreaming as TurnCreateParamsStreaming,
+    type TurnResumeParams as TurnResumeParams,
+    type TurnResumeParamsNonStreaming as TurnResumeParamsNonStreaming,
+    type TurnResumeParamsStreaming as TurnResumeParamsStreaming,
   };
 }

package/src/resources/agents/index.ts

@@ -27,4 +27,7 @@ export {
   type TurnCreateParams,
   type TurnCreateParamsNonStreaming,
   type TurnCreateParamsStreaming,
+  type TurnResumeParams,
+  type TurnResumeParamsNonStreaming,
+  type TurnResumeParamsStreaming,
 } from './turn';

package/src/resources/agents/turn.ts

@@ -48,6 +48,47 @@ export class TurnResource extends APIResource {
   ): Core.APIPromise<Turn> {
     return this._client.get(`/v1/agents/${agentId}/session/${sessionId}/turn/${turnId}`, options);
   }
+
+  /**
+   * Resume an agent turn with executed tool call responses. When a Turn has the
+   * status `awaiting_input` due to pending input from client side tool calls, this
+   * endpoint can be used to submit the outputs from the tool calls once they are
+   * ready.
+   */
+  resume(
+    agentId: string,
+    sessionId: string,
+    turnId: string,
+    body: TurnResumeParamsNonStreaming,
+    options?: Core.RequestOptions,
+  ): APIPromise<Turn>;
+  resume(
+    agentId: string,
+    sessionId: string,
+    turnId: string,
+    body: TurnResumeParamsStreaming,
+    options?: Core.RequestOptions,
+  ): APIPromise<Stream<AgentTurnResponseStreamChunk>>;
+  resume(
+    agentId: string,
+    sessionId: string,
+    turnId: string,
+    body: TurnResumeParamsBase,
+    options?: Core.RequestOptions,
+  ): APIPromise<Stream<AgentTurnResponseStreamChunk> | Turn>;
+  resume(
+    agentId: string,
+    sessionId: string,
+    turnId: string,
+    body: TurnResumeParams,
+    options?: Core.RequestOptions,
+  ): APIPromise<Turn> | APIPromise<Stream<AgentTurnResponseStreamChunk>> {
+    return this._client.post(`/v1/agents/${agentId}/session/${sessionId}/turn/${turnId}/resume`, {
+      body,
+      ...options,
+      stream: body.stream ?? false,
+    }) as APIPromise<Turn> | APIPromise<Stream<AgentTurnResponseStreamChunk>>;
+  }
 }
 
 /**
@@ -63,8 +104,6 @@ export interface AgentTurnResponseStreamChunk {
 export interface Turn {
   input_messages: Array<Shared.UserMessage | Shared.ToolResponseMessage>;
 
-  output_attachments: Array<Turn.OutputAttachment>;
-
   /**
    * A message containing the model's (assistant) response in a chat conversation.
    */
@@ -84,6 +123,8 @@ export interface Turn {
   turn_id: string;
 
   completed_at?: string;
+
+  output_attachments?: Array<Turn.OutputAttachment>;
 }
 
 export namespace Turn {
@@ -96,7 +137,7 @@ export namespace Turn {
       | OutputAttachment.ImageContentItem
       | OutputAttachment.TextContentItem
       | Array<Shared.InterleavedContentItem>
-      | Shared.URL;
+      | OutputAttachment.URL;
 
     mime_type: string;
   }
@@ -131,7 +172,17 @@ export namespace Turn {
          * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
          * Note that URL could have length limits.
          */
-        url?: Shared.URL;
+        url?: Image.URL;
+      }
+
+      export namespace Image {
+        /**
+         * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
+         * Note that URL could have length limits.
+         */
+        export interface URL {
+          uri: string;
+        }
       }
     }
 
@@ -149,6 +200,10 @@ export namespace Turn {
        */
       type: 'text';
     }
+
+    export interface URL {
+      uri: string;
+    }
   }
 }
 
@@ -161,7 +216,8 @@ export type TurnResponseEventPayload =
   | TurnResponseEventPayload.AgentTurnResponseStepProgressPayload
   | TurnResponseEventPayload.AgentTurnResponseStepCompletePayload
   | TurnResponseEventPayload.AgentTurnResponseTurnStartPayload
-  | TurnResponseEventPayload.AgentTurnResponseTurnCompletePayload;
+  | TurnResponseEventPayload.AgentTurnResponseTurnCompletePayload
+  | TurnResponseEventPayload.AgentTurnResponseTurnAwaitingInputPayload;
 
 export namespace TurnResponseEventPayload {
   export interface AgentTurnResponseStepStartPayload {
@@ -212,6 +268,15 @@ export namespace TurnResponseEventPayload {
      */
     turn: TurnAPI.Turn;
   }
+
+  export interface AgentTurnResponseTurnAwaitingInputPayload {
+    event_type: 'turn_awaiting_input';
+
+    /**
+     * A single turn in an interaction with an Agentic System.
+     */
+    turn: TurnAPI.Turn;
+  }
 }
 
 export type TurnCreateParams = TurnCreateParamsNonStreaming | TurnCreateParamsStreaming;
@@ -219,11 +284,18 @@ export type TurnCreateParams = TurnCreateParamsNonStreaming | TurnCreateParamsSt
 export interface TurnCreateParamsBase {
   messages: Array<Shared.UserMessage | Shared.ToolResponseMessage>;
 
+  allow_turn_resume?: boolean;
+
   documents?: Array<TurnCreateParams.Document>;
 
   stream?: boolean;
 
-  toolgroups?: Array<string | TurnCreateParams.UnionMember1>;
+  /**
+   * Configuration for tool use.
+   */
+  tool_config?: TurnCreateParams.ToolConfig;
+
+  toolgroups?: Array<string | TurnCreateParams.AgentToolGroupWithArgs>;
 }
 
 export namespace TurnCreateParams {
@@ -236,7 +308,7 @@ export namespace TurnCreateParams {
       | Document.ImageContentItem
       | Document.TextContentItem
       | Array<Shared.InterleavedContentItem>
-      | Shared.URL;
+      | Document.URL;
 
     mime_type: string;
   }
@@ -271,7 +343,17 @@ export namespace TurnCreateParams {
          * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
          * Note that URL could have length limits.
          */
-        url?: Shared.URL;
+        url?: Image.URL;
+      }
+
+      export namespace Image {
+        /**
+         * A URL of the image or data URL in the format of data:image/{type};base64,{data}.
+         * Note that URL could have length limits.
+         */
+        export interface URL {
+          uri: string;
+        }
       }
     }
 
@@ -289,9 +371,44 @@ export namespace TurnCreateParams {
        */
       type: 'text';
     }
+
+    export interface URL {
+      uri: string;
+    }
   }
 
-  export interface UnionMember1 {
+  /**
+   * Configuration for tool use.
+   */
+  export interface ToolConfig {
+    /**
+     * (Optional) Config for how to override the default system prompt. -
+     * `SystemMessageBehavior.append`: Appends the provided system message to the
+     * default system prompt. - `SystemMessageBehavior.replace`: Replaces the default
+     * system prompt with the provided system message. The system message can include
+     * the string '{{function_definitions}}' to indicate where the function definitions
+     * should be inserted.
+     */
+    system_message_behavior?: 'append' | 'replace';
+
+    /**
+     * (Optional) Whether tool use is automatic, required, or none. Can also specify a
+     * tool name to use a specific tool. Defaults to ToolChoice.auto.
+     */
+    tool_choice?: 'auto' | 'required' | 'none' | (string & {});
+
+    /**
+     * (Optional) Instructs the model how to format tool calls. By default, Llama Stack
+     * will attempt to use a format that is best adapted to the model. -
+     * `ToolPromptFormat.json`: The tool calls are formatted as a JSON object. -
+     * `ToolPromptFormat.function_tag`: The tool calls are enclosed in a
+     * <function=function_name> tag. - `ToolPromptFormat.python_list`: The tool calls
+     * are output as Python syntax -- a list of function calls.
+     */
+    tool_prompt_format?: 'json' | 'function_tag' | 'python_list';
+  }
+
+  export interface AgentToolGroupWithArgs {
     args: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
 
     name: string;
@@ -309,6 +426,39 @@ export interface TurnCreateParamsStreaming extends TurnCreateParamsBase {
   stream: true;
 }
 
+export type TurnResumeParams = TurnResumeParamsNonStreaming | TurnResumeParamsStreaming;
+
+export interface TurnResumeParamsBase {
+  /**
+   * The tool call responses to resume the turn with.
+   */
+  tool_responses: Array<Shared.ToolResponseMessage>;
+
+  /**
+   * Whether to stream the response.
+   */
+  stream?: boolean;
+}
+
+export namespace TurnResumeParams {
+  export type TurnResumeParamsNonStreaming = TurnAPI.TurnResumeParamsNonStreaming;
+  export type TurnResumeParamsStreaming = TurnAPI.TurnResumeParamsStreaming;
+}
+
+export interface TurnResumeParamsNonStreaming extends TurnResumeParamsBase {
+  /**
+   * Whether to stream the response.
+   */
+  stream?: false;
+}
+
+export interface TurnResumeParamsStreaming extends TurnResumeParamsBase {
+  /**
+   * Whether to stream the response.
+   */
+  stream: true;
+}
+
 export declare namespace TurnResource {
   export {
     type AgentTurnResponseStreamChunk as AgentTurnResponseStreamChunk,
@@ -318,5 +468,8 @@ export declare namespace TurnResource {
     type TurnCreateParams as TurnCreateParams,
     type TurnCreateParamsNonStreaming as TurnCreateParamsNonStreaming,
     type TurnCreateParamsStreaming as TurnCreateParamsStreaming,
+    type TurnResumeParams as TurnResumeParams,
+    type TurnResumeParamsNonStreaming as TurnResumeParamsNonStreaming,
+    type TurnResumeParamsStreaming as TurnResumeParamsStreaming,
   };
 }

package/src/resources/batch-inference.ts

@@ -43,7 +43,7 @@ export interface BatchInferenceChatCompletionParams {
    * not be followed. It depends on the Instruction Following capabilities of the
    * model.
    */
-  tool_choice?: 'auto' | 'required';
+  tool_choice?: 'auto' | 'required' | 'none';
 
   /**
    * Prompt format for calling custom / zero shot tools.

package/src/resources/benchmarks.ts

@@ -0,0 +1,69 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../resource';
+import * as Core from '../core';
+
+export class Benchmarks extends APIResource {
+  retrieve(benchmarkId: string, options?: Core.RequestOptions): Core.APIPromise<Benchmark | null> {
+    return this._client.get(`/v1/eval/benchmarks/${benchmarkId}`, options);
+  }
+
+  list(options?: Core.RequestOptions): Core.APIPromise<BenchmarkListResponse> {
+    return (
+      this._client.get('/v1/eval/benchmarks', options) as Core.APIPromise<{ data: BenchmarkListResponse }>
+    )._thenUnwrap((obj) => obj.data);
+  }
+
+  register(body: BenchmarkRegisterParams, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.post('/v1/eval/benchmarks', {
+      body,
+      ...options,
+      headers: { Accept: '*/*', ...options?.headers },
+    });
+  }
+}
+
+export interface Benchmark {
+  dataset_id: string;
+
+  identifier: string;
+
+  metadata: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
+
+  provider_id: string;
+
+  provider_resource_id: string;
+
+  scoring_functions: Array<string>;
+
+  type: 'benchmark';
+}
+
+export interface ListBenchmarksResponse {
+  data: BenchmarkListResponse;
+}
+
+export type BenchmarkListResponse = Array<Benchmark>;
+
+export interface BenchmarkRegisterParams {
+  benchmark_id: string;
+
+  dataset_id: string;
+
+  scoring_functions: Array<string>;
+
+  metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
+
+  provider_benchmark_id?: string;
+
+  provider_id?: string;
+}
+
+export declare namespace Benchmarks {
+  export {
+    type Benchmark as Benchmark,
+    type ListBenchmarksResponse as ListBenchmarksResponse,
+    type BenchmarkListResponse as BenchmarkListResponse,
+    type BenchmarkRegisterParams as BenchmarkRegisterParams,
+  };
+}