llama-stack-client 0.4.3 → 0.4.4

package/dist/src/resources/scoring.ts

@@ -0,0 +1,197 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../resource';
+import * as Core from '../core';
+import * as Shared from './shared';
+
+export class Scoring extends APIResource {
+  /**
+   * Score a list of rows.
+   */
+  score(body: ScoringScoreParams, options?: Core.RequestOptions): Core.APIPromise<ScoringScoreResponse> {
+    return this._client.post('/v1/scoring/score', { body, ...options });
+  }
+
+  /**
+   * Score a batch of rows.
+   */
+  scoreBatch(
+    body: ScoringScoreBatchParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<ScoringScoreBatchResponse> {
+    return this._client.post('/v1/scoring/score-batch', { body, ...options });
+  }
+}
+
+/**
+ * The response from scoring.
+ */
+export interface ScoringScoreResponse {
+  results: { [key: string]: Shared.ScoringResult };
+}
+
+/**
+ * Response from batch scoring operations on datasets.
+ */
+export interface ScoringScoreBatchResponse {
+  results: { [key: string]: Shared.ScoringResult };
+
+  dataset_id?: string | null;
+}
+
+export interface ScoringScoreParams {
+  input_rows: Array<{ [key: string]: unknown }>;
+
+  scoring_functions: {
+    [key: string]:
+      | ScoringScoreParams.LlmAsJudgeScoringFnParams
+      | ScoringScoreParams.RegexParserScoringFnParams
+      | ScoringScoreParams.BasicScoringFnParams
+      | null;
+  };
+}
+
+export namespace ScoringScoreParams {
+  /**
+   * Parameters for LLM-as-judge scoring function configuration.
+   */
+  export interface LlmAsJudgeScoringFnParams {
+    judge_model: string;
+
+    /**
+     * Aggregation functions to apply to the scores of each row
+     */
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
+
+    /**
+     * Regexes to extract the answer from generated response
+     */
+    judge_score_regexes?: Array<string>;
+
+    prompt_template?: string | null;
+
+    type?: 'llm_as_judge';
+  }
+
+  /**
+   * Parameters for regex parser scoring function configuration.
+   */
+  export interface RegexParserScoringFnParams {
+    /**
+     * Aggregation functions to apply to the scores of each row
+     */
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
+
+    /**
+     * Regex to extract the answer from generated response
+     */
+    parsing_regexes?: Array<string>;
+
+    type?: 'regex_parser';
+  }
+
+  /**
+   * Parameters for basic scoring function configuration.
+   */
+  export interface BasicScoringFnParams {
+    /**
+     * Aggregation functions to apply to the scores of each row
+     */
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
+
+    type?: 'basic';
+  }
+}
+
+export interface ScoringScoreBatchParams {
+  dataset_id: string;
+
+  scoring_functions: {
+    [key: string]:
+      | ScoringScoreBatchParams.LlmAsJudgeScoringFnParams
+      | ScoringScoreBatchParams.RegexParserScoringFnParams
+      | ScoringScoreBatchParams.BasicScoringFnParams
+      | null;
+  };
+
+  save_results_dataset?: boolean;
+}
+
+export namespace ScoringScoreBatchParams {
+  /**
+   * Parameters for LLM-as-judge scoring function configuration.
+   */
+  export interface LlmAsJudgeScoringFnParams {
+    judge_model: string;
+
+    /**
+     * Aggregation functions to apply to the scores of each row
+     */
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
+
+    /**
+     * Regexes to extract the answer from generated response
+     */
+    judge_score_regexes?: Array<string>;
+
+    prompt_template?: string | null;
+
+    type?: 'llm_as_judge';
+  }
+
+  /**
+   * Parameters for regex parser scoring function configuration.
+   */
+  export interface RegexParserScoringFnParams {
+    /**
+     * Aggregation functions to apply to the scores of each row
+     */
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
+
+    /**
+     * Regex to extract the answer from generated response
+     */
+    parsing_regexes?: Array<string>;
+
+    type?: 'regex_parser';
+  }
+
+  /**
+   * Parameters for basic scoring function configuration.
+   */
+  export interface BasicScoringFnParams {
+    /**
+     * Aggregation functions to apply to the scores of each row
+     */
+    aggregation_functions?: Array<
+      'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'
+    >;
+
+    type?: 'basic';
+  }
+}
+
+export declare namespace Scoring {
+  export {
+    type ScoringScoreResponse as ScoringScoreResponse,
+    type ScoringScoreBatchResponse as ScoringScoreBatchResponse,
+    type ScoringScoreParams as ScoringScoreParams,
+    type ScoringScoreBatchParams as ScoringScoreBatchParams,
+  };
+}

package/dist/src/resources/shared.ts

@@ -0,0 +1,418 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+/**
+ * A image content item
+ */
+export type InterleavedContent =
+  | string
+  | InterleavedContent.ImageContentItem
+  | InterleavedContent.TextContentItem
+  | Array<InterleavedContent.ImageContentItem | InterleavedContent.TextContentItem>;
+
+export namespace InterleavedContent {
+  /**
+   * A image content item
+   */
+  export interface ImageContentItem {
+    /**
+     * A URL or a base64 encoded string
+     */
+    image: ImageContentItem.Image;
+
+    type?: 'image';
+  }
+
+  export namespace ImageContentItem {
+    /**
+     * A URL or a base64 encoded string
+     */
+    export interface Image {
+      data?: string | null;
+
+      /**
+       * A URL reference to external content.
+       */
+      url?: Image.URL | null;
+    }
+
+    export namespace Image {
+      /**
+       * A URL reference to external content.
+       */
+      export interface URL {
+        uri: string;
+      }
+    }
+  }
+
+  /**
+   * A text content item
+   */
+  export interface TextContentItem {
+    text: string;
+
+    type?: 'text';
+  }
+
+  /**
+   * A image content item
+   */
+  export interface ImageContentItem {
+    /**
+     * A URL or a base64 encoded string
+     */
+    image: ImageContentItem.Image;
+
+    type?: 'image';
+  }
+
+  export namespace ImageContentItem {
+    /**
+     * A URL or a base64 encoded string
+     */
+    export interface Image {
+      data?: string | null;
+
+      /**
+       * A URL reference to external content.
+       */
+      url?: Image.URL | null;
+    }
+
+    export namespace Image {
+      /**
+       * A URL reference to external content.
+       */
+      export interface URL {
+        uri: string;
+      }
+    }
+  }
+
+  /**
+   * A text content item
+   */
+  export interface TextContentItem {
+    text: string;
+
+    type?: 'text';
+  }
+}
+
+/**
+ * A image content item
+ */
+export type InterleavedContentItem =
+  | InterleavedContentItem.ImageContentItem
+  | InterleavedContentItem.TextContentItem;
+
+export namespace InterleavedContentItem {
+  /**
+   * A image content item
+   */
+  export interface ImageContentItem {
+    /**
+     * A URL or a base64 encoded string
+     */
+    image: ImageContentItem.Image;
+
+    type?: 'image';
+  }
+
+  export namespace ImageContentItem {
+    /**
+     * A URL or a base64 encoded string
+     */
+    export interface Image {
+      data?: string | null;
+
+      /**
+       * A URL reference to external content.
+       */
+      url?: Image.URL | null;
+    }
+
+    export namespace Image {
+      /**
+       * A URL reference to external content.
+       */
+      export interface URL {
+        uri: string;
+      }
+    }
+  }
+
+  /**
+   * A text content item
+   */
+  export interface TextContentItem {
+    text: string;
+
+    type?: 'text';
+  }
+}
+
+/**
+ * Parameter type for string values.
+ */
+export type ParamType =
+  | ParamType.StringType
+  | ParamType.NumberType
+  | ParamType.BooleanType
+  | ParamType.ArrayType
+  | ParamType.ObjectType
+  | ParamType.JsonType
+  | ParamType.UnionType
+  | ParamType.ChatCompletionInputType
+  | ParamType.CompletionInputType;
+
+export namespace ParamType {
+  /**
+   * Parameter type for string values.
+   */
+  export interface StringType {
+    type?: 'string';
+  }
+
+  /**
+   * Parameter type for numeric values.
+   */
+  export interface NumberType {
+    type?: 'number';
+  }
+
+  /**
+   * Parameter type for boolean values.
+   */
+  export interface BooleanType {
+    type?: 'boolean';
+  }
+
+  /**
+   * Parameter type for array values.
+   */
+  export interface ArrayType {
+    type?: 'array';
+  }
+
+  /**
+   * Parameter type for object values.
+   */
+  export interface ObjectType {
+    type?: 'object';
+  }
+
+  /**
+   * Parameter type for JSON values.
+   */
+  export interface JsonType {
+    type?: 'json';
+  }
+
+  /**
+   * Parameter type for union values.
+   */
+  export interface UnionType {
+    type?: 'union';
+  }
+
+  /**
+   * Parameter type for chat completion input.
+   */
+  export interface ChatCompletionInputType {
+    type?: 'chat_completion_input';
+  }
+
+  /**
+   * Parameter type for completion input.
+   */
+  export interface CompletionInputType {
+    type?: 'completion_input';
+  }
+}
+
+/**
+ * Details of a safety violation detected by content moderation.
+ */
+export interface SafetyViolation {
+  /**
+   * Severity level of a safety violation.
+   */
+  violation_level: 'info' | 'warn' | 'error';
+
+  metadata?: { [key: string]: unknown };
+
+  user_message?: string | null;
+}
+
+/**
+ * Sampling parameters.
+ */
+export interface SamplingParams {
+  max_tokens?: number | null;
+
+  repetition_penalty?: number | null;
+
+  stop?: Array<string> | null;
+
+  /**
+   * Greedy sampling strategy that selects the highest probability token at each
+   * step.
+   */
+  strategy?:
+    | SamplingParams.GreedySamplingStrategy
+    | SamplingParams.TopPSamplingStrategy
+    | SamplingParams.TopKSamplingStrategy;
+}
+
+export namespace SamplingParams {
+  /**
+   * Greedy sampling strategy that selects the highest probability token at each
+   * step.
+   */
+  export interface GreedySamplingStrategy {
+    type?: 'greedy';
+  }
+
+  /**
+   * Top-p (nucleus) sampling strategy that samples from the smallest set of tokens
+   * with cumulative probability >= p.
+   */
+  export interface TopPSamplingStrategy {
+    temperature: number | null;
+
+    top_p?: number | null;
+
+    type?: 'top_p';
+  }
+
+  /**
+   * Top-k sampling strategy that restricts sampling to the k most likely tokens.
+   */
+  export interface TopKSamplingStrategy {
+    top_k: number;
+
+    type?: 'top_k';
+  }
+}
+
+/**
+ * A scoring result for a single row.
+ */
+export interface ScoringResult {
+  aggregated_results: { [key: string]: unknown };
+
+  score_rows: Array<{ [key: string]: unknown }>;
+}
+
+/**
+ * A system message providing instructions or context to the model.
+ */
+export interface SystemMessage {
+  /**
+   * A image content item
+   */
+  content:
+    | string
+    | SystemMessage.ImageContentItemInput
+    | SystemMessage.TextContentItem
+    | Array<SystemMessage.ImageContentItemInput | SystemMessage.TextContentItem>;
+
+  role?: 'system';
+}
+
+export namespace SystemMessage {
+  /**
+   * A image content item
+   */
+  export interface ImageContentItemInput {
+    /**
+     * A URL or a base64 encoded string
+     */
+    image: ImageContentItemInput.Image;
+
+    type?: 'image';
+  }
+
+  export namespace ImageContentItemInput {
+    /**
+     * A URL or a base64 encoded string
+     */
+    export interface Image {
+      data?: string | null;
+
+      /**
+       * A URL reference to external content.
+       */
+      url?: Image.URL | null;
+    }
+
+    export namespace Image {
+      /**
+       * A URL reference to external content.
+       */
+      export interface URL {
+        uri: string;
+      }
+    }
+  }
+
+  /**
+   * A text content item
+   */
+  export interface TextContentItem {
+    text: string;
+
+    type?: 'text';
+  }
+
+  /**
+   * A image content item
+   */
+  export interface ImageContentItemInput {
+    /**
+     * A URL or a base64 encoded string
+     */
+    image: ImageContentItemInput.Image;
+
+    type?: 'image';
+  }
+
+  export namespace ImageContentItemInput {
+    /**
+     * A URL or a base64 encoded string
+     */
+    export interface Image {
+      data?: string | null;
+
+      /**
+       * A URL reference to external content.
+       */
+      url?: Image.URL | null;
+    }
+
+    export namespace Image {
+      /**
+       * A URL reference to external content.
+       */
+      export interface URL {
+        uri: string;
+      }
+    }
+  }
+
+  /**
+   * A text content item
+   */
+  export interface TextContentItem {
+    text: string;
+
+    type?: 'text';
+  }
+}

package/dist/src/resources/shields.ts

@@ -0,0 +1,98 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../resource';
+import * as Core from '../core';
+
+export class Shields extends APIResource {
+  /**
+   * Get a shield by its identifier.
+   */
+  retrieve(identifier: string, options?: Core.RequestOptions): Core.APIPromise<Shield> {
+    return this._client.get(`/v1/shields/${identifier}`, options);
+  }
+
+  /**
+   * List all shields.
+   */
+  list(options?: Core.RequestOptions): Core.APIPromise<ShieldListResponse> {
+    return (
+      this._client.get('/v1/shields', options) as Core.APIPromise<{ data: ShieldListResponse }>
+    )._thenUnwrap((obj) => obj.data);
+  }
+
+  /**
+   * Unregister a shield.
+   *
+   * @deprecated
+   */
+  delete(identifier: string, options?: Core.RequestOptions): Core.APIPromise<void> {
+    return this._client.delete(`/v1/shields/${identifier}`, {
+      ...options,
+      headers: { Accept: '*/*', ...options?.headers },
+    });
+  }
+
+  /**
+   * Register a shield.
+   *
+   * @deprecated
+   */
+  register(body: ShieldRegisterParams, options?: Core.RequestOptions): Core.APIPromise<Shield> {
+    return this._client.post('/v1/shields', { body, ...options });
+  }
+}
+
+export interface ListShieldsResponse {
+  data: ShieldListResponse;
+}
+
+/**
+ * A safety shield resource that can be used to check content.
+ */
+export interface Shield {
+  /**
+   * Unique identifier for this resource in llama stack
+   */
+  identifier: string;
+
+  /**
+   * ID of the provider that owns this resource
+   */
+  provider_id: string;
+
+  params?: { [key: string]: unknown } | null;
+
+  /**
+   * Unique identifier for this resource in the provider
+   */
+  provider_resource_id?: string | null;
+
+  type?: 'shield';
+}
+
+export type ShieldListResponse = Array<Shield>;
+
+export interface ShieldRegisterParams {
+  shield_id: string;
+
+  params?: { [key: string]: unknown } | null;
+
+  provider_id?: string | null;
+
+  provider_shield_id?: string | null;
+}
+
+export declare namespace Shields {
+  export {
+    type ListShieldsResponse as ListShieldsResponse,
+    type Shield as Shield,
+    type ShieldListResponse as ShieldListResponse,
+    type ShieldRegisterParams as ShieldRegisterParams,
+  };
+}