llama-stack-client 0.3.4-rc3 → 0.4.0-rc1

package/resources/scoring.d.ts

@@ -1,6 +1,5 @@
 import { APIResource } from "../resource.js";
 import * as Core from "../core.js";
-import * as ScoringFunctionsAPI from "./scoring-functions.js";
 import * as Shared from "./shared.js";
 export declare class Scoring extends APIResource {
     /**
@@ -16,9 +15,6 @@ export declare class Scoring extends APIResource {
  * The response from scoring.
  */
 export interface ScoringScoreResponse {
-    /**
-     * A map of scoring function name to ScoringResult.
-     */
     results: {
         [key: string]: Shared.ScoringResult;
     };
@@ -27,46 +23,109 @@ export interface ScoringScoreResponse {
  * Response from batch scoring operations on datasets.
  */
 export interface ScoringScoreBatchResponse {
-    /**
-     * A map of scoring function name to ScoringResult
-     */
     results: {
         [key: string]: Shared.ScoringResult;
     };
-    /**
-     * (Optional) The identifier of the dataset that was scored
-     */
-    dataset_id?: string;
+    dataset_id?: string | null;
 }
 export interface ScoringScoreParams {
-    /**
-     * The rows to score.
-     */
     input_rows: Array<{
-        [key: string]: boolean | number | string | Array<unknown> | unknown | null;
+        [key: string]: unknown;
     }>;
-    /**
-     * The scoring functions to use for the scoring.
-     */
     scoring_functions: {
-        [key: string]: ScoringFunctionsAPI.ScoringFnParams | null;
+        [key: string]: ScoringScoreParams.LlmAsJudgeScoringFnParams | ScoringScoreParams.RegexParserScoringFnParams | ScoringScoreParams.BasicScoringFnParams | null;
     };
 }
-export interface ScoringScoreBatchParams {
+export declare namespace ScoringScoreParams {
     /**
-     * The ID of the dataset to score.
+     * Parameters for LLM-as-judge scoring function configuration.
      */
-    dataset_id: string;
+    interface LlmAsJudgeScoringFnParams {
+        judge_model: string;
+        /**
+         * Aggregation functions to apply to the scores of each row
+         */
+        aggregation_functions?: Array<'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'>;
+        /**
+         * Regexes to extract the answer from generated response
+         */
+        judge_score_regexes?: Array<string>;
+        prompt_template?: string | null;
+        type?: 'llm_as_judge';
+    }
     /**
-     * Whether to save the results to a dataset.
+     * Parameters for regex parser scoring function configuration.
      */
-    save_results_dataset: boolean;
+    interface RegexParserScoringFnParams {
+        /**
+         * Aggregation functions to apply to the scores of each row
+         */
+        aggregation_functions?: Array<'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'>;
+        /**
+         * Regex to extract the answer from generated response
+         */
+        parsing_regexes?: Array<string>;
+        type?: 'regex_parser';
+    }
     /**
-     * The scoring functions to use for the scoring.
+     * Parameters for basic scoring function configuration.
      */
+    interface BasicScoringFnParams {
+        /**
+         * Aggregation functions to apply to the scores of each row
+         */
+        aggregation_functions?: Array<'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'>;
+        type?: 'basic';
+    }
+}
+export interface ScoringScoreBatchParams {
+    dataset_id: string;
     scoring_functions: {
-        [key: string]: ScoringFunctionsAPI.ScoringFnParams | null;
+        [key: string]: ScoringScoreBatchParams.LlmAsJudgeScoringFnParams | ScoringScoreBatchParams.RegexParserScoringFnParams | ScoringScoreBatchParams.BasicScoringFnParams | null;
     };
+    save_results_dataset?: boolean;
+}
+export declare namespace ScoringScoreBatchParams {
+    /**
+     * Parameters for LLM-as-judge scoring function configuration.
+     */
+    interface LlmAsJudgeScoringFnParams {
+        judge_model: string;
+        /**
+         * Aggregation functions to apply to the scores of each row
+         */
+        aggregation_functions?: Array<'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'>;
+        /**
+         * Regexes to extract the answer from generated response
+         */
+        judge_score_regexes?: Array<string>;
+        prompt_template?: string | null;
+        type?: 'llm_as_judge';
+    }
+    /**
+     * Parameters for regex parser scoring function configuration.
+     */
+    interface RegexParserScoringFnParams {
+        /**
+         * Aggregation functions to apply to the scores of each row
+         */
+        aggregation_functions?: Array<'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'>;
+        /**
+         * Regex to extract the answer from generated response
+         */
+        parsing_regexes?: Array<string>;
+        type?: 'regex_parser';
+    }
+    /**
+     * Parameters for basic scoring function configuration.
+     */
+    interface BasicScoringFnParams {
+        /**
+         * Aggregation functions to apply to the scores of each row
+         */
+        aggregation_functions?: Array<'average' | 'weighted_average' | 'median' | 'categorical_count' | 'accuracy'>;
+        type?: 'basic';
+    }
 }
 export declare namespace Scoring {
     export { type ScoringScoreResponse as ScoringScoreResponse, type ScoringScoreBatchResponse as ScoringScoreBatchResponse, type ScoringScoreParams as ScoringScoreParams, type ScoringScoreBatchParams as ScoringScoreBatchParams, };

package/resources/scoring.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../src/resources/scoring.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAChC,OAAO,KAAK,mBAAmB,MAAM,qBAAqB,CAAC;AAC3D,OAAO,KAAK,MAAM,MAAM,UAAU,CAAC;AAEnC,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC;IAIrG;;OAEG;IACH,UAAU,CACR,IAAI,EAAE,uBAAuB,EAC7B,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,yBAAyB,CAAC;CAG9C;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,OAAO,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,aAAa,CAAA;KAAE,CAAC;CAClD;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC;;OAEG;IACH,OAAO,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,aAAa,CAAA;KAAE,CAAC;IAEjD;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,UAAU,EAAE,KAAK,CAAC;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;IAElG;;OAEG;IACH,iBAAiB,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAAC,eAAe,GAAG,IAAI,CAAA;KAAE,CAAC;CAClF;AAED,MAAM,WAAW,uBAAuB;IACtC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,oBAAoB,EAAE,OAAO,CAAC;IAE9B;;OAEG;IACH,iBAAiB,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAAC,eAAe,GAAG,IAAI,CAAA;KAAE,CAAC;CAClF;AAED,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,yBAAyB,IAAI,yBAAyB,EAC3D,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,uBAAuB,IAAI,uBAAuB,GACxD,CAAC;CACH"}
+{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../src/resources/scoring.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAChC,OAAO,KAAK,MAAM,MAAM,UAAU,CAAC;AAEnC,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC;IAIrG;;OAEG;IACH,UAAU,CACR,IAAI,EAAE,uBAAuB,EAC7B,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,yBAAyB,CAAC;CAG9C;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,aAAa,CAAA;KAAE,CAAC;CAClD;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,aAAa,CAAA;KAAE,CAAC;IAEjD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,KAAK,CAAC;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAC;IAE9C,iBAAiB,EAAE;QACjB,CAAC,GAAG,EAAE,MAAM,GACR,kBAAkB,CAAC,yBAAyB,GAC5C,kBAAkB,CAAC,0BAA0B,GAC7C,kBAAkB,CAAC,oBAAoB,GACvC,IAAI,CAAC;KACV,CAAC;CACH;AAED,yBAAiB,kBAAkB,CAAC;IAClC;;OAEG;IACH,UAAiB,yBAAyB;QACxC,WAAW,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,qBAAqB,CAAC,EAAE,KAAK,CAC3B,SAAS,GAAG,kBAAkB,GAAG,QAAQ,GAAG,mBAAmB,GAAG,UAAU,CAC7E,CAAC;QAEF;;WAEG;QACH,mBAAmB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEpC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAEhC,IAAI,CAAC,EAAE,cAAc,CAAC;KACvB;IAED;;OAEG;IACH,UAAiB,0BAA0B;QACzC;;WAEG;QACH,qBAAqB,CAAC,EAAE,KAAK,CAC3B,SAAS,GAAG,kBAAkB,GAAG,QAAQ,GAAG,mBAAmB,GAAG,UAAU,CAC7E,CAAC;QAEF;;WAEG;QACH,eAAe,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEhC,IAAI,CAAC,EAAE,cAAc,CAAC;KACvB;IAED;;OAEG;IACH,UAAiB,oBAAoB;QACnC;;WAEG;QACH,qBAAqB,CAAC,EAAE,KAAK,CAC3B,SAAS,GAAG,kBAAkB,GAAG,QAAQ,GAAG,mBAAmB,GAAG,UAAU,CAC7E,CAAC;QAEF,IAAI,CAAC,EAAE,OAAO,CAAC;KAChB;CACF;AAED,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,MAAM,CAAC;IAEnB,iBAAiB,EAAE;QACjB,CAAC,GAAG,EAAE,MAAM,GACR,uBAAuB,CAAC,yBAAyB,GACjD,uBAAuB,CAAC,0BAA0B,GAClD,uBAAuB,CAAC,oBAAoB,GAC5C,IAAI,CAAC;KACV,CAAC;IAEF,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,yBAAiB,uBAAuB,CAAC;IACvC;;OAEG;IACH,UAAiB,yBAAyB;QACxC,WAAW,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,qBAAqB,CAAC,EAAE,KAAK,CAC3B,SAAS,GAAG,kBAAkB,GAAG,QAAQ,GAAG,mBAAmB,GAAG,UAAU,CAC7E,CAAC;QAEF;;WAEG;QACH,mBAAmB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEpC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAEhC,IAAI,CAAC,EAAE,cAAc,CAAC;KACvB;IAED;;OAEG;IACH,UAAiB,0BAA0B;QACzC;;WAEG;QACH,qBAAqB,CAAC,EAAE,KAAK,CAC3B,SAAS,GAAG,kBAAkB,GAAG,QAAQ,GAAG,mBAAmB,GAAG,UAAU,CAC7E,CAAC;QAEF;;WAEG;QACH,eAAe,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEhC,IAAI,CAAC,EAAE,cAAc,CAAC;KACvB;IAED;;OAEG;IACH,UAAiB,oBAAoB;QACnC;;WAEG;QACH,qBAAqB,CAAC,EAAE,KAAK,CAC3B,SAAS,GAAG,kBAAkB,GAAG,QAAQ,GAAG,mBAAmB,GAAG,UAAU,CAC7E,CAAC;QAEF,IAAI,CAAC,EAAE,OAAO,CAAC;KAChB;CACF;AAED,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,yBAAyB,IAAI,yBAAyB,EAC3D,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,uBAAuB,IAAI,uBAAuB,GACxD,CAAC;CACH"}

package/resources/scoring.js

@@ -4,9 +4,10 @@
 //
 // This source code is licensed under the terms described in the LICENSE file in
 // the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Scoring = void 0;
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 const resource_1 = require("../resource.js");
 class Scoring extends resource_1.APIResource {
     /**

package/resources/scoring.js.map

@@ -1 +1 @@
-{"version":3,"file":"scoring.js","sourceRoot":"","sources":["../src/resources/scoring.ts"],"names":[],"mappings":";AAAA,qDAAqD;AACrD,uBAAuB;AACvB,EAAE;AACF,gFAAgF;AAChF,0CAA0C;;;AAE1C,sFAAsF;AAEtF,6CAA0C;AAK1C,MAAa,OAAQ,SAAQ,sBAAW;IACtC;;OAEG;IACH,KAAK,CAAC,IAAwB,EAAE,OAA6B;QAC3D,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,UAAU,CACR,IAA6B,EAC7B,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;CACF;AAjBD,0BAiBC"}
+{"version":3,"file":"scoring.js","sourceRoot":"","sources":["../src/resources/scoring.ts"],"names":[],"mappings":";AAAA,qDAAqD;AACrD,uBAAuB;AACvB,EAAE;AACF,gFAAgF;AAChF,0CAA0C;AAC1C,EAAE;AACF,sFAAsF;;;AAEtF,6CAA0C;AAI1C,MAAa,OAAQ,SAAQ,sBAAW;IACtC;;OAEG;IACH,KAAK,CAAC,IAAwB,EAAE,OAA6B;QAC3D,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,UAAU,CACR,IAA6B,EAC7B,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;CACF;AAjBD,0BAiBC"}

package/resources/scoring.mjs

@@ -3,6 +3,8 @@
 //
 // This source code is licensed under the terms described in the LICENSE file in
 // the root directory of this source tree.
+//
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 import { APIResource } from "../resource.mjs";
 export class Scoring extends APIResource {
     /**

package/resources/scoring.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"scoring.mjs","sourceRoot":"","sources":["../src/resources/scoring.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,uBAAuB;AACvB,EAAE;AACF,gFAAgF;AAChF,0CAA0C;OAInC,EAAE,WAAW,EAAE;AAKtB,MAAM,OAAO,OAAQ,SAAQ,WAAW;IACtC;;OAEG;IACH,KAAK,CAAC,IAAwB,EAAE,OAA6B;QAC3D,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,UAAU,CACR,IAA6B,EAC7B,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;CACF"}
+{"version":3,"file":"scoring.mjs","sourceRoot":"","sources":["../src/resources/scoring.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,uBAAuB;AACvB,EAAE;AACF,gFAAgF;AAChF,0CAA0C;AAC1C,EAAE;AACF,sFAAsF;OAE/E,EAAE,WAAW,EAAE;AAItB,MAAM,OAAO,OAAQ,SAAQ,WAAW;IACtC;;OAEG;IACH,KAAK,CAAC,IAAwB,EAAE,OAA6B;QAC3D,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,UAAU,CACR,IAA6B,EAC7B,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;CACF"}