llama-stack-client 0.1.2 → 0.1.4

package/resources/benchmarks.js

@@ -0,0 +1,22 @@
+"use strict";
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Benchmarks = void 0;
+const resource_1 = require("../resource.js");
+class Benchmarks extends resource_1.APIResource {
+    retrieve(benchmarkId, options) {
+        return this._client.get(`/v1/eval/benchmarks/${benchmarkId}`, options);
+    }
+    list(options) {
+        return this._client.get('/v1/eval/benchmarks', options)._thenUnwrap((obj) => obj.data);
+    }
+    register(body, options) {
+        return this._client.post('/v1/eval/benchmarks', {
+            body,
+            ...options,
+            headers: { Accept: '*/*', ...options?.headers },
+        });
+    }
+}
+exports.Benchmarks = Benchmarks;
+//# sourceMappingURL=benchmarks.js.map

package/resources/benchmarks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmarks.js","sourceRoot":"","sources":["../src/resources/benchmarks.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,6CAA0C;AAG1C,MAAa,UAAW,SAAQ,sBAAW;IACzC,QAAQ,CAAC,WAAmB,EAAE,OAA6B;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,WAAW,EAAE,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,CAAC,OAA6B;QAChC,OACE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,OAAO,CAChD,CAAC,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,QAAQ,CAAC,IAA6B,EAAE,OAA6B;QACnE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,qBAAqB,EAAE;YAC9C,IAAI;YACJ,GAAG,OAAO;YACV,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE;SAChD,CAAC,CAAC;IACL,CAAC;CACF;AAlBD,gCAkBC"}

package/resources/benchmarks.mjs

@@ -0,0 +1,18 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from "../resource.mjs";
+export class Benchmarks extends APIResource {
+    retrieve(benchmarkId, options) {
+        return this._client.get(`/v1/eval/benchmarks/${benchmarkId}`, options);
+    }
+    list(options) {
+        return this._client.get('/v1/eval/benchmarks', options)._thenUnwrap((obj) => obj.data);
+    }
+    register(body, options) {
+        return this._client.post('/v1/eval/benchmarks', {
+            body,
+            ...options,
+            headers: { Accept: '*/*', ...options?.headers },
+        });
+    }
+}
+//# sourceMappingURL=benchmarks.mjs.map

package/resources/benchmarks.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmarks.mjs","sourceRoot":"","sources":["../src/resources/benchmarks.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;AAGtB,MAAM,OAAO,UAAW,SAAQ,WAAW;IACzC,QAAQ,CAAC,WAAmB,EAAE,OAA6B;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,WAAW,EAAE,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,CAAC,OAA6B;QAChC,OACE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,OAAO,CAChD,CAAC,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,QAAQ,CAAC,IAA6B,EAAE,OAA6B;QACnE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,qBAAqB,EAAE;YAC9C,IAAI;YACJ,GAAG,OAAO;YACV,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE;SAChD,CAAC,CAAC;IACL,CAAC;CACF"}

package/resources/datasets.d.ts

@@ -17,7 +17,12 @@ export interface DatasetRetrieveResponse {
     provider_id: string;
     provider_resource_id: string;
     type: 'dataset';
-    url: Shared.URL;
+    url: DatasetRetrieveResponse.URL;
+}
+export declare namespace DatasetRetrieveResponse {
+    interface URL {
+        uri: string;
+    }
 }
 export type DatasetListResponse = Array<DatasetListResponse.DatasetListResponseItem>;
 export declare namespace DatasetListResponse {
@@ -28,17 +33,27 @@ export declare namespace DatasetListResponse {
         provider_id: string;
         provider_resource_id: string;
         type: 'dataset';
-        url: Shared.URL;
+        url: DatasetListResponseItem.URL;
+    }
+    namespace DatasetListResponseItem {
+        interface URL {
+            uri: string;
+        }
     }
 }
 export interface DatasetRegisterParams {
     dataset_id: string;
     dataset_schema: Record<string, Shared.ParamType>;
-    url: Shared.URL;
+    url: DatasetRegisterParams.URL;
     metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
     provider_dataset_id?: string;
     provider_id?: string;
 }
+export declare namespace DatasetRegisterParams {
+    interface URL {
+        uri: string;
+    }
+}
 export declare namespace Datasets {
     export { type ListDatasetsResponse as ListDatasetsResponse, type DatasetRetrieveResponse as DatasetRetrieveResponse, type DatasetListResponse as DatasetListResponse, type DatasetRegisterParams as DatasetRegisterParams, };
 }

package/resources/datasets.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"datasets.d.ts","sourceRoot":"","sources":["../src/resources/datasets.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAChC,OAAO,KAAK,MAAM,MAAM,UAAU,CAAC;AAEnC,qBAAa,QAAS,SAAQ,WAAW;IACvC,QAAQ,CACN,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,uBAAuB,GAAG,IAAI,CAAC;IAIlD,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC;IAMzE,QAAQ,CAAC,IAAI,EAAE,qBAAqB,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAQ3F,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;CAMpF;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,mBAAmB,CAAC;CAC3B;AAED,MAAM,WAAW,uBAAuB;IACtC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAEjD,UAAU,EAAE,MAAM,CAAC;IAEnB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;IAEtF,WAAW,EAAE,MAAM,CAAC;IAEpB,oBAAoB,EAAE,MAAM,CAAC;IAE7B,IAAI,EAAE,SAAS,CAAC;IAEhB,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC;CACjB;AAED,MAAM,MAAM,mBAAmB,GAAG,KAAK,CAAC,mBAAmB,CAAC,uBAAuB,CAAC,CAAC;AAErF,yBAAiB,mBAAmB,CAAC;IACnC,UAAiB,uBAAuB;QACtC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEjD,UAAU,EAAE,MAAM,CAAC;QAEnB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;QAEtF,WAAW,EAAE,MAAM,CAAC;QAEpB,oBAAoB,EAAE,MAAM,CAAC;QAE7B,IAAI,EAAE,SAAS,CAAC;QAEhB,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC;KACjB;CACF;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IAEnB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAEjD,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC;IAEhB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;IAEvF,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,CAAC,OAAO,WAAW,QAAQ,CAAC;IAChC,OAAO,EACL,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,uBAAuB,IAAI,uBAAuB,EACvD,KAAK,mBAAmB,IAAI,mBAAmB,EAC/C,KAAK,qBAAqB,IAAI,qBAAqB,GACpD,CAAC;CACH"}
+{"version":3,"file":"datasets.d.ts","sourceRoot":"","sources":["../src/resources/datasets.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAChC,OAAO,KAAK,MAAM,MAAM,UAAU,CAAC;AAEnC,qBAAa,QAAS,SAAQ,WAAW;IACvC,QAAQ,CACN,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,uBAAuB,GAAG,IAAI,CAAC;IAIlD,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC;IAMzE,QAAQ,CAAC,IAAI,EAAE,qBAAqB,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAQ3F,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;CAMpF;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,mBAAmB,CAAC;CAC3B;AAED,MAAM,WAAW,uBAAuB;IACtC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAEjD,UAAU,EAAE,MAAM,CAAC;IAEnB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;IAEtF,WAAW,EAAE,MAAM,CAAC;IAEpB,oBAAoB,EAAE,MAAM,CAAC;IAE7B,IAAI,EAAE,SAAS,CAAC;IAEhB,GAAG,EAAE,uBAAuB,CAAC,GAAG,CAAC;CAClC;AAED,yBAAiB,uBAAuB,CAAC;IACvC,UAAiB,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC;KACb;CACF;AAED,MAAM,MAAM,mBAAmB,GAAG,KAAK,CAAC,mBAAmB,CAAC,uBAAuB,CAAC,CAAC;AAErF,yBAAiB,mBAAmB,CAAC;IACnC,UAAiB,uBAAuB;QACtC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEjD,UAAU,EAAE,MAAM,CAAC;QAEnB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;QAEtF,WAAW,EAAE,MAAM,CAAC;QAEpB,oBAAoB,EAAE,MAAM,CAAC;QAE7B,IAAI,EAAE,SAAS,CAAC;QAEhB,GAAG,EAAE,uBAAuB,CAAC,GAAG,CAAC;KAClC;IAED,UAAiB,uBAAuB,CAAC;QACvC,UAAiB,GAAG;YAClB,GAAG,EAAE,MAAM,CAAC;SACb;KACF;CACF;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IAEnB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAEjD,GAAG,EAAE,qBAAqB,CAAC,GAAG,CAAC;IAE/B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;IAEvF,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,yBAAiB,qBAAqB,CAAC;IACrC,UAAiB,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC;KACb;CACF;AAED,MAAM,CAAC,OAAO,WAAW,QAAQ,CAAC;IAChC,OAAO,EACL,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,uBAAuB,IAAI,uBAAuB,EACvD,KAAK,mBAAmB,IAAI,mBAAmB,EAC/C,KAAK,qBAAqB,IAAI,qBAAqB,GACpD,CAAC;CACH"}

package/resources/eval/eval.d.ts

@@ -1,14 +1,20 @@
 import { APIResource } from "../../resource.js";
 import * as Core from "../../core.js";
-import * as EvalAPI from "./eval.js";
 import * as ScoringFunctionsAPI from "../scoring-functions.js";
 import * as Shared from "../shared.js";
 import * as JobsAPI from "./jobs.js";
 import { JobStatusResponse, Jobs } from "./jobs.js";
 export declare class Eval extends APIResource {
     jobs: JobsAPI.Jobs;
-    evaluateRows(taskId: string, body: EvalEvaluateRowsParams, options?: Core.RequestOptions): Core.APIPromise<EvaluateResponse>;
-    runEval(taskId: string, body: EvalRunEvalParams, options?: Core.RequestOptions): Core.APIPromise<Job>;
+    evaluateRows(benchmarkId: string, body: EvalEvaluateRowsParams, options?: Core.RequestOptions): Core.APIPromise<EvaluateResponse>;
+    evaluateRowsAlpha(benchmarkId: string, body: EvalEvaluateRowsAlphaParams, options?: Core.RequestOptions): Core.APIPromise<EvaluateResponse>;
+    runEval(benchmarkId: string, body: EvalRunEvalParams, options?: Core.RequestOptions): Core.APIPromise<Job>;
+    runEvalAlpha(benchmarkId: string, body: EvalRunEvalAlphaParams, options?: Core.RequestOptions): Core.APIPromise<Job>;
+}
+export interface BenchmarkConfig {
+    eval_candidate: EvalCandidate;
+    scoring_params: Record<string, ScoringFunctionsAPI.ScoringFnParams>;
+    num_examples?: number;
 }
 export type EvalCandidate = EvalCandidate.ModelCandidate | EvalCandidate.AgentCandidate;
 export declare namespace EvalCandidate {
@@ -26,20 +32,6 @@ export declare namespace EvalCandidate {
         type: 'agent';
     }
 }
-export type EvalTaskConfig = EvalTaskConfig.BenchmarkEvalTaskConfig | EvalTaskConfig.AppEvalTaskConfig;
-export declare namespace EvalTaskConfig {
-    interface BenchmarkEvalTaskConfig {
-        eval_candidate: EvalAPI.EvalCandidate;
-        type: 'benchmark';
-        num_examples?: number;
-    }
-    interface AppEvalTaskConfig {
-        eval_candidate: EvalAPI.EvalCandidate;
-        scoring_params: Record<string, ScoringFunctionsAPI.ScoringFnParams>;
-        type: 'app';
-        num_examples?: number;
-    }
-}
 export interface EvaluateResponse {
     generations: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;
     scores: Record<string, Shared.ScoringResult>;
@@ -50,13 +42,21 @@ export interface Job {
 export interface EvalEvaluateRowsParams {
     input_rows: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;
     scoring_functions: Array<string>;
-    task_config: EvalTaskConfig;
+    task_config: BenchmarkConfig;
+}
+export interface EvalEvaluateRowsAlphaParams {
+    input_rows: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;
+    scoring_functions: Array<string>;
+    task_config: BenchmarkConfig;
 }
 export interface EvalRunEvalParams {
-    task_config: EvalTaskConfig;
+    task_config: BenchmarkConfig;
+}
+export interface EvalRunEvalAlphaParams {
+    task_config: BenchmarkConfig;
 }
 export declare namespace Eval {
-    export { type EvalCandidate as EvalCandidate, type EvalTaskConfig as EvalTaskConfig, type EvaluateResponse as EvaluateResponse, type Job as Job, type EvalEvaluateRowsParams as EvalEvaluateRowsParams, type EvalRunEvalParams as EvalRunEvalParams, };
+    export { type BenchmarkConfig as BenchmarkConfig, type EvalCandidate as EvalCandidate, type EvaluateResponse as EvaluateResponse, type Job as Job, type EvalEvaluateRowsParams as EvalEvaluateRowsParams, type EvalEvaluateRowsAlphaParams as EvalEvaluateRowsAlphaParams, type EvalRunEvalParams as EvalRunEvalParams, type EvalRunEvalAlphaParams as EvalRunEvalAlphaParams, };
     export { Jobs as Jobs, type JobStatusResponse as JobStatusResponse };
 }
 //# sourceMappingURL=eval.d.ts.map

package/resources/eval/eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/resources/eval/eval.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,KAAK,IAAI,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,OAAO,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,mBAAmB,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,MAAM,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,OAAO,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAEjD,qBAAa,IAAK,SAAQ,WAAW;IACnC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAkC;IAEpD,YAAY,CACV,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,sBAAsB,EAC5B,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC;IAIpC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;CAGtG;AAED,MAAM,MAAM,aAAa,GAAG,aAAa,CAAC,cAAc,GAAG,aAAa,CAAC,cAAc,CAAC;AAExF,yBAAiB,aAAa,CAAC;IAC7B,UAAiB,cAAc;QAC7B,KAAK,EAAE,MAAM,CAAC;QAEd,eAAe,EAAE,MAAM,CAAC,cAAc,CAAC;QAEvC,IAAI,EAAE,OAAO,CAAC;QAEd;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC;KACvC;IAED,UAAiB,cAAc;QAC7B,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC;QAE3B,IAAI,EAAE,OAAO,CAAC;KACf;CACF;AAED,MAAM,MAAM,cAAc,GAAG,cAAc,CAAC,uBAAuB,GAAG,cAAc,CAAC,iBAAiB,CAAC;AAEvG,yBAAiB,cAAc,CAAC;IAC9B,UAAiB,uBAAuB;QACtC,cAAc,EAAE,OAAO,CAAC,aAAa,CAAC;QAEtC,IAAI,EAAE,WAAW,CAAC;QAElB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB;IAED,UAAiB,iBAAiB;QAChC,cAAc,EAAE,OAAO,CAAC,aAAa,CAAC;QAEtC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,eAAe,CAAC,CAAC;QAEpE,IAAI,EAAE,KAAK,CAAC;QAEZ,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB;CACF;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;IAEhG,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,GAAG;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;IAE/F,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEjC,WAAW,EAAE,cAAc,CAAC;CAC7B;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,cAAc,CAAC;CAC7B;AAID,MAAM,CAAC,OAAO,WAAW,IAAI,CAAC;IAC5B,OAAO,EACL,KAAK,aAAa,IAAI,aAAa,EACnC,KAAK,cAAc,IAAI,cAAc,EACrC,KAAK,gBAAgB,IAAI,gBAAgB,EACzC,KAAK,GAAG,IAAI,GAAG,EACf,KAAK,sBAAsB,IAAI,sBAAsB,EACrD,KAAK,iBAAiB,IAAI,iBAAiB,GAC5C,CAAC;IAEF,OAAO,EAAE,IAAI,IAAI,IAAI,EAAE,KAAK,iBAAiB,IAAI,iBAAiB,EAAE,CAAC;CACtE"}
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/resources/eval/eval.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,KAAK,IAAI,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,mBAAmB,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,MAAM,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,OAAO,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAEjD,qBAAa,IAAK,SAAQ,WAAW;IACnC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAkC;IAEpD,YAAY,CACV,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,sBAAsB,EAC5B,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC;IAIpC,iBAAiB,CACf,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,2BAA2B,EACjC,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC;IAIpC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAI1G,YAAY,CACV,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,sBAAsB,EAC5B,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;CAGxB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,aAAa,CAAC;IAE9B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAEpE,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,aAAa,GAAG,aAAa,CAAC,cAAc,GAAG,aAAa,CAAC,cAAc,CAAC;AAExF,yBAAiB,aAAa,CAAC;IAC7B,UAAiB,cAAc;QAC7B,KAAK,EAAE,MAAM,CAAC;QAEd,eAAe,EAAE,MAAM,CAAC,cAAc,CAAC;QAEvC,IAAI,EAAE,OAAO,CAAC;QAEd;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC;KACvC;IAED,UAAiB,cAAc;QAC7B,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC;QAE3B,IAAI,EAAE,OAAO,CAAC;KACf;CACF;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;IAEhG,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,GAAG;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;IAE/F,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEjC,WAAW,EAAE,eAAe,CAAC;CAC9B;AAED,MAAM,WAAW,2BAA2B;IAC1C,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;IAE/F,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEjC,WAAW,EAAE,eAAe,CAAC;CAC9B;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,eAAe,CAAC;CAC9B;AAED,MAAM,WAAW,sBAAsB;IACrC,WAAW,EAAE,eAAe,CAAC;CAC9B;AAID,MAAM,CAAC,OAAO,WAAW,IAAI,CAAC;IAC5B,OAAO,EACL,KAAK,eAAe,IAAI,eAAe,EACvC,KAAK,aAAa,IAAI,aAAa,EACnC,KAAK,gBAAgB,IAAI,gBAAgB,EACzC,KAAK,GAAG,IAAI,GAAG,EACf,KAAK,sBAAsB,IAAI,sBAAsB,EACrD,KAAK,2BAA2B,IAAI,2BAA2B,EAC/D,KAAK,iBAAiB,IAAI,iBAAiB,EAC3C,KAAK,sBAAsB,IAAI,sBAAsB,GACtD,CAAC;IAEF,OAAO,EAAE,IAAI,IAAI,IAAI,EAAE,KAAK,iBAAiB,IAAI,iBAAiB,EAAE,CAAC;CACtE"}

package/resources/eval/eval.js

@@ -33,11 +33,17 @@ class Eval extends resource_1.APIResource {
         super(...arguments);
         this.jobs = new JobsAPI.Jobs(this._client);
     }
-    evaluateRows(taskId, body, options) {
-        return this._client.post(`/v1/eval/tasks/${taskId}/evaluations`, { body, ...options });
+    evaluateRows(benchmarkId, body, options) {
+        return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
     }
-    runEval(taskId, body, options) {
-        return this._client.post(`/v1/eval/tasks/${taskId}/jobs`, { body, ...options });
+    evaluateRowsAlpha(benchmarkId, body, options) {
+        return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
+    }
+    runEval(benchmarkId, body, options) {
+        return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
+    }
+    runEvalAlpha(benchmarkId, body, options) {
+        return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
     }
 }
 exports.Eval = Eval;

package/resources/eval/eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../src/resources/eval/eval.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;;;;;;;;;;;;;;;;;;;;;;;AAEtF,gDAA6C;AAK7C,mDAAkC;AAClC,oCAAiD;AAEjD,MAAa,IAAK,SAAQ,sBAAW;IAArC;;QACE,SAAI,GAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAatD,CAAC;IAXC,YAAY,CACV,MAAc,EACd,IAA4B,EAC5B,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,MAAM,cAAc,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACzF,CAAC;IAED,OAAO,CAAC,MAAc,EAAE,IAAuB,EAAE,OAA6B;QAC5E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,MAAM,OAAO,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAClF,CAAC;CACF;AAdD,oBAcC;AAqED,IAAI,CAAC,IAAI,GAAG,WAAI,CAAC"}
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../src/resources/eval/eval.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;;;;;;;;;;;;;;;;;;;;;;;AAEtF,gDAA6C;AAI7C,mDAAkC;AAClC,oCAAiD;AAEjD,MAAa,IAAK,SAAQ,sBAAW;IAArC;;QACE,SAAI,GAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IA6BtD,CAAC;IA3BC,YAAY,CACV,WAAmB,EACnB,IAA4B,EAC5B,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,WAAW,cAAc,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnG,CAAC;IAED,iBAAiB,CACf,WAAmB,EACnB,IAAiC,EACjC,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,WAAW,cAAc,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnG,CAAC;IAED,OAAO,CAAC,WAAmB,EAAE,IAAuB,EAAE,OAA6B;QACjF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,WAAW,OAAO,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,YAAY,CACV,WAAmB,EACnB,IAA4B,EAC5B,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,WAAW,OAAO,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC5F,CAAC;CACF;AA9BD,oBA8BC;AAmED,IAAI,CAAC,IAAI,GAAG,WAAI,CAAC"}

package/resources/eval/eval.mjs

@@ -7,11 +7,17 @@ export class Eval extends APIResource {
         super(...arguments);
         this.jobs = new JobsAPI.Jobs(this._client);
     }
-    evaluateRows(taskId, body, options) {
-        return this._client.post(`/v1/eval/tasks/${taskId}/evaluations`, { body, ...options });
+    evaluateRows(benchmarkId, body, options) {
+        return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
     }
-    runEval(taskId, body, options) {
-        return this._client.post(`/v1/eval/tasks/${taskId}/jobs`, { body, ...options });
+    evaluateRowsAlpha(benchmarkId, body, options) {
+        return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/evaluations`, { body, ...options });
+    }
+    runEval(benchmarkId, body, options) {
+        return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
+    }
+    runEvalAlpha(benchmarkId, body, options) {
+        return this._client.post(`/v1/eval/benchmarks/${benchmarkId}/jobs`, { body, ...options });
     }
 }
 Eval.Jobs = Jobs;

package/resources/eval/eval.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"eval.mjs","sourceRoot":"","sources":["../../src/resources/eval/eval.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAKf,KAAK,OAAO;OACZ,EAAqB,IAAI,EAAE;AAElC,MAAM,OAAO,IAAK,SAAQ,WAAW;IAArC;;QACE,SAAI,GAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAatD,CAAC;IAXC,YAAY,CACV,MAAc,EACd,IAA4B,EAC5B,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,MAAM,cAAc,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACzF,CAAC;IAED,OAAO,CAAC,MAAc,EAAE,IAAuB,EAAE,OAA6B;QAC5E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,MAAM,OAAO,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAClF,CAAC;CACF;AAqED,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC"}
+{"version":3,"file":"eval.mjs","sourceRoot":"","sources":["../../src/resources/eval/eval.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAIf,KAAK,OAAO;OACZ,EAAqB,IAAI,EAAE;AAElC,MAAM,OAAO,IAAK,SAAQ,WAAW;IAArC;;QACE,SAAI,GAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IA6BtD,CAAC;IA3BC,YAAY,CACV,WAAmB,EACnB,IAA4B,EAC5B,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,WAAW,cAAc,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnG,CAAC;IAED,iBAAiB,CACf,WAAmB,EACnB,IAAiC,EACjC,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,WAAW,cAAc,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnG,CAAC;IAED,OAAO,CAAC,WAAmB,EAAE,IAAuB,EAAE,OAA6B;QACjF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,WAAW,OAAO,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,YAAY,CACV,WAAmB,EACnB,IAA4B,EAC5B,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,WAAW,OAAO,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC5F,CAAC;CACF;AAmED,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC"}

package/resources/eval/index.d.ts

@@ -1,3 +1,3 @@
-export { Eval, type EvalCandidate, type EvalTaskConfig, type EvaluateResponse, type Job, type EvalEvaluateRowsParams, type EvalRunEvalParams, } from "./eval.js";
+export { Eval, type BenchmarkConfig, type EvalCandidate, type EvaluateResponse, type Job, type EvalEvaluateRowsParams, type EvalEvaluateRowsAlphaParams, type EvalRunEvalParams, type EvalRunEvalAlphaParams, } from "./eval.js";
 export { Jobs, type JobStatusResponse } from "./jobs.js";
 //# sourceMappingURL=index.d.ts.map

package/resources/eval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/resources/eval/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,IAAI,EACJ,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,GAAG,EACR,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,GACvB,MAAM,QAAQ,CAAC;AAChB,OAAO,EAAE,IAAI,EAAE,KAAK,iBAAiB,EAAE,MAAM,QAAQ,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/resources/eval/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,IAAI,EACJ,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,GAAG,EACR,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,GAC5B,MAAM,QAAQ,CAAC;AAChB,OAAO,EAAE,IAAI,EAAE,KAAK,iBAAiB,EAAE,MAAM,QAAQ,CAAC"}

package/resources/eval/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/resources/eval/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kCAQgB;AAPd,4FAAA,IAAI,OAAA;AAQN,kCAAsD;AAA7C,4FAAA,IAAI,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/resources/eval/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kCAUgB;AATd,4FAAA,IAAI,OAAA;AAUN,kCAAsD;AAA7C,4FAAA,IAAI,OAAA"}

package/resources/eval/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../../src/resources/eval/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EACL,IAAI,GAOL;OACM,EAAE,IAAI,EAA0B"}
+{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../../src/resources/eval/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EACL,IAAI,GASL;OACM,EAAE,IAAI,EAA0B"}

package/resources/eval/jobs.d.ts

@@ -2,9 +2,9 @@ import { APIResource } from "../../resource.js";
 import * as Core from "../../core.js";
 import * as EvalAPI from "./eval.js";
 export declare class Jobs extends APIResource {
-    retrieve(taskId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<EvalAPI.EvaluateResponse>;
-    cancel(taskId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<void>;
-    status(taskId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<JobStatusResponse | null>;
+    retrieve(benchmarkId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<EvalAPI.EvaluateResponse>;
+    cancel(benchmarkId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<void>;
+    status(benchmarkId: string, jobId: string, options?: Core.RequestOptions): Core.APIPromise<JobStatusResponse | null>;
 }
 export type JobStatusResponse = 'completed' | 'in_progress' | 'failed' | 'scheduled' | null;
 export declare namespace Jobs {

package/resources/eval/jobs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jobs.d.ts","sourceRoot":"","sources":["../../src/resources/eval/jobs.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,KAAK,IAAI,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,OAAO,MAAM,QAAQ,CAAC;AAElC,qBAAa,IAAK,SAAQ,WAAW;IACnC,QAAQ,CACN,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,gBAAgB,CAAC;IAI5C,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAO3F,MAAM,CACJ,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,iBAAiB,GAAG,IAAI,CAAC;CAG7C;AAED,MAAM,MAAM,iBAAiB,GAAG,WAAW,GAAG,aAAa,GAAG,QAAQ,GAAG,WAAW,GAAG,IAAI,CAAC;AAE5F,MAAM,CAAC,OAAO,WAAW,IAAI,CAAC;IAC5B,OAAO,EAAE,KAAK,iBAAiB,IAAI,iBAAiB,EAAE,CAAC;CACxD"}
+{"version":3,"file":"jobs.d.ts","sourceRoot":"","sources":["../../src/resources/eval/jobs.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,KAAK,IAAI,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,OAAO,MAAM,QAAQ,CAAC;AAElC,qBAAa,IAAK,SAAQ,WAAW;IACnC,QAAQ,CACN,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,gBAAgB,CAAC;IAI5C,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAOhG,MAAM,CACJ,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,iBAAiB,GAAG,IAAI,CAAC;CAG7C;AAED,MAAM,MAAM,iBAAiB,GAAG,WAAW,GAAG,aAAa,GAAG,QAAQ,GAAG,WAAW,GAAG,IAAI,CAAC;AAE5F,MAAM,CAAC,OAAO,WAAW,IAAI,CAAC;IAC5B,OAAO,EAAE,KAAK,iBAAiB,IAAI,iBAAiB,EAAE,CAAC;CACxD"}

package/resources/eval/jobs.js

@@ -4,17 +4,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Jobs = void 0;
 const resource_1 = require("../../resource.js");
 class Jobs extends resource_1.APIResource {
-    retrieve(taskId, jobId, options) {
-        return this._client.get(`/v1/eval/tasks/${taskId}/jobs/${jobId}/result`, options);
+    retrieve(benchmarkId, jobId, options) {
+        return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}/result`, options);
     }
-    cancel(taskId, jobId, options) {
-        return this._client.delete(`/v1/eval/tasks/${taskId}/jobs/${jobId}`, {
+    cancel(benchmarkId, jobId, options) {
+        return this._client.delete(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, {
             ...options,
             headers: { Accept: '*/*', ...options?.headers },
         });
     }
-    status(taskId, jobId, options) {
-        return this._client.get(`/v1/eval/tasks/${taskId}/jobs/${jobId}`, options);
+    status(benchmarkId, jobId, options) {
+        return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, options);
     }
 }
 exports.Jobs = Jobs;

package/resources/eval/jobs.js.map

@@ -1 +1 @@
-{"version":3,"file":"jobs.js","sourceRoot":"","sources":["../../src/resources/eval/jobs.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,gDAA6C;AAI7C,MAAa,IAAK,SAAQ,sBAAW;IACnC,QAAQ,CACN,MAAc,EACd,KAAa,EACb,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,SAAS,KAAK,SAAS,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,CAAC,MAAc,EAAE,KAAa,EAAE,OAA6B;QACjE,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,MAAM,SAAS,KAAK,EAAE,EAAE;YACnE,GAAG,OAAO;YACV,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE;SAChD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CACJ,MAAc,EACd,KAAa,EACb,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,SAAS,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IAC7E,CAAC;CACF;AAvBD,oBAuBC"}
+{"version":3,"file":"jobs.js","sourceRoot":"","sources":["../../src/resources/eval/jobs.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,gDAA6C;AAI7C,MAAa,IAAK,SAAQ,sBAAW;IACnC,QAAQ,CACN,WAAmB,EACnB,KAAa,EACb,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,WAAW,SAAS,KAAK,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,CAAC,WAAmB,EAAE,KAAa,EAAE,OAA6B;QACtE,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,uBAAuB,WAAW,SAAS,KAAK,EAAE,EAAE;YAC7E,GAAG,OAAO;YACV,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE;SAChD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CACJ,WAAmB,EACnB,KAAa,EACb,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,WAAW,SAAS,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IACvF,CAAC;CACF;AAvBD,oBAuBC"}

package/resources/eval/jobs.mjs

@@ -1,17 +1,17 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 import { APIResource } from "../../resource.mjs";
 export class Jobs extends APIResource {
-    retrieve(taskId, jobId, options) {
-        return this._client.get(`/v1/eval/tasks/${taskId}/jobs/${jobId}/result`, options);
+    retrieve(benchmarkId, jobId, options) {
+        return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}/result`, options);
     }
-    cancel(taskId, jobId, options) {
-        return this._client.delete(`/v1/eval/tasks/${taskId}/jobs/${jobId}`, {
+    cancel(benchmarkId, jobId, options) {
+        return this._client.delete(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, {
             ...options,
             headers: { Accept: '*/*', ...options?.headers },
         });
     }
-    status(taskId, jobId, options) {
-        return this._client.get(`/v1/eval/tasks/${taskId}/jobs/${jobId}`, options);
+    status(benchmarkId, jobId, options) {
+        return this._client.get(`/v1/eval/benchmarks/${benchmarkId}/jobs/${jobId}`, options);
     }
 }
 //# sourceMappingURL=jobs.mjs.map

package/resources/eval/jobs.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"jobs.mjs","sourceRoot":"","sources":["../../src/resources/eval/jobs.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;AAItB,MAAM,OAAO,IAAK,SAAQ,WAAW;IACnC,QAAQ,CACN,MAAc,EACd,KAAa,EACb,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,SAAS,KAAK,SAAS,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,CAAC,MAAc,EAAE,KAAa,EAAE,OAA6B;QACjE,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,MAAM,SAAS,KAAK,EAAE,EAAE;YACnE,GAAG,OAAO;YACV,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE;SAChD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CACJ,MAAc,EACd,KAAa,EACb,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,SAAS,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IAC7E,CAAC;CACF"}
+{"version":3,"file":"jobs.mjs","sourceRoot":"","sources":["../../src/resources/eval/jobs.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;AAItB,MAAM,OAAO,IAAK,SAAQ,WAAW;IACnC,QAAQ,CACN,WAAmB,EACnB,KAAa,EACb,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,WAAW,SAAS,KAAK,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,CAAC,WAAmB,EAAE,KAAa,EAAE,OAA6B;QACtE,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,uBAAuB,WAAW,SAAS,KAAK,EAAE,EAAE;YAC7E,GAAG,OAAO;YACV,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE;SAChD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CACJ,WAAmB,EACnB,KAAa,EACb,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,WAAW,SAAS,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IACvF,CAAC;CACF"}

package/resources/eval-tasks.d.ts

@@ -1,32 +1,21 @@
 import { APIResource } from "../resource.js";
 import * as Core from "../core.js";
+import * as BenchmarksAPI from "./benchmarks.js";
 export declare class EvalTasks extends APIResource {
-    retrieve(evalTaskId: string, options?: Core.RequestOptions): Core.APIPromise<EvalTask | null>;
-    list(options?: Core.RequestOptions): Core.APIPromise<EvalTaskListResponse>;
+    retrieve(evalTaskId: string, options?: Core.RequestOptions): Core.APIPromise<BenchmarksAPI.Benchmark | null>;
+    list(options?: Core.RequestOptions): Core.APIPromise<BenchmarksAPI.BenchmarkListResponse>;
     register(body: EvalTaskRegisterParams, options?: Core.RequestOptions): Core.APIPromise<void>;
 }
-export interface EvalTask {
-    dataset_id: string;
-    identifier: string;
-    metadata: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
-    provider_id: string;
-    provider_resource_id: string;
-    scoring_functions: Array<string>;
-    type: 'eval_task';
-}
-export interface ListEvalTasksResponse {
-    data: EvalTaskListResponse;
-}
-export type EvalTaskListResponse = Array<EvalTask>;
+export type EvalTaskListResponse = Array<BenchmarksAPI.Benchmark>;
 export interface EvalTaskRegisterParams {
     dataset_id: string;
     eval_task_id: string;
     scoring_functions: Array<string>;
     metadata?: Record<string, boolean | number | string | Array<unknown> | unknown | null>;
-    provider_eval_task_id?: string;
+    provider_benchmark_id?: string;
     provider_id?: string;
 }
 export declare namespace EvalTasks {
-    export { type EvalTask as EvalTask, type ListEvalTasksResponse as ListEvalTasksResponse, type EvalTaskListResponse as EvalTaskListResponse, type EvalTaskRegisterParams as EvalTaskRegisterParams, };
+    export { type EvalTaskListResponse as EvalTaskListResponse, type EvalTaskRegisterParams as EvalTaskRegisterParams, };
 }
 //# sourceMappingURL=eval-tasks.d.ts.map

package/resources/eval-tasks.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-tasks.d.ts","sourceRoot":"","sources":["../src/resources/eval-tasks.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAEhC,qBAAa,SAAU,SAAQ,WAAW;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC;IAI7F,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC;IAM1E,QAAQ,CAAC,IAAI,EAAE,sBAAsB,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;CAO7F;AAED,MAAM,WAAW,QAAQ;IACvB,UAAU,EAAE,MAAM,CAAC;IAEnB,UAAU,EAAE,MAAM,CAAC;IAEnB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;IAEtF,WAAW,EAAE,MAAM,CAAC;IAEpB,oBAAoB,EAAE,MAAM,CAAC;IAE7B,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEjC,IAAI,EAAE,WAAW,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,oBAAoB,CAAC;CAC5B;AAED,MAAM,MAAM,oBAAoB,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;AAEnD,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,MAAM,CAAC;IAEnB,YAAY,EAAE,MAAM,CAAC;IAErB,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;IAEvF,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,CAAC,OAAO,WAAW,SAAS,CAAC;IACjC,OAAO,EACL,KAAK,QAAQ,IAAI,QAAQ,EACzB,KAAK,qBAAqB,IAAI,qBAAqB,EACnD,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,sBAAsB,IAAI,sBAAsB,GACtD,CAAC;CACH"}
+{"version":3,"file":"eval-tasks.d.ts","sourceRoot":"","sources":["../src/resources/eval-tasks.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAChC,OAAO,KAAK,aAAa,MAAM,cAAc,CAAC;AAE9C,qBAAa,SAAU,SAAQ,WAAW;IACxC,QAAQ,CACN,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAC5B,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,SAAS,GAAG,IAAI,CAAC;IAIlD,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,qBAAqB,CAAC;IAQzF,QAAQ,CAAC,IAAI,EAAE,sBAAsB,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;CAO7F;AAED,MAAM,MAAM,oBAAoB,GAAG,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;AAElE,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,MAAM,CAAC;IAEnB,YAAY,EAAE,MAAM,CAAC;IAErB,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;IAEvF,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,CAAC,OAAO,WAAW,SAAS,CAAC;IACjC,OAAO,EACL,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,sBAAsB,IAAI,sBAAsB,GACtD,CAAC;CACH"}

package/resources/eval-tasks.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-tasks.js","sourceRoot":"","sources":["../src/resources/eval-tasks.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,6CAA0C;AAG1C,MAAa,SAAU,SAAQ,sBAAW;IACxC,QAAQ,CAAC,UAAkB,EAAE,OAA6B;QACxD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,CAAC,OAA6B;QAChC,OACE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAC3C,CAAC,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,QAAQ,CAAC,IAA4B,EAAE,OAA6B;QAClE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE;YACzC,IAAI;YACJ,GAAG,OAAO;YACV,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE;SAChD,CAAC,CAAC;IACL,CAAC;CACF;AAlBD,8BAkBC"}
+{"version":3,"file":"eval-tasks.js","sourceRoot":"","sources":["../src/resources/eval-tasks.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,6CAA0C;AAI1C,MAAa,SAAU,SAAQ,sBAAW;IACxC,QAAQ,CACN,UAAkB,EAClB,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,CAAC,OAA6B;QAChC,OACE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAG3C,CAAC,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,QAAQ,CAAC,IAA4B,EAAE,OAA6B;QAClE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE;YACzC,IAAI;YACJ,GAAG,OAAO;YACV,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE;SAChD,CAAC,CAAC;IACL,CAAC;CACF;AAvBD,8BAuBC"}

package/resources/eval-tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"eval-tasks.mjs","sourceRoot":"","sources":["../src/resources/eval-tasks.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;AAGtB,MAAM,OAAO,SAAU,SAAQ,WAAW;IACxC,QAAQ,CAAC,UAAkB,EAAE,OAA6B;QACxD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,CAAC,OAA6B;QAChC,OACE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAC3C,CAAC,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,QAAQ,CAAC,IAA4B,EAAE,OAA6B;QAClE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE;YACzC,IAAI;YACJ,GAAG,OAAO;YACV,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE;SAChD,CAAC,CAAC;IACL,CAAC;CACF"}
+{"version":3,"file":"eval-tasks.mjs","sourceRoot":"","sources":["../src/resources/eval-tasks.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;AAItB,MAAM,OAAO,SAAU,SAAQ,WAAW;IACxC,QAAQ,CACN,UAAkB,EAClB,OAA6B;QAE7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,CAAC,OAA6B;QAChC,OACE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAG3C,CAAC,WAAW,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,QAAQ,CAAC,IAA4B,EAAE,OAA6B;QAClE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE;YACzC,IAAI;YACJ,GAAG,OAAO;YACV,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE;SAChD,CAAC,CAAC;IACL,CAAC;CACF"}

package/resources/index.d.ts

@@ -1,10 +1,10 @@
 export * from "./shared.js";
 export { Agents, type InferenceStep, type MemoryRetrievalStep, type ShieldCallStep, type ToolExecutionStep, type ToolResponse, type AgentCreateResponse, type AgentCreateParams, } from "./agents/agents.js";
 export { BatchInference, type BatchInferenceChatCompletionResponse, type BatchInferenceChatCompletionParams, type BatchInferenceCompletionParams, } from "./batch-inference.js";
+export { Benchmarks, type Benchmark, type ListBenchmarksResponse, type BenchmarkListResponse, type BenchmarkRegisterParams, } from "./benchmarks.js";
 export { Datasetio, type PaginatedRowsResult, type DatasetioAppendRowsParams, type DatasetioGetRowsPaginatedParams, } from "./datasetio.js";
 export { Datasets, type ListDatasetsResponse, type DatasetRetrieveResponse, type DatasetListResponse, type DatasetRegisterParams, } from "./datasets.js";
-export { Eval, type EvalCandidate, type EvalTaskConfig, type EvaluateResponse, type Job, type EvalEvaluateRowsParams, type EvalRunEvalParams, } from "./eval/eval.js";
-export { EvalTasks, type EvalTask, type ListEvalTasksResponse, type EvalTaskListResponse, type EvalTaskRegisterParams, } from "./eval-tasks.js";
+export { Eval, type BenchmarkConfig, type EvalCandidate, type EvaluateResponse, type Job, type EvalEvaluateRowsParams, type EvalEvaluateRowsAlphaParams, type EvalRunEvalParams, type EvalRunEvalAlphaParams, } from "./eval/eval.js";
 export { Inference, type ChatCompletionResponseStreamChunk, type CompletionResponse, type EmbeddingsResponse, type TokenLogProbs, type InferenceChatCompletionParams, type InferenceChatCompletionParamsNonStreaming, type InferenceChatCompletionParamsStreaming, type InferenceCompletionParams, type InferenceCompletionParamsNonStreaming, type InferenceCompletionParamsStreaming, type InferenceEmbeddingsParams, } from "./inference.js";
 export { Inspect, type HealthInfo, type ProviderInfo, type RouteInfo, type VersionInfo } from "./inspect.js";
 export { Models, type ListModelsResponse, type Model, type ModelListResponse, type ModelRegisterParams, } from "./models.js";

package/resources/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":"AAEA,cAAc,UAAU,CAAC;AACzB,OAAO,EACL,MAAM,EACN,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,GACvB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,cAAc,EACd,KAAK,oCAAoC,EACzC,KAAK,kCAAkC,EACvC,KAAK,8BAA8B,GACpC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,SAAS,EACT,KAAK,mBAAmB,EACxB,KAAK,yBAAyB,EAC9B,KAAK,+BAA+B,GACrC,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,QAAQ,EACR,KAAK,oBAAoB,EACzB,KAAK,uBAAuB,EAC5B,KAAK,mBAAmB,EACxB,KAAK,qBAAqB,GAC3B,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,IAAI,EACJ,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,GAAG,EACR,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,GACvB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,sBAAsB,GAC5B,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,SAAS,EACT,KAAK,iCAAiC,EACtC,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,6BAA6B,EAClC,KAAK,yCAAyC,EAC9C,KAAK,sCAAsC,EAC3C,KAAK,yBAAyB,EAC9B,KAAK,qCAAqC,EAC1C,KAAK,kCAAkC,EACvC,KAAK,yBAAyB,GAC/B,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,YAAY,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAC1G,OAAO,EACL,MAAM,EACN,KAAK,kBAAkB,EACvB,KAAK,KAAK,EACV,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,GACzB,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,YAAY,EACZ,KAAK,eAAe,EACpB,KAAK,4BAA4B,EACjC,KAAK,eAAe,EACpB,KAAK,oCAAoC,EACzC,KAAK,oCAAoC,GAC1C,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,KAAK,qBAAqB,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAC/F,OAAO,EAAE,MAAM,EAAE,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAE,MAAM,UAAU,CAAC;AACnF,OAAO,EAAE,MAAM,EAAE,KAAK,iBAAiB,EAAE,KAAK,qBAAqB,EAAE,MAAM,UAAU,CAAC;AACtF,OAAO,EACL,OAAO,EACP,KAAK,oBAAoB,EACzB,KAAK,yBAAyB,EAC9B,KAAK,kBAAkB,EACvB,KAAK,uBAAuB,GAC7B,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,gBAAgB,EAChB,KAAK,4BAA4B,EACjC,KAAK,SAAS,EACd,KAAK,eAAe,EACpB,KAAK,2BAA2B,EAChC,KAAK,6BAA6B,GACnC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,OAAO,EACP,KAAK,mBAAmB,EACxB,KAAK,MAAM,EACX,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,uBAAuB,EACvB,KAAK,+BAA+B,EACpC,KAAK,qCAAqC,GAC3C,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,SAAS,EACT,KAAK,KAAK,EACV,KAAK,cAAc,EACnB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,KAAK,EACV,KAAK,wBAAwB,EAC7B,KAAK,4BAA4B,EACjC,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,0BAA0B,EAC/B,KAAK,uBAAuB,EAC5B,KAAK,yBAAyB,EAC9B,KAAK,0BAA0B,EAC/B,KAAK,iCAAiC,GACvC,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,WAAW,EACX,KAAK,OAAO,EACZ,KAAK,oBAAoB,EACzB,KAAK,2BAA2B,EAChC,KAAK,0BAA0B,GAChC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,UAAU,EACV,KAAK,sBAAsB,EAC3B,KAAK,SAAS,EACd,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,GAC7B,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,KAAK,EACL,KAAK,iBAAiB,EACtB,KAAK,IAAI,EACT,KAAK,gBAAgB,EACrB,KAAK,cAAc,GACpB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,SAAS,EACT,KAAK,qBAAqB,EAC1B,KAAK,wBAAwB,EAC7B,KAAK,oBAAoB,EACzB,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,GAC5B,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,QAAQ,EACR,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EACzB,KAAK,mBAAmB,GACzB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":"AAEA,cAAc,UAAU,CAAC;AACzB,OAAO,EACL,MAAM,EACN,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,GACvB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,cAAc,EACd,KAAK,oCAAoC,EACzC,KAAK,kCAAkC,EACvC,KAAK,8BAA8B,GACpC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,UAAU,EACV,KAAK,SAAS,EACd,KAAK,sBAAsB,EAC3B,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,GAC7B,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,SAAS,EACT,KAAK,mBAAmB,EACxB,KAAK,yBAAyB,EAC9B,KAAK,+BAA+B,GACrC,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,QAAQ,EACR,KAAK,oBAAoB,EACzB,KAAK,uBAAuB,EAC5B,KAAK,mBAAmB,EACxB,KAAK,qBAAqB,GAC3B,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,IAAI,EACJ,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,GAAG,EACR,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,GAC5B,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,SAAS,EACT,KAAK,iCAAiC,EACtC,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,6BAA6B,EAClC,KAAK,yCAAyC,EAC9C,KAAK,sCAAsC,EAC3C,KAAK,yBAAyB,EAC9B,KAAK,qCAAqC,EAC1C,KAAK,kCAAkC,EACvC,KAAK,yBAAyB,GAC/B,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,YAAY,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAC1G,OAAO,EACL,MAAM,EACN,KAAK,kBAAkB,EACvB,KAAK,KAAK,EACV,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,GACzB,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,YAAY,EACZ,KAAK,eAAe,EACpB,KAAK,4BAA4B,EACjC,KAAK,eAAe,EACpB,KAAK,oCAAoC,EACzC,KAAK,oCAAoC,GAC1C,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,KAAK,qBAAqB,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAC/F,OAAO,EAAE,MAAM,EAAE,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAE,MAAM,UAAU,CAAC;AACnF,OAAO,EAAE,MAAM,EAAE,KAAK,iBAAiB,EAAE,KAAK,qBAAqB,EAAE,MAAM,UAAU,CAAC;AACtF,OAAO,EACL,OAAO,EACP,KAAK,oBAAoB,EACzB,KAAK,yBAAyB,EAC9B,KAAK,kBAAkB,EACvB,KAAK,uBAAuB,GAC7B,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,gBAAgB,EAChB,KAAK,4BAA4B,EACjC,KAAK,SAAS,EACd,KAAK,eAAe,EACpB,KAAK,2BAA2B,EAChC,KAAK,6BAA6B,GACnC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,OAAO,EACP,KAAK,mBAAmB,EACxB,KAAK,MAAM,EACX,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,uBAAuB,EACvB,KAAK,+BAA+B,EACpC,KAAK,qCAAqC,GAC3C,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,SAAS,EACT,KAAK,KAAK,EACV,KAAK,cAAc,EACnB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,KAAK,EACV,KAAK,wBAAwB,EAC7B,KAAK,4BAA4B,EACjC,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,0BAA0B,EAC/B,KAAK,uBAAuB,EAC5B,KAAK,yBAAyB,EAC9B,KAAK,0BAA0B,EAC/B,KAAK,iCAAiC,GACvC,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,WAAW,EACX,KAAK,OAAO,EACZ,KAAK,oBAAoB,EACzB,KAAK,2BAA2B,EAChC,KAAK,0BAA0B,GAChC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,UAAU,EACV,KAAK,sBAAsB,EAC3B,KAAK,SAAS,EACd,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,GAC7B,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,KAAK,EACL,KAAK,iBAAiB,EACtB,KAAK,IAAI,EACT,KAAK,gBAAgB,EACrB,KAAK,cAAc,GACpB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,SAAS,EACT,KAAK,qBAAqB,EAC1B,KAAK,wBAAwB,EAC7B,KAAK,oBAAoB,EACzB,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,GAC5B,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,QAAQ,EACR,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EACzB,KAAK,mBAAmB,GACzB,MAAM,aAAa,CAAC"}

package/resources/index.js

@@ -15,20 +15,20 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.VectorIo = exports.VectorDBs = exports.Tools = exports.Toolgroups = exports.ToolRuntime = exports.Telemetry = exports.SyntheticDataGeneration = exports.Shields = exports.ScoringFunctions = exports.Scoring = exports.Safety = exports.Routes = exports.Providers = exports.PostTraining = exports.Models = exports.Inspect = exports.Inference = exports.EvalTasks = exports.Eval = exports.Datasets = exports.Datasetio = exports.BatchInference = exports.Agents = void 0;
+exports.VectorIo = exports.VectorDBs = exports.Tools = exports.Toolgroups = exports.ToolRuntime = exports.Telemetry = exports.SyntheticDataGeneration = exports.Shields = exports.ScoringFunctions = exports.Scoring = exports.Safety = exports.Routes = exports.Providers = exports.PostTraining = exports.Models = exports.Inspect = exports.Inference = exports.Eval = exports.Datasets = exports.Datasetio = exports.Benchmarks = exports.BatchInference = exports.Agents = void 0;
 __exportStar(require("./shared.js"), exports);
 var agents_1 = require("./agents/agents.js");
 Object.defineProperty(exports, "Agents", { enumerable: true, get: function () { return agents_1.Agents; } });
 var batch_inference_1 = require("./batch-inference.js");
 Object.defineProperty(exports, "BatchInference", { enumerable: true, get: function () { return batch_inference_1.BatchInference; } });
+var benchmarks_1 = require("./benchmarks.js");
+Object.defineProperty(exports, "Benchmarks", { enumerable: true, get: function () { return benchmarks_1.Benchmarks; } });
 var datasetio_1 = require("./datasetio.js");
 Object.defineProperty(exports, "Datasetio", { enumerable: true, get: function () { return datasetio_1.Datasetio; } });
 var datasets_1 = require("./datasets.js");
 Object.defineProperty(exports, "Datasets", { enumerable: true, get: function () { return datasets_1.Datasets; } });
 var eval_1 = require("./eval/eval.js");
 Object.defineProperty(exports, "Eval", { enumerable: true, get: function () { return eval_1.Eval; } });
-var eval_tasks_1 = require("./eval-tasks.js");
-Object.defineProperty(exports, "EvalTasks", { enumerable: true, get: function () { return eval_tasks_1.EvalTasks; } });
 var inference_1 = require("./inference.js");
 Object.defineProperty(exports, "Inference", { enumerable: true, get: function () { return inference_1.Inference; } });
 var inspect_1 = require("./inspect.js");

package/resources/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;;;;;;;;;;;;;;AAEtF,8CAAyB;AACzB,6CASyB;AARvB,gGAAA,MAAM,OAAA;AASR,wDAK2B;AAJzB,iHAAA,cAAc,OAAA;AAKhB,4CAKqB;AAJnB,sGAAA,SAAS,OAAA;AAKX,0CAMoB;AALlB,oGAAA,QAAQ,OAAA;AAMV,uCAQqB;AAPnB,4FAAA,IAAI,OAAA;AAQN,8CAMsB;AALpB,uGAAA,SAAS,OAAA;AAMX,4CAaqB;AAZnB,sGAAA,SAAS,OAAA;AAaX,wCAA0G;AAAjG,kGAAA,OAAO,OAAA;AAChB,sCAMkB;AALhB,gGAAA,MAAM,OAAA;AAMR,kEAOuC;AANrC,6GAAA,YAAY,OAAA;AAOd,4CAA+F;AAAtF,sGAAA,SAAS,OAAA;AAClB,sCAAmF;AAA1E,gGAAA,MAAM,OAAA;AACf,sCAAsF;AAA7E,gGAAA,MAAM,OAAA;AACf,wCAMmB;AALjB,kGAAA,OAAO,OAAA;AAMT,4DAO6B;AAN3B,qHAAA,gBAAgB,OAAA;AAOlB,wCAMmB;AALjB,kGAAA,OAAO,OAAA;AAMT,4EAIqC;AAHnC,oIAAA,uBAAuB,OAAA;AAIzB,4CAgBqB;AAfnB,sGAAA,SAAS,OAAA;AAgBX,+DAMqC;AALnC,2GAAA,WAAW,OAAA;AAMb,8CAMsB;AALpB,wGAAA,UAAU,OAAA;AAMZ,oCAMiB;AALf,8FAAA,KAAK,OAAA;AAMP,8CAOsB;AANpB,uGAAA,SAAS,OAAA;AAOX,4CAKqB;AAJnB,qGAAA,QAAQ,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;;;;;;;;;;;;;;AAEtF,8CAAyB;AACzB,6CASyB;AARvB,gGAAA,MAAM,OAAA;AASR,wDAK2B;AAJzB,iHAAA,cAAc,OAAA;AAKhB,8CAMsB;AALpB,wGAAA,UAAU,OAAA;AAMZ,4CAKqB;AAJnB,sGAAA,SAAS,OAAA;AAKX,0CAMoB;AALlB,oGAAA,QAAQ,OAAA;AAMV,uCAUqB;AATnB,4FAAA,IAAI,OAAA;AAUN,4CAaqB;AAZnB,sGAAA,SAAS,OAAA;AAaX,wCAA0G;AAAjG,kGAAA,OAAO,OAAA;AAChB,sCAMkB;AALhB,gGAAA,MAAM,OAAA;AAMR,kEAOuC;AANrC,6GAAA,YAAY,OAAA;AAOd,4CAA+F;AAAtF,sGAAA,SAAS,OAAA;AAClB,sCAAmF;AAA1E,gGAAA,MAAM,OAAA;AACf,sCAAsF;AAA7E,gGAAA,MAAM,OAAA;AACf,wCAMmB;AALjB,kGAAA,OAAO,OAAA;AAMT,4DAO6B;AAN3B,qHAAA,gBAAgB,OAAA;AAOlB,wCAMmB;AALjB,kGAAA,OAAO,OAAA;AAMT,4EAIqC;AAHnC,oIAAA,uBAAuB,OAAA;AAIzB,4CAgBqB;AAfnB,sGAAA,SAAS,OAAA;AAgBX,+DAMqC;AALnC,2GAAA,WAAW,OAAA;AAMb,8CAMsB;AALpB,wGAAA,UAAU,OAAA;AAMZ,oCAMiB;AALf,8FAAA,KAAK,OAAA;AAMP,8CAOsB;AANpB,uGAAA,SAAS,OAAA;AAOX,4CAKqB;AAJnB,qGAAA,QAAQ,OAAA"}

package/resources/index.mjs

@@ -2,10 +2,10 @@
 export * from "./shared.mjs";
 export { Agents, } from "./agents/agents.mjs";
 export { BatchInference, } from "./batch-inference.mjs";
+export { Benchmarks, } from "./benchmarks.mjs";
 export { Datasetio, } from "./datasetio.mjs";
 export { Datasets, } from "./datasets.mjs";
 export { Eval, } from "./eval/eval.mjs";
-export { EvalTasks, } from "./eval-tasks.mjs";
 export { Inference, } from "./inference.mjs";
 export { Inspect } from "./inspect.mjs";
 export { Models, } from "./models.mjs";

package/resources/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;;OAG/E,EACL,MAAM,GAQP;OACM,EACL,cAAc,GAIf;OACM,EACL,SAAS,GAIV;OACM,EACL,QAAQ,GAKT;OACM,EACL,IAAI,GAOL;OACM,EACL,SAAS,GAKV;OACM,EACL,SAAS,GAYV;OACM,EAAE,OAAO,EAAwE;OACjF,EACL,MAAM,GAKP;OACM,EACL,YAAY,GAMb;OACM,EAAE,SAAS,EAAyD;OACpE,EAAE,MAAM,EAAmD;OAC3D,EAAE,MAAM,EAAsD;OAC9D,EACL,OAAO,GAKR;OACM,EACL,gBAAgB,GAMjB;OACM,EACL,OAAO,GAKR;OACM,EACL,uBAAuB,GAGxB;OACM,EACL,SAAS,GAeV;OACM,EACL,WAAW,GAKZ;OACM,EACL,UAAU,GAKX;OACM,EACL,KAAK,GAKN;OACM,EACL,SAAS,GAMV;OACM,EACL,QAAQ,GAIT"}
+{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../src/resources/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;;OAG/E,EACL,MAAM,GAQP;OACM,EACL,cAAc,GAIf;OACM,EACL,UAAU,GAKX;OACM,EACL,SAAS,GAIV;OACM,EACL,QAAQ,GAKT;OACM,EACL,IAAI,GASL;OACM,EACL,SAAS,GAYV;OACM,EAAE,OAAO,EAAwE;OACjF,EACL,MAAM,GAKP;OACM,EACL,YAAY,GAMb;OACM,EAAE,SAAS,EAAyD;OACpE,EAAE,MAAM,EAAmD;OAC3D,EAAE,MAAM,EAAsD;OAC9D,EACL,OAAO,GAKR;OACM,EACL,gBAAgB,GAMjB;OACM,EACL,OAAO,GAKR;OACM,EACL,uBAAuB,GAGxB;OACM,EACL,SAAS,GAeV;OACM,EACL,WAAW,GAKZ;OACM,EACL,UAAU,GAKX;OACM,EACL,KAAK,GAKN;OACM,EACL,SAAS,GAMV;OACM,EACL,QAAQ,GAIT"}

package/resources/inference.d.ts

@@ -30,6 +30,7 @@ export interface ChatCompletionResponseStreamChunk {
      * The event containing the new content
      */
     event: ChatCompletionResponseStreamChunk.Event;
+    metrics?: Array<ChatCompletionResponseStreamChunk.Metric>;
 }
 export declare namespace ChatCompletionResponseStreamChunk {
     /**
@@ -54,6 +55,16 @@ export declare namespace ChatCompletionResponseStreamChunk {
          */
         stop_reason?: 'end_of_turn' | 'end_of_message' | 'out_of_tokens';
     }
+    interface Metric {
+        metric: string;
+        span_id: string;
+        timestamp: string;
+        trace_id: string;
+        type: 'metric';
+        unit: string;
+        value: number;
+        attributes?: Record<string, string | number | boolean | null>;
+    }
 }
 /**
  * Response from a completion request.
@@ -128,7 +139,7 @@ export interface InferenceChatCompletionParamsBase {
      * (Optional) Whether tool use is required or automatic. Defaults to
      * ToolChoice.auto. .. deprecated:: Use tool_config instead.
      */
-    tool_choice?: 'auto' | 'required';
+    tool_choice?: 'auto' | 'required' | 'none';
     /**
      * (Optional) Configuration for tool use.
      */
@@ -171,12 +182,12 @@ export declare namespace InferenceChatCompletionParams {
          * the string '{{function_definitions}}' to indicate where the function definitions
          * should be inserted.
          */
-        system_message_behavior: 'append' | 'replace';
+        system_message_behavior?: 'append' | 'replace';
         /**
-         * (Optional) Whether tool use is required or automatic. Defaults to
-         * ToolChoice.auto.
+         * (Optional) Whether tool use is automatic, required, or none. Can also specify a
+         * tool name to use a specific tool. Defaults to ToolChoice.auto.
          */
-        tool_choice?: 'auto' | 'required';
+        tool_choice?: 'auto' | 'required' | 'none' | (string & {});
         /**
          * (Optional) Instructs the model how to format tool calls. By default, Llama Stack
          * will attempt to use a format that is best adapted to the model. -
@@ -269,16 +280,31 @@ export interface InferenceCompletionParamsStreaming extends InferenceCompletionP
 }
 export interface InferenceEmbeddingsParams {
     /**
-     * List of contents to generate embeddings for. Note that content can be
-     * multimodal. The behavior depends on the model and provider. Some models may only
-     * support text.
+     * List of contents to generate embeddings for. Each content can be a string or an
+     * InterleavedContentItem (and hence can be multimodal). The behavior depends on
+     * the model and provider. Some models may only support text.
      */
-    contents: Array<Shared.InterleavedContent>;
+    contents: Array<string> | Array<Shared.InterleavedContentItem>;
     /**
      * The identifier of the model to use. The model must be an embedding model
      * registered with Llama Stack and available via the /models endpoint.
      */
     model_id: string;
+    /**
+     * (Optional) Output dimensionality for the embeddings. Only supported by
+     * Matryoshka models.
+     */
+    output_dimension?: number;
+    /**
+     * (Optional) How is the embedding being used? This is only supported by asymmetric
+     * embedding models.
+     */
+    task_type?: 'query' | 'document';
+    /**
+     * (Optional) Config for how to truncate text for embedding when text is longer
+     * than the model's max sequence length.
+     */
+    text_truncation?: 'none' | 'start' | 'end';
 }
 export declare namespace Inference {
     export { type ChatCompletionResponseStreamChunk as ChatCompletionResponseStreamChunk, type CompletionResponse as CompletionResponse, type EmbeddingsResponse as EmbeddingsResponse, type TokenLogProbs as TokenLogProbs, type InferenceChatCompletionParams as InferenceChatCompletionParams, type InferenceChatCompletionParamsNonStreaming as InferenceChatCompletionParamsNonStreaming, type InferenceChatCompletionParamsStreaming as InferenceChatCompletionParamsStreaming, type InferenceCompletionParams as InferenceCompletionParams, type InferenceCompletionParamsNonStreaming as InferenceCompletionParamsNonStreaming, type InferenceCompletionParamsStreaming as InferenceCompletionParamsStreaming, type InferenceEmbeddingsParams as InferenceEmbeddingsParams, };