ai-capability-framework 1.0.3 → 1.1.0

package/dist/evals-live/live-eval-runner.js

@@ -7,6 +7,8 @@ import { emitTraceEvent } from "../observability/index.js";
 export async function runLiveEvalSuite(options) {
     const results = [];
     const runner = resolveLiveEvalRunner(options);
+    const issuedRuns = new WeakSet();
+    const issuedRunIds = new Set();
     for (const testCase of options.cases) {
         const loadedEval = options.registry.evalById.get(testCase.evalId);
         if (!loadedEval) {
@@ -35,7 +37,15 @@ export async function runLiveEvalSuite(options) {
                 sink: options.traceSink,
                 type: "eval.score"
             });
+            const providerRun = options.providerRunFactory?.(testCase);
+            if (providerRun && (issuedRuns.has(providerRun) || issuedRunIds.has(providerRun.runId) || providerRun.runId !== testCase.runtimeContext.runId))
+                throw new Error("Live eval cases require independent provider-run invocations.");
+            if (providerRun) {
+                issuedRuns.add(providerRun);
+                issuedRunIds.add(providerRun.runId);
+            }
             const providerResult = await runner.runCase({
+                providerRun,
                 caseId: testCase.evalId,
                 contextBuilder: options.contextBuilderFactory(testCase),
                 executor: options.executor,
@@ -107,7 +117,7 @@ export function createOpenAILiveEvalRunner(options) {
         runtimeName: options.runtimeName ?? "openai-responses",
         async runCase(input) {
             const runResult = await runOpenAIResponses({
-                client: options.client,
+                client: options.client, providerRun: input.providerRun, providerExecution: options.providerExecution, signal: input.abortSignal,
                 contextBuilder: input.contextBuilder,
                 executor: input.executor,
                 maxToolCalls: options.maxToolCalls,
@@ -333,6 +343,7 @@ function normalizeOpenAIRunResult(runResult, runtimeName) {
         })),
         toolResults: runResult.toolResults,
         traceEvents: runResult.traceEvents,
+        providerUsage: runResult.providerUsage,
         usage: runResult.usage
     };
 }

package/dist/evals-live/types.d.ts

@@ -11,6 +11,7 @@ export interface AicfLiveEvalCaseInput {
     userInput: AicfRuntimeUserInput;
 }
 export interface AicfNormalizedLiveEvalRunResult {
+    providerUsage?: import("../runtime/provider-types.js").ProviderUsage;
     errors: Array<{
         code: string;
         message: string;
@@ -33,6 +34,7 @@ export interface AicfNormalizedLiveEvalRunResult {
     usage?: unknown;
 }
 export interface AicfLiveEvalCaseRunInput<TProviderConfig = unknown> {
+    providerRun?: import("../runtime/provider-types.js").AicfProviderRun;
     abortSignal?: AbortSignal;
     caseId: string;
     contextBuilder: AicfContextBuilder;
@@ -66,6 +68,7 @@ export interface AicfLiveEvalScorer {
     }): EvalScorerResult;
 }
 export interface AicfLiveEvalOptions {
+    providerRunFactory?: (testCase: AicfLiveEvalCaseInput) => import("../runtime/provider-types.js").AicfProviderRun;
     cases: AicfLiveEvalCaseInput[];
     contextBuilderFactory: (testCase: AicfLiveEvalCaseInput) => AicfContextBuilder;
     executor: AicfToolExecutor;

package/dist/evals-live/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evals-live/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,KAAK,EACV,oBAAoB,EACpB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,6BAA6B,EAC7B,oBAAoB,EACpB,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EACjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACnG,OAAO,KAAK,EAAE,qBAAqB,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAEtF,MAAM,MAAM,sBAAsB,GAC9B,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,QAAQ,GACR,WAAW,GACX,WAAW,GACX,KAAK,GACL,4BAA4B,GAC5B,MAAM,CAAC;AAEX,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;IAChC,mBAAmB,CAAC,EAAE,eAAe,EAAE,CAAC;IACxC,cAAc,EAAE,kBAAkB,CAAC;IACnC,SAAS,EAAE,oBAAoB,CAAC;CACjC;AAED,MAAM,WAAW,+BAA+B;IAC9C,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,sBAAsB,CAAC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,eAAe,CAAC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,KAAK,CAAC;QACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC,CAAC;IACH,WAAW,EAAE,6BAA6B,EAAE,CAAC;IAC7C,WAAW,EAAE,qBAAqB,EAAE,CAAC;IACrC,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB,CAAC,eAAe,GAAG,OAAO;IACjE,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,kBAAkB,CAAC;IACnC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,UAAU,EAAE,cAAc,CAAC;IAC3B,cAAc,CAAC,EAAE,eAAe,CAAC;IACjC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,MAAM,EAAE,oBAAoB,CAAC;IAC7B,cAAc,EAAE,kBAAkB,CAAC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,qBAAqB,CAAC;IAChC,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B,SAAS,EAAE,oBAAoB,CAAC;CACjC;AAED,MAAM,WAAW,yBAAyB;IACxC,UAAU,EAAE,sBAAsB,CAAC;IACnC,SAAS,EAAE,+BAA+B,CAAC;IAC3C,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB,CAAC,eAAe,GAAG,OAAO;IAC3D,QAAQ,CAAC,UAAU,EAAE,sBAAsB,CAAC;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,OAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;CAC/F;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,KAAK,EAAE;QACX,SAAS,EAAE,mBAAmB,CAAC;QAC/B,SAAS,EAAE,+BAA+B,CAAC;QAC3C,QAAQ,EAAE,qBAAqB,CAAC;KACjC,GAAG,gBAAgB,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,qBAAqB,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,kBAAkB,CAAC;IAC/E,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,MAAM,EAAE,oBAAoB,CAAC;IAC7B,OAAO,CAAC,EAAE,kBAAkB,EAAE,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,aAAa,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,SAAS,CAAC,EAAE,+BAA+B,CAAC;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,KAAK,CAAC;QACZ,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,OAAO,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;CACzC;AAED,MAAM,WAAW,YAAY;IAC3B,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAC7B;AAED,MAAM,WAAW,4BAA4B;IAC3C,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,eAAe,GAAG,eAAe,GAAG,oBAAoB,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC1F,SAAS,EAAE,+BAA+B,CAAC;CAC5C"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evals-live/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,KAAK,EACV,oBAAoB,EACpB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,6BAA6B,EAC7B,oBAAoB,EACpB,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EACjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACnG,OAAO,KAAK,EAAE,qBAAqB,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAEtF,MAAM,MAAM,sBAAsB,GAC9B,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,QAAQ,GACR,WAAW,GACX,WAAW,GACX,KAAK,GACL,4BAA4B,GAC5B,MAAM,CAAC;AAEX,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;IAChC,mBAAmB,CAAC,EAAE,eAAe,EAAE,CAAC;IACxC,cAAc,EAAE,kBAAkB,CAAC;IACnC,SAAS,EAAE,oBAAoB,CAAC;CACjC;AAED,MAAM,WAAW,+BAA+B;IAC9C,aAAa,CAAC,EAAE,OAAO,8BAA8B,EAAE,aAAa,CAAC;IACrE,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,sBAAsB,CAAC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,eAAe,CAAC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,KAAK,CAAC;QACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC,CAAC;IACH,WAAW,EAAE,6BAA6B,EAAE,CAAC;IAC7C,WAAW,EAAE,qBAAqB,EAAE,CAAC;IACrC,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB,CAAC,eAAe,GAAG,OAAO;IACjE,WAAW,CAAC,EAAE,OAAO,8BAA8B,EAAE,eAAe,CAAC;IACrE,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,kBAAkB,CAAC;IACnC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,UAAU,EAAE,cAAc,CAAC;IAC3B,cAAc,CAAC,EAAE,eAAe,CAAC;IACjC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,MAAM,EAAE,oBAAoB,CAAC;IAC7B,cAAc,EAAE,kBAAkB,CAAC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,qBAAqB,CAAC;IAChC,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B,SAAS,EAAE,oBAAoB,CAAC;CACjC;AAED,MAAM,WAAW,yBAAyB;IACxC,UAAU,EAAE,sBAAsB,CAAC;IACnC,SAAS,EAAE,+BAA+B,CAAC;IAC3C,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB,CAAC,eAAe,GAAG,OAAO;IAC3D,QAAQ,CAAC,UAAU,EAAE,sBAAsB,CAAC;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,OAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;CAC/F;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,KAAK,EAAE;QACX,SAAS,EAAE,mBAAmB,CAAC;QAC/B,SAAS,EAAE,+BAA+B,CAAC;QAC3C,QAAQ,EAAE,qBAAqB,CAAC;KACjC,GAAG,gBAAgB,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,kBAAkB,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,OAAO,8BAA8B,EAAE,eAAe,CAAC;IACjH,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,qBAAqB,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,kBAAkB,CAAC;IAC/E,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,MAAM,EAAE,oBAAoB,CAAC;IAC7B,OAAO,CAAC,EAAE,kBAAkB,EAAE,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,aAAa,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,SAAS,CAAC,EAAE,+BAA+B,CAAC;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,KAAK,CAAC;QACZ,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,OAAO,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;CACzC;AAED,MAAM,WAAW,YAAY;IAC3B,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAC7B;AAED,MAAM,WAAW,4BAA4B;IAC3C,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,eAAe,GAAG,eAAe,GAAG,oBAAoB,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC1F,SAAS,EAAE,+BAA+B,CAAC;CAC5C"}