ai-capability-framework 1.0.0-rc.3 → 1.0.0

package/dist/cli.js

@@ -16,13 +16,15 @@ import { buildLangChainToolDescriptors } from "./langchain.js";
 import { loadManifests } from "./loader.js";
 import { buildMcpToolDescriptors } from "./mcp.js";
 import { createDefaultOpenAIResponsesClient } from "./openai/index.js";
+import { createDefaultAnthropicMessagesClient } from "./providers/anthropic/index.js";
+import { createDefaultGeminiClient } from "./providers/gemini/index.js";
 import { buildOpenAIResponsesTools } from "./openai-responses.js";
 import { exportPromptfooSuite } from "./promptfoo/index.js";
 import { buildRegistry, formatInspection, inspectRegistry } from "./registry.js";
 import { createGoldenFromTrace, runReplay, validateReplayTrace } from "./replay/index.js";
 import { exportPromptfooSecurityPackSuite, generateSecurityCases, getSecurityPack, listSecurityPacks } from "./security-packs/index.js";
 import { AicfActionLifecycleManager, AicfHandlerRegistry, AicfToolExecutor, DefaultCapabilityRouter, DefaultContextBuilder, DefaultPolicyBroker, InMemoryApprovalStore, InMemoryAuditSink, InMemoryIdempotencyStore, InMemoryPreparedActionStore } from "./runtime/index.js";
-import { evaluateGate, runLiveEvalSuite } from "./evals-live/index.js";
+import { createAnthropicLiveEvalRunner, createGeminiLiveEvalRunner, createOpenAILiveEvalRunner, evaluateGate, runLiveEvalSuite } from "./evals-live/index.js";
 import { buildSemanticKernelFunctions } from "./semantic-kernel.js";
 import { validateManifests, validatePublicFixtures } from "./validator.js";
 const adapterCommands = new Set([
@@ -207,12 +209,21 @@ export async function runCli(argv, options = {}) {
             stderr.write("Missing required --cases <cases.json>.\n");
             return 1;
         }
+        if (!parsedArgs.providerName) {
+            stderr.write("Missing required --provider <openai|anthropic|gemini|ai-sdk> for live evals.\n");
+            return 1;
+        }
         if (!parsedArgs.model) {
             stderr.write("Missing required --model <model> for live evals.\n");
             return 1;
         }
-        if (!process.env.OPENAI_API_KEY) {
-            stderr.write("OPENAI_API_KEY is required for eval-live. Use the TypeScript API with a mock client for API-key-free tests.\n");
+        if (process.env.RUN_REAL_AICF_LIVE_EVALS !== "1") {
+            stderr.write("RUN_REAL_AICF_LIVE_EVALS=1 is required for CLI live evals. Use the TypeScript API with a mock runner for API-key-free tests.\n");
+            return 1;
+        }
+        const runner = await createCliLiveEvalRunner(parsedArgs.providerName, parsedArgs.model);
+        if (!runner.ok) {
+            stderr.write(`${runner.error}\n`);
             return 1;
         }
         const cases = await readLiveEvalCases(parsedArgs.casesPath);
@@ -243,9 +254,8 @@ export async function runCli(argv, options = {}) {
             cases: cases.value,
             contextBuilderFactory: () => new DefaultContextBuilder(),
             executor,
-            model: parsedArgs.model,
-            openAIClient: await createDefaultOpenAIResponsesClient(),
             registry,
+            runner: runner.value,
             router: new DefaultCapabilityRouter()
         });
         const gate = evaluateGate(results);
@@ -266,7 +276,7 @@ function helpText() {
         "Commands:",
         "  decide <path> --request <file>  Evaluate a decision request.",
         "  eval <path> --results <file> [--format text|json]  Run deterministic evals.",
-        "  eval-live <path> --cases <file> --model <model> [--format text|json]  Run opt-in live evals.",
+        "  eval-live <path> --cases <file> --provider <provider> --model <model> [--format text|json]  Run opt-in live evals.",
         "  export promptfoo <path> --out <dir> [--provider <provider>]  Generate Promptfoo suite files.",
         "  governance risk <path> [--capability <id>] [--format text|json]  Compile governance risk.",
         "  governance lifecycle <path> --capability <id> --to <status> --reason <text>  Evaluate lifecycle transition.",
@@ -2042,6 +2052,65 @@ function adapterLabel(command) {
             return "Semantic Kernel";
     }
 }
+async function createCliLiveEvalRunner(providerName, model) {
+    const provider = normalizeLiveEvalProvider(providerName);
+    if (!provider) {
+        return {
+            error: "Unsupported live eval provider. Use one of: openai, anthropic, gemini, ai-sdk.",
+            ok: false
+        };
+    }
+    if (provider === "openai") {
+        if (!process.env.OPENAI_API_KEY) {
+            return { error: "OPENAI_API_KEY is required for OpenAI CLI live evals.", ok: false };
+        }
+        return {
+            ok: true,
+            value: createOpenAILiveEvalRunner({
+                client: await createDefaultOpenAIResponsesClient(),
+                model
+            })
+        };
+    }
+    if (provider === "anthropic") {
+        if (!process.env.ANTHROPIC_API_KEY) {
+            return { error: "ANTHROPIC_API_KEY is required for Anthropic CLI live evals.", ok: false };
+        }
+        return {
+            ok: true,
+            value: createAnthropicLiveEvalRunner({
+                client: await createDefaultAnthropicMessagesClient(),
+                model
+            })
+        };
+    }
+    if (provider === "gemini") {
+        if (!process.env.GEMINI_API_KEY) {
+            return { error: "GEMINI_API_KEY is required for Gemini CLI live evals.", ok: false };
+        }
+        return {
+            ok: true,
+            value: createGeminiLiveEvalRunner({
+                client: await createDefaultGeminiClient(),
+                model
+            })
+        };
+    }
+    return {
+        error: "AI SDK CLI live evals require a host-supplied generateText function. Use createAiSdkLiveEvalRunner() from the TypeScript API.",
+        ok: false
+    };
+}
+function normalizeLiveEvalProvider(value) {
+    const normalized = value.toLowerCase().replace(/_/g, "-");
+    if (normalized === "openai" || normalized === "anthropic" || normalized === "gemini" || normalized === "ai-sdk") {
+        return normalized;
+    }
+    if (normalized === "vercel-ai-sdk") {
+        return "ai-sdk";
+    }
+    return undefined;
+}
 function formatDiagnostics(errors) {
     return errors
         .map((error) => {
@@ -2057,7 +2126,8 @@ function formatLiveEvalResults(results, gate) {
         `Live eval gate ${gate.status}: average score ${gate.averageScore.toFixed(3)}.`
     ];
     for (const result of results) {
-        lines.push(`- ${result.status}: ${result.evalId}`);
+        const provider = result.providerId ? ` (${result.providerId}${result.runtimeName ? `/${result.runtimeName}` : ""})` : "";
+        lines.push(`- ${result.status}: ${result.evalId}${provider}`);
     }
     return `${lines.join("\n")}\n`;
 }

package/dist/evals-live/index.d.ts

@@ -1,3 +1,3 @@
-export { createEvalCandidateFromRunResult, createEvalCaseFromTrace, evaluateGate, noRawInternalDetailsScorer, runLiveEvalSuite } from "./live-eval-runner.js";
-export type { AicfEvalGate, AicfEvalGateResult, AicfLiveEvalCaseInput, AicfLiveEvalOptions, AicfLiveEvalResult, AicfLiveEvalScorer, CreateEvalCaseFromTraceInput } from "./types.js";
+export { createEvalCandidateFromRunResult, createEvalCaseFromTrace, createAiSdkLiveEvalRunner, createAnthropicLiveEvalRunner, createGeminiLiveEvalRunner, createMockLiveEvalRunner, createOpenAILiveEvalRunner, evaluateGate, noRawInternalDetailsScorer, runOpenAILiveEvalSuite, runLiveEvalSuite } from "./live-eval-runner.js";
+export type { AicfEvalGate, AicfEvalGateResult, AicfLiveEvalCaseInput, AicfLiveEvalCaseRunInput, AicfLiveEvalCaseRunResult, AicfLiveEvalOptions, AicfLiveEvalProviderId, AicfLiveEvalResult, AicfLiveEvalRunner, AicfLiveEvalScorer, AicfNormalizedLiveEvalRunResult, CreateEvalCaseFromTraceInput } from "./types.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/evals-live/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evals-live/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gCAAgC,EAChC,uBAAuB,EACvB,YAAY,EACZ,0BAA0B,EAC1B,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,YAAY,EACZ,kBAAkB,EAClB,qBAAqB,EACrB,mBAAmB,EACnB,kBAAkB,EAClB,kBAAkB,EAClB,4BAA4B,EAC7B,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evals-live/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gCAAgC,EAChC,uBAAuB,EACvB,yBAAyB,EACzB,6BAA6B,EAC7B,0BAA0B,EAC1B,wBAAwB,EACxB,0BAA0B,EAC1B,YAAY,EACZ,0BAA0B,EAC1B,sBAAsB,EACtB,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,YAAY,EACZ,kBAAkB,EAClB,qBAAqB,EACrB,wBAAwB,EACxB,yBAAyB,EACzB,mBAAmB,EACnB,sBAAsB,EACtB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,+BAA+B,EAC/B,4BAA4B,EAC7B,MAAM,YAAY,CAAC"}

package/dist/evals-live/index.js

@@ -1 +1 @@
-export { createEvalCandidateFromRunResult, createEvalCaseFromTrace, evaluateGate, noRawInternalDetailsScorer, runLiveEvalSuite } from "./live-eval-runner.js";
+export { createEvalCandidateFromRunResult, createEvalCaseFromTrace, createAiSdkLiveEvalRunner, createAnthropicLiveEvalRunner, createGeminiLiveEvalRunner, createMockLiveEvalRunner, createOpenAILiveEvalRunner, evaluateGate, noRawInternalDetailsScorer, runOpenAILiveEvalSuite, runLiveEvalSuite } from "./live-eval-runner.js";

package/dist/evals-live/live-eval-runner.d.ts

@@ -1,6 +1,54 @@
+import { type AicfOpenAIResponsesClient } from "../openai/index.js";
+import { type AicfAnthropicMessagesClient } from "../providers/anthropic/index.js";
+import { type AicfGeminiClient } from "../providers/gemini/index.js";
+import { type AicfAiSdkGenerateTextLike, type AicfAiSdkToolFactories } from "../providers/ai-sdk/index.js";
 import type { EvalCandidateResult, EvalCase, EvalScorerResult } from "../types.js";
-import type { AicfEvalGate, AicfEvalGateResult, AicfLiveEvalOptions, AicfLiveEvalResult, CreateEvalCaseFromTraceInput } from "./types.js";
+import type { AicfEvalGate, AicfEvalGateResult, AicfLiveEvalCaseInput, AicfLiveEvalOptions, AicfLiveEvalProviderId, AicfLiveEvalResult, AicfLiveEvalRunner, AicfNormalizedLiveEvalRunResult, CreateEvalCaseFromTraceInput } from "./types.js";
 export declare function runLiveEvalSuite(options: AicfLiveEvalOptions): Promise<AicfLiveEvalResult[]>;
+export declare function runOpenAILiveEvalSuite(options: Omit<AicfLiveEvalOptions, "runner"> & {
+    model: string;
+    openAIClient: AicfOpenAIResponsesClient;
+}): Promise<AicfLiveEvalResult[]>;
+export declare function createOpenAILiveEvalRunner(options: {
+    client: AicfOpenAIResponsesClient;
+    maxToolCalls?: number;
+    maxTurns?: number;
+    model: string;
+    runtimeName?: string;
+    systemInstructions?: string;
+    temperature?: number;
+}): AicfLiveEvalRunner;
+export declare function createAnthropicLiveEvalRunner(options: {
+    client: AicfAnthropicMessagesClient;
+    maxTokens?: number;
+    maxToolCalls?: number;
+    maxToolIterations?: number;
+    model: string;
+    runtimeName?: string;
+    strictTools?: boolean;
+    system?: string;
+}): AicfLiveEvalRunner;
+export declare function createGeminiLiveEvalRunner(options: {
+    client: AicfGeminiClient;
+    maxToolCalls?: number;
+    maxToolIterations?: number;
+    model: string;
+    runtimeName?: string;
+    systemInstruction?: string;
+}): AicfLiveEvalRunner;
+export declare function createAiSdkLiveEvalRunner(options: {
+    generateText: AicfAiSdkGenerateTextLike;
+    maxSteps?: number;
+    model: unknown;
+    runtimeName?: string;
+    system?: string;
+    toolFactories?: AicfAiSdkToolFactories;
+}): AicfLiveEvalRunner;
+export declare function createMockLiveEvalRunner(input: {
+    providerId?: AicfLiveEvalProviderId;
+    result: AicfNormalizedLiveEvalRunResult | ((testCase: AicfLiveEvalCaseInput) => AicfNormalizedLiveEvalRunResult);
+    runtimeName?: string;
+}): AicfLiveEvalRunner;
 export declare function evaluateGate(results: AicfLiveEvalResult[], gate?: AicfEvalGate): AicfEvalGateResult;
 export declare function createEvalCandidateFromRunResult(evalId: string, runResult: {
     finalText: string;

package/dist/evals-live/live-eval-runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"live-eval-runner.d.ts","sourceRoot":"","sources":["../../src/evals-live/live-eval-runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,mBAAmB,EACnB,QAAQ,EACR,gBAAgB,EAEjB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACV,YAAY,EACZ,kBAAkB,EAElB,mBAAmB,EACnB,kBAAkB,EAClB,4BAA4B,EAC7B,MAAM,YAAY,CAAC;AAEpB,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,mBAAmB,GAC3B,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAiF/B;AAED,wBAAgB,YAAY,CAC1B,OAAO,EAAE,kBAAkB,EAAE,EAC7B,IAAI,GAAE,YAAiB,GACtB,kBAAkB,CAsCpB;AAED,wBAAgB,gCAAgC,CAC9C,MAAM,EAAE,MAAM,EACd,SAAS,EAAE;IACT,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE;QAAE,KAAK,EAAE,KAAK,CAAC;YAAE,YAAY,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IACjE,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3E,WAAW,EAAE,KAAK,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE;YAAE,MAAM,EAAE,SAAS,GAAG,mBAAmB,GAAG,QAAQ,CAAA;SAAE,CAAC;QAChE,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ,GACA,mBAAmB,CAoBrB;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,4BAA4B,GAAG,QAAQ,CAsBrF;AAED,wBAAgB,0BAA0B,IAAI;IAC5C,IAAI,EAAE,yBAAyB,CAAC;IAChC,KAAK,CAAC,KAAK,EAAE;QAAE,SAAS,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,OAAO,EAAE,CAAA;SAAE,CAAA;KAAE,GAAG,gBAAgB,CAAC;CAC9F,CAoBA"}
+{"version":3,"file":"live-eval-runner.d.ts","sourceRoot":"","sources":["../../src/evals-live/live-eval-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,KAAK,yBAAyB,EAA4B,MAAM,oBAAoB,CAAC;AAClH,OAAO,EAAwB,KAAK,2BAA2B,EAA+B,MAAM,iCAAiC,CAAC;AACtI,OAAO,EAA4B,KAAK,gBAAgB,EAA4B,MAAM,8BAA8B,CAAC;AACzH,OAAO,EAAuD,KAAK,yBAAyB,EAA2B,KAAK,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAIzL,OAAO,KAAK,EACV,mBAAmB,EACnB,QAAQ,EACR,gBAAgB,EAEjB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACV,YAAY,EACZ,kBAAkB,EAClB,qBAAqB,EAErB,mBAAmB,EACnB,sBAAsB,EACtB,kBAAkB,EAClB,kBAAkB,EAClB,+BAA+B,EAC/B,4BAA4B,EAC7B,MAAM,YAAY,CAAC;AAEpB,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,mBAAmB,GAC3B,OAAO,CAAC,kBAAkB,EAAE,CAAC,CA0F/B;AAED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,IAAI,CAAC,mBAAmB,EAAE,QAAQ,CAAC,GAAG;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,yBAAyB,CAAC;CACzC,GACA,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAQ/B;AAED,wBAAgB,0BAA0B,CAAC,OAAO,EAAE;IAClD,MAAM,EAAE,yBAAyB,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,kBAAkB,CA2BrB;AAED,wBAAgB,6BAA6B,CAAC,OAAO,EAAE;IACrD,MAAM,EAAE,2BAA2B,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,GAAG,kBAAkB,CA6BrB;AAED,wBAAgB,0BAA0B,CAAC,OAAO,EAAE;IAClD,MAAM,EAAE,gBAAgB,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,kBAAkB,CA2BrB;AAED,wBAAgB,yBAAyB,CAAC,OAAO,EAAE;IACjD,YAAY,EAAE,yBAAyB,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,OAAO,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,sBAAsB,CAAC;CACxC,GAAG,kBAAkB,CA2BrB;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAC9C,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,MAAM,EAAE,+BAA+B,GAAG,CAAC,CAAC,QAAQ,EAAE,qBAAqB,KAAK,+BAA+B,CAAC,CAAC;IACjH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,kBAAkB,CAmBrB;AAED,wBAAgB,YAAY,CAC1B,OAAO,EAAE,kBAAkB,EAAE,EAC7B,IAAI,GAAE,YAAiB,GACtB,kBAAkB,CAsCpB;AAED,wBAAgB,gCAAgC,CAC9C,MAAM,EAAE,MAAM,EACd,SAAS,EAAE;IACT,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE;QAAE,KAAK,EAAE,KAAK,CAAC;YAAE,YAAY,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IACjE,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3E,WAAW,EAAE,KAAK,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE;YAAE,MAAM,EAAE,SAAS,GAAG,mBAAmB,GAAG,QAAQ,CAAA;SAAE,CAAC;QAChE,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ,GACA,mBAAmB,CAoBrB;AAgHD,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,4BAA4B,GAAG,QAAQ,CAsBrF;AAED,wBAAgB,0BAA0B,IAAI;IAC5C,IAAI,EAAE,yBAAyB,CAAC;IAChC,KAAK,CAAC,KAAK,EAAE;QAAE,SAAS,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,OAAO,EAAE,CAAA;SAAE,CAAA;KAAE,GAAG,gBAAgB,CAAC;CAC9F,CAoBA"}

package/dist/evals-live/live-eval-runner.js

@@ -1,8 +1,12 @@
 import { runOpenAIResponses } from "../openai/index.js";
+import { runAnthropicMessages } from "../providers/anthropic/index.js";
+import { runGeminiGenerateContent } from "../providers/gemini/index.js";
+import { createPlainAiSdkToolFactories, runAiSdkGenerateText } from "../providers/ai-sdk/index.js";
 import { scoreEvalCase } from "../eval-runner.js";
 import { emitTraceEvent } from "../observability/index.js";
 export async function runLiveEvalSuite(options) {
     const results = [];
+    const runner = resolveLiveEvalRunner(options);
     for (const testCase of options.cases) {
         const loadedEval = options.registry.evalById.get(testCase.evalId);
         if (!loadedEval) {
@@ -21,7 +25,9 @@ export async function runLiveEvalSuite(options) {
         try {
             await emitTraceEvent({
                 attributes: {
-                    evalId: testCase.evalId
+                    evalId: testCase.evalId,
+                    provider: runner.providerId,
+                    runtimeName: runner.runtimeName
                 },
                 events: undefined,
                 requestId: testCase.runtimeContext.requestId,
@@ -29,17 +35,21 @@ export async function runLiveEvalSuite(options) {
                 sink: options.traceSink,
                 type: "eval.score"
             });
-            const runResult = await runOpenAIResponses({
-                client: options.openAIClient,
+            const providerResult = await runner.runCase({
+                caseId: testCase.evalId,
                 contextBuilder: options.contextBuilderFactory(testCase),
                 executor: options.executor,
-                model: options.model,
+                loadedEval,
+                providerConfig: options.providerConfig,
                 registry: options.registry,
                 router: options.router,
                 runtimeContext: testCase.runtimeContext,
+                suiteId: options.suiteId ?? "live_eval_suite",
+                testCase,
                 traceSink: options.traceSink,
                 userInput: testCase.userInput
             });
+            const runResult = providerResult.runResult;
             const candidate = createEvalCandidateFromRunResult(testCase.evalId, runResult);
             const evalForScoring = testCase.expected
                 ? loadedEvalWithExpected(loadedEval, testCase.expected)
@@ -60,7 +70,9 @@ export async function runLiveEvalSuite(options) {
             results.push({
                 candidate,
                 evalId: testCase.evalId,
+                providerId: providerResult.providerId,
                 runResult,
+                runtimeName: providerResult.runtimeName,
                 scores,
                 status: passed ? "passed" : "failed"
             });
@@ -80,6 +92,149 @@ export async function runLiveEvalSuite(options) {
     }
     return results;
 }
+export function runOpenAILiveEvalSuite(options) {
+    return runLiveEvalSuite({
+        ...options,
+        runner: createOpenAILiveEvalRunner({
+            client: options.openAIClient,
+            model: options.model
+        })
+    });
+}
+export function createOpenAILiveEvalRunner(options) {
+    return {
+        providerId: "openai",
+        runtimeName: options.runtimeName ?? "openai-responses",
+        async runCase(input) {
+            const runResult = await runOpenAIResponses({
+                client: options.client,
+                contextBuilder: input.contextBuilder,
+                executor: input.executor,
+                maxToolCalls: options.maxToolCalls,
+                maxTurns: options.maxTurns,
+                model: options.model,
+                registry: input.registry,
+                router: input.router,
+                runtimeContext: input.runtimeContext,
+                systemInstructions: options.systemInstructions,
+                temperature: options.temperature,
+                traceSink: input.traceSink,
+                userInput: input.userInput
+            });
+            return {
+                providerId: "openai",
+                runtimeName: options.runtimeName ?? "openai-responses",
+                runResult: normalizeOpenAIRunResult(runResult, options.runtimeName ?? "openai-responses")
+            };
+        }
+    };
+}
+export function createAnthropicLiveEvalRunner(options) {
+    return {
+        providerId: "anthropic",
+        runtimeName: options.runtimeName ?? "anthropic-messages",
+        async runCase(input) {
+            const { builtContext, slice } = await buildLiveEvalContext(input);
+            const runResult = await runAnthropicMessages({
+                builtContext,
+                client: options.client,
+                executor: input.executor,
+                maxTokens: options.maxTokens,
+                maxToolCalls: options.maxToolCalls,
+                maxToolIterations: options.maxToolIterations,
+                messages: [{ content: input.userInput.text, role: "user" }],
+                model: options.model,
+                registry: input.registry,
+                runtimeContext: input.runtimeContext,
+                slice,
+                strictTools: options.strictTools,
+                system: options.system,
+                traceSink: input.traceSink
+            });
+            return {
+                providerId: "anthropic",
+                runtimeName: options.runtimeName ?? "anthropic-messages",
+                runResult: normalizeProviderRuntimeResult("anthropic", options.runtimeName ?? "anthropic-messages", runResult, slice)
+            };
+        }
+    };
+}
+export function createGeminiLiveEvalRunner(options) {
+    return {
+        providerId: "gemini",
+        runtimeName: options.runtimeName ?? "gemini-generate-content",
+        async runCase(input) {
+            const { builtContext, slice } = await buildLiveEvalContext(input);
+            const runResult = await runGeminiGenerateContent({
+                builtContext,
+                client: options.client,
+                contents: input.userInput.text,
+                executor: input.executor,
+                maxToolCalls: options.maxToolCalls,
+                maxToolIterations: options.maxToolIterations,
+                model: options.model,
+                registry: input.registry,
+                runtimeContext: input.runtimeContext,
+                slice,
+                systemInstruction: options.systemInstruction,
+                traceSink: input.traceSink
+            });
+            return {
+                providerId: "gemini",
+                runtimeName: options.runtimeName ?? "gemini-generate-content",
+                runResult: normalizeProviderRuntimeResult("gemini", options.runtimeName ?? "gemini-generate-content", runResult, slice)
+            };
+        }
+    };
+}
+export function createAiSdkLiveEvalRunner(options) {
+    return {
+        providerId: "ai-sdk",
+        runtimeName: options.runtimeName ?? "ai-sdk-generate-text",
+        async runCase(input) {
+            const { builtContext, slice } = await buildLiveEvalContext(input);
+            const runResult = await runAiSdkGenerateText({
+                builtContext,
+                executor: input.executor,
+                generateText: options.generateText,
+                maxSteps: options.maxSteps,
+                model: options.model,
+                prompt: input.userInput.text,
+                registry: input.registry,
+                runtimeContext: input.runtimeContext,
+                slice,
+                system: options.system,
+                toolFactories: options.toolFactories ?? createPlainAiSdkToolFactories(),
+                traceSink: input.traceSink
+            });
+            return {
+                providerId: "ai-sdk",
+                runtimeName: options.runtimeName ?? "ai-sdk-generate-text",
+                runResult: normalizeAiSdkRunResult(runResult, options.runtimeName ?? "ai-sdk-generate-text", slice)
+            };
+        }
+    };
+}
+export function createMockLiveEvalRunner(input) {
+    const providerId = input.providerId ?? "mock";
+    const runtimeName = input.runtimeName ?? "mock-live-eval";
+    return {
+        providerId,
+        runtimeName,
+        async runCase(runInput) {
+            const result = typeof input.result === "function" ? input.result(runInput.testCase) : input.result;
+            return {
+                providerId,
+                runtimeName,
+                runResult: {
+                    ...result,
+                    providerId,
+                    runtimeName
+                }
+            };
+        }
+    };
+}
 export function evaluateGate(results, gate = {}) {
     const minAverageScore = gate.minAverageScore ?? 0.9;
     const forbiddenFailedScorers = gate.forbiddenFailedScorers ?? [
@@ -135,6 +290,93 @@ export function createEvalCandidateFromRunResult(evalId, runResult) {
         }))
     };
 }
+function resolveLiveEvalRunner(options) {
+    if (options.runner) {
+        return options.runner;
+    }
+    if (options.openAIClient && options.model) {
+        return createOpenAILiveEvalRunner({
+            client: options.openAIClient,
+            model: options.model
+        });
+    }
+    throw new Error("A live eval runner is required. Pass runner or legacy openAIClient plus model.");
+}
+async function buildLiveEvalContext(input) {
+    const builtContext = await input.contextBuilder.build({
+        baseContext: input.runtimeContext,
+        registry: input.registry,
+        userInput: input.userInput
+    });
+    const slice = await input.router.route({
+        builtContext,
+        registry: input.registry,
+        userInput: input.userInput
+    });
+    return { builtContext, slice };
+}
+function normalizeOpenAIRunResult(runResult, runtimeName) {
+    return {
+        errors: runResult.errors,
+        finalText: runResult.finalText,
+        providerId: "openai",
+        responseId: runResult.responseId,
+        runId: runResult.runId,
+        runtimeName,
+        selectedCapabilities: runResult.selectedCapabilities,
+        status: runResult.status,
+        toolCalls: runResult.toolCalls.map((call) => ({
+            args: call.args,
+            callId: call.callId,
+            capabilityId: call.capabilityId,
+            toolName: call.toolName
+        })),
+        toolResults: runResult.toolResults,
+        traceEvents: runResult.traceEvents,
+        usage: runResult.usage
+    };
+}
+function normalizeProviderRuntimeResult(providerId, runtimeName, runResult, selectedCapabilities) {
+    return {
+        errors: runResult.errors,
+        finalText: runResult.finalText,
+        providerId,
+        responseId: runResult.responseId,
+        runId: runResult.traceEvents[0]?.runId ?? runResult.toolResults[0]?.runId ?? "unknown",
+        runtimeName,
+        selectedCapabilities,
+        status: runResult.status,
+        toolCalls: runResult.toolCalls.map((call) => ({
+            args: call.args,
+            callId: call.callId,
+            capabilityId: call.capabilityId,
+            toolName: call.providerToolName
+        })),
+        toolResults: runResult.toolResults,
+        traceEvents: runResult.traceEvents,
+        usage: runResult.usage
+    };
+}
+function normalizeAiSdkRunResult(runResult, runtimeName, selectedCapabilities) {
+    return {
+        errors: runResult.errors,
+        finalText: runResult.text ?? "",
+        providerId: "ai-sdk",
+        runId: runResult.traceEvents[0]?.runId ?? runResult.toolResults[0]?.runId ?? "unknown",
+        runtimeName,
+        selectedCapabilities,
+        status: runResult.status,
+        toolCalls: runResult.toolCalls.map((call) => ({
+            args: {},
+            callId: call.toolCallId,
+            capabilityId: call.capabilityId ?? "unknown",
+            toolName: call.toolName
+        })),
+        toolResults: runResult.toolResults,
+        traceEvents: runResult.traceEvents,
+        usage: runResult.usage
+    };
+}
 export function createEvalCaseFromTrace(input) {
     const runResult = input.runResult;
     return {

package/dist/evals-live/types.d.ts

@@ -1,7 +1,8 @@
-import type { AicfOpenAIResponsesClient, AicfOpenAIRunResult } from "../openai/index.js";
-import type { AicfCapabilityRouter, AicfContextBuilder, AicfContextItem, AicfRuntimeContext, AicfRuntimeUserInput, AicfToolExecutor, ManifestRegistry } from "../runtime/index.js";
-import type { EvalCandidateResult, EvalCase, EvalScorerResult } from "../types.js";
-import type { AicfTraceSink } from "../observability/index.js";
+import type { AicfOpenAIResponsesClient } from "../openai/index.js";
+import type { AicfCapabilityRouter, AicfContextBuilder, AicfContextItem, AicfRuntimeContext, AicfRuntimeToolResultEnvelope, AicfRuntimeUserInput, AicfToolExecutor, CapabilitySlice, ManifestRegistry } from "../runtime/index.js";
+import type { EvalCandidateResult, EvalCase, EvalScorerResult, LoadedEvalCase } from "../types.js";
+import type { AicfRuntimeTraceEvent, AicfTraceSink } from "../observability/index.js";
+export type AicfLiveEvalProviderId = "openai" | "anthropic" | "gemini" | "ai-sdk" | "langchain" | "langgraph" | "mcp" | "semantic-kernel-compatible" | string;
 export interface AicfLiveEvalCaseInput {
     evalId: string;
     expected?: EvalCase["expected"];
@@ -9,11 +10,58 @@ export interface AicfLiveEvalCaseInput {
     runtimeContext: AicfRuntimeContext;
     userInput: AicfRuntimeUserInput;
 }
+export interface AicfNormalizedLiveEvalRunResult {
+    errors: Array<{
+        code: string;
+        message: string;
+    }>;
+    finalText: string;
+    providerId: AicfLiveEvalProviderId;
+    responseId?: string;
+    runId: string;
+    runtimeName: string;
+    selectedCapabilities: CapabilitySlice;
+    status: string;
+    toolCalls: Array<{
+        args: Record<string, unknown>;
+        callId?: string;
+        capabilityId: string;
+        toolName?: string;
+    }>;
+    toolResults: AicfRuntimeToolResultEnvelope[];
+    traceEvents: AicfRuntimeTraceEvent[];
+    usage?: unknown;
+}
+export interface AicfLiveEvalCaseRunInput<TProviderConfig = unknown> {
+    abortSignal?: AbortSignal;
+    caseId: string;
+    contextBuilder: AicfContextBuilder;
+    executor: AicfToolExecutor;
+    loadedEval: LoadedEvalCase;
+    providerConfig?: TProviderConfig;
+    registry: ManifestRegistry;
+    router: AicfCapabilityRouter;
+    runtimeContext: AicfRuntimeContext;
+    suiteId: string;
+    testCase: AicfLiveEvalCaseInput;
+    traceSink?: AicfTraceSink;
+    userInput: AicfRuntimeUserInput;
+}
+export interface AicfLiveEvalCaseRunResult {
+    providerId: AicfLiveEvalProviderId;
+    runResult: AicfNormalizedLiveEvalRunResult;
+    runtimeName: string;
+}
+export interface AicfLiveEvalRunner<TProviderConfig = unknown> {
+    readonly providerId: AicfLiveEvalProviderId;
+    readonly runtimeName: string;
+    runCase(input: AicfLiveEvalCaseRunInput<TProviderConfig>): Promise<AicfLiveEvalCaseRunResult>;
+}
 export interface AicfLiveEvalScorer {
     name: string;
     score(input: {
         candidate: EvalCandidateResult;
-        runResult: AicfOpenAIRunResult;
+        runResult: AicfNormalizedLiveEvalRunResult;
         testCase: AicfLiveEvalCaseInput;
     }): EvalScorerResult;
 }
@@ -22,17 +70,22 @@ export interface AicfLiveEvalOptions {
     contextBuilderFactory: (testCase: AicfLiveEvalCaseInput) => AicfContextBuilder;
     executor: AicfToolExecutor;
     maxConcurrency?: number;
-    model: string;
-    openAIClient: AicfOpenAIResponsesClient;
+    model?: string;
+    openAIClient?: AicfOpenAIResponsesClient;
+    providerConfig?: unknown;
     registry: ManifestRegistry;
+    runner?: AicfLiveEvalRunner;
     router: AicfCapabilityRouter;
     scorers?: AicfLiveEvalScorer[];
+    suiteId?: string;
     traceSink?: AicfTraceSink;
 }
 export interface AicfLiveEvalResult {
     candidate?: EvalCandidateResult;
     evalId: string;
-    runResult?: AicfOpenAIRunResult;
+    providerId?: AicfLiveEvalProviderId;
+    runResult?: AicfNormalizedLiveEvalRunResult;
+    runtimeName?: string;
     scores: Array<{
         message?: string;
         passed: boolean;
@@ -55,6 +108,6 @@ export interface AicfEvalGateResult {
 export interface CreateEvalCaseFromTraceInput {
     includeModelOutput?: boolean;
     reason: "user_feedback" | "policy_denial" | "approval_rejection" | "low_score" | "manual";
-    runResult: AicfOpenAIRunResult;
+    runResult: AicfNormalizedLiveEvalRunResult;
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/evals-live/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evals-live/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzF,OAAO,KAAK,EACV,oBAAoB,EACpB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,oBAAoB,EACpB,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACnF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;IAChC,mBAAmB,CAAC,EAAE,eAAe,EAAE,CAAC;IACxC,cAAc,EAAE,kBAAkB,CAAC;IACnC,SAAS,EAAE,oBAAoB,CAAC;CACjC;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,KAAK,EAAE;QACX,SAAS,EAAE,mBAAmB,CAAC;QAC/B,SAAS,EAAE,mBAAmB,CAAC;QAC/B,QAAQ,EAAE,qBAAqB,CAAC;KACjC,GAAG,gBAAgB,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,qBAAqB,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,kBAAkB,CAAC;IAC/E,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,yBAAyB,CAAC;IACxC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,MAAM,EAAE,oBAAoB,CAAC;IAC7B,OAAO,CAAC,EAAE,kBAAkB,EAAE,CAAC;IAC/B,SAAS,CAAC,EAAE,aAAa,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,MAAM,EAAE,KAAK,CAAC;QACZ,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,OAAO,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;CACzC;AAED,MAAM,WAAW,YAAY;IAC3B,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAC7B;AAED,MAAM,WAAW,4BAA4B;IAC3C,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,eAAe,GAAG,eAAe,GAAG,oBAAoB,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC1F,SAAS,EAAE,mBAAmB,CAAC;CAChC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evals-live/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,KAAK,EACV,oBAAoB,EACpB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,6BAA6B,EAC7B,oBAAoB,EACpB,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EACjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACnG,OAAO,KAAK,EAAE,qBAAqB,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAEtF,MAAM,MAAM,sBAAsB,GAC9B,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,QAAQ,GACR,WAAW,GACX,WAAW,GACX,KAAK,GACL,4BAA4B,GAC5B,MAAM,CAAC;AAEX,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;IAChC,mBAAmB,CAAC,EAAE,eAAe,EAAE,CAAC;IACxC,cAAc,EAAE,kBAAkB,CAAC;IACnC,SAAS,EAAE,oBAAoB,CAAC;CACjC;AAED,MAAM,WAAW,+BAA+B;IAC9C,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,sBAAsB,CAAC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,eAAe,CAAC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,KAAK,CAAC;QACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC,CAAC;IACH,WAAW,EAAE,6BAA6B,EAAE,CAAC;IAC7C,WAAW,EAAE,qBAAqB,EAAE,CAAC;IACrC,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB,CAAC,eAAe,GAAG,OAAO;IACjE,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,kBAAkB,CAAC;IACnC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,UAAU,EAAE,cAAc,CAAC;IAC3B,cAAc,CAAC,EAAE,eAAe,CAAC;IACjC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,MAAM,EAAE,oBAAoB,CAAC;IAC7B,cAAc,EAAE,kBAAkB,CAAC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,qBAAqB,CAAC;IAChC,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B,SAAS,EAAE,oBAAoB,CAAC;CACjC;AAED,MAAM,WAAW,yBAAyB;IACxC,UAAU,EAAE,sBAAsB,CAAC;IACnC,SAAS,EAAE,+BAA+B,CAAC;IAC3C,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB,CAAC,eAAe,GAAG,OAAO;IAC3D,QAAQ,CAAC,UAAU,EAAE,sBAAsB,CAAC;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,OAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;CAC/F;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,KAAK,EAAE;QACX,SAAS,EAAE,mBAAmB,CAAC;QAC/B,SAAS,EAAE,+BAA+B,CAAC;QAC3C,QAAQ,EAAE,qBAAqB,CAAC;KACjC,GAAG,gBAAgB,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,qBAAqB,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,kBAAkB,CAAC;IAC/E,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,MAAM,EAAE,oBAAoB,CAAC;IAC7B,OAAO,CAAC,EAAE,kBAAkB,EAAE,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,aAAa,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,SAAS,CAAC,EAAE,+BAA+B,CAAC;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,KAAK,CAAC;QACZ,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,OAAO,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;CACzC;AAED,MAAM,WAAW,YAAY;IAC3B,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAC7B;AAED,MAAM,WAAW,4BAA4B;IAC3C,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,eAAe,GAAG,eAAe,GAAG,oBAAoB,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC1F,SAAS,EAAE,+BAA+B,CAAC;CAC5C"}

package/dist/index.d.ts

@@ -9,7 +9,6 @@ export { loadManifests, kindFromPath } from "./loader.js";
 export { buildMcpToolDescriptors, parseMcpToolCall, toMcpToolName } from "./mcp.js";
 export { buildOpenAIResponsesTools, parseOpenAIResponsesToolCall, toOpenAIResponsesToolName } from "./openai-responses.js";
 export { buildRegistry, formatInspection, inspectRegistry } from "./registry.js";
-export { runCli } from "./cli.js";
 export { buildSemanticKernelFunctions, parseSemanticKernelFunctionCall, toSemanticKernelFunctionName } from "./semantic-kernel.js";
 export { approvalRequiredToolResult, deniedToolResult, errorToolResult, okToolResult, toModelFacingToolResult, unavailableToolResult } from "./tool-result.js";
 export { validateCapabilityInvariants, validateManifests, validatePublicFixtures } from "./validator.js";