@launchdarkly/server-sdk-ai 0.12.3 → 0.14.0

package/dist/src/api/LDAIClient.d.ts

@@ -0,0 +1,258 @@
+import { LDContext } from '@launchdarkly/js-server-sdk-common';
+import { TrackedChat } from './chat';
+import { LDAIAgentConfig, LDAIAgentConfigDefault, LDAIAgentRequestConfig, LDAICompletionConfig, LDAICompletionConfigDefault, LDAIJudgeConfig, LDAIJudgeConfigDefault } from './config';
+import { Judge } from './judge/Judge';
+import { SupportedAIProvider } from './providers';
+/**
+ * Interface for performing AI operations using LaunchDarkly.
+ */
+export interface LDAIClient {
+    /**
+     * Retrieves and processes a completion AI Config based on the provided key, LaunchDarkly context,
+     * and variables. This includes the model configuration and the customized messages.
+     *
+     * @param key The key of the AI Config.
+     * @param context The LaunchDarkly context object that contains relevant information about the
+     * current environment, user, or session. This context may influence how the configuration is
+     * processed or personalized.
+     * @param defaultValue A fallback value containing model configuration and messages. This will
+     * be used if the configuration is not available from LaunchDarkly.
+     * @param variables A map of key-value pairs representing dynamic variables to be injected into
+     * the message content. The keys correspond to placeholders within the template, and the values
+     * are the corresponding replacements.
+     *
+     * @returns The AI `config`, customized `messages`, and a `tracker`. If the configuration cannot be accessed from
+     * LaunchDarkly, then the return value will include information from the `defaultValue`. The returned `tracker` can
+     * be used to track AI operation metrics (latency, token usage, etc.).
+     *
+     * @example
+     * ```
+     * const key = "welcome_prompt";
+     * const context = {...};
+     * const variables = {username: 'john'};
+     * const defaultValue = {
+     *  enabled: false,
+     *  model: { name: 'gpt-4' },
+     *  provider: { name: 'openai' },
+     * };
+     *
+     * const result = completionConfig(key, context, defaultValue, variables);
+     * // Output:
+     * {
+     *   enabled: true,
+     *   config: {
+     *     modelId: "gpt-4o",
+     *     temperature: 0.2,
+     *     maxTokens: 4096,
+     *     userDefinedKey: "myValue",
+     *   },
+     *   messages: [
+     *     {
+     *       role: "system",
+     *       content: "You are an amazing GPT."
+     *     },
+     *     {
+     *       role: "user",
+     *       content: "Explain how you're an amazing GPT."
+     *     }
+     *   ],
+     *   tracker: ...
+     * }
+     * ```
+     */
+    completionConfig(key: string, context: LDContext, defaultValue: LDAICompletionConfigDefault, variables?: Record<string, unknown>): Promise<LDAICompletionConfig>;
+    /**
+     * @deprecated Use `completionConfig` instead. This method will be removed in a future version.
+     */
+    config(key: string, context: LDContext, defaultValue: LDAICompletionConfigDefault, variables?: Record<string, unknown>): Promise<LDAICompletionConfig>;
+    /**
+     * Retrieves and processes a single AI Config agent based on the provided key, LaunchDarkly context,
+     * and variables. This includes the model configuration and the customized instructions.
+     *
+     * @param key The key of the AI Config agent.
+     * @param context The LaunchDarkly context object that contains relevant information about the
+     * current environment, user, or session. This context may influence how the configuration is
+     * processed or personalized.
+     * @param defaultValue A fallback value containing model configuration and instructions.
+     * @param variables A map of key-value pairs representing dynamic variables to be injected into
+     * the instructions. The keys correspond to placeholders within the template, and the values
+     * are the corresponding replacements.
+     *
+     * @returns An AI agent with customized `instructions` and a `tracker`. If the configuration
+     * cannot be accessed from LaunchDarkly, then the return value will include information from the
+     * `defaultValue`. The returned `tracker` can be used to track AI operation metrics (latency, token usage, etc.).
+     *
+     * @example
+     * ```
+     * const key = "research_agent";
+     * const context = {...};
+     * const variables = { topic: 'climate change' };
+     * const agentConfig = await client.agentConfig(key, context, {
+     *   enabled: true,
+     *   model: { name: 'gpt-4' },
+     *   provider: { name: 'openai' },
+     *   instructions: 'You are a research assistant.',
+     * }, variables);
+     *
+     * const researchResult = agentConfig.instructions; // Interpolated instructions
+     * agentConfig.tracker.trackSuccess();
+     * ```
+     */
+    agentConfig(key: string, context: LDContext, defaultValue: LDAIAgentConfigDefault, variables?: Record<string, unknown>): Promise<LDAIAgentConfig>;
+    /**
+     * @deprecated Use `agentConfig` instead. This method will be removed in a future version.
+     */
+    agent(key: string, context: LDContext, defaultValue: LDAIAgentConfigDefault, variables?: Record<string, unknown>): Promise<LDAIAgentConfig>;
+    /**
+     * Retrieves and processes a Judge AI Config based on the provided key, LaunchDarkly context,
+     * and variables. This includes the model configuration and the customized messages for evaluation.
+     *
+     * @param key The key of the Judge AI Config.
+     * @param context The LaunchDarkly context object that contains relevant information about the
+     * current environment, user, or session. This context may influence how the configuration is
+     * processed or personalized.
+     * @param defaultValue A fallback value containing model configuration and messages. This will
+     * be used if the configuration is not available from LaunchDarkly.
+     * @param variables Optional variables for template interpolation in messages and instructions.
+     * @returns A promise that resolves to a tracked judge configuration.
+     *
+     * @example
+     * ```typescript
+     * const judgeConf = await client.judgeConfig(key, context, {
+     *   enabled: true,
+     *   model: { name: 'gpt-4' },
+     *   provider: { name: 'openai' },
+     *   evaluationMetricKeys: ['$ld:ai:judge:relevance'],
+     *   messages: [{ role: 'system', content: 'You are a relevance judge.' }]
+     * }, variables);
+     *
+     * const config = judgeConf.config; // Interpolated configuration
+     * judgeConf.tracker.trackSuccess();
+     * ```
+     */
+    judgeConfig(key: string, context: LDContext, defaultValue: LDAIJudgeConfigDefault, variables?: Record<string, unknown>): Promise<LDAIJudgeConfig>;
+    /**
+     * Retrieves and processes multiple AI Config agents based on the provided agent configurations
+     * and LaunchDarkly context. This includes the model configuration and the customized instructions.
+     *
+     * @param agentConfigs An array of agent configurations, each containing the agent key, default configuration,
+     * and variables for instructions interpolation.
+     * @param context The LaunchDarkly context object that contains relevant information about the
+     * current environment, user, or session. This context may influence how the configuration is
+     * processed or personalized.
+     *
+     * @returns A map of agent keys to their respective AI agents with customized `instructions` and `tracker`.
+     * If a configuration cannot be accessed from LaunchDarkly, then the return value will include information
+     * from the respective `defaultValue`. The returned `tracker` can be used to track AI operation metrics
+     * (latency, token usage, etc.).
+     *
+     * @example
+     * ```
+     * const agentConfigsList = [
+     *   {
+     *     key: 'research_agent',
+     *     defaultValue: {
+     *       enabled: true,
+     *       model: { name: 'gpt-4' },
+     *       provider: { name: 'openai' },
+     *       instructions: 'You are a research assistant.'
+     *     },
+     *     variables: { topic: 'climate change' }
+     *   },
+     *   {
+     *     key: 'writing_agent',
+     *     defaultValue: {
+     *       enabled: true,
+     *       model: { name: 'gpt-4' },
+     *       provider: { name: 'openai' },
+     *       instructions: 'You are a writing assistant.'
+     *     },
+     *     variables: { style: 'academic' }
+     *   }
+     * ] as const;
+     * const context = {...};
+     *
+     * const configs = await client.agentConfigs(agentConfigsList, context);
+     * const researchResult = configs["research_agent"].instructions; // Interpolated instructions
+     * configs["research_agent"].tracker.trackSuccess();
+     * ```
+     */
+    agentConfigs<const T extends readonly LDAIAgentRequestConfig[]>(agentConfigs: T, context: LDContext): Promise<Record<T[number]['key'], LDAIAgentConfig>>;
+    /**
+     * @deprecated Use `agentConfigs` instead. This method will be removed in a future version.
+     */
+    agents<const T extends readonly LDAIAgentRequestConfig[]>(agentConfigs: T, context: LDContext): Promise<Record<T[number]['key'], LDAIAgentConfig>>;
+    /**
+     * Returns a TrackedChat instance for chat interactions.
+     * This method serves as the primary entry point for creating TrackedChat instances from configuration.
+     *
+     * @param key The key identifying the AI chat configuration to use.
+     * @param context The standard LDContext used when evaluating flags.
+     * @param defaultValue A default value representing a standard AI chat config result.
+     * @param variables Dictionary of values for instruction interpolation.
+     * The variables will also be used for judge evaluation. For the judge only, the variables
+     * `message_history` and `response_to_evaluate` are reserved and will be ignored.
+     * @param defaultAiProvider Optional default AI provider to use.
+     * @returns A promise that resolves to the TrackedChat instance, or null if the configuration is disabled.
+     *
+     * @example
+     * ```
+     * const key = "customer_support_chat";
+     * const context = {...};
+     * const defaultValue = {
+     *   enabled: false,
+     *   model: { name: "gpt-4" },
+     *   provider: { name: "openai" },
+     *   messages: [
+     *     { role: "system", content: "You are a helpful customer support agent." }
+     *   ]
+     * };
+     * const variables = { customerName: 'John' };
+     *
+     * const chat = await client.createChat(key, context, defaultValue, variables);
+     * if (chat) {
+     *   const response = await chat.invoke("I need help with my order");
+     *   console.log(response.message.content);
+     * }
+     * ```
+     */
+    createChat(key: string, context: LDContext, defaultValue: LDAICompletionConfigDefault, variables?: Record<string, unknown>, defaultAiProvider?: SupportedAIProvider): Promise<TrackedChat | undefined>;
+    /**
+     * @deprecated Use `createChat` instead. This method will be removed in a future version.
+     */
+    initChat(key: string, context: LDContext, defaultValue: LDAICompletionConfigDefault, variables?: Record<string, unknown>, defaultAiProvider?: SupportedAIProvider): Promise<TrackedChat | undefined>;
+    /**
+     * Creates and returns a new Judge instance for AI evaluation.
+     *
+     * @param key The key identifying the AI judge configuration to use
+     * @param context Standard LDContext used when evaluating flags
+     * @param defaultValue A default value representing a standard AI config result
+     * @param variables Dictionary of values for instruction interpolation.
+     * The variables `message_history` and `response_to_evaluate` are reserved for the judge and will be ignored.
+     * @param defaultAiProvider Optional default AI provider to use.
+     * @returns Promise that resolves to a Judge instance or undefined if disabled/unsupported
+     *
+     * @example
+     * ```
+     * const judge = await client.createJudge(
+     *   "relevance-judge",
+     *   context,
+     *   {
+     *     enabled: true,
+     *     model: { name: "gpt-4" },
+     *     provider: { name: "openai" },
+     *     evaluationMetricKeys: ['$ld:ai:judge:relevance'],
+     *     messages: [{ role: 'system', content: 'You are a relevance judge.' }]
+     *   },
+     *   { metric: "relevance" }
+     * );
+     *
+     * if (judge) {
+     *   const result = await judge.evaluate("User question", "AI response");
+     *   console.log('Relevance score:', result.evals.relevance?.score);
+     * }
+     * ```
+     */
+    createJudge(key: string, context: LDContext, defaultValue: LDAIJudgeConfigDefault, variables?: Record<string, unknown>, defaultAiProvider?: SupportedAIProvider): Promise<Judge | undefined>;
+}
+//# sourceMappingURL=LDAIClient.d.ts.map

package/dist/src/api/LDAIClient.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LDAIClient.d.ts","sourceRoot":"","sources":["../../../src/api/LDAIClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,oCAAoC,CAAC;AAE/D,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,sBAAsB,EACtB,oBAAoB,EACpB,2BAA2B,EAC3B,eAAe,EACf,sBAAsB,EACvB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;IACH,gBAAgB,CACd,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,2BAA2B,EACzC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;OAEG;IACH,MAAM,CACJ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,2BAA2B,EACzC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,WAAW,CACT,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,sBAAsB,EACpC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAClC,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5B;;OAEG;IACH,KAAK,CACH,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,sBAAsB,EACpC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAClC,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5B;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,WAAW,CACT,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,sBAAsB,EACpC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAClC,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6CG;IACH,YAAY,CAAC,KAAK,CAAC,CAAC,SAAS,SAAS,sBAAsB,EAAE,EAC5D,YAAY,EAAE,CAAC,EACf,OAAO,EAAE,SAAS,GACjB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC;IAEtD;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,SAAS,sBAAsB,EAAE,EACtD,YAAY,EAAE,CAAC,EACf,OAAO,EAAE,SAAS,GACjB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC;IAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiCG;IACH,UAAU,CACR,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,2BAA2B,EACzC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnC,iBAAiB,CAAC,EAAE,mBAAmB,GACtC,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC;IAEpC;;OAEG;IACH,QAAQ,CACN,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,2BAA2B,EACzC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnC,iBAAiB,CAAC,EAAE,mBAAmB,GACtC,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC;IAEpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,WAAW,CACT,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,sBAAsB,EACpC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnC,iBAAiB,CAAC,EAAE,mBAAmB,GACtC,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC;CAC/B"}

package/dist/{api → src/api}/LDAIClient.js.map

@@ -1 +1 @@
-{"version":3,"file":"LDAIClient.js","sourceRoot":"","sources":["../../src/api/LDAIClient.ts"],"names":[],"mappings":""}
+{"version":3,"file":"LDAIClient.js","sourceRoot":"","sources":["../../../src/api/LDAIClient.ts"],"names":[],"mappings":""}

package/dist/{api → src/api}/chat/TrackedChat.d.ts

@@ -1,5 +1,7 @@
-import { LDAIConfig, LDMessage } from '../config/LDAIConfig';
+import { LDLogger } from '@launchdarkly/js-server-sdk-common';
 import { LDAIConfigTracker } from '../config/LDAIConfigTracker';
+import { LDAICompletionConfig, LDMessage } from '../config/types';
+import { Judge } from '../judge/Judge';
 import { AIProvider } from '../providers/AIProvider';
 import { ChatResponse } from './types';
 /**
@@ -9,20 +11,31 @@ import { ChatResponse } from './types';
  * the actual model invocation to the provider.
  */
 export declare class TrackedChat {
-    protected readonly aiConfig: LDAIConfig;
+    protected readonly aiConfig: LDAICompletionConfig;
     protected readonly tracker: LDAIConfigTracker;
     protected readonly provider: AIProvider;
+    protected readonly judges: Record<string, Judge>;
+    private readonly _logger?;
     protected messages: LDMessage[];
-    constructor(aiConfig: LDAIConfig, tracker: LDAIConfigTracker, provider: AIProvider);
+    constructor(aiConfig: LDAICompletionConfig, tracker: LDAIConfigTracker, provider: AIProvider, judges?: Record<string, Judge>, _logger?: LDLogger | undefined);
     /**
      * Invoke the chat model with a prompt string.
      * This method handles conversation management and tracking, delegating to the provider's invokeModel method.
      */
     invoke(prompt: string): Promise<ChatResponse>;
+    /**
+     * Evaluates the response with all configured judges.
+     * Returns a promise that resolves to an array of evaluation results.
+     *
+     * @param messages Array of messages representing the conversation history
+     * @param response The AI response to be evaluated
+     * @returns Promise resolving to array of judge evaluation results
+     */
+    private _evaluateWithJudges;
     /**
      * Get the underlying AI configuration used to initialize this TrackedChat.
      */
-    getConfig(): LDAIConfig;
+    getConfig(): LDAICompletionConfig;
     /**
      * Get the underlying AI configuration tracker used to initialize this TrackedChat.
      */
@@ -32,6 +45,11 @@ export declare class TrackedChat {
      * This provides direct access to the provider for advanced use cases.
      */
     getProvider(): AIProvider;
+    /**
+     * Get the judges associated with this TrackedChat.
+     * Returns a record of judge instances keyed by their configuration keys.
+     */
+    getJudges(): Record<string, Judge>;
     /**
      * Append messages to the conversation history.
      * Adds messages to the conversation history without invoking the model,

package/dist/src/api/chat/TrackedChat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TrackedChat.d.ts","sourceRoot":"","sources":["../../../../src/api/chat/TrackedChat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oCAAoC,CAAC;AAE9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC;;;;;GAKG;AACH,qBAAa,WAAW;IAIpB,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,oBAAoB;IACjD,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,iBAAiB;IAC7C,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU;IACvC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;IAP3B,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC;gBAGX,QAAQ,EAAE,oBAAoB,EAC9B,OAAO,EAAE,iBAAiB,EAC1B,QAAQ,EAAE,UAAU,EACpB,MAAM,GAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAM,EACpC,OAAO,CAAC,sBAAU;IAKrC;;;OAGG;IACG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA6BnD;;;;;;;OAOG;YACW,mBAAmB;IAgCjC;;OAEG;IACH,SAAS,IAAI,oBAAoB;IAIjC;;OAEG;IACH,UAAU,IAAI,iBAAiB;IAI/B;;;OAGG;IACH,WAAW,IAAI,UAAU;IAIzB;;;OAGG;IACH,SAAS,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC;IAIlC;;;;;;OAMG;IACH,cAAc,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI;IAI3C;;;;;;;;OAQG;IACH,WAAW,CAAC,qBAAqB,GAAE,OAAe,GAAG,SAAS,EAAE;CAOjE"}

package/dist/{api → src/api}/chat/TrackedChat.js

@@ -8,10 +8,12 @@ exports.TrackedChat = void 0;
  * the actual model invocation to the provider.
  */
 class TrackedChat {
-    constructor(aiConfig, tracker, provider) {
+    constructor(aiConfig, tracker, provider, judges = {}, _logger) {
         this.aiConfig = aiConfig;
         this.tracker = tracker;
         this.provider = provider;
+        this.judges = judges;
+        this._logger = _logger;
         this.messages = [];
     }
     /**
@@ -19,6 +21,7 @@ class TrackedChat {
      * This method handles conversation management and tracking, delegating to the provider's invokeModel method.
      */
     async invoke(prompt) {
+        var _a;
         // Convert prompt string to LDMessage with role 'user' and add to conversation history
         const userMessage = {
             role: 'user',
@@ -30,10 +33,41 @@ class TrackedChat {
         const allMessages = [...configMessages, ...this.messages];
         // Delegate to provider-specific implementation with tracking
         const response = await this.tracker.trackMetricsOf((result) => result.metrics, () => this.provider.invokeModel(allMessages));
-        // Add the assistant response to the conversation history
+        if (((_a = this.aiConfig.judgeConfiguration) === null || _a === void 0 ? void 0 : _a.judges) &&
+            this.aiConfig.judgeConfiguration.judges.length > 0) {
+            response.evaluations = this._evaluateWithJudges(this.messages, response);
+        }
         this.messages.push(response.message);
         return response;
     }
+    /**
+     * Evaluates the response with all configured judges.
+     * Returns a promise that resolves to an array of evaluation results.
+     *
+     * @param messages Array of messages representing the conversation history
+     * @param response The AI response to be evaluated
+     * @returns Promise resolving to array of judge evaluation results
+     */
+    async _evaluateWithJudges(messages, response) {
+        const judgeConfigs = this.aiConfig.judgeConfiguration.judges;
+        // Start all judge evaluations in parallel
+        const evaluationPromises = judgeConfigs.map(async (judgeConfig) => {
+            var _a;
+            const judge = this.judges[judgeConfig.key];
+            if (!judge) {
+                (_a = this._logger) === null || _a === void 0 ? void 0 : _a.warn(`Judge configuration is not enabled: ${judgeConfig.key}`, this.tracker.getTrackData());
+                return undefined;
+            }
+            const evalResult = await judge.evaluateMessages(messages, response, judgeConfig.samplingRate);
+            if (evalResult && evalResult.success) {
+                this.tracker.trackEvalScores(evalResult.evals);
+            }
+            return evalResult;
+        });
+        // ensure all evaluations complete even if some fail
+        const results = await Promise.allSettled(evaluationPromises);
+        return results.map((result) => (result.status === 'fulfilled' ? result.value : undefined));
+    }
     /**
      * Get the underlying AI configuration used to initialize this TrackedChat.
      */
@@ -53,6 +87,13 @@ class TrackedChat {
     getProvider() {
         return this.provider;
     }
+    /**
+     * Get the judges associated with this TrackedChat.
+     * Returns a record of judge instances keyed by their configuration keys.
+     */
+    getJudges() {
+        return this.judges;
+    }
     /**
      * Append messages to the conversation history.
      * Adds messages to the conversation history without invoking the model,

package/dist/src/api/chat/TrackedChat.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TrackedChat.js","sourceRoot":"","sources":["../../../../src/api/chat/TrackedChat.ts"],"names":[],"mappings":";;;AASA;;;;;GAKG;AACH,MAAa,WAAW;IAGtB,YACqB,QAA8B,EAC9B,OAA0B,EAC1B,QAAoB,EACpB,SAAgC,EAAE,EACpC,OAAkB;QAJhB,aAAQ,GAAR,QAAQ,CAAsB;QAC9B,YAAO,GAAP,OAAO,CAAmB;QAC1B,aAAQ,GAAR,QAAQ,CAAY;QACpB,WAAM,GAAN,MAAM,CAA4B;QACpC,YAAO,GAAP,OAAO,CAAW;QAEnC,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,MAAc;;QACzB,sFAAsF;QACtF,MAAM,WAAW,GAAc;YAC7B,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,MAAM;SAChB,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEhC,uEAAuE;QACvE,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;QACpD,MAAM,WAAW,GAAG,CAAC,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1D,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAChD,CAAC,MAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,EACxC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,WAAW,CAAC,CAC7C,CAAC;QAEF,IACE,CAAA,MAAA,IAAI,CAAC,QAAQ,CAAC,kBAAkB,0CAAE,MAAM;YACxC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAClD;YACA,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;SAC1E;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACrC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,mBAAmB,CAC/B,QAAqB,EACrB,QAAsB;QAEtB,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAmB,CAAC,MAAM,CAAC;QAE9D,0CAA0C;QAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;;YAChE,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,CAAC,KAAK,EAAE;gBACV,MAAA,IAAI,CAAC,OAAO,0CAAE,IAAI,CAChB,uCAAuC,WAAW,CAAC,GAAG,EAAE,EACxD,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAC5B,CAAC;gBACF,OAAO,SAAS,CAAC;aAClB;YAED,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC;YAE9F,IAAI,UAAU,IAAI,UAAU,CAAC,OAAO,EAAE;gBACpC,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;aAChD;YAED,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC,CAAC;QAEH,oDAAoD;QACpD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;QAE7D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAC7F,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;;OAGG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,cAAc,CAAC,QAAqB;QAClC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;;;;;;;OAQG;IACH,WAAW,CAAC,wBAAiC,KAAK;QAChD,IAAI,qBAAqB,EAAE;YACzB,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;YACpD,OAAO,CAAC,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC9C;QACD,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;CACF;AA/ID,kCA+IC"}

package/dist/src/api/chat/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/api/chat/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,eAAe,CAAC"}

package/dist/src/api/chat/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/api/chat/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0CAAwB;AACxB,gDAA8B"}

package/dist/src/api/chat/types.d.ts

@@ -0,0 +1,22 @@
+import { LDMessage } from '../config/types';
+import { JudgeResponse } from '../judge/types';
+import { LDAIMetrics } from '../metrics/LDAIMetrics';
+/**
+ * Chat response structure.
+ */
+export interface ChatResponse {
+    /**
+     * The response message from the AI.
+     */
+    message: LDMessage;
+    /**
+     * Metrics information including success status and token usage.
+     */
+    metrics: LDAIMetrics;
+    /**
+     * Promise that resolves to judge evaluation results.
+     * Only present when judges are configured for evaluation.
+     */
+    evaluations?: Promise<Array<JudgeResponse | undefined>>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/src/api/chat/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/api/chat/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,OAAO,EAAE,SAAS,CAAC;IAEnB;;OAEG;IACH,OAAO,EAAE,WAAW,CAAC;IAErB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC,CAAC;CACzD"}

package/dist/{api → src/api}/chat/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/api/chat/types.ts"],"names":[],"mappings":""}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/api/chat/types.ts"],"names":[],"mappings":""}

package/dist/{api → src/api}/config/LDAIConfigTracker.d.ts

@@ -1,3 +1,4 @@
+import { EvalScore } from '../judge/types';
 import { LDAIMetrics, LDFeedbackKind, LDTokenUsage } from '../metrics';
 /**
  * Metrics which have been tracked.
@@ -30,6 +31,18 @@ export interface LDAIMetricSummary {
  * The LDAIConfigTracker is used to track various details about AI operations.
  */
 export interface LDAIConfigTracker {
+    /**
+     * Get the data for tracking.
+     */
+    getTrackData(): {
+        variationKey: string;
+        configKey: string;
+        version: number;
+        modelName: string;
+        providerName: string;
+        aiSdkName: string;
+        aiSdkVersion: string;
+    };
     /**
      * Track the duration of generation.
      *
@@ -66,6 +79,12 @@ export interface LDAIConfigTracker {
      * @param timeToFirstTokenMs The duration in milliseconds.
      */
     trackTimeToFirstToken(timeToFirstTokenMs: number): void;
+    /**
+     * Track evaluation scores for multiple metrics.
+     *
+     * @param scores Record mapping metric keys to their evaluation scores
+     */
+    trackEvalScores(scores: Record<string, EvalScore>): void;
     /**
      * Track the duration of execution of the provided function.
      *
@@ -93,6 +112,28 @@ export interface LDAIConfigTracker {
      * @returns The result of the operation
      */
     trackMetricsOf<TRes>(metricsExtractor: (result: TRes) => LDAIMetrics, func: () => Promise<TRes>): Promise<TRes>;
+    /**
+     * Track metrics for a streaming AI operation.
+     *
+     * This function will track the duration of the operation, extract metrics using the provided
+     * metrics extractor function, and track success or error status accordingly.
+     *
+     * Unlike trackMetricsOf, this method is designed for streaming operations where:
+     * - The stream is created and returned immediately (synchronously)
+     * - Metrics are extracted asynchronously in the background once the stream completes
+     * - Duration is tracked from stream creation to metrics extraction completion
+     *
+     * The stream is returned immediately so the caller can begin consuming it without waiting.
+     * Metrics extraction happens in the background and does not block stream consumption.
+     *
+     * If the stream creator throws, then this method will also throw and record an error.
+     * If metrics extraction fails, the error is logged but does not affect stream consumption.
+     *
+     * @param streamCreator Function that creates and returns the stream (synchronous)
+     * @param metricsExtractor Function that asynchronously extracts metrics from the stream
+     * @returns The stream result (returned immediately, not a Promise)
+     */
+    trackStreamMetricsOf<TStream>(streamCreator: () => TStream, metricsExtractor: (stream: TStream) => Promise<LDAIMetrics>): TStream;
     /**
      * Track an OpenAI operation.
      *
@@ -154,28 +195,6 @@ export interface LDAIConfigTracker {
             completionTokens?: number;
         };
     }>(func: () => Promise<TRes>): Promise<TRes>;
-    /**
-     * Track a Vercel AI SDK streamText operation.
-     *
-     * This function will track the duration of the operation, the token usage, and the success or error status.
-     *
-     * If the provided function throws, then this method will also throw.
-     * In the case the provided function throws, this function will record the duration and an error.
-     * A failed operation will not have any token usage data.
-     *
-     * @param func Function which executes the operation.
-     * @returns The result of the operation.
-     */
-    trackVercelAISDKStreamTextMetrics<TRes extends {
-        finishReason?: Promise<string>;
-        usage?: Promise<{
-            totalTokens?: number;
-            inputTokens?: number;
-            promptTokens?: number;
-            outputTokens?: number;
-            completionTokens?: number;
-        }>;
-    }>(func: () => TRes): TRes;
     /**
      * Get a summary of the tracked metrics.
      */

package/dist/src/api/config/LDAIConfigTracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LDAIConfigTracker.d.ts","sourceRoot":"","sources":["../../../../src/api/config/LDAIConfigTracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,MAAM,CAAC,EAAE,YAAY,CAAC;IAEtB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;OAEG;IACH,QAAQ,CAAC,EAAE;QAAE,IAAI,EAAE,cAAc,CAAA;KAAE,CAAC;IAEpC;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,YAAY,IAAI;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF;;;;;;OAMG;IACH,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAExC;;;;OAIG;IACH,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAExC;;OAEG;IACH,YAAY,IAAI,IAAI,CAAC;IAErB;;OAEG;IACH,UAAU,IAAI,IAAI,CAAC;IAEnB;;;;OAIG;IACH,aAAa,CAAC,QAAQ,EAAE;QAAE,IAAI,EAAE,cAAc,CAAA;KAAE,GAAG,IAAI,CAAC;IAExD;;;;OAIG;IACH,qBAAqB,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAExD;;;;OAIG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,IAAI,CAAC;IAEzD;;;;;;;;;;OAUG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAExD;;;;;;;;;;;;;OAaG;IACH,cAAc,CAAC,IAAI,EACjB,gBAAgB,EAAE,CAAC,MAAM,EAAE,IAAI,KAAK,WAAW,EAC/C,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GACxB,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,oBAAoB,CAAC,OAAO,EAC1B,aAAa,EAAE,MAAM,OAAO,EAC5B,gBAAgB,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,WAAW,CAAC,GAC1D,OAAO,CAAC;IAEX;;;;;;;;;;;OAWG;IACH,kBAAkB,CAChB,IAAI,SAAS;QACX,KAAK,CAAC,EAAE;YACN,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,aAAa,CAAC,EAAE,MAAM,CAAC;YACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;SAC5B,CAAC;KACH,EAED,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GACxB,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;OAOG;IACH,2BAA2B,CACzB,IAAI,SAAS;QACX,SAAS,EAAE;YAAE,cAAc,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACvC,OAAO,CAAC,EAAE;YAAE,SAAS,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACjC,KAAK,CAAC,EAAE;YACN,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,WAAW,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;KACH,EAED,GAAG,EAAE,IAAI,GACR,IAAI,CAAC;IAER;;;;;;;;;;;OAWG;IACH,mCAAmC,CACjC,IAAI,SAAS;QACX,KAAK,CAAC,EAAE;YACN,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,EAED,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GACxB,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;OAEG;IACH,UAAU,IAAI,iBAAiB,CAAC;CACjC"}

package/dist/src/api/config/LDAIConfigTracker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LDAIConfigTracker.js","sourceRoot":"","sources":["../../../../src/api/config/LDAIConfigTracker.ts"],"names":[],"mappings":""}

package/dist/src/api/config/LDAIConfigUtils.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=LDAIConfigUtils.d.ts.map

package/dist/src/api/config/LDAIConfigUtils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LDAIConfigUtils.d.ts","sourceRoot":"","sources":["../../../../src/api/config/LDAIConfigUtils.ts"],"names":[],"mappings":""}