@launchdarkly/server-sdk-ai 0.12.3 → 0.14.0

package/src/LDAIConfigTrackerImpl.ts

@@ -1,7 +1,9 @@
 import { LDContext } from '@launchdarkly/js-server-sdk-common';
 
+import { name as aiSdkName, version as aiSdkVersion } from '../package.json';
 import { LDAIConfigTracker } from './api/config';
 import { LDAIMetricSummary } from './api/config/LDAIConfigTracker';
+import { EvalScore } from './api/judge/types';
 import {
   createBedrockTokenUsage,
   createOpenAiUsage,
@@ -25,12 +27,14 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
     private _context: LDContext,
   ) {}
 
-  private _getTrackData(): {
+  getTrackData(): {
     variationKey: string;
     configKey: string;
     version: number;
     modelName: string;
     providerName: string;
+    aiSdkName: string;
+    aiSdkVersion: string;
   } {
     return {
       variationKey: this._variationKey,
@@ -38,12 +42,14 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
       version: this._version,
       modelName: this._modelName,
       providerName: this._providerName,
+      aiSdkName,
+      aiSdkVersion,
     };
   }
 
   trackDuration(duration: number): void {
     this._trackedMetrics.durationMs = duration;
-    this._ldClient.track('$ld:ai:duration:total', this._context, this._getTrackData(), duration);
+    this._ldClient.track('$ld:ai:duration:total', this._context, this.getTrackData(), duration);
   }
 
   async trackDurationOf<TRes>(func: () => Promise<TRes>): Promise<TRes> {
@@ -64,28 +70,35 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
     this._ldClient.track(
       '$ld:ai:tokens:ttf',
       this._context,
-      this._getTrackData(),
+      this.getTrackData(),
       timeToFirstTokenMs,
     );
   }
 
+  trackEvalScores(scores: Record<string, EvalScore>) {
+    // Track each evaluation score individually
+    Object.entries(scores).forEach(([metricKey, evalScore]) => {
+      this._ldClient.track(metricKey, this._context, this.getTrackData(), evalScore.score);
+    });
+  }
+
   trackFeedback(feedback: { kind: LDFeedbackKind }): void {
     this._trackedMetrics.feedback = feedback;
     if (feedback.kind === LDFeedbackKind.Positive) {
-      this._ldClient.track('$ld:ai:feedback:user:positive', this._context, this._getTrackData(), 1);
+      this._ldClient.track('$ld:ai:feedback:user:positive', this._context, this.getTrackData(), 1);
     } else if (feedback.kind === LDFeedbackKind.Negative) {
-      this._ldClient.track('$ld:ai:feedback:user:negative', this._context, this._getTrackData(), 1);
+      this._ldClient.track('$ld:ai:feedback:user:negative', this._context, this.getTrackData(), 1);
     }
   }
 
   trackSuccess(): void {
     this._trackedMetrics.success = true;
-    this._ldClient.track('$ld:ai:generation:success', this._context, this._getTrackData(), 1);
+    this._ldClient.track('$ld:ai:generation:success', this._context, this.getTrackData(), 1);
   }
 
   trackError(): void {
     this._trackedMetrics.success = false;
-    this._ldClient.track('$ld:ai:generation:error', this._context, this._getTrackData(), 1);
+    this._ldClient.track('$ld:ai:generation:error', this._context, this.getTrackData(), 1);
   }
 
   async trackMetricsOf<TRes>(
@@ -119,6 +132,58 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
     return result;
   }
 
+  trackStreamMetricsOf<TStream>(
+    streamCreator: () => TStream,
+    metricsExtractor: (stream: TStream) => Promise<LDAIMetrics>,
+  ): TStream {
+    const startTime = Date.now();
+
+    try {
+      // Create the stream synchronously
+      const stream = streamCreator();
+
+      // Start background metrics tracking (fire and forget)
+      this._trackStreamMetricsInBackground(stream, metricsExtractor, startTime);
+
+      // Return stream immediately for consumption
+      return stream;
+    } catch (error) {
+      // Track error if stream creation fails
+      this.trackDuration(Date.now() - startTime);
+      this.trackError();
+      throw error;
+    }
+  }
+
+  private async _trackStreamMetricsInBackground<TStream>(
+    stream: TStream,
+    metricsExtractor: (stream: TStream) => Promise<LDAIMetrics>,
+    startTime: number,
+  ): Promise<void> {
+    try {
+      // Wait for metrics to be available
+      const metrics = await metricsExtractor(stream);
+
+      // Track success/error based on metrics
+      if (metrics.success) {
+        this.trackSuccess();
+      } else {
+        this.trackError();
+      }
+
+      // Track token usage if available
+      if (metrics.usage) {
+        this.trackTokens(metrics.usage);
+      }
+    } catch (error) {
+      // If metrics extraction fails, track error
+      this.trackError();
+    } finally {
+      // Track duration regardless of success/error
+      this.trackDuration(Date.now() - startTime);
+    }
+  }
+
   async trackOpenAIMetrics<
     TRes extends {
       usage?: {
@@ -190,55 +255,9 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
     }
   }
 
-  trackVercelAISDKStreamTextMetrics<
-    TRes extends {
-      finishReason?: Promise<string>;
-      usage?: Promise<{
-        totalTokens?: number;
-        inputTokens?: number;
-        promptTokens?: number;
-        outputTokens?: number;
-        completionTokens?: number;
-      }>;
-    },
-  >(func: () => TRes): TRes {
-    const startTime = Date.now();
-    try {
-      const result = func();
-      result.finishReason
-        ?.then(async (finishReason) => {
-          const endTime = Date.now();
-          this.trackDuration(endTime - startTime);
-          if (finishReason === 'error') {
-            this.trackError();
-          } else {
-            this.trackSuccess();
-            if (result.usage) {
-              try {
-                this.trackTokens(createVercelAISDKTokenUsage(await result.usage));
-              } catch {
-                // Intentionally squashing this error
-              }
-            }
-          }
-        })
-        .catch(() => {
-          const endTime = Date.now();
-          this.trackDuration(endTime - startTime);
-          this.trackError();
-        });
-      return result;
-    } catch (err) {
-      const endTime = Date.now();
-      this.trackDuration(endTime - startTime);
-      this.trackError();
-      throw err;
-    }
-  }
-
   trackTokens(tokens: LDTokenUsage): void {
     this._trackedMetrics.tokens = tokens;
-    const trackData = this._getTrackData();
+    const trackData = this.getTrackData();
     if (tokens.total > 0) {
       this._ldClient.track('$ld:ai:tokens:total', this._context, trackData, tokens.total);
     }

package/src/api/LDAIClient.ts

@@ -1,8 +1,16 @@
 import { LDContext } from '@launchdarkly/js-server-sdk-common';
 
-import { LDAIAgent, LDAIAgentConfig, LDAIAgentDefaults } from './agents';
 import { TrackedChat } from './chat';
-import { LDAIConfig, LDAIDefaults } from './config/LDAIConfig';
+import {
+  LDAIAgentConfig,
+  LDAIAgentConfigDefault,
+  LDAIAgentRequestConfig,
+  LDAICompletionConfig,
+  LDAICompletionConfigDefault,
+  LDAIJudgeConfig,
+  LDAIJudgeConfigDefault,
+} from './config';
+import { Judge } from './judge/Judge';
 import { SupportedAIProvider } from './providers';
 
 /**
@@ -10,7 +18,7 @@ import { SupportedAIProvider } from './providers';
  */
 export interface LDAIClient {
   /**
-   * Retrieves and processes an AI Config based on the provided key, LaunchDarkly context,
+   * Retrieves and processes a completion AI Config based on the provided key, LaunchDarkly context,
    * and variables. This includes the model configuration and the customized messages.
    *
    * @param key The key of the AI Config.
@@ -34,9 +42,11 @@ export interface LDAIClient {
    * const variables = {username: 'john'};
    * const defaultValue = {
    *  enabled: false,
+   *  model: { name: 'gpt-4' },
+   *  provider: { name: 'openai' },
    * };
    *
-   * const result = config(key, context, defaultValue, variables);
+   * const result = completionConfig(key, context, defaultValue, variables);
    * // Output:
    * {
    *   enabled: true,
@@ -60,12 +70,22 @@ export interface LDAIClient {
    * }
    * ```
    */
+  completionConfig(
+    key: string,
+    context: LDContext,
+    defaultValue: LDAICompletionConfigDefault,
+    variables?: Record<string, unknown>,
+  ): Promise<LDAICompletionConfig>;
+
+  /**
+   * @deprecated Use `completionConfig` instead. This method will be removed in a future version.
+   */
   config(
     key: string,
     context: LDContext,
-    defaultValue: LDAIDefaults,
+    defaultValue: LDAICompletionConfigDefault,
     variables?: Record<string, unknown>,
-  ): Promise<LDAIConfig>;
+  ): Promise<LDAICompletionConfig>;
 
   /**
    * Retrieves and processes a single AI Config agent based on the provided key, LaunchDarkly context,
@@ -89,21 +109,67 @@ export interface LDAIClient {
    * const key = "research_agent";
    * const context = {...};
    * const variables = { topic: 'climate change' };
-   * const agent = await client.agent(key, context, {
+   * const agentConfig = await client.agentConfig(key, context, {
    *   enabled: true,
+   *   model: { name: 'gpt-4' },
+   *   provider: { name: 'openai' },
    *   instructions: 'You are a research assistant.',
    * }, variables);
    *
-   * const researchResult = agent.instructions; // Interpolated instructions
-   * agent.tracker.trackSuccess();
+   * const researchResult = agentConfig.instructions; // Interpolated instructions
+   * agentConfig.tracker.trackSuccess();
    * ```
    */
+  agentConfig(
+    key: string,
+    context: LDContext,
+    defaultValue: LDAIAgentConfigDefault,
+    variables?: Record<string, unknown>,
+  ): Promise<LDAIAgentConfig>;
+
+  /**
+   * @deprecated Use `agentConfig` instead. This method will be removed in a future version.
+   */
   agent(
     key: string,
     context: LDContext,
-    defaultValue: LDAIAgentDefaults,
+    defaultValue: LDAIAgentConfigDefault,
+    variables?: Record<string, unknown>,
+  ): Promise<LDAIAgentConfig>;
+
+  /**
+   * Retrieves and processes a Judge AI Config based on the provided key, LaunchDarkly context,
+   * and variables. This includes the model configuration and the customized messages for evaluation.
+   *
+   * @param key The key of the Judge AI Config.
+   * @param context The LaunchDarkly context object that contains relevant information about the
+   * current environment, user, or session. This context may influence how the configuration is
+   * processed or personalized.
+   * @param defaultValue A fallback value containing model configuration and messages. This will
+   * be used if the configuration is not available from LaunchDarkly.
+   * @param variables Optional variables for template interpolation in messages and instructions.
+   * @returns A promise that resolves to a tracked judge configuration.
+   *
+   * @example
+   * ```typescript
+   * const judgeConf = await client.judgeConfig(key, context, {
+   *   enabled: true,
+   *   model: { name: 'gpt-4' },
+   *   provider: { name: 'openai' },
+   *   evaluationMetricKeys: ['$ld:ai:judge:relevance'],
+   *   messages: [{ role: 'system', content: 'You are a relevance judge.' }]
+   * }, variables);
+   *
+   * const config = judgeConf.config; // Interpolated configuration
+   * judgeConf.tracker.trackSuccess();
+   * ```
+   */
+  judgeConfig(
+    key: string,
+    context: LDContext,
+    defaultValue: LDAIJudgeConfigDefault,
     variables?: Record<string, unknown>,
-  ): Promise<LDAIAgent>;
+  ): Promise<LDAIJudgeConfig>;
 
   /**
    * Retrieves and processes multiple AI Config agents based on the provided agent configurations
@@ -122,38 +188,59 @@ export interface LDAIClient {
    *
    * @example
    * ```
-   * const agentConfigs = [
+   * const agentConfigsList = [
    *   {
    *     key: 'research_agent',
-   *     defaultValue: { enabled: true, instructions: 'You are a research assistant.' },
+   *     defaultValue: {
+   *       enabled: true,
+   *       model: { name: 'gpt-4' },
+   *       provider: { name: 'openai' },
+   *       instructions: 'You are a research assistant.'
+   *     },
    *     variables: { topic: 'climate change' }
    *   },
    *   {
    *     key: 'writing_agent',
-   *     defaultValue: { enabled: true, instructions: 'You are a writing assistant.' },
+   *     defaultValue: {
+   *       enabled: true,
+   *       model: { name: 'gpt-4' },
+   *       provider: { name: 'openai' },
+   *       instructions: 'You are a writing assistant.'
+   *     },
    *     variables: { style: 'academic' }
    *   }
    * ] as const;
    * const context = {...};
    *
-   * const agents = await client.agents(agentConfigs, context);
-   * const researchResult = agents["research_agent"].instructions; // Interpolated instructions
-   * agents["research_agent"].tracker.trackSuccess();
+   * const configs = await client.agentConfigs(agentConfigsList, context);
+   * const researchResult = configs["research_agent"].instructions; // Interpolated instructions
+   * configs["research_agent"].tracker.trackSuccess();
    * ```
    */
-  agents<const T extends readonly LDAIAgentConfig[]>(
+  agentConfigs<const T extends readonly LDAIAgentRequestConfig[]>(
+    agentConfigs: T,
+    context: LDContext,
+  ): Promise<Record<T[number]['key'], LDAIAgentConfig>>;
+
+  /**
+   * @deprecated Use `agentConfigs` instead. This method will be removed in a future version.
+   */
+  agents<const T extends readonly LDAIAgentRequestConfig[]>(
     agentConfigs: T,
     context: LDContext,
-  ): Promise<Record<T[number]['key'], LDAIAgent>>;
+  ): Promise<Record<T[number]['key'], LDAIAgentConfig>>;
 
   /**
-   * Initializes and returns a new TrackedChat instance for chat interactions.
+   * Returns a TrackedChat instance for chat interactions.
    * This method serves as the primary entry point for creating TrackedChat instances from configuration.
    *
    * @param key The key identifying the AI chat configuration to use.
    * @param context The standard LDContext used when evaluating flags.
    * @param defaultValue A default value representing a standard AI chat config result.
    * @param variables Dictionary of values for instruction interpolation.
+   * The variables will also be used for judge evaluation. For the judge only, the variables
+   * `message_history` and `response_to_evaluate` are reserved and will be ignored.
+   * @param defaultAiProvider Optional default AI provider to use.
    * @returns A promise that resolves to the TrackedChat instance, or null if the configuration is disabled.
    *
    * @example
@@ -161,32 +248,78 @@ export interface LDAIClient {
    * const key = "customer_support_chat";
    * const context = {...};
    * const defaultValue = {
-   *   config: {
-   *     enabled: false,
-   *     model: { name: "gpt-4" },
-   *     messages: [
-   *       { role: "system", content: "You are a helpful customer support agent." }
-   *     ]
-   *   }
+   *   enabled: false,
+   *   model: { name: "gpt-4" },
+   *   provider: { name: "openai" },
+   *   messages: [
+   *     { role: "system", content: "You are a helpful customer support agent." }
+   *   ]
    * };
    * const variables = { customerName: 'John' };
    *
-   * const chat = await client.initChat(key, context, defaultValue, variables);
+   * const chat = await client.createChat(key, context, defaultValue, variables);
    * if (chat) {
    *   const response = await chat.invoke("I need help with my order");
    *   console.log(response.message.content);
-   *
-   *   // Access configuration and tracker if needed
-   *   console.log('Model:', chat.getConfig().model?.name);
-   *   chat.getTracker().trackSuccess();
    * }
    * ```
    */
+  createChat(
+    key: string,
+    context: LDContext,
+    defaultValue: LDAICompletionConfigDefault,
+    variables?: Record<string, unknown>,
+    defaultAiProvider?: SupportedAIProvider,
+  ): Promise<TrackedChat | undefined>;
+
+  /**
+   * @deprecated Use `createChat` instead. This method will be removed in a future version.
+   */
   initChat(
     key: string,
     context: LDContext,
-    defaultValue: LDAIDefaults,
+    defaultValue: LDAICompletionConfigDefault,
     variables?: Record<string, unknown>,
     defaultAiProvider?: SupportedAIProvider,
   ): Promise<TrackedChat | undefined>;
+
+  /**
+   * Creates and returns a new Judge instance for AI evaluation.
+   *
+   * @param key The key identifying the AI judge configuration to use
+   * @param context Standard LDContext used when evaluating flags
+   * @param defaultValue A default value representing a standard AI config result
+   * @param variables Dictionary of values for instruction interpolation.
+   * The variables `message_history` and `response_to_evaluate` are reserved for the judge and will be ignored.
+   * @param defaultAiProvider Optional default AI provider to use.
+   * @returns Promise that resolves to a Judge instance or undefined if disabled/unsupported
+   *
+   * @example
+   * ```
+   * const judge = await client.createJudge(
+   *   "relevance-judge",
+   *   context,
+   *   {
+   *     enabled: true,
+   *     model: { name: "gpt-4" },
+   *     provider: { name: "openai" },
+   *     evaluationMetricKeys: ['$ld:ai:judge:relevance'],
+   *     messages: [{ role: 'system', content: 'You are a relevance judge.' }]
+   *   },
+   *   { metric: "relevance" }
+   * );
+   *
+   * if (judge) {
+   *   const result = await judge.evaluate("User question", "AI response");
+   *   console.log('Relevance score:', result.evals.relevance?.score);
+   * }
+   * ```
+   */
+  createJudge(
+    key: string,
+    context: LDContext,
+    defaultValue: LDAIJudgeConfigDefault,
+    variables?: Record<string, unknown>,
+    defaultAiProvider?: SupportedAIProvider,
+  ): Promise<Judge | undefined>;
 }

package/src/api/chat/TrackedChat.ts

@@ -1,5 +1,9 @@
-import { LDAIConfig, LDMessage } from '../config/LDAIConfig';
+import { LDLogger } from '@launchdarkly/js-server-sdk-common';
+
 import { LDAIConfigTracker } from '../config/LDAIConfigTracker';
+import { LDAICompletionConfig, LDMessage } from '../config/types';
+import { Judge } from '../judge/Judge';
+import { JudgeResponse } from '../judge/types';
 import { AIProvider } from '../providers/AIProvider';
 import { ChatResponse } from './types';
 
@@ -13,9 +17,11 @@ export class TrackedChat {
   protected messages: LDMessage[];
 
   constructor(
-    protected readonly aiConfig: LDAIConfig,
+    protected readonly aiConfig: LDAICompletionConfig,
     protected readonly tracker: LDAIConfigTracker,
     protected readonly provider: AIProvider,
+    protected readonly judges: Record<string, Judge> = {},
+    private readonly _logger?: LDLogger,
   ) {
     this.messages = [];
   }
@@ -42,16 +48,61 @@ export class TrackedChat {
       () => this.provider.invokeModel(allMessages),
     );
 
-    // Add the assistant response to the conversation history
-    this.messages.push(response.message);
+    if (
+      this.aiConfig.judgeConfiguration?.judges &&
+      this.aiConfig.judgeConfiguration.judges.length > 0
+    ) {
+      response.evaluations = this._evaluateWithJudges(this.messages, response);
+    }
 
+    this.messages.push(response.message);
     return response;
   }
 
+  /**
+   * Evaluates the response with all configured judges.
+   * Returns a promise that resolves to an array of evaluation results.
+   *
+   * @param messages Array of messages representing the conversation history
+   * @param response The AI response to be evaluated
+   * @returns Promise resolving to array of judge evaluation results
+   */
+  private async _evaluateWithJudges(
+    messages: LDMessage[],
+    response: ChatResponse,
+  ): Promise<Array<JudgeResponse | undefined>> {
+    const judgeConfigs = this.aiConfig.judgeConfiguration!.judges;
+
+    // Start all judge evaluations in parallel
+    const evaluationPromises = judgeConfigs.map(async (judgeConfig) => {
+      const judge = this.judges[judgeConfig.key];
+      if (!judge) {
+        this._logger?.warn(
+          `Judge configuration is not enabled: ${judgeConfig.key}`,
+          this.tracker.getTrackData(),
+        );
+        return undefined;
+      }
+
+      const evalResult = await judge.evaluateMessages(messages, response, judgeConfig.samplingRate);
+
+      if (evalResult && evalResult.success) {
+        this.tracker.trackEvalScores(evalResult.evals);
+      }
+
+      return evalResult;
+    });
+
+    // ensure all evaluations complete even if some fail
+    const results = await Promise.allSettled(evaluationPromises);
+
+    return results.map((result) => (result.status === 'fulfilled' ? result.value : undefined));
+  }
+
   /**
    * Get the underlying AI configuration used to initialize this TrackedChat.
    */
-  getConfig(): LDAIConfig {
+  getConfig(): LDAICompletionConfig {
     return this.aiConfig;
   }
 
@@ -70,6 +121,14 @@ export class TrackedChat {
     return this.provider;
   }
 
+  /**
+   * Get the judges associated with this TrackedChat.
+   * Returns a record of judge instances keyed by their configuration keys.
+   */
+  getJudges(): Record<string, Judge> {
+    return this.judges;
+  }
+
   /**
    * Append messages to the conversation history.
    * Adds messages to the conversation history without invoking the model,

package/src/api/chat/types.ts

@@ -1,4 +1,5 @@
-import { LDMessage } from '../config/LDAIConfig';
+import { LDMessage } from '../config/types';
+import { JudgeResponse } from '../judge/types';
 import { LDAIMetrics } from '../metrics/LDAIMetrics';
 
 /**
@@ -14,4 +15,10 @@ export interface ChatResponse {
    * Metrics information including success status and token usage.
    */
   metrics: LDAIMetrics;
+
+  /**
+   * Promise that resolves to judge evaluation results.
+   * Only present when judges are configured for evaluation.
+   */
+  evaluations?: Promise<Array<JudgeResponse | undefined>>;
 }