@launchdarkly/server-sdk-ai 0.14.0 → 0.14.1

package/src/LDAIConfigTrackerImpl.ts

@@ -3,7 +3,7 @@ import { LDContext } from '@launchdarkly/js-server-sdk-common';
 import { name as aiSdkName, version as aiSdkVersion } from '../package.json';
 import { LDAIConfigTracker } from './api/config';
 import { LDAIMetricSummary } from './api/config/LDAIConfigTracker';
-import { EvalScore } from './api/judge/types';
+import { EvalScore, JudgeResponse } from './api/judge/types';
 import {
   createBedrockTokenUsage,
   createOpenAiUsage,
@@ -76,12 +76,22 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
   }
 
   trackEvalScores(scores: Record<string, EvalScore>) {
-    // Track each evaluation score individually
     Object.entries(scores).forEach(([metricKey, evalScore]) => {
       this._ldClient.track(metricKey, this._context, this.getTrackData(), evalScore.score);
     });
   }
 
+  trackJudgeResponse(response: JudgeResponse) {
+    Object.entries(response.evals).forEach(([metricKey, evalScore]) => {
+      this._ldClient.track(
+        metricKey,
+        this._context,
+        { ...this.getTrackData(), judgeConfigKey: response.judgeConfigKey },
+        evalScore.score,
+      );
+    });
+  }
+
   trackFeedback(feedback: { kind: LDFeedbackKind }): void {
     this._trackedMetrics.feedback = feedback;
     if (feedback.kind === LDFeedbackKind.Positive) {

package/src/api/chat/TrackedChat.ts

@@ -84,13 +84,17 @@ export class TrackedChat {
         return undefined;
       }
 
-      const evalResult = await judge.evaluateMessages(messages, response, judgeConfig.samplingRate);
-
-      if (evalResult && evalResult.success) {
-        this.tracker.trackEvalScores(evalResult.evals);
+      const judgeResponse = await judge.evaluateMessages(
+        messages,
+        response,
+        judgeConfig.samplingRate,
+      );
+
+      if (judgeResponse && judgeResponse.success) {
+        this.tracker.trackJudgeResponse(judgeResponse);
       }
 
-      return evalResult;
+      return judgeResponse;
     });
 
     // ensure all evaluations complete even if some fail

package/src/api/config/LDAIConfigTracker.ts

@@ -1,4 +1,4 @@
-import { EvalScore } from '../judge/types';
+import { EvalScore, JudgeResponse } from '../judge/types';
 import { LDAIMetrics, LDFeedbackKind, LDTokenUsage } from '../metrics';
 
 /**
@@ -94,6 +94,13 @@ export interface LDAIConfigTracker {
    */
   trackEvalScores(scores: Record<string, EvalScore>): void;
 
+  /**
+   * Track a judge response containing evaluation scores and judge configuration key.
+   *
+   * @param response Judge response containing evaluation scores and judge configuration key
+   */
+  trackJudgeResponse(response: JudgeResponse): void;
+
   /**
    * Track the duration of execution of the provided function.
    *

package/src/api/config/LDAIConfigUtils.ts

@@ -82,19 +82,23 @@ export class LDAIConfigUtils {
    * @param tracker The tracker to add to the config
    * @returns The appropriate AI configuration type
    */
-  static fromFlagValue(flagValue: LDAIConfigFlagValue, tracker: LDAIConfigTracker): LDAIConfigKind {
+  static fromFlagValue(
+    key: string,
+    flagValue: LDAIConfigFlagValue,
+    tracker: LDAIConfigTracker,
+  ): LDAIConfigKind {
     // Determine the actual mode from flag value
     // eslint-disable-next-line no-underscore-dangle
     const flagValueMode = flagValue._ldMeta?.mode;
 
     switch (flagValueMode) {
       case 'agent':
-        return this.toAgentConfig(flagValue, tracker);
+        return this.toAgentConfig(key, flagValue, tracker);
       case 'judge':
-        return this.toJudgeConfig(flagValue, tracker);
+        return this.toJudgeConfig(key, flagValue, tracker);
       case 'completion':
       default:
-        return this.toCompletionConfig(flagValue, tracker);
+        return this.toCompletionConfig(key, flagValue, tracker);
     }
   }
 
@@ -104,15 +108,17 @@ export class LDAIConfigUtils {
    * @param mode The mode for the disabled config
    * @returns A disabled config of the appropriate type
    */
-  static createDisabledConfig(mode: LDAIConfigMode): LDAIConfigKind {
+  static createDisabledConfig(key: string, mode: LDAIConfigMode): LDAIConfigKind {
     switch (mode) {
       case 'agent':
         return {
+          key,
           enabled: false,
           tracker: undefined,
         } as LDAIAgentConfig;
       case 'judge':
         return {
+          key,
           enabled: false,
           tracker: undefined,
           evaluationMetricKeys: [],
@@ -121,6 +127,7 @@ export class LDAIConfigUtils {
       default:
         // Default to completion config for completion mode or any unexpected mode
         return {
+          key,
           enabled: false,
           tracker: undefined,
         } as LDAICompletionConfig;
@@ -133,8 +140,9 @@ export class LDAIConfigUtils {
    * @param flagValue The flag value from LaunchDarkly
    * @returns Base configuration object
    */
-  private static _toBaseConfig(flagValue: LDAIConfigFlagValue) {
+  private static _toBaseConfig(key: string, flagValue: LDAIConfigFlagValue) {
     return {
+      key,
       // eslint-disable-next-line no-underscore-dangle
       enabled: flagValue._ldMeta?.enabled ?? false,
       model: flagValue.model,
@@ -150,11 +158,12 @@ export class LDAIConfigUtils {
    * @returns A completion configuration
    */
   static toCompletionConfig(
+    key: string,
     flagValue: LDAIConfigFlagValue,
     tracker: LDAIConfigTracker,
   ): LDAICompletionConfig {
     return {
-      ...this._toBaseConfig(flagValue),
+      ...this._toBaseConfig(key, flagValue),
       tracker,
       messages: flagValue.messages,
       judgeConfiguration: flagValue.judgeConfiguration,
@@ -169,11 +178,12 @@ export class LDAIConfigUtils {
    * @returns An agent configuration
    */
   static toAgentConfig(
+    key: string,
     flagValue: LDAIConfigFlagValue,
     tracker: LDAIConfigTracker,
   ): LDAIAgentConfig {
     return {
-      ...this._toBaseConfig(flagValue),
+      ...this._toBaseConfig(key, flagValue),
       tracker,
       instructions: flagValue.instructions,
       judgeConfiguration: flagValue.judgeConfiguration,
@@ -188,11 +198,12 @@ export class LDAIConfigUtils {
    * @returns A judge configuration
    */
   static toJudgeConfig(
+    key: string,
     flagValue: LDAIConfigFlagValue,
     tracker: LDAIConfigTracker,
   ): LDAIJudgeConfig {
     return {
-      ...this._toBaseConfig(flagValue),
+      ...this._toBaseConfig(key, flagValue),
       tracker,
       messages: flagValue.messages,
       evaluationMetricKeys: flagValue.evaluationMetricKeys || [],

package/src/api/config/types.ts

@@ -95,6 +95,10 @@ export interface LDAIConfigDefault {
  * Base AI Config interface without mode-specific fields.
  */
 export interface LDAIConfig extends Omit<LDAIConfigDefault, 'enabled'> {
+  /**
+   * The key of the AI Config.
+   */
+  key: string;
   /**
    * Whether the configuration is enabled.
    */

package/src/api/judge/Judge.ts

@@ -91,6 +91,7 @@ export class Judge {
       return {
         evals,
         success,
+        judgeConfigKey: this._aiConfig.key,
       };
     } catch (error) {
       this._logger?.error('Judge evaluation failed:', error);
@@ -98,6 +99,7 @@ export class Judge {
         evals: {},
         success: false,
         error: error instanceof Error ? error.message : 'Unknown error',
+        judgeConfigKey: this._aiConfig.key,
       };
     }
   }

package/src/api/judge/types.ts

@@ -30,6 +30,8 @@ export interface EvalScore {
  * Response from a judge evaluation containing scores and reasoning for multiple metrics.
  */
 export interface JudgeResponse {
+  /** The key of the judge configuration that was used to generate this response */
+  judgeConfigKey?: string;
   /** Dictionary where keys are metric names and values contain score and reasoning */
   evals: Record<string, EvalScore>;
   /** Whether the evaluation completed successfully */