@launchdarkly/server-sdk-ai 0.12.3 → 0.14.0

package/src/api/judge/Judge.ts

@@ -0,0 +1,216 @@
+import * as Mustache from 'mustache';
+
+import { LDLogger } from '@launchdarkly/js-server-sdk-common';
+
+import { ChatResponse } from '../chat/types';
+import { LDAIConfigTracker } from '../config/LDAIConfigTracker';
+import { LDAIJudgeConfig, LDMessage } from '../config/types';
+import { AIProvider } from '../providers/AIProvider';
+import { EvaluationSchemaBuilder } from './EvaluationSchemaBuilder';
+import { EvalScore, JudgeResponse, StructuredResponse } from './types';
+
+/**
+ * Judge implementation that handles evaluation functionality and conversation management.
+ *
+ * According to the AIEval spec, judges are AI Configs with mode: "judge" that evaluate
+ * other AI Configs using structured output.
+ */
+export class Judge {
+  private readonly _logger?: LDLogger;
+  private readonly _evaluationResponseStructure: Record<string, unknown>;
+
+  constructor(
+    private readonly _aiConfig: LDAIJudgeConfig,
+    private readonly _aiConfigTracker: LDAIConfigTracker,
+    private readonly _aiProvider: AIProvider,
+    logger?: LDLogger,
+  ) {
+    this._logger = logger;
+    this._evaluationResponseStructure = EvaluationSchemaBuilder.build(
+      this._aiConfig.evaluationMetricKeys,
+    );
+  }
+
+  /**
+   * Evaluates an AI response using the judge's configuration.
+   *
+   * @param input The input prompt or question that was provided to the AI
+   * @param output The AI-generated response to be evaluated
+   * @param samplingRate Sampling rate (0-1) to determine if evaluation should be processed (defaults to 1)
+   * @returns Promise that resolves to evaluation results or undefined if not sampled
+   */
+  async evaluate(
+    input: string,
+    output: string,
+    samplingRate: number = 1,
+  ): Promise<JudgeResponse | undefined> {
+    try {
+      if (
+        !this._aiConfig.evaluationMetricKeys ||
+        this._aiConfig.evaluationMetricKeys.length === 0
+      ) {
+        this._logger?.warn(
+          'Judge configuration is missing required evaluationMetricKeys',
+          this._aiConfigTracker.getTrackData(),
+        );
+        return undefined;
+      }
+
+      if (!this._aiConfig.messages) {
+        this._logger?.warn(
+          'Judge configuration must include messages',
+          this._aiConfigTracker.getTrackData(),
+        );
+        return undefined;
+      }
+
+      if (Math.random() > samplingRate) {
+        this._logger?.debug(`Judge evaluation skipped due to sampling rate: ${samplingRate}`);
+        return undefined;
+      }
+
+      const messages = this._constructEvaluationMessages(input, output);
+
+      const response = await this._aiConfigTracker.trackMetricsOf(
+        (result: StructuredResponse) => result.metrics,
+        () => this._aiProvider.invokeStructuredModel(messages, this._evaluationResponseStructure),
+      );
+
+      let { success } = response.metrics;
+
+      const evals = this._parseEvaluationResponse(response.data);
+
+      if (Object.keys(evals).length !== this._aiConfig.evaluationMetricKeys.length) {
+        this._logger?.warn(
+          'Judge evaluation did not return all evaluations',
+          this._aiConfigTracker.getTrackData(),
+        );
+        success = false;
+      }
+
+      return {
+        evals,
+        success,
+      };
+    } catch (error) {
+      this._logger?.error('Judge evaluation failed:', error);
+      return {
+        evals: {},
+        success: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+      };
+    }
+  }
+
+  /**
+   * Evaluates an AI response from chat messages and response.
+   *
+   * @param messages Array of messages representing the conversation history
+   * @param response The AI response to be evaluated
+   * @param samplingRatio Sampling ratio (0-1) to determine if evaluation should be processed (defaults to 1)
+   * @returns Promise that resolves to evaluation results or undefined if not sampled
+   */
+  async evaluateMessages(
+    messages: LDMessage[],
+    response: ChatResponse,
+    samplingRatio: number = 1,
+  ): Promise<JudgeResponse | undefined> {
+    const input = messages.length === 0 ? '' : messages.map((msg) => msg.content).join('\r\n');
+    const output = response.message.content;
+
+    return this.evaluate(input, output, samplingRatio);
+  }
+
+  /**
+   * Returns the AI Config used by this judge.
+   */
+  getAIConfig(): LDAIJudgeConfig {
+    return this._aiConfig;
+  }
+
+  /**
+   * Returns the tracker associated with this judge.
+   */
+  getTracker(): LDAIConfigTracker {
+    return this._aiConfigTracker;
+  }
+
+  /**
+   * Returns the AI provider used by this judge.
+   */
+  getProvider(): AIProvider {
+    return this._aiProvider;
+  }
+
+  /**
+   * Constructs evaluation messages by combining judge's config messages with input/output.
+   */
+  private _constructEvaluationMessages(input: string, output: string): LDMessage[] {
+    const messages: LDMessage[] = this._aiConfig.messages!.map((msg) => ({
+      ...msg,
+      content: this._interpolateMessage(msg.content, {
+        message_history: input,
+        response_to_evaluate: output,
+      }),
+    }));
+
+    return messages;
+  }
+
+  /**
+   * Interpolates message content with variables using Mustache templating.
+   */
+  private _interpolateMessage(content: string, variables: Record<string, string>): string {
+    return Mustache.render(content, variables, undefined, { escape: (item: any) => item });
+  }
+
+  /**
+   * Parses the structured evaluation response from the AI provider.
+   */
+  private _parseEvaluationResponse(data: Record<string, unknown>): Record<string, EvalScore> {
+    const evaluations = data.evaluations as Record<string, unknown>;
+    const results: Record<string, EvalScore> = {};
+
+    if (!data.evaluations || typeof data.evaluations !== 'object') {
+      this._logger?.warn('Invalid response: missing or invalid evaluations object');
+      return results;
+    }
+
+    this._aiConfig.evaluationMetricKeys.forEach((metricKey) => {
+      const evaluation = evaluations[metricKey];
+
+      if (!evaluation || typeof evaluation !== 'object') {
+        this._logger?.warn(
+          `Missing evaluation for metric key: ${metricKey}`,
+          this._aiConfigTracker.getTrackData(),
+        );
+        return;
+      }
+
+      const evalData = evaluation as Record<string, unknown>;
+
+      if (typeof evalData.score !== 'number' || evalData.score < 0 || evalData.score > 1) {
+        this._logger?.warn(
+          `Invalid score evaluated for ${metricKey}: ${evalData.score}. Score must be a number between 0 and 1 inclusive`,
+          this._aiConfigTracker.getTrackData(),
+        );
+        return;
+      }
+
+      if (typeof evalData.reasoning !== 'string') {
+        this._logger?.warn(
+          `Invalid reasoning evaluated for ${metricKey}: ${evalData.reasoning}. Reasoning must be a string`,
+          this._aiConfigTracker.getTrackData(),
+        );
+        return;
+      }
+
+      results[metricKey] = {
+        score: evalData.score,
+        reasoning: evalData.reasoning,
+      };
+    });
+
+    return results;
+  }
+}

package/src/api/judge/index.ts

@@ -0,0 +1,2 @@
+export { Judge } from './Judge';
+export type { EvalScore, JudgeResponse, StructuredResponse } from './types';

package/src/api/judge/types.ts

@@ -0,0 +1,39 @@
+import { LDAIMetrics } from '../metrics/LDAIMetrics';
+
+/**
+ * Structured response from AI models.
+ */
+export interface StructuredResponse {
+  /** The structured data returned by the model */
+  data: Record<string, unknown>;
+
+  /** The raw response from the model */
+  rawResponse: string;
+
+  /**
+   * Metrics information including success status and token usage.
+   */
+  metrics: LDAIMetrics;
+}
+
+/**
+ * Score and reasoning for a single evaluation metric.
+ */
+export interface EvalScore {
+  /** Score between 0.0 and 1.0 indicating the evaluation result for this metric */
+  score: number;
+  /** Reasoning behind the provided score for this metric */
+  reasoning: string;
+}
+
+/**
+ * Response from a judge evaluation containing scores and reasoning for multiple metrics.
+ */
+export interface JudgeResponse {
+  /** Dictionary where keys are metric names and values contain score and reasoning */
+  evals: Record<string, EvalScore>;
+  /** Whether the evaluation completed successfully */
+  success: boolean;
+  /** Error message if evaluation failed */
+  error?: string;
+}

package/src/api/providers/AIProvider.ts

@@ -1,7 +1,8 @@
 import { LDLogger } from '@launchdarkly/js-server-sdk-common';
 
 import { ChatResponse } from '../chat/types';
-import { LDAIConfig, LDMessage } from '../config/LDAIConfig';
+import { LDAIConfigKind, LDMessage } from '../config/types';
+import { StructuredResponse } from '../judge/types';
 
 /**
  * Abstract base class for AI providers that implement chat model functionality.
@@ -22,10 +23,60 @@ export abstract class AIProvider {
    * This method should convert messages to provider format, invoke the model,
    * and return a ChatResponse with the result and metrics.
    *
+   * Default implementation takes no action and returns a placeholder response.
+   * Provider implementations should override this method.
+   *
    * @param messages Array of LDMessage objects representing the conversation
    * @returns Promise that resolves to a ChatResponse containing the model's response
    */
-  abstract invokeModel(messages: LDMessage[]): Promise<ChatResponse>;
+  async invokeModel(_messages: LDMessage[]): Promise<ChatResponse> {
+    this.logger?.warn('invokeModel not implemented by this provider');
+    return {
+      message: {
+        role: 'assistant',
+        content: '',
+      },
+      metrics: {
+        success: false,
+        usage: {
+          total: 0,
+          input: 0,
+          output: 0,
+        },
+      },
+    };
+  }
+
+  /**
+   * Invoke the chat model with structured output support.
+   * This method should convert messages to provider format, invoke the model with
+   * structured output configuration, and return a structured response.
+   *
+   * Default implementation takes no action and returns a placeholder response.
+   * Provider implementations should override this method.
+   *
+   * @param messages Array of LDMessage objects representing the conversation
+   * @param responseStructure Dictionary of output configurations keyed by output name
+   * @returns Promise that resolves to a structured response
+   */
+  async invokeStructuredModel(
+    _messages: LDMessage[],
+    _responseStructure: Record<string, unknown>,
+  ): Promise<StructuredResponse> {
+    this.logger?.warn('invokeStructuredModel not implemented by this provider');
+    return {
+      data: {},
+      rawResponse: '',
+      metrics: {
+        success: false,
+        usage: {
+          total: 0,
+          input: 0,
+          output: 0,
+        },
+      },
+    };
+  }
 
   /**
    * Static method that constructs an instance of the provider.
@@ -37,7 +88,7 @@ export abstract class AIProvider {
    * @returns Promise that resolves to a configured provider instance
    */
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  static async create(aiConfig: LDAIConfig, logger?: LDLogger): Promise<AIProvider> {
+  static async create(aiConfig: LDAIConfigKind, logger?: LDLogger): Promise<AIProvider> {
     throw new Error('Provider implementations must override the static create method');
   }
 }

package/src/api/providers/AIProviderFactory.ts

@@ -1,6 +1,6 @@
 import { LDLogger } from '@launchdarkly/js-server-sdk-common';
 
-import { LDAIConfig } from '../config/LDAIConfig';
+import { LDAIConfigKind } from '../config/types';
 import { AIProvider } from './AIProvider';
 
 /**
@@ -32,7 +32,7 @@ export class AIProviderFactory {
    * @param defaultAiProvider Optional default AI provider to use
    */
   static async create(
-    aiConfig: LDAIConfig,
+    aiConfig: LDAIConfigKind,
     logger?: LDLogger,
     defaultAiProvider?: SupportedAIProvider,
   ): Promise<AIProvider | undefined> {
@@ -91,7 +91,7 @@ export class AIProviderFactory {
    */
   private static async _tryCreateProvider(
     providerType: SupportedAIProvider,
-    aiConfig: LDAIConfig,
+    aiConfig: LDAIConfigKind,
     logger?: LDLogger,
   ): Promise<AIProvider | undefined> {
     switch (providerType) {
@@ -127,7 +127,7 @@ export class AIProviderFactory {
   private static async _createProvider(
     packageName: string,
     providerClassName: string,
-    aiConfig: LDAIConfig,
+    aiConfig: LDAIConfigKind,
     logger?: LDLogger,
   ): Promise<AIProvider | undefined> {
     try {

package/tsconfig.json

@@ -1,6 +1,7 @@
 {
   "compilerOptions": {
-    "rootDir": "src",
+    // Uses "." so it can load package.json.
+    "rootDir": ".",
     "outDir": "dist",
     "target": "es2017",
     "lib": ["es6"],
@@ -16,6 +17,5 @@
     "stripInternal": true,
     "moduleResolution": "node"
   },
-  "include": ["src"],
-  "exclude": ["**/*.test.ts", "dist", "node_modules", "__tests__"]
+  "exclude": ["**/*.test.ts", "dist", "node_modules", "__tests__", "examples"]
 }

package/tsconfig.ref.json

@@ -1,6 +1,6 @@
 {
   "extends": "./tsconfig.json",
-  "include": ["src/**/*"],
+  "include": ["src/**/*", "package.json"],
   "compilerOptions": {
     "composite": true
   }

package/__tests__/LDAIConfigMapper.test.ts

@@ -1,159 +0,0 @@
-import { LDMessage, VercelAISDKMapOptions } from '../src/api/config';
-import { LDAIConfigMapper } from '../src/LDAIConfigMapper';
-
-describe('_findParameter', () => {
-  it('handles undefined model and messages', () => {
-    const mapper = new LDAIConfigMapper();
-    // eslint-disable-next-line @typescript-eslint/dot-notation
-    expect(mapper['_findParameter']<number>('test-param')).toBeUndefined();
-  });
-
-  it('handles parameter not found', () => {
-    const mapper = new LDAIConfigMapper({
-      name: 'test-ai-model',
-      parameters: {
-        'test-param': 123,
-      },
-      custom: {
-        'test-param': 456,
-      },
-    });
-    // eslint-disable-next-line @typescript-eslint/dot-notation
-    expect(mapper['_findParameter']<number>('other-param')).toBeUndefined();
-  });
-
-  it('finds parameter from single model parameter', () => {
-    const mapper = new LDAIConfigMapper({
-      name: 'test-ai-model',
-      parameters: {
-        'test-param': 123,
-      },
-    });
-    // eslint-disable-next-line @typescript-eslint/dot-notation
-    expect(mapper['_findParameter']<number>('test-param')).toEqual(123);
-  });
-
-  it('finds parameter from multiple model parameters', () => {
-    const mapper = new LDAIConfigMapper({
-      name: 'test-ai-model',
-      parameters: {
-        testParam: 123,
-      },
-    });
-    // eslint-disable-next-line @typescript-eslint/dot-notation
-    expect(mapper['_findParameter']<number>('test-param', 'testParam')).toEqual(123);
-  });
-
-  it('finds parameter from single model custom parameter', () => {
-    const mapper = new LDAIConfigMapper({
-      name: 'test-ai-model',
-      custom: {
-        'test-param': 123,
-      },
-    });
-    // eslint-disable-next-line @typescript-eslint/dot-notation
-    expect(mapper['_findParameter']<number>('test-param')).toEqual(123);
-  });
-
-  it('finds parameter from multiple model custom parameters', () => {
-    const mapper = new LDAIConfigMapper({
-      name: 'test-ai-model',
-      custom: {
-        testParam: 123,
-      },
-    });
-    // eslint-disable-next-line @typescript-eslint/dot-notation
-    expect(mapper['_findParameter']<number>('test-param', 'testParam')).toEqual(123);
-  });
-
-  it('gives precedence to model parameters over model custom parameters', () => {
-    const mapper = new LDAIConfigMapper({
-      name: 'test-ai-model',
-      parameters: {
-        'test-param': 123,
-      },
-      custom: {
-        'test-param': 456,
-      },
-    });
-    // eslint-disable-next-line @typescript-eslint/dot-notation
-    expect(mapper['_findParameter']<number>('test-param', 'testParam')).toEqual(123);
-  });
-});
-
-describe('toVercelAIAISDK', () => {
-  const mockModel = { name: 'mockModel' };
-  const mockMessages: LDMessage[] = [
-    { role: 'user', content: 'test prompt' },
-    { role: 'system', content: 'test instruction' },
-  ];
-  const mockOptions: VercelAISDKMapOptions = {
-    nonInterpolatedMessages: [{ role: 'assistant', content: 'test assistant instruction' }],
-  };
-  const mockProvider = jest.fn().mockReturnValue(mockModel);
-
-  beforeEach(() => {
-    jest.clearAllMocks();
-  });
-
-  it('handles undefined model and messages', () => {
-    const mapper = new LDAIConfigMapper();
-    const result = mapper.toVercelAISDK(mockProvider);
-
-    expect(mockProvider).toHaveBeenCalledWith('');
-    expect(result).toEqual(
-      expect.objectContaining({
-        model: mockModel,
-        messages: undefined,
-      }),
-    );
-  });
-
-  it('uses additional messages', () => {
-    const mapper = new LDAIConfigMapper({ name: 'test-ai-model' });
-    const result = mapper.toVercelAISDK(mockProvider, mockOptions);
-
-    expect(mockProvider).toHaveBeenCalledWith('test-ai-model');
-    expect(result).toEqual(
-      expect.objectContaining({
-        model: mockModel,
-        messages: mockOptions.nonInterpolatedMessages,
-      }),
-    );
-  });
-
-  it('combines config messages and additional messages', () => {
-    const mapper = new LDAIConfigMapper({ name: 'test-ai-model' }, undefined, mockMessages);
-    const result = mapper.toVercelAISDK(mockProvider, mockOptions);
-
-    expect(mockProvider).toHaveBeenCalledWith('test-ai-model');
-    expect(result).toEqual(
-      expect.objectContaining({
-        model: mockModel,
-        messages: [...mockMessages, ...(mockOptions.nonInterpolatedMessages ?? [])],
-      }),
-    );
-  });
-
-  it('requests parameters correctly', () => {
-    const mapper = new LDAIConfigMapper({ name: 'test-ai-model' }, undefined, mockMessages);
-    const findParameterMock = jest.spyOn(mapper as any, '_findParameter');
-    const result = mapper.toVercelAISDK(mockProvider);
-
-    expect(mockProvider).toHaveBeenCalledWith('test-ai-model');
-    expect(result).toEqual(
-      expect.objectContaining({
-        model: mockModel,
-        messages: mockMessages,
-      }),
-    );
-    expect(findParameterMock).toHaveBeenCalledWith('max_tokens', 'maxTokens');
-    expect(findParameterMock).toHaveBeenCalledWith('temperature');
-    expect(findParameterMock).toHaveBeenCalledWith('top_p', 'topP');
-    expect(findParameterMock).toHaveBeenCalledWith('top_k', 'topK');
-    expect(findParameterMock).toHaveBeenCalledWith('presence_penalty', 'presencePenalty');
-    expect(findParameterMock).toHaveBeenCalledWith('frequency_penalty', 'frequencyPenalty');
-    expect(findParameterMock).toHaveBeenCalledWith('stop', 'stop_sequences', 'stopSequences');
-    expect(findParameterMock).toHaveBeenCalledWith('seed');
-  });
-});

package/dist/LDAIClientImpl.d.ts

@@ -1,23 +0,0 @@
-import { LDContext } from '@launchdarkly/js-server-sdk-common';
-import { LDAIAgent, LDAIAgentConfig, LDAIAgentDefaults } from './api/agents';
-import { TrackedChat } from './api/chat';
-import { LDAIConfig, LDAIDefaults } from './api/config';
-import { LDAIClient } from './api/LDAIClient';
-import { SupportedAIProvider } from './api/providers';
-import { LDClientMin } from './LDClientMin';
-export declare class LDAIClientImpl implements LDAIClient {
-    private _ldClient;
-    private _logger?;
-    constructor(_ldClient: LDClientMin);
-    private _interpolateTemplate;
-    private static _toLDFlagValue;
-    private _evaluate;
-    private _evaluateAgent;
-    config(key: string, context: LDContext, defaultValue: LDAIDefaults, variables?: Record<string, unknown>): Promise<LDAIConfig>;
-    agent(key: string, context: LDContext, defaultValue: LDAIAgentDefaults, variables?: Record<string, unknown>): Promise<LDAIAgent>;
-    agents<const T extends readonly (LDAIAgentConfig & {
-        defaultValue: LDAIAgentDefaults;
-    })[]>(agentConfigs: T, context: LDContext): Promise<Record<T[number]['key'], LDAIAgent>>;
-    initChat(key: string, context: LDContext, defaultValue: LDAIDefaults, variables?: Record<string, unknown>, defaultAiProvider?: SupportedAIProvider): Promise<TrackedChat | undefined>;
-}
-//# sourceMappingURL=LDAIClientImpl.d.ts.map

package/dist/LDAIClientImpl.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"LDAIClientImpl.d.ts","sourceRoot":"","sources":["../src/LDAIClientImpl.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAY,MAAM,oCAAoC,CAAC;AAEzE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EACL,UAAU,EAEV,YAAY,EAOb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAqB,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAGzE,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAuC5C,qBAAa,cAAe,YAAW,UAAU;IAGnC,OAAO,CAAC,SAAS;IAF7B,OAAO,CAAC,OAAO,CAAC,CAAW;gBAEP,SAAS,EAAE,WAAW;IAI1C,OAAO,CAAC,oBAAoB;IAI5B,OAAO,CAAC,MAAM,CAAC,cAAc;YAgBf,SAAS;YAsCT,cAAc;IAsCtB,MAAM,CACV,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAClC,OAAO,CAAC,UAAU,CAAC;IA4ChB,KAAK,CACT,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,iBAAiB,EAC/B,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAClC,OAAO,CAAC,SAAS,CAAC;IAOf,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,SAAS,CAAC,eAAe,GAAG;QAAE,YAAY,EAAE,iBAAiB,CAAA;KAAE,CAAC,EAAE,EAC7F,YAAY,EAAE,CAAC,EACf,OAAO,EAAE,SAAS,GACjB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;IA0BzC,QAAQ,CACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,SAAS,EAClB,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnC,iBAAiB,CAAC,EAAE,mBAAmB,GACtC,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;CAqBpC"}