@launchdarkly/server-sdk-ai 0.12.3 → 0.14.0

package/CHANGELOG.md

@@ -1,5 +1,49 @@
 # Changelog
 
+## [0.14.0](https://github.com/launchdarkly/js-core/compare/server-sdk-ai-v0.13.0...server-sdk-ai-v0.14.0) (2025-11-06)
+
+
+### ⚠ BREAKING CHANGES
+
+* Removed deprecated Vercel methods ([#983](https://github.com/launchdarkly/js-core/issues/983))
+* Add support for real time judge evals ([#969](https://github.com/launchdarkly/js-core/issues/969))
+* AI Config defaults require the "enabled" attribute
+* Renamed LDAIAgentConfig to LDAIAgentConfigRequest for clarity
+* Renamed LDAIAgent to LDAIAgentConfig *note the previous use of this name
+* Renamed LDAIAgentDefault to LDAIAgentConfigDefault for clarity
+* Renamed LDAIDefaults to LDAICompletionConfigDefault for clarity
+
+### Features
+
+* Add support for real time judge evals ([#969](https://github.com/launchdarkly/js-core/issues/969)) ([6ecd9ab](https://github.com/launchdarkly/js-core/commit/6ecd9ab4d97f6445adfd377709f14d7f3b420363))
+* Added createJudge method ([6ecd9ab](https://github.com/launchdarkly/js-core/commit/6ecd9ab4d97f6445adfd377709f14d7f3b420363))
+* Added judgeConfig method to AI SDK to retrieve an AI Judge Config ([6ecd9ab](https://github.com/launchdarkly/js-core/commit/6ecd9ab4d97f6445adfd377709f14d7f3b420363))
+* Added trackEvalScores method to config tracker ([6ecd9ab](https://github.com/launchdarkly/js-core/commit/6ecd9ab4d97f6445adfd377709f14d7f3b420363))
+* Chat will evaluate responses with configured judges ([6ecd9ab](https://github.com/launchdarkly/js-core/commit/6ecd9ab4d97f6445adfd377709f14d7f3b420363))
+* Include AI SDK version in tracking information ([#985](https://github.com/launchdarkly/js-core/issues/985)) ([ef90564](https://github.com/launchdarkly/js-core/commit/ef90564ee1ed9411e77b836d2b5b8037ff671b07))
+* Removed deprecated Vercel methods ([#983](https://github.com/launchdarkly/js-core/issues/983)) ([960a499](https://github.com/launchdarkly/js-core/commit/960a49927e795890e5093b0156ec6d721c3066fd))
+
+
+### Bug Fixes
+
+* AI Config defaults require the "enabled" attribute ([6ecd9ab](https://github.com/launchdarkly/js-core/commit/6ecd9ab4d97f6445adfd377709f14d7f3b420363))
+* Renamed LDAIAgent to LDAIAgentConfig *note the previous use of this name ([6ecd9ab](https://github.com/launchdarkly/js-core/commit/6ecd9ab4d97f6445adfd377709f14d7f3b420363))
+* Renamed LDAIAgentConfig to LDAIAgentConfigRequest for clarity ([6ecd9ab](https://github.com/launchdarkly/js-core/commit/6ecd9ab4d97f6445adfd377709f14d7f3b420363))
+* Renamed LDAIAgentDefault to LDAIAgentConfigDefault for clarity ([6ecd9ab](https://github.com/launchdarkly/js-core/commit/6ecd9ab4d97f6445adfd377709f14d7f3b420363))
+* Renamed LDAIDefaults to LDAICompletionConfigDefault for clarity ([6ecd9ab](https://github.com/launchdarkly/js-core/commit/6ecd9ab4d97f6445adfd377709f14d7f3b420363))
+
+## [0.13.0](https://github.com/launchdarkly/js-core/compare/server-sdk-ai-v0.12.3...server-sdk-ai-v0.13.0) (2025-11-04)
+
+
+### Features
+
+* Add support for trackStreamMetricsOf method ([#971](https://github.com/launchdarkly/js-core/issues/971)) ([e18979e](https://github.com/launchdarkly/js-core/commit/e18979e27f4542552762a30a390749541daa3749))
+
+
+### Bug Fixes
+
+* Deprecated toVercelAISDK, trackVercelAISDKStreamTextMetrics, use `@launchdarkly/server-sdk-ai-vercel` package ([e18979e](https://github.com/launchdarkly/js-core/commit/e18979e27f4542552762a30a390749541daa3749))
+
 ## [0.12.3](https://github.com/launchdarkly/js-core/compare/server-sdk-ai-v0.12.2...server-sdk-ai-v0.12.3) (2025-10-24)
 
 

package/README.md

@@ -97,7 +97,7 @@ if (aiConfig.enabled) {
 
 ```typescript
 // Use the same defaultConfig from the retrieval section above
-const chat = await aiClient.initChat(
+const chat = await aiClient.createChat(
   'customer-support-chat',
   context,
   defaultConfig,
@@ -134,7 +134,7 @@ const llm = await LangChainProvider.createLangChainModel(aiConfig);
 
 // Use with tracking
 const response = await aiConfig.tracker.trackMetricsOf(
-  (result) => LangChainProvider.createAIMetrics(result),
+  LangChainProvider.getAIMetricsFromResponse,
   () => llm.invoke(messages)
 );
 

package/__tests__/Judge.test.ts

@@ -0,0 +1,496 @@
+import { LDLogger } from '@launchdarkly/js-server-sdk-common';
+
+import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker';
+import { LDAIJudgeConfig, LDMessage } from '../src/api/config/types';
+import { Judge } from '../src/api/judge/Judge';
+import { StructuredResponse } from '../src/api/judge/types';
+import { AIProvider } from '../src/api/providers/AIProvider';
+
+describe('Judge', () => {
+  let mockProvider: jest.Mocked<AIProvider>;
+  let mockTracker: jest.Mocked<LDAIConfigTracker>;
+  let mockLogger: jest.Mocked<LDLogger>;
+  let judgeConfig: LDAIJudgeConfig;
+
+  const mockTrackData = {
+    variationKey: 'test-variation',
+    configKey: 'test-config',
+    version: 1,
+  };
+
+  beforeEach(() => {
+    // Mock the AIProvider - only mock what's actually used
+    mockProvider = {
+      invokeStructuredModel: jest.fn(),
+    } as any;
+
+    // Mock the LDAIConfigTracker - only mock what's actually used
+    mockTracker = {
+      trackMetricsOf: jest.fn(),
+      getTrackData: jest.fn().mockReturnValue(mockTrackData),
+    } as any;
+
+    // Mock the logger - only mock what's actually used
+    mockLogger = {
+      debug: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn(),
+    } as any;
+
+    // Create a basic judge config
+    judgeConfig = {
+      enabled: true,
+      messages: [
+        { role: 'system', content: 'You are a helpful judge that evaluates AI responses.' },
+        {
+          role: 'user',
+          content:
+            'Evaluate and report scores for important metrics: Input: {{message_history}}, Output: {{response_to_evaluate}}',
+        },
+      ],
+      model: { name: 'gpt-4' },
+      provider: { name: 'openai' },
+      tracker: mockTracker,
+      evaluationMetricKeys: ['relevance', 'accuracy', 'helpfulness'],
+    };
+  });
+
+  describe('constructor', () => {
+    it('initializes with proper configuration', () => {
+      const judge = new Judge(judgeConfig, mockTracker, mockProvider, mockLogger);
+
+      expect(judge).toBeDefined();
+    });
+  });
+
+  describe('evaluate', () => {
+    let judge: Judge;
+
+    beforeEach(() => {
+      judge = new Judge(judgeConfig, mockTracker, mockProvider, mockLogger);
+    });
+
+    it('evaluates AI response successfully', async () => {
+      const mockStructuredResponse: StructuredResponse = {
+        data: {
+          evaluations: {
+            relevance: { score: 0.8, reasoning: 'The response is relevant to the question' },
+            accuracy: { score: 0.9, reasoning: 'The response is factually accurate' },
+            helpfulness: { score: 0.7, reasoning: 'The response provides helpful information' },
+          },
+        },
+        rawResponse: JSON.stringify({
+          evaluations: {
+            relevance: { score: 0.8, reasoning: 'The response is relevant to the question' },
+            accuracy: { score: 0.9, reasoning: 'The response is factually accurate' },
+            helpfulness: { score: 0.7, reasoning: 'The response provides helpful information' },
+          },
+        }),
+        metrics: {
+          success: true,
+          usage: {
+            total: 100,
+            input: 50,
+            output: 50,
+          },
+        },
+      };
+
+      mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func());
+      mockProvider.invokeStructuredModel.mockResolvedValue(mockStructuredResponse);
+
+      const result = await judge.evaluate(
+        'What is the capital of France?',
+        'Paris is the capital of France.',
+      );
+
+      expect(result).toEqual({
+        evals: {
+          relevance: { score: 0.8, reasoning: 'The response is relevant to the question' },
+          accuracy: { score: 0.9, reasoning: 'The response is factually accurate' },
+          helpfulness: { score: 0.7, reasoning: 'The response provides helpful information' },
+        },
+        success: true,
+      });
+
+      expect(mockProvider.invokeStructuredModel).toHaveBeenCalledWith(
+        expect.arrayContaining([
+          expect.objectContaining({
+            role: 'system',
+            content: 'You are a helpful judge that evaluates AI responses.',
+          }),
+          expect.objectContaining({
+            role: 'user',
+            content:
+              'Evaluate and report scores for important metrics: Input: What is the capital of France?, Output: Paris is the capital of France.',
+          }),
+        ]),
+        expect.any(Object), // evaluation response structure
+      );
+    });
+
+    it('handles sampling rate correctly', async () => {
+      // Mock Math.random to return 0.3 (should be sampled with rate 0.5 since 0.3 <= 0.5)
+      const originalRandom = Math.random;
+      Math.random = jest.fn().mockReturnValue(0.3);
+
+      // Mock the structured response
+      const mockStructuredResponse: StructuredResponse = {
+        data: {
+          evaluations: {
+            relevance: { score: 0.8, reasoning: 'Good' },
+            accuracy: { score: 0.9, reasoning: 'Accurate' },
+            helpfulness: { score: 0.7, reasoning: 'Helpful' },
+          },
+        },
+        rawResponse: JSON.stringify({
+          evaluations: {
+            relevance: { score: 0.8, reasoning: 'Good' },
+            accuracy: { score: 0.9, reasoning: 'Accurate' },
+            helpfulness: { score: 0.7, reasoning: 'Helpful' },
+          },
+        }),
+        metrics: {
+          success: true,
+          usage: { total: 100, input: 50, output: 50 },
+        },
+      };
+
+      mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func());
+      mockProvider.invokeStructuredModel.mockResolvedValue(mockStructuredResponse);
+
+      const result = await judge.evaluate('test input', 'test output', 0.5);
+
+      expect(result).toBeDefined();
+      expect(mockProvider.invokeStructuredModel).toHaveBeenCalled();
+
+      Math.random = originalRandom;
+    });
+
+    it('returns undefined when not sampled', async () => {
+      // Mock Math.random to return 0.8 (should not be sampled with rate 0.5 since 0.8 > 0.5)
+      const originalRandom = Math.random;
+      Math.random = jest.fn().mockReturnValue(0.8);
+
+      const result = await judge.evaluate('test input', 'test output', 0.5);
+
+      expect(result).toBeUndefined();
+      expect(mockProvider.invokeStructuredModel).not.toHaveBeenCalled();
+      expect(mockLogger.debug).toHaveBeenCalledWith(
+        'Judge evaluation skipped due to sampling rate: 0.5',
+      );
+
+      Math.random = originalRandom;
+    });
+
+    it('returns undefined when evaluationMetricKeys is empty', async () => {
+      const configWithoutMetrics: LDAIJudgeConfig = {
+        ...judgeConfig,
+        evaluationMetricKeys: [],
+      };
+      const judgeWithoutMetrics = new Judge(
+        configWithoutMetrics,
+        mockTracker,
+        mockProvider,
+        mockLogger,
+      );
+
+      const result = await judgeWithoutMetrics.evaluate('test input', 'test output');
+
+      expect(result).toBeUndefined();
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Judge configuration is missing required evaluationMetricKeys',
+        mockTrackData,
+      );
+    });
+
+    it('returns undefined when messages are missing', async () => {
+      const configWithoutMessages: LDAIJudgeConfig = {
+        ...judgeConfig,
+        messages: undefined,
+      };
+      const judgeWithoutMessages = new Judge(
+        configWithoutMessages,
+        mockTracker,
+        mockProvider,
+        mockLogger,
+      );
+
+      const result = await judgeWithoutMessages.evaluate('test input', 'test output');
+
+      expect(result).toBeUndefined();
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Judge configuration must include messages',
+        mockTrackData,
+      );
+    });
+
+    it('returns partial evaluations when some metrics are missing', async () => {
+      const mockStructuredResponse: StructuredResponse = {
+        data: {
+          evaluations: {
+            relevance: { score: 0.8, reasoning: 'Good' },
+            // accuracy is missing
+            helpfulness: { score: 0.7, reasoning: 'Helpful' },
+          },
+        },
+        rawResponse: JSON.stringify({
+          evaluations: {
+            relevance: { score: 0.8, reasoning: 'Good' },
+            helpfulness: { score: 0.7, reasoning: 'Helpful' },
+          },
+        }),
+        metrics: {
+          success: true,
+          usage: { total: 100, input: 50, output: 50 },
+        },
+      };
+
+      mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func());
+      mockProvider.invokeStructuredModel.mockResolvedValue(mockStructuredResponse);
+
+      const result = await judge.evaluate('test input', 'test output');
+
+      // When one metric is missing, it returns the partial evals it has with success: false
+      expect(result).toEqual({
+        evals: {
+          relevance: { score: 0.8, reasoning: 'Good' },
+          helpfulness: { score: 0.7, reasoning: 'Helpful' },
+        },
+        success: false,
+      });
+    });
+
+    it('returns empty evaluations when response structure is malformed', async () => {
+      const mockStructuredResponse: StructuredResponse = {
+        data: {
+          // Missing 'evaluations' wrapper - malformed structure
+          relevance: { score: 0.8, reasoning: 'Good' },
+          accuracy: { score: 0.9, reasoning: 'Accurate' },
+          helpfulness: { score: 0.7, reasoning: 'Helpful' },
+        },
+        rawResponse: JSON.stringify({
+          relevance: { score: 0.8, reasoning: 'Good' },
+          accuracy: { score: 0.9, reasoning: 'Accurate' },
+          helpfulness: { score: 0.7, reasoning: 'Helpful' },
+        }),
+        metrics: {
+          success: true,
+          usage: { total: 100, input: 50, output: 50 },
+        },
+      };
+
+      mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func());
+      mockProvider.invokeStructuredModel.mockResolvedValue(mockStructuredResponse);
+
+      const result = await judge.evaluate('test input', 'test output');
+
+      // When the structure is completely wrong, returns empty evals with success: false
+      expect(result).toEqual({
+        evals: {},
+        success: false,
+      });
+    });
+
+    it('handles provider errors gracefully', async () => {
+      const error = new Error('Provider error');
+      mockTracker.trackMetricsOf.mockRejectedValue(error);
+
+      const result = await judge.evaluate('test input', 'test output');
+
+      expect(result).toEqual({
+        evals: {},
+        success: false,
+        error: 'Provider error',
+      });
+      expect(mockLogger.error).toHaveBeenCalledWith('Judge evaluation failed:', error);
+    });
+
+    it('handles non-Error exceptions', async () => {
+      mockTracker.trackMetricsOf.mockRejectedValue('String error');
+
+      const result = await judge.evaluate('test input', 'test output');
+
+      expect(result).toEqual({
+        evals: {},
+        success: false,
+        error: 'Unknown error',
+      });
+    });
+  });
+
+  describe('evaluateMessages', () => {
+    let judge: Judge;
+
+    beforeEach(() => {
+      judge = new Judge(judgeConfig, mockTracker, mockProvider, mockLogger);
+    });
+
+    it('evaluates messages and response successfully', async () => {
+      const messages: LDMessage[] = [
+        { role: 'user', content: 'What is the capital of France?' },
+        { role: 'assistant', content: 'Paris is the capital of France.' },
+      ];
+      const response = {
+        message: { role: 'assistant' as const, content: 'Paris is the capital of France.' },
+        metrics: { success: true },
+      };
+
+      const mockStructuredResponse: StructuredResponse = {
+        data: {
+          evaluations: {
+            relevance: { score: 0.8, reasoning: 'The response is relevant to the question' },
+            accuracy: { score: 0.9, reasoning: 'The response is factually accurate' },
+            helpfulness: { score: 0.7, reasoning: 'The response provides helpful information' },
+          },
+        },
+        rawResponse: JSON.stringify({
+          evaluations: {
+            relevance: { score: 0.8, reasoning: 'The response is relevant to the question' },
+            accuracy: { score: 0.9, reasoning: 'The response is factually accurate' },
+            helpfulness: { score: 0.7, reasoning: 'The response provides helpful information' },
+          },
+        }),
+        metrics: {
+          success: true,
+          usage: { total: 100, input: 50, output: 50 },
+        },
+      };
+
+      mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func());
+      mockProvider.invokeStructuredModel.mockResolvedValue(mockStructuredResponse);
+
+      const result = await judge.evaluateMessages(messages, response);
+
+      expect(result).toEqual({
+        evals: {
+          relevance: { score: 0.8, reasoning: 'The response is relevant to the question' },
+          accuracy: { score: 0.9, reasoning: 'The response is factually accurate' },
+          helpfulness: { score: 0.7, reasoning: 'The response provides helpful information' },
+        },
+        success: true,
+      });
+
+      expect(mockProvider.invokeStructuredModel).toHaveBeenCalledWith(
+        expect.arrayContaining([
+          expect.objectContaining({
+            role: 'system',
+            content: 'You are a helpful judge that evaluates AI responses.',
+          }),
+          expect.objectContaining({
+            role: 'user',
+            content:
+              'Evaluate and report scores for important metrics: Input: What is the capital of France?\r\nParis is the capital of France., Output: Paris is the capital of France.',
+          }),
+        ]),
+        expect.any(Object), // evaluation response structure
+      );
+    });
+
+    it('handles sampling rate correctly', async () => {
+      const messages: LDMessage[] = [{ role: 'user', content: 'test' }];
+      const response = {
+        message: { role: 'assistant' as const, content: 'test response' },
+        metrics: { success: true },
+      };
+
+      // Mock Math.random to return 0.8 (should not be sampled with rate 0.5 since 0.8 > 0.5)
+      const originalRandom = Math.random;
+      Math.random = jest.fn().mockReturnValue(0.8);
+
+      const result = await judge.evaluateMessages(messages, response, 0.5);
+
+      expect(result).toBeUndefined();
+      expect(mockProvider.invokeStructuredModel).not.toHaveBeenCalled();
+
+      Math.random = originalRandom;
+    });
+  });
+
+  describe('_constructEvaluationMessages', () => {
+    let judge: Judge;
+
+    beforeEach(() => {
+      judge = new Judge(judgeConfig, mockTracker, mockProvider, mockLogger);
+    });
+
+    it('constructs evaluation messages correctly', () => {
+      // Access private method for testing
+      // eslint-disable-next-line no-underscore-dangle
+      const constructMessages = (judge as any)._constructEvaluationMessages.bind(judge);
+      const messages = constructMessages('test input', 'test output');
+
+      expect(messages).toHaveLength(2);
+      expect(messages[0]).toEqual({
+        role: 'system',
+        content: 'You are a helpful judge that evaluates AI responses.',
+      });
+      expect(messages[1]).toEqual({
+        role: 'user',
+        content:
+          'Evaluate and report scores for important metrics: Input: test input, Output: test output',
+      });
+    });
+  });
+
+  describe('_parseEvaluationResponse', () => {
+    let judge: Judge;
+
+    beforeEach(() => {
+      judge = new Judge(judgeConfig, mockTracker, mockProvider, mockLogger);
+    });
+
+    it('parses valid evaluation response correctly', () => {
+      // eslint-disable-next-line no-underscore-dangle
+      const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge);
+      const responseData = {
+        evaluations: {
+          relevance: { score: 0.8, reasoning: 'Good' },
+          accuracy: { score: 0.9, reasoning: 'Accurate' },
+          helpfulness: { score: 0.7, reasoning: 'Helpful' },
+        },
+      };
+
+      const result = parseResponse(responseData);
+
+      expect(result).toEqual({
+        relevance: { score: 0.8, reasoning: 'Good' },
+        accuracy: { score: 0.9, reasoning: 'Accurate' },
+        helpfulness: { score: 0.7, reasoning: 'Helpful' },
+      });
+    });
+
+    it('returns empty object for invalid response data', () => {
+      // eslint-disable-next-line no-underscore-dangle
+      const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge);
+      const responseData = {
+        relevance: { score: 0.8, reasoning: 'Good' },
+        // Missing evaluations wrapper - invalid structure
+      };
+
+      const result = parseResponse(responseData);
+
+      // Returns empty object when evaluations structure is missing
+      expect(result).toEqual({});
+    });
+
+    it('handles missing score or reasoning fields', () => {
+      // eslint-disable-next-line no-underscore-dangle
+      const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge);
+      const responseData = {
+        evaluations: {
+          relevance: { score: 0.8 }, // Missing reasoning
+          accuracy: { reasoning: 'Accurate' }, // Missing score
+          helpfulness: { score: 0.7, reasoning: 'Helpful' },
+        },
+      };
+
+      const result = parseResponse(responseData);
+
+      // Only helpfulness passes validation, relevance and accuracy are skipped
+      expect(result).toEqual({
+        helpfulness: { score: 0.7, reasoning: 'Helpful' },
+      });
+    });
+  });
+});