@launchdarkly/server-sdk-ai 0.14.1 → 0.15.0

package/src/api/config/types.ts

@@ -1,260 +0,0 @@
-import { LDAIConfigTracker } from './LDAIConfigTracker';
-
-// ============================================================================
-// Foundation Types
-// ============================================================================
-
-/**
- * Information about prompts.
- */
-export interface LDMessage {
-  /**
-   * The role of the prompt.
-   */
-  role: 'user' | 'assistant' | 'system';
-  /**
-   * Content for the prompt.
-   */
-  content: string;
-}
-
-/**
- * Configuration related to the model.
- */
-export interface LDModelConfig {
-  /**
-   * The ID of the model.
-   */
-  name: string;
-
-  /**
-   * Model specific parameters.
-   */
-  parameters?: { [index: string]: unknown };
-
-  /**
-   * Additional user-specified parameters.
-   */
-  custom?: { [index: string]: unknown };
-}
-
-export interface LDProviderConfig {
-  /**
-   * The name of the provider.
-   */
-  name: string;
-}
-
-// ============================================================================
-// Judge Types
-// ============================================================================
-
-/**
- * Configuration for a single judge attachment.
- */
-export interface LDJudge {
-  /** The key of the judge AI Config */
-  key: string;
-  /** Sampling rate for evaluation (0.0 to 1.0) */
-  samplingRate: number;
-}
-
-/**
- * Configuration for judge attachment to AI Configs.
- */
-export interface LDJudgeConfiguration {
-  /** Array of judge configurations */
-  judges: LDJudge[];
-}
-
-// ============================================================================
-// Base AI Config Types
-// ============================================================================
-
-/**
- * Base AI Config interface for default implementations with optional enabled property.
- */
-export interface LDAIConfigDefault {
-  /**
-   * Optional model configuration.
-   */
-  model?: LDModelConfig;
-
-  /**
-   * Optional configuration for the provider.
-   */
-  provider?: LDProviderConfig;
-
-  /**
-   * Whether the configuration is enabled. Defaults to false when not provided.
-   */
-  enabled?: boolean;
-}
-
-/**
- * Base AI Config interface without mode-specific fields.
- */
-export interface LDAIConfig extends Omit<LDAIConfigDefault, 'enabled'> {
-  /**
-   * The key of the AI Config.
-   */
-  key: string;
-  /**
-   * Whether the configuration is enabled.
-   */
-  enabled: boolean;
-
-  /**
-   * A tracker which can be used to generate analytics.
-   * Undefined for disabled configs.
-   */
-  tracker?: LDAIConfigTracker;
-}
-
-// ============================================================================
-// Default AI Config Implementation Types
-// ============================================================================
-
-/**
- * Default Agent-specific AI Config with instructions.
- */
-export interface LDAIAgentConfigDefault extends LDAIConfigDefault {
-  /**
-   * Instructions for the agent.
-   */
-  instructions?: string;
-  /**
-   * Judge configuration for AI Configs being evaluated.
-   * References judge AI Configs that should evaluate this AI Config.
-   */
-  judgeConfiguration?: LDJudgeConfiguration;
-}
-
-/**
- * Default Completion AI Config (default mode).
- */
-export interface LDAICompletionConfigDefault extends LDAIConfigDefault {
-  /**
-   * Optional prompt data for completion configurations.
-   */
-  messages?: LDMessage[];
-  /**
-   * Judge configuration for AI Configs being evaluated.
-   * References judge AI Configs that should evaluate this AI Config.
-   */
-  judgeConfiguration?: LDJudgeConfiguration;
-}
-
-/**
- * Default Judge-specific AI Config with required evaluation metric key.
- */
-export interface LDAIJudgeConfigDefault extends LDAIConfigDefault {
-  /**
-   * Optional prompt data for judge configurations.
-   */
-  messages?: LDMessage[];
-  /**
-   * Evaluation metric keys for judge configurations.
-   * The keys of the metrics that this judge can evaluate.
-   */
-  evaluationMetricKeys?: string[];
-}
-
-/**
- * Union type for all default AI Config variants.
- */
-export type LDAIConfigDefaultKind =
-  | LDAIAgentConfigDefault
-  | LDAICompletionConfigDefault
-  | LDAIJudgeConfigDefault;
-
-// ============================================================================
-// AI Config Implementation Types
-// ============================================================================
-
-/**
- * Agent-specific AI Config with instructions.
- */
-export interface LDAIAgentConfig extends LDAIConfig {
-  /**
-   * Instructions for the agent.
-   */
-  instructions?: string;
-  /**
-   * Judge configuration for AI Configs being evaluated.
-   * References judge AI Configs that should evaluate this AI Config.
-   */
-  judgeConfiguration?: LDJudgeConfiguration;
-}
-
-/**
- * Completion AI Config (default mode).
- */
-export interface LDAICompletionConfig extends LDAIConfig {
-  /**
-   * Optional prompt data for completion configurations.
-   */
-  messages?: LDMessage[];
-  /**
-   * Judge configuration for AI Configs being evaluated.
-   * References judge AI Configs that should evaluate this AI Config.
-   */
-  judgeConfiguration?: LDJudgeConfiguration;
-}
-
-/**
- * Judge-specific AI Config with required evaluation metric key.
- */
-export interface LDAIJudgeConfig extends LDAIConfig {
-  /**
-   * Optional prompt data for judge configurations.
-   */
-  messages?: LDMessage[];
-  /**
-   * Evaluation metric keys for judge configurations.
-   * The keys of the metrics that this judge can evaluate.
-   */
-  evaluationMetricKeys: string[];
-}
-
-// ============================================================================
-// Union Types
-// ============================================================================
-
-/**
- * Union type for all AI Config variants.
- */
-export type LDAIConfigKind = LDAIAgentConfig | LDAICompletionConfig | LDAIJudgeConfig;
-
-// ============================================================================
-// Agent-Specific Request Type
-// ============================================================================
-
-/**
- * Configuration for a single agent request.
- */
-export interface LDAIAgentRequestConfig {
-  /**
-   * The agent key to retrieve.
-   */
-  key: string;
-
-  /**
-   * Default configuration for the agent.
-   */
-  defaultValue: LDAIAgentConfigDefault;
-
-  /**
-   * Variables for instructions interpolation.
-   */
-  variables?: Record<string, unknown>;
-}
-
-// ============================================================================
-// Mode Type
-// ============================================================================
-
-/**
- * Mode type for AI configurations.
- */
-export type LDAIConfigMode = 'completion' | 'agent' | 'judge';

package/src/api/index.ts

@@ -1,6 +0,0 @@
-export * from './config';
-export * from './chat';
-export * from './judge';
-export * from './metrics';
-export * from './LDAIClient';
-export * from './providers';

package/src/api/judge/EvaluationSchemaBuilder.ts

@@ -1,54 +0,0 @@
-/**
- * Internal class for building dynamic evaluation response schemas.
- * Not exported - only used internally by TrackedJudge.
- */
-class EvaluationSchemaBuilder {
-  static build(evaluationMetricKeys: string[]): Record<string, unknown> {
-    return {
-      type: 'object',
-      properties: {
-        evaluations: {
-          type: 'object',
-          description: `Object containing evaluation results for ${evaluationMetricKeys.join(', ')} metrics`,
-          properties: this._buildKeyProperties(evaluationMetricKeys),
-          required: evaluationMetricKeys,
-          additionalProperties: false,
-        },
-      },
-      required: ['evaluations'],
-      additionalProperties: false,
-    } as const;
-  }
-
-  private static _buildKeyProperties(evaluationMetricKeys: string[]) {
-    return evaluationMetricKeys.reduce(
-      (acc, key) => {
-        acc[key] = this._buildKeySchema(key);
-        return acc;
-      },
-      {} as Record<string, unknown>,
-    );
-  }
-
-  private static _buildKeySchema(key: string) {
-    return {
-      type: 'object',
-      properties: {
-        score: {
-          type: 'number',
-          minimum: 0,
-          maximum: 1,
-          description: `Score between 0.0 and 1.0 for ${key}`,
-        },
-        reasoning: {
-          type: 'string',
-          description: `Reasoning behind the score for ${key}`,
-        },
-      },
-      required: ['score', 'reasoning'],
-      additionalProperties: false,
-    };
-  }
-}
-
-export { EvaluationSchemaBuilder };

package/src/api/judge/Judge.ts

@@ -1,218 +0,0 @@
-import * as Mustache from 'mustache';
-
-import { LDLogger } from '@launchdarkly/js-server-sdk-common';
-
-import { ChatResponse } from '../chat/types';
-import { LDAIConfigTracker } from '../config/LDAIConfigTracker';
-import { LDAIJudgeConfig, LDMessage } from '../config/types';
-import { AIProvider } from '../providers/AIProvider';
-import { EvaluationSchemaBuilder } from './EvaluationSchemaBuilder';
-import { EvalScore, JudgeResponse, StructuredResponse } from './types';
-
-/**
- * Judge implementation that handles evaluation functionality and conversation management.
- *
- * According to the AIEval spec, judges are AI Configs with mode: "judge" that evaluate
- * other AI Configs using structured output.
- */
-export class Judge {
-  private readonly _logger?: LDLogger;
-  private readonly _evaluationResponseStructure: Record<string, unknown>;
-
-  constructor(
-    private readonly _aiConfig: LDAIJudgeConfig,
-    private readonly _aiConfigTracker: LDAIConfigTracker,
-    private readonly _aiProvider: AIProvider,
-    logger?: LDLogger,
-  ) {
-    this._logger = logger;
-    this._evaluationResponseStructure = EvaluationSchemaBuilder.build(
-      this._aiConfig.evaluationMetricKeys,
-    );
-  }
-
-  /**
-   * Evaluates an AI response using the judge's configuration.
-   *
-   * @param input The input prompt or question that was provided to the AI
-   * @param output The AI-generated response to be evaluated
-   * @param samplingRate Sampling rate (0-1) to determine if evaluation should be processed (defaults to 1)
-   * @returns Promise that resolves to evaluation results or undefined if not sampled
-   */
-  async evaluate(
-    input: string,
-    output: string,
-    samplingRate: number = 1,
-  ): Promise<JudgeResponse | undefined> {
-    try {
-      if (
-        !this._aiConfig.evaluationMetricKeys ||
-        this._aiConfig.evaluationMetricKeys.length === 0
-      ) {
-        this._logger?.warn(
-          'Judge configuration is missing required evaluationMetricKeys',
-          this._aiConfigTracker.getTrackData(),
-        );
-        return undefined;
-      }
-
-      if (!this._aiConfig.messages) {
-        this._logger?.warn(
-          'Judge configuration must include messages',
-          this._aiConfigTracker.getTrackData(),
-        );
-        return undefined;
-      }
-
-      if (Math.random() > samplingRate) {
-        this._logger?.debug(`Judge evaluation skipped due to sampling rate: ${samplingRate}`);
-        return undefined;
-      }
-
-      const messages = this._constructEvaluationMessages(input, output);
-
-      const response = await this._aiConfigTracker.trackMetricsOf(
-        (result: StructuredResponse) => result.metrics,
-        () => this._aiProvider.invokeStructuredModel(messages, this._evaluationResponseStructure),
-      );
-
-      let { success } = response.metrics;
-
-      const evals = this._parseEvaluationResponse(response.data);
-
-      if (Object.keys(evals).length !== this._aiConfig.evaluationMetricKeys.length) {
-        this._logger?.warn(
-          'Judge evaluation did not return all evaluations',
-          this._aiConfigTracker.getTrackData(),
-        );
-        success = false;
-      }
-
-      return {
-        evals,
-        success,
-        judgeConfigKey: this._aiConfig.key,
-      };
-    } catch (error) {
-      this._logger?.error('Judge evaluation failed:', error);
-      return {
-        evals: {},
-        success: false,
-        error: error instanceof Error ? error.message : 'Unknown error',
-        judgeConfigKey: this._aiConfig.key,
-      };
-    }
-  }
-
-  /**
-   * Evaluates an AI response from chat messages and response.
-   *
-   * @param messages Array of messages representing the conversation history
-   * @param response The AI response to be evaluated
-   * @param samplingRatio Sampling ratio (0-1) to determine if evaluation should be processed (defaults to 1)
-   * @returns Promise that resolves to evaluation results or undefined if not sampled
-   */
-  async evaluateMessages(
-    messages: LDMessage[],
-    response: ChatResponse,
-    samplingRatio: number = 1,
-  ): Promise<JudgeResponse | undefined> {
-    const input = messages.length === 0 ? '' : messages.map((msg) => msg.content).join('\r\n');
-    const output = response.message.content;
-
-    return this.evaluate(input, output, samplingRatio);
-  }
-
-  /**
-   * Returns the AI Config used by this judge.
-   */
-  getAIConfig(): LDAIJudgeConfig {
-    return this._aiConfig;
-  }
-
-  /**
-   * Returns the tracker associated with this judge.
-   */
-  getTracker(): LDAIConfigTracker {
-    return this._aiConfigTracker;
-  }
-
-  /**
-   * Returns the AI provider used by this judge.
-   */
-  getProvider(): AIProvider {
-    return this._aiProvider;
-  }
-
-  /**
-   * Constructs evaluation messages by combining judge's config messages with input/output.
-   */
-  private _constructEvaluationMessages(input: string, output: string): LDMessage[] {
-    const messages: LDMessage[] = this._aiConfig.messages!.map((msg) => ({
-      ...msg,
-      content: this._interpolateMessage(msg.content, {
-        message_history: input,
-        response_to_evaluate: output,
-      }),
-    }));
-
-    return messages;
-  }
-
-  /**
-   * Interpolates message content with variables using Mustache templating.
-   */
-  private _interpolateMessage(content: string, variables: Record<string, string>): string {
-    return Mustache.render(content, variables, undefined, { escape: (item: any) => item });
-  }
-
-  /**
-   * Parses the structured evaluation response from the AI provider.
-   */
-  private _parseEvaluationResponse(data: Record<string, unknown>): Record<string, EvalScore> {
-    const evaluations = data.evaluations as Record<string, unknown>;
-    const results: Record<string, EvalScore> = {};
-
-    if (!data.evaluations || typeof data.evaluations !== 'object') {
-      this._logger?.warn('Invalid response: missing or invalid evaluations object');
-      return results;
-    }
-
-    this._aiConfig.evaluationMetricKeys.forEach((metricKey) => {
-      const evaluation = evaluations[metricKey];
-
-      if (!evaluation || typeof evaluation !== 'object') {
-        this._logger?.warn(
-          `Missing evaluation for metric key: ${metricKey}`,
-          this._aiConfigTracker.getTrackData(),
-        );
-        return;
-      }
-
-      const evalData = evaluation as Record<string, unknown>;
-
-      if (typeof evalData.score !== 'number' || evalData.score < 0 || evalData.score > 1) {
-        this._logger?.warn(
-          `Invalid score evaluated for ${metricKey}: ${evalData.score}. Score must be a number between 0 and 1 inclusive`,
-          this._aiConfigTracker.getTrackData(),
-        );
-        return;
-      }
-
-      if (typeof evalData.reasoning !== 'string') {
-        this._logger?.warn(
-          `Invalid reasoning evaluated for ${metricKey}: ${evalData.reasoning}. Reasoning must be a string`,
-          this._aiConfigTracker.getTrackData(),
-        );
-        return;
-      }
-
-      results[metricKey] = {
-        score: evalData.score,
-        reasoning: evalData.reasoning,
-      };
-    });
-
-    return results;
-  }
-}

package/src/api/judge/index.ts

@@ -1,2 +0,0 @@
-export { Judge } from './Judge';
-export type { EvalScore, JudgeResponse, StructuredResponse } from './types';

package/src/api/judge/types.ts

@@ -1,41 +0,0 @@
-import { LDAIMetrics } from '../metrics/LDAIMetrics';
-
-/**
- * Structured response from AI models.
- */
-export interface StructuredResponse {
-  /** The structured data returned by the model */
-  data: Record<string, unknown>;
-
-  /** The raw response from the model */
-  rawResponse: string;
-
-  /**
-   * Metrics information including success status and token usage.
-   */
-  metrics: LDAIMetrics;
-}
-
-/**
- * Score and reasoning for a single evaluation metric.
- */
-export interface EvalScore {
-  /** Score between 0.0 and 1.0 indicating the evaluation result for this metric */
-  score: number;
-  /** Reasoning behind the provided score for this metric */
-  reasoning: string;
-}
-
-/**
- * Response from a judge evaluation containing scores and reasoning for multiple metrics.
- */
-export interface JudgeResponse {
-  /** The key of the judge configuration that was used to generate this response */
-  judgeConfigKey?: string;
-  /** Dictionary where keys are metric names and values contain score and reasoning */
-  evals: Record<string, EvalScore>;
-  /** Whether the evaluation completed successfully */
-  success: boolean;
-  /** Error message if evaluation failed */
-  error?: string;
-}

package/src/api/metrics/BedrockTokenUsage.ts

@@ -1,13 +0,0 @@
-import { LDTokenUsage } from './LDTokenUsage';
-
-export function createBedrockTokenUsage(data: {
-  totalTokens?: number;
-  inputTokens?: number;
-  outputTokens?: number;
-}): LDTokenUsage {
-  return {
-    total: data.totalTokens || 0,
-    input: data.inputTokens || 0,
-    output: data.outputTokens || 0,
-  };
-}

package/src/api/metrics/LDAIMetrics.ts

@@ -1,18 +0,0 @@
-import { LDTokenUsage } from './LDTokenUsage';
-
-/**
- * Metrics information for AI operations that includes success status and token usage.
- * This class combines success/failure tracking with token usage metrics.
- */
-export interface LDAIMetrics {
-  /**
-   * Whether the AI operation was successful.
-   */
-  success: boolean;
-
-  /**
-   * Token usage information for the operation.
-   * This will be undefined if no token usage data is available.
-   */
-  usage?: LDTokenUsage;
-}

package/src/api/metrics/LDFeedbackKind.ts

@@ -1,13 +0,0 @@
-/**
- * Feedback about the generated content.
- */
-export enum LDFeedbackKind {
-  /**
-   * The sentiment was positive.
-   */
-  Positive = 'positive',
-  /**
-   * The sentiment is negative.
-   */
-  Negative = 'negative',
-}

package/src/api/metrics/LDTokenUsage.ts

@@ -1,19 +0,0 @@
-/**
- * Information about token usage.
- */
-export interface LDTokenUsage {
-  /**
-   * Combined token usage.
-   */
-  total: number;
-
-  /**
-   * Number of tokens in the input.
-   */
-  input: number;
-
-  /**
-   * Number of tokens in the output.
-   */
-  output: number;
-}

package/src/api/metrics/OpenAiUsage.ts

@@ -1,13 +0,0 @@
-import { LDTokenUsage } from './LDTokenUsage';
-
-export function createOpenAiUsage(data: {
-  total_tokens?: number;
-  prompt_tokens?: number;
-  completion_tokens?: number;
-}): LDTokenUsage {
-  return {
-    total: data.total_tokens ?? 0,
-    input: data.prompt_tokens ?? 0,
-    output: data.completion_tokens ?? 0,
-  };
-}

package/src/api/metrics/VercelAISDKTokenUsage.ts

@@ -1,15 +0,0 @@
-import { LDTokenUsage } from './LDTokenUsage';
-
-export function createVercelAISDKTokenUsage(data: {
-  totalTokens?: number;
-  inputTokens?: number;
-  promptTokens?: number;
-  outputTokens?: number;
-  completionTokens?: number;
-}): LDTokenUsage {
-  return {
-    total: data.totalTokens ?? 0,
-    input: data.inputTokens ?? data.promptTokens ?? 0,
-    output: data.outputTokens ?? data.completionTokens ?? 0,
-  };
-}

package/src/api/metrics/index.ts

@@ -1,6 +0,0 @@
-export * from './BedrockTokenUsage';
-export * from './OpenAiUsage';
-export * from './LDFeedbackKind';
-export * from './LDAIMetrics';
-export * from './LDTokenUsage';
-export * from './VercelAISDKTokenUsage';