@launchdarkly/server-sdk-ai 0.12.3 → 0.14.0

package/src/api/config/LDAIConfigTracker.ts

@@ -1,3 +1,4 @@
+import { EvalScore } from '../judge/types';
 import { LDAIMetrics, LDFeedbackKind, LDTokenUsage } from '../metrics';
 
 /**
@@ -34,6 +35,18 @@ export interface LDAIMetricSummary {
  * The LDAIConfigTracker is used to track various details about AI operations.
  */
 export interface LDAIConfigTracker {
+  /**
+   * Get the data for tracking.
+   */
+  getTrackData(): {
+    variationKey: string;
+    configKey: string;
+    version: number;
+    modelName: string;
+    providerName: string;
+    aiSdkName: string;
+    aiSdkVersion: string;
+  };
   /**
    * Track the duration of generation.
    *
@@ -74,6 +87,13 @@ export interface LDAIConfigTracker {
    */
   trackTimeToFirstToken(timeToFirstTokenMs: number): void;
 
+  /**
+   * Track evaluation scores for multiple metrics.
+   *
+   * @param scores Record mapping metric keys to their evaluation scores
+   */
+  trackEvalScores(scores: Record<string, EvalScore>): void;
+
   /**
    * Track the duration of execution of the provided function.
    *
@@ -106,6 +126,32 @@ export interface LDAIConfigTracker {
     func: () => Promise<TRes>,
   ): Promise<TRes>;
 
+  /**
+   * Track metrics for a streaming AI operation.
+   *
+   * This function will track the duration of the operation, extract metrics using the provided
+   * metrics extractor function, and track success or error status accordingly.
+   *
+   * Unlike trackMetricsOf, this method is designed for streaming operations where:
+   * - The stream is created and returned immediately (synchronously)
+   * - Metrics are extracted asynchronously in the background once the stream completes
+   * - Duration is tracked from stream creation to metrics extraction completion
+   *
+   * The stream is returned immediately so the caller can begin consuming it without waiting.
+   * Metrics extraction happens in the background and does not block stream consumption.
+   *
+   * If the stream creator throws, then this method will also throw and record an error.
+   * If metrics extraction fails, the error is logged but does not affect stream consumption.
+   *
+   * @param streamCreator Function that creates and returns the stream (synchronous)
+   * @param metricsExtractor Function that asynchronously extracts metrics from the stream
+   * @returns The stream result (returned immediately, not a Promise)
+   */
+  trackStreamMetricsOf<TStream>(
+    streamCreator: () => TStream,
+    metricsExtractor: (stream: TStream) => Promise<LDAIMetrics>,
+  ): TStream;
+
   /**
    * Track an OpenAI operation.
    *
@@ -178,33 +224,6 @@ export interface LDAIConfigTracker {
     func: () => Promise<TRes>,
   ): Promise<TRes>;
 
-  /**
-   * Track a Vercel AI SDK streamText operation.
-   *
-   * This function will track the duration of the operation, the token usage, and the success or error status.
-   *
-   * If the provided function throws, then this method will also throw.
-   * In the case the provided function throws, this function will record the duration and an error.
-   * A failed operation will not have any token usage data.
-   *
-   * @param func Function which executes the operation.
-   * @returns The result of the operation.
-   */
-  trackVercelAISDKStreamTextMetrics<
-    TRes extends {
-      finishReason?: Promise<string>;
-      usage?: Promise<{
-        totalTokens?: number;
-        inputTokens?: number;
-        promptTokens?: number;
-        outputTokens?: number;
-        completionTokens?: number;
-      }>;
-    },
-  >(
-    func: () => TRes,
-  ): TRes;
-
   /**
    * Get a summary of the tracked metrics.
    */

package/src/api/config/LDAIConfigUtils.ts

@@ -0,0 +1,201 @@
+import { LDAIConfigTracker } from './LDAIConfigTracker';
+import {
+  LDAIAgentConfig,
+  LDAICompletionConfig,
+  LDAIConfigDefaultKind,
+  LDAIConfigKind,
+  LDAIConfigMode,
+  LDAIJudgeConfig,
+  LDJudgeConfiguration,
+  LDMessage,
+  LDModelConfig,
+  LDProviderConfig,
+} from './types';
+
+/**
+ * Internal flag value structure returned by LaunchDarkly.
+ * This represents the raw data structure that LaunchDarkly returns for AI configuration flags.
+ *
+ * @internal - Not meant for external use
+ */
+export interface LDAIConfigFlagValue {
+  _ldMeta?: {
+    variationKey?: string;
+    enabled: boolean;
+    version?: number;
+    mode?: LDAIConfigMode;
+  };
+  model?: LDModelConfig;
+  messages?: LDMessage[];
+  provider?: LDProviderConfig;
+  instructions?: string;
+  evaluationMetricKeys?: string[];
+  judgeConfiguration?: LDJudgeConfiguration;
+}
+
+/**
+ * Utility class for converting between AI configuration types and LaunchDarkly flag values.
+ *
+ * @internal - This class and its types are internal implementation details and should not be used by SDK consumers.
+ */
+export class LDAIConfigUtils {
+  /**
+   * Converts a default AI configuration to a LaunchDarkly flag value format.
+   *
+   * @param config The default AI configuration to convert
+   * @param mode The mode for the configuration
+   * @returns The flag value structure for LaunchDarkly
+   */
+  static toFlagValue(config: LDAIConfigDefaultKind, mode: LDAIConfigMode): LDAIConfigFlagValue {
+    const flagValue: LDAIConfigFlagValue = {
+      _ldMeta: {
+        variationKey: '', // Not available when converting from config
+        enabled: config.enabled ?? false,
+        mode,
+      },
+      model: config.model,
+    };
+
+    if ('messages' in config && config.messages !== undefined) {
+      flagValue.messages = config.messages;
+    }
+    if (config.provider !== undefined) {
+      flagValue.provider = config.provider;
+    }
+    if ('instructions' in config && config.instructions !== undefined) {
+      flagValue.instructions = config.instructions;
+    }
+    if ('evaluationMetricKeys' in config && config.evaluationMetricKeys !== undefined) {
+      flagValue.evaluationMetricKeys = config.evaluationMetricKeys;
+    }
+    if ('judgeConfiguration' in config && config.judgeConfiguration !== undefined) {
+      flagValue.judgeConfiguration = config.judgeConfiguration;
+    }
+
+    return flagValue;
+  }
+
+  /**
+   * Converts a LaunchDarkly flag value to the appropriate AI configuration type.
+   *
+   * @param flagValue The flag value from LaunchDarkly
+   * @param tracker The tracker to add to the config
+   * @returns The appropriate AI configuration type
+   */
+  static fromFlagValue(flagValue: LDAIConfigFlagValue, tracker: LDAIConfigTracker): LDAIConfigKind {
+    // Determine the actual mode from flag value
+    // eslint-disable-next-line no-underscore-dangle
+    const flagValueMode = flagValue._ldMeta?.mode;
+
+    switch (flagValueMode) {
+      case 'agent':
+        return this.toAgentConfig(flagValue, tracker);
+      case 'judge':
+        return this.toJudgeConfig(flagValue, tracker);
+      case 'completion':
+      default:
+        return this.toCompletionConfig(flagValue, tracker);
+    }
+  }
+
+  /**
+   * Creates a disabled configuration of the specified mode.
+   *
+   * @param mode The mode for the disabled config
+   * @returns A disabled config of the appropriate type
+   */
+  static createDisabledConfig(mode: LDAIConfigMode): LDAIConfigKind {
+    switch (mode) {
+      case 'agent':
+        return {
+          enabled: false,
+          tracker: undefined,
+        } as LDAIAgentConfig;
+      case 'judge':
+        return {
+          enabled: false,
+          tracker: undefined,
+          evaluationMetricKeys: [],
+        } as LDAIJudgeConfig;
+      case 'completion':
+      default:
+        // Default to completion config for completion mode or any unexpected mode
+        return {
+          enabled: false,
+          tracker: undefined,
+        } as LDAICompletionConfig;
+    }
+  }
+
+  /**
+   * Creates the base configuration that all config types share.
+   *
+   * @param flagValue The flag value from LaunchDarkly
+   * @returns Base configuration object
+   */
+  private static _toBaseConfig(flagValue: LDAIConfigFlagValue) {
+    return {
+      // eslint-disable-next-line no-underscore-dangle
+      enabled: flagValue._ldMeta?.enabled ?? false,
+      model: flagValue.model,
+      provider: flagValue.provider,
+    };
+  }
+
+  /**
+   * Creates a completion config from flag value data.
+   *
+   * @param flagValue The flag value from LaunchDarkly
+   * @param tracker The tracker to add to the config
+   * @returns A completion configuration
+   */
+  static toCompletionConfig(
+    flagValue: LDAIConfigFlagValue,
+    tracker: LDAIConfigTracker,
+  ): LDAICompletionConfig {
+    return {
+      ...this._toBaseConfig(flagValue),
+      tracker,
+      messages: flagValue.messages,
+      judgeConfiguration: flagValue.judgeConfiguration,
+    };
+  }
+
+  /**
+   * Creates an agent config from flag value data.
+   *
+   * @param flagValue The flag value from LaunchDarkly
+   * @param tracker The tracker to add to the config
+   * @returns An agent configuration
+   */
+  static toAgentConfig(
+    flagValue: LDAIConfigFlagValue,
+    tracker: LDAIConfigTracker,
+  ): LDAIAgentConfig {
+    return {
+      ...this._toBaseConfig(flagValue),
+      tracker,
+      instructions: flagValue.instructions,
+      judgeConfiguration: flagValue.judgeConfiguration,
+    };
+  }
+
+  /**
+   * Creates a judge config from flag value data.
+   *
+   * @param flagValue The flag value from LaunchDarkly
+   * @param tracker The tracker to add to the config
+   * @returns A judge configuration
+   */
+  static toJudgeConfig(
+    flagValue: LDAIConfigFlagValue,
+    tracker: LDAIConfigTracker,
+  ): LDAIJudgeConfig {
+    return {
+      ...this._toBaseConfig(flagValue),
+      tracker,
+      messages: flagValue.messages,
+      evaluationMetricKeys: flagValue.evaluationMetricKeys || [],
+    };
+  }
+}

package/src/api/config/index.ts

@@ -1,3 +1,3 @@
-export * from './LDAIConfig';
-export * from './VercelAISDK';
+export * from './types';
+// LDAIConfigUtils is intentionally not exported - it's an internal utility class
 export { LDAIConfigTracker } from './LDAIConfigTracker';

package/src/api/config/types.ts

@@ -0,0 +1,256 @@
+import { LDAIConfigTracker } from './LDAIConfigTracker';
+
+// ============================================================================
+// Foundation Types
+// ============================================================================
+
+/**
+ * Information about prompts.
+ */
+export interface LDMessage {
+  /**
+   * The role of the prompt.
+   */
+  role: 'user' | 'assistant' | 'system';
+  /**
+   * Content for the prompt.
+   */
+  content: string;
+}
+
+/**
+ * Configuration related to the model.
+ */
+export interface LDModelConfig {
+  /**
+   * The ID of the model.
+   */
+  name: string;
+
+  /**
+   * Model specific parameters.
+   */
+  parameters?: { [index: string]: unknown };
+
+  /**
+   * Additional user-specified parameters.
+   */
+  custom?: { [index: string]: unknown };
+}
+
+export interface LDProviderConfig {
+  /**
+   * The name of the provider.
+   */
+  name: string;
+}
+
+// ============================================================================
+// Judge Types
+// ============================================================================
+
+/**
+ * Configuration for a single judge attachment.
+ */
+export interface LDJudge {
+  /** The key of the judge AI Config */
+  key: string;
+  /** Sampling rate for evaluation (0.0 to 1.0) */
+  samplingRate: number;
+}
+
+/**
+ * Configuration for judge attachment to AI Configs.
+ */
+export interface LDJudgeConfiguration {
+  /** Array of judge configurations */
+  judges: LDJudge[];
+}
+
+// ============================================================================
+// Base AI Config Types
+// ============================================================================
+
+/**
+ * Base AI Config interface for default implementations with optional enabled property.
+ */
+export interface LDAIConfigDefault {
+  /**
+   * Optional model configuration.
+   */
+  model?: LDModelConfig;
+
+  /**
+   * Optional configuration for the provider.
+   */
+  provider?: LDProviderConfig;
+
+  /**
+   * Whether the configuration is enabled. Defaults to false when not provided.
+   */
+  enabled?: boolean;
+}
+
+/**
+ * Base AI Config interface without mode-specific fields.
+ */
+export interface LDAIConfig extends Omit<LDAIConfigDefault, 'enabled'> {
+  /**
+   * Whether the configuration is enabled.
+   */
+  enabled: boolean;
+
+  /**
+   * A tracker which can be used to generate analytics.
+   * Undefined for disabled configs.
+   */
+  tracker?: LDAIConfigTracker;
+}
+
+// ============================================================================
+// Default AI Config Implementation Types
+// ============================================================================
+
+/**
+ * Default Agent-specific AI Config with instructions.
+ */
+export interface LDAIAgentConfigDefault extends LDAIConfigDefault {
+  /**
+   * Instructions for the agent.
+   */
+  instructions?: string;
+  /**
+   * Judge configuration for AI Configs being evaluated.
+   * References judge AI Configs that should evaluate this AI Config.
+   */
+  judgeConfiguration?: LDJudgeConfiguration;
+}
+
+/**
+ * Default Completion AI Config (default mode).
+ */
+export interface LDAICompletionConfigDefault extends LDAIConfigDefault {
+  /**
+   * Optional prompt data for completion configurations.
+   */
+  messages?: LDMessage[];
+  /**
+   * Judge configuration for AI Configs being evaluated.
+   * References judge AI Configs that should evaluate this AI Config.
+   */
+  judgeConfiguration?: LDJudgeConfiguration;
+}
+
+/**
+ * Default Judge-specific AI Config with required evaluation metric key.
+ */
+export interface LDAIJudgeConfigDefault extends LDAIConfigDefault {
+  /**
+   * Optional prompt data for judge configurations.
+   */
+  messages?: LDMessage[];
+  /**
+   * Evaluation metric keys for judge configurations.
+   * The keys of the metrics that this judge can evaluate.
+   */
+  evaluationMetricKeys?: string[];
+}
+
+/**
+ * Union type for all default AI Config variants.
+ */
+export type LDAIConfigDefaultKind =
+  | LDAIAgentConfigDefault
+  | LDAICompletionConfigDefault
+  | LDAIJudgeConfigDefault;
+
+// ============================================================================
+// AI Config Implementation Types
+// ============================================================================
+
+/**
+ * Agent-specific AI Config with instructions.
+ */
+export interface LDAIAgentConfig extends LDAIConfig {
+  /**
+   * Instructions for the agent.
+   */
+  instructions?: string;
+  /**
+   * Judge configuration for AI Configs being evaluated.
+   * References judge AI Configs that should evaluate this AI Config.
+   */
+  judgeConfiguration?: LDJudgeConfiguration;
+}
+
+/**
+ * Completion AI Config (default mode).
+ */
+export interface LDAICompletionConfig extends LDAIConfig {
+  /**
+   * Optional prompt data for completion configurations.
+   */
+  messages?: LDMessage[];
+  /**
+   * Judge configuration for AI Configs being evaluated.
+   * References judge AI Configs that should evaluate this AI Config.
+   */
+  judgeConfiguration?: LDJudgeConfiguration;
+}
+
+/**
+ * Judge-specific AI Config with required evaluation metric key.
+ */
+export interface LDAIJudgeConfig extends LDAIConfig {
+  /**
+   * Optional prompt data for judge configurations.
+   */
+  messages?: LDMessage[];
+  /**
+   * Evaluation metric keys for judge configurations.
+   * The keys of the metrics that this judge can evaluate.
+   */
+  evaluationMetricKeys: string[];
+}
+
+// ============================================================================
+// Union Types
+// ============================================================================
+
+/**
+ * Union type for all AI Config variants.
+ */
+export type LDAIConfigKind = LDAIAgentConfig | LDAICompletionConfig | LDAIJudgeConfig;
+
+// ============================================================================
+// Agent-Specific Request Type
+// ============================================================================
+
+/**
+ * Configuration for a single agent request.
+ */
+export interface LDAIAgentRequestConfig {
+  /**
+   * The agent key to retrieve.
+   */
+  key: string;
+
+  /**
+   * Default configuration for the agent.
+   */
+  defaultValue: LDAIAgentConfigDefault;
+
+  /**
+   * Variables for instructions interpolation.
+   */
+  variables?: Record<string, unknown>;
+}
+
+// ============================================================================
+// Mode Type
+// ============================================================================
+
+/**
+ * Mode type for AI configurations.
+ */
+export type LDAIConfigMode = 'completion' | 'agent' | 'judge';

package/src/api/index.ts

@@ -1,6 +1,6 @@
 export * from './config';
-export * from './agents';
 export * from './chat';
+export * from './judge';
 export * from './metrics';
 export * from './LDAIClient';
 export * from './providers';

package/src/api/judge/EvaluationSchemaBuilder.ts

@@ -0,0 +1,54 @@
+/**
+ * Internal class for building dynamic evaluation response schemas.
+ * Not exported - only used internally by TrackedJudge.
+ */
+class EvaluationSchemaBuilder {
+  static build(evaluationMetricKeys: string[]): Record<string, unknown> {
+    return {
+      type: 'object',
+      properties: {
+        evaluations: {
+          type: 'object',
+          description: `Object containing evaluation results for ${evaluationMetricKeys.join(', ')} metrics`,
+          properties: this._buildKeyProperties(evaluationMetricKeys),
+          required: evaluationMetricKeys,
+          additionalProperties: false,
+        },
+      },
+      required: ['evaluations'],
+      additionalProperties: false,
+    } as const;
+  }
+
+  private static _buildKeyProperties(evaluationMetricKeys: string[]) {
+    return evaluationMetricKeys.reduce(
+      (acc, key) => {
+        acc[key] = this._buildKeySchema(key);
+        return acc;
+      },
+      {} as Record<string, unknown>,
+    );
+  }
+
+  private static _buildKeySchema(key: string) {
+    return {
+      type: 'object',
+      properties: {
+        score: {
+          type: 'number',
+          minimum: 0,
+          maximum: 1,
+          description: `Score between 0.0 and 1.0 for ${key}`,
+        },
+        reasoning: {
+          type: 'string',
+          description: `Reasoning behind the score for ${key}`,
+        },
+      },
+      required: ['score', 'reasoning'],
+      additionalProperties: false,
+    };
+  }
+}
+
+export { EvaluationSchemaBuilder };