@launchdarkly/server-sdk-ai 0.19.0 → 0.20.0

package/dist/index.js

@@ -1,129 +1,110 @@
 // src/LDAIClientImpl.ts
-import Mustache2 from "mustache";
+import Mustache from "mustache";
 import { randomUUID } from "crypto";
 
-// src/api/chat/TrackedChat.ts
-var TrackedChat = class {
-  constructor(aiConfig, provider, judges = {}, _logger) {
-    this.aiConfig = aiConfig;
-    this.provider = provider;
-    this.judges = judges;
+// src/api/ManagedAgent.ts
+var ManagedAgent = class {
+  constructor(aiAgentConfig, runner, _logger) {
+    this.aiAgentConfig = aiAgentConfig;
+    this.runner = runner;
     this._logger = _logger;
-    this.messages = [];
   }
   /**
-   * Invoke the chat model with a prompt string.
-   * This method handles conversation management and tracking, delegating to the provider's invokeModel method.
+   * Invoke the agent with a prompt string and return a ManagedResult.
+   *
+   * `run()` resolves before `ManagedResult.evaluations` resolves. Awaiting
+   * `evaluations` guarantees both judge evaluation and tracker.trackJudgeResult()
+   * are complete.
+   *
+   * @param prompt The user input to send to the agent.
+   * @returns Promise resolving to ManagedResult (before evaluations settle).
    */
-  async invoke(prompt) {
-    const tracker = this.aiConfig.createTracker();
-    const userMessage = {
-      role: "user",
-      content: prompt
-    };
-    this.messages.push(userMessage);
-    const configMessages = this.aiConfig.messages || [];
-    const allMessages = [...configMessages, ...this.messages];
-    const response = await tracker.trackMetricsOf(
-      (result) => result.metrics,
-      () => this.provider.invokeModel(allMessages)
+  async run(prompt) {
+    const tracker = this.aiAgentConfig.createTracker();
+    const result = await tracker.trackMetricsOf(
+      (r) => r.metrics,
+      () => this.runner.run(prompt)
     );
-    if (this.aiConfig.judgeConfiguration?.judges && this.aiConfig.judgeConfiguration.judges.length > 0) {
-      response.evaluations = this._evaluateWithJudges(this.messages, response).then(
-        (evaluations) => {
-          evaluations.forEach((judgeResult) => {
-            tracker.trackJudgeResult(judgeResult);
-          });
-          return evaluations;
+    const metrics = tracker.getSummary();
+    const output = result.content;
+    const evaluations = this.aiAgentConfig.evaluator.evaluate(prompt, output).then((results) => {
+      results.forEach((judgeResult) => {
+        if (!judgeResult.sampled) {
+          return;
         }
-      );
-    }
-    this.messages.push(response.message);
-    return response;
-  }
-  /**
-   * Evaluates the response with all configured judges.
-   * Returns a promise that resolves to an array of evaluation results.
-   *
-   * @param messages Array of messages representing the conversation history
-   * @param response The AI response to be evaluated
-   * @returns Promise resolving to array of judge evaluation results
-   */
-  async _evaluateWithJudges(messages, response) {
-    const judgeConfigs = this.aiConfig.judgeConfiguration.judges;
-    const evaluationPromises = judgeConfigs.map(async (judgeConfig) => {
-      const judge = this.judges[judgeConfig.key];
-      if (!judge) {
-        this._logger?.warn(
-          `Judge configuration is not enabled for ${judgeConfig.key} in ${this.aiConfig.key}`
-        );
-        const result = {
-          success: false,
-          sampled: true,
-          errorMessage: `Judge configuration is not enabled for ${judgeConfig.key}`
-        };
-        return result;
-      }
-      return judge.evaluateMessages(messages, response, judgeConfig.samplingRate);
-    });
-    const results = await Promise.allSettled(evaluationPromises);
-    return results.map((settled) => {
-      if (settled.status === "fulfilled") {
-        return settled.value;
-      }
-      const result = {
-        success: false,
-        sampled: true,
-        errorMessage: "Judge evaluation failed"
-      };
-      return result;
+        tracker.trackJudgeResult(judgeResult);
+      });
+      return results;
+    }).catch((err) => {
+      this._logger?.warn("Judge evaluation failed unexpectedly:", err);
+      return [];
     });
+    return {
+      content: output,
+      metrics,
+      raw: result.raw,
+      parsed: result.parsed,
+      evaluations
+    };
   }
   /**
-   * Get the underlying AI configuration used to initialize this TrackedChat.
+   * Get the underlying AI agent configuration used to initialize this ManagedAgent.
    */
   getConfig() {
-    return this.aiConfig;
+    return this.aiAgentConfig;
   }
-  /**
-   * Get the underlying AI provider instance.
-   * This provides direct access to the provider for advanced use cases.
-   */
-  getProvider() {
-    return this.provider;
-  }
-  /**
-   * Get the judges associated with this TrackedChat.
-   * Returns a record of judge instances keyed by their configuration keys.
-   */
-  getJudges() {
-    return this.judges;
+};
+
+// src/api/ManagedModel.ts
+var ManagedModel = class {
+  constructor(aiConfig, runner, _logger) {
+    this.aiConfig = aiConfig;
+    this.runner = runner;
+    this._logger = _logger;
   }
   /**
-   * Append messages to the conversation history.
-   * Adds messages to the conversation history without invoking the model,
-   * which is useful for managing multi-turn conversations or injecting context.
+   * Invoke the model with a prompt string and return a ManagedResult.
    *
-   * @param messages Array of messages to append to the conversation history
+   * `run()` resolves before `ManagedResult.evaluations` resolves. Awaiting
+   * `evaluations` guarantees both judge evaluation and tracker.trackJudgeResult()
+   * are complete.
+   *
+   * @param prompt The user input to send to the model.
+   * @returns Promise resolving to ManagedResult (before evaluations settle).
    */
-  appendMessages(messages) {
-    this.messages.push(...messages);
+  async run(prompt) {
+    const tracker = this.aiConfig.createTracker();
+    const result = await tracker.trackMetricsOf(
+      (r) => r.metrics,
+      () => this.runner.run(prompt)
+    );
+    const metrics = tracker.getSummary();
+    const output = result.content;
+    const evaluations = this.aiConfig.evaluator.evaluate(prompt, output).then((results) => {
+      results.forEach((judgeResult) => {
+        if (!judgeResult.sampled) {
+          return;
+        }
+        tracker.trackJudgeResult(judgeResult);
+      });
+      return results;
+    }).catch((err) => {
+      this._logger?.warn("Judge evaluation failed unexpectedly:", err);
+      return [];
+    });
+    return {
+      content: output,
+      metrics,
+      raw: result.raw,
+      parsed: result.parsed,
+      evaluations
+    };
   }
   /**
-   * Get all messages in the conversation history.
-   *
-   * @param includeConfigMessages Whether to include the config messages from the AIConfig.
-   *                              Defaults to false.
-   * @returns Array of messages. When includeConfigMessages is true, returns both config
-   *          messages and conversation history with config messages prepended. When false,
-   *          returns only the conversation history messages.
+   * Get the underlying AI configuration used to initialize this ManagedModel.
    */
-  getMessages(includeConfigMessages = false) {
-    if (includeConfigMessages) {
-      const configMessages = this.aiConfig.messages || [];
-      return [...configMessages, ...this.messages];
-    }
-    return [...this.messages];
+  getConfig() {
+    return this.aiConfig;
   }
 };
 
@@ -175,46 +156,52 @@ var LDAIConfigUtils = class {
    * @param key The configuration key
    * @param flagValue The flag value from LaunchDarkly
    * @param trackerFactory A factory function that creates a new tracker for each execution
+   * @param evaluator The evaluator to attach to completion and agent configs
    * @returns The appropriate AI configuration type
    */
-  static fromFlagValue(key, flagValue, trackerFactory) {
+  static fromFlagValue(key, flagValue, trackerFactory, evaluator) {
     const flagValueMode = flagValue._ldMeta?.mode;
     switch (flagValueMode) {
       case "agent":
-        return this.toAgentConfig(key, flagValue, trackerFactory);
+        return this.toAgentConfig(key, flagValue, trackerFactory, evaluator);
       case "judge":
         return this.toJudgeConfig(key, flagValue, trackerFactory);
       case "completion":
       default:
-        return this.toCompletionConfig(key, flagValue, trackerFactory);
+        return this.toCompletionConfig(key, flagValue, trackerFactory, evaluator);
     }
   }
   /**
    * Creates a disabled configuration of the specified mode.
    *
+   * @param key The configuration key
    * @param mode The mode for the disabled config
+   * @param createTracker A factory function that creates a new tracker for each execution
+   * @param evaluator The evaluator to attach to completion and agent configs
    * @returns A disabled config of the appropriate type
    */
-  static createDisabledConfig(key, mode) {
+  static createDisabledConfig(key, mode, createTracker, evaluator) {
     switch (mode) {
       case "agent":
         return {
           key,
           enabled: false,
-          createTracker: void 0
+          createTracker,
+          evaluator
         };
       case "judge":
         return {
           key,
           enabled: false,
-          createTracker: void 0
+          createTracker
         };
       case "completion":
       default:
         return {
           key,
           enabled: false,
-          createTracker: void 0
+          createTracker,
+          evaluator
         };
     }
   }
@@ -256,12 +243,14 @@ var LDAIConfigUtils = class {
    * @param key The configuration key
    * @param flagValue The flag value from LaunchDarkly
    * @param trackerFactory A factory function that creates a new tracker for each execution
+   * @param evaluator The evaluator for this completion config
    * @returns A completion configuration
    */
-  static toCompletionConfig(key, flagValue, trackerFactory) {
+  static toCompletionConfig(key, flagValue, trackerFactory, evaluator) {
     return {
       ...this._toBaseConfig(key, flagValue),
       createTracker: trackerFactory,
+      evaluator,
       messages: flagValue.messages,
       judgeConfiguration: flagValue.judgeConfiguration,
       tools: this._resolveTools(flagValue)
@@ -273,12 +262,14 @@ var LDAIConfigUtils = class {
    * @param key The configuration key
    * @param flagValue The flag value from LaunchDarkly
    * @param trackerFactory A factory function that creates a new tracker for each execution
+   * @param evaluator The evaluator for this agent config
    * @returns An agent configuration
    */
-  static toAgentConfig(key, flagValue, trackerFactory) {
+  static toAgentConfig(key, flagValue, trackerFactory, evaluator) {
     return {
       ...this._toBaseConfig(key, flagValue),
       createTracker: trackerFactory,
+      evaluator,
       instructions: flagValue.instructions,
       judgeConfiguration: flagValue.judgeConfiguration,
       tools: this._resolveTools(flagValue)
@@ -543,8 +534,98 @@ var AgentGraphDefinition = class _AgentGraphDefinition {
   }
 };
 
+// src/api/graph/ManagedAgentGraph.ts
+var ManagedAgentGraph = class {
+  constructor(_graphDefinition, _logger) {
+    this._graphDefinition = _graphDefinition;
+    this._logger = _logger;
+  }
+  /**
+   * Runs the agent graph using the provided runner function and returns a ManagedGraphResult.
+   *
+   * The runner function receives the graph tracker and AgentGraphDefinition,
+   * executes the graph, and returns an AgentGraphRunnerResult.
+   *
+   * run() returns before ManagedGraphResult.evaluations resolves.
+   *
+   * @param runner Async function that executes the graph and returns AgentGraphRunnerResult.
+   * @returns ManagedGraphResult with LDAIGraphMetricSummary and evaluations promise.
+   */
+  async run(runner) {
+    const graphTracker = this._graphDefinition.createTracker();
+    const runnerResult = await runner(this._graphDefinition, graphTracker);
+    const metrics = {
+      success: runnerResult.metrics.success,
+      path: runnerResult.metrics.path,
+      durationMs: runnerResult.metrics.durationMs,
+      tokens: runnerResult.metrics.tokens,
+      nodeMetrics: this._trackNodeMetrics(runnerResult.metrics.nodeMetrics),
+      resumptionToken: graphTracker.resumptionToken
+    };
+    const evaluations = Promise.resolve([]);
+    return {
+      content: runnerResult.content,
+      metrics,
+      raw: runnerResult.raw,
+      evaluations
+    };
+  }
+  /**
+   * Converts per-node LDAIMetrics from the runner into LDAIMetricSummary by
+   * creating a per-node tracker, firing tracking events, and calling getSummary().
+   */
+  _trackNodeMetrics(nodeMetrics) {
+    const summaries = {};
+    for (const [nodeKey, metrics] of Object.entries(nodeMetrics)) {
+      const node = this._graphDefinition.getNode(nodeKey);
+      if (!node) {
+        this._logger?.warn(`ManagedAgentGraph: no node found for key "${nodeKey}", skipping metrics`);
+        continue;
+      }
+      const tracker = node.getConfig().createTracker();
+      if (metrics.tokens) {
+        tracker.trackTokens(metrics.tokens);
+      }
+      if (metrics.durationMs !== void 0) {
+        tracker.trackDuration(metrics.durationMs);
+      }
+      if (metrics.toolCalls?.length) {
+        tracker.trackToolCalls(metrics.toolCalls);
+      }
+      if (metrics.success) {
+        tracker.trackSuccess();
+      } else {
+        tracker.trackError();
+      }
+      summaries[nodeKey] = tracker.getSummary();
+    }
+    return summaries;
+  }
+  /**
+   * Returns the underlying AgentGraphDefinition.
+   */
+  getGraphDefinition() {
+    return this._graphDefinition;
+  }
+};
+
+// src/api/judge/Evaluator.ts
+var Evaluator = class _Evaluator {
+  constructor(_judges) {
+    this._judges = _judges;
+  }
+  static noop() {
+    return new _Evaluator([]);
+  }
+  async evaluate(input, output) {
+    if (this._judges.length === 0) {
+      return [];
+    }
+    return Promise.all(this._judges.map((judge) => judge.evaluate(input, output)));
+  }
+};
+
 // src/api/judge/Judge.ts
-import Mustache from "mustache";
 var EVALUATION_SCHEMA = {
   type: "object",
   properties: {
@@ -562,12 +643,25 @@ var EVALUATION_SCHEMA = {
   required: ["score", "reasoning"],
   additionalProperties: false
 };
+function stripLegacyJudgeMessages(messages) {
+  return messages.filter(
+    (msg) => msg.role === "system" || !msg.content.includes("{{message_history}}") && !msg.content.includes("{{response_to_evaluate}}")
+  );
+}
 var Judge = class {
-  constructor(_aiConfig, _aiProvider, logger) {
+  constructor(_aiConfig, _runner, _sampleRate = 1, logger) {
     this._aiConfig = _aiConfig;
-    this._aiProvider = _aiProvider;
+    this._runner = _runner;
+    this._sampleRate = _sampleRate;
     this._logger = logger;
   }
+  /**
+   * The default sampling rate baked in at construction. Used by `evaluate` /
+   * `evaluateMessages` when no per-call rate is supplied.
+   */
+  get sampleRate() {
+    return this._sampleRate;
+  }
   /**
    * Gets the evaluation metric key, prioritizing evaluationMetricKey over evaluationMetricKeys.
    * Falls back to the first valid (non-empty, non-whitespace) value in evaluationMetricKeys if evaluationMetricKey is not provided.
@@ -591,10 +685,13 @@ var Judge = class {
    *
    * @param input The input prompt or question that was provided to the AI
    * @param output The AI-generated response to be evaluated
-   * @param samplingRate Sampling rate (0-1) to determine if evaluation should be processed (defaults to 1)
+   * @param samplingRate Sampling rate (0-1) to determine if evaluation should be processed.
+   *   When omitted, the Judge's constructor-default rate is used. An explicit `0` overrides
+   *   the default — only `undefined` falls through.
    * @returns Promise that resolves to evaluation results
    */
-  async evaluate(input, output, samplingRate = 1) {
+  async evaluate(input, output, samplingRate) {
+    const effectiveRate = samplingRate ?? this._sampleRate;
     const result = {
       success: false,
       sampled: false,
@@ -612,26 +709,20 @@ var Judge = class {
         result.errorMessage = "Judge configuration is missing required evaluation metric key";
         return result;
       }
-      if (!this._aiConfig.messages) {
-        this._logger?.warn("Judge configuration must include messages", tracker.getTrackData());
-        result.sampled = true;
-        result.errorMessage = "Judge configuration must include messages";
-        return result;
-      }
-      if (Math.random() > samplingRate) {
-        this._logger?.debug(`Judge evaluation skipped due to sampling rate: ${samplingRate}`);
+      if (Math.random() > effectiveRate) {
+        this._logger?.debug(`Judge evaluation skipped due to sampling rate: ${effectiveRate}`);
         return result;
       }
       result.sampled = true;
-      const messages = this._constructEvaluationMessages(input, output);
+      const evaluationInput = this._buildEvaluationInput(input, output);
       const response = await tracker.trackMetricsOf(
         (r) => r.metrics,
-        () => this._aiProvider.invokeStructuredModel(messages, EVALUATION_SCHEMA)
+        () => this._runner.run(evaluationInput, EVALUATION_SCHEMA)
       );
-      const evalResult = this._parseEvaluationResponse(response.data);
+      const evalResult = this._parseEvaluationResponse(response.parsed);
       if (!evalResult) {
         this._logger?.warn(
-          `Could not parse evaluation response: ${JSON.stringify(response.data)}`,
+          `Could not parse evaluation response: ${JSON.stringify(response.parsed)}`,
           tracker.getTrackData()
         );
         return result;
@@ -655,10 +746,11 @@ var Judge = class {
    *
    * @param messages Array of messages representing the conversation history
    * @param response The AI response to be evaluated
-   * @param samplingRatio Sampling ratio (0-1) to determine if evaluation should be processed (defaults to 1)
+   * @param samplingRatio Sampling ratio (0-1). When omitted, the Judge's
+   *   constructor-default rate is used.
    * @returns Promise that resolves to evaluation results
    */
-  async evaluateMessages(messages, response, samplingRatio = 1) {
+  async evaluateMessages(messages, response, samplingRatio) {
     const input = messages.length === 0 ? "" : messages.map((msg) => msg.content).join("\r\n");
     const output = response.message.content;
     return this.evaluate(input, output, samplingRatio);
@@ -670,29 +762,23 @@ var Judge = class {
     return this._aiConfig;
   }
   /**
-   * Returns the AI provider used by this judge.
+   * Returns the runner used by this judge.
    */
-  getProvider() {
-    return this._aiProvider;
-  }
-  /**
-   * Constructs evaluation messages by combining judge's config messages with input/output.
-   */
-  _constructEvaluationMessages(input, output) {
-    const messages = this._aiConfig.messages.map((msg) => ({
-      ...msg,
-      content: this._interpolateMessage(msg.content, {
-        message_history: input,
-        response_to_evaluate: output
-      })
-    }));
-    return messages;
+  getRunner() {
+    return this._runner;
   }
   /**
-   * Interpolates message content with variables using Mustache templating.
+   * Builds the evaluation input string passed to the runner.
+   *
+   * Combines the original prompt and the response into a single, well-known
+   * format the judge model is expected to evaluate.
    */
-  _interpolateMessage(content, variables) {
-    return Mustache.render(content, variables, void 0, { escape: (item) => item });
+  _buildEvaluationInput(input, output) {
+    return `MESSAGE HISTORY:
+${input}
+
+RESPONSE TO EVALUATE:
+${output}`;
   }
   /**
    * Parses the structured evaluation response. Expects top-level {score, reasoning}.
@@ -718,115 +804,101 @@ var Judge = class {
 // src/api/providers/AIProvider.ts
 var AIProvider = class {
   constructor(logger) {
-    this.logger = logger;
+    this._logger = logger;
   }
   /**
-   * Invoke the chat model with an array of messages.
-   * This method should convert messages to provider format, invoke the model,
-   * and return a ChatResponse with the result and metrics.
+   * Create a Runner for a completion or judge AI Config.
    *
-   * Default implementation takes no action and returns a placeholder response.
-   * Provider implementations should override this method.
+   * Override in provider subclasses to return a configured {@link Runner}.
+   * Default implementation returns `undefined`.
    *
-   * @param messages Array of LDMessage objects representing the conversation
-   * @returns Promise that resolves to a ChatResponse containing the model's response
+   * @param config The completion or judge AI configuration.
+   * @returns Promise resolving to a {@link Runner}, or `undefined` if this
+   *   provider does not support model creation.
    */
-  async invokeModel(_messages) {
-    this.logger?.warn("invokeModel not implemented by this provider");
-    return {
-      message: {
-        role: "assistant",
-        content: ""
-      },
-      metrics: {
-        success: false,
-        usage: {
-          total: 0,
-          input: 0,
-          output: 0
-        }
-      }
-    };
+  async createModel(_config) {
+    return void 0;
   }
   /**
-   * Invoke the chat model with structured output support.
-   * This method should convert messages to provider format, invoke the model with
-   * structured output configuration, and return a structured response.
+   * Create a Runner for an agent AI Config.
    *
-   * Default implementation takes no action and returns a placeholder response.
-   * Provider implementations should override this method.
+   * Override in provider subclasses to return a configured {@link Runner}.
+   * Default implementation returns `undefined`.
    *
-   * @param messages Array of LDMessage objects representing the conversation
-   * @param responseStructure Dictionary of output configurations keyed by output name
-   * @returns Promise that resolves to a structured response
+   * @param config The agent AI configuration.
+   * @param tools Optional registry of callable tools.
+   * @returns Promise resolving to a {@link Runner}, or `undefined` if this
+   *   provider does not support agent creation.
    */
-  async invokeStructuredModel(_messages, _responseStructure) {
-    this.logger?.warn("invokeStructuredModel not implemented by this provider");
-    return {
-      data: {},
-      rawResponse: "",
-      metrics: {
-        success: false,
-        usage: {
-          total: 0,
-          input: 0,
-          output: 0
-        }
-      }
-    };
+  async createAgent(_config, _tools) {
+    return void 0;
   }
   /**
-   * Static method that constructs an instance of the provider.
-   * Each provider implementation must provide their own static create method
-   * that accepts an AIConfig and returns a configured instance.
+   * Create an AgentGraphRunner for an agent graph definition.
    *
-   * @param aiConfig The LaunchDarkly AI configuration
-   * @param logger Optional logger for the provider
-   * @returns Promise that resolves to a configured provider instance
+   * Override in provider subclasses to return a configured {@link AgentGraphRunner}.
+   * Default implementation returns `undefined`.
+   *
+   * @param graphDef The agent graph definition.
+   * @param tools Optional registry of callable tools.
+   * @returns Promise resolving to an {@link AgentGraphRunner}, or `undefined` if
+   *   this provider does not support graph execution.
    */
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  static async create(aiConfig, logger) {
-    throw new Error("Provider implementations must override the static create method");
+  async createAgentGraph(_graphDef, _tools) {
+    return void 0;
   }
 };
 
-// src/api/providers/AIProviderFactory.ts
+// src/api/providers/RunnerFactory.ts
 var SUPPORTED_AI_PROVIDERS = [
   "openai",
   // Multi-provider packages should be last in the list
   "langchain",
   "vercel"
 ];
-var AIProviderFactory = class {
+var RunnerFactory = class _RunnerFactory {
   /**
-   * Create an AIProvider instance based on the AI configuration.
-   * This method attempts to load provider-specific implementations dynamically.
-   * Returns undefined if the provider is not supported.
+   * Load and return the AIProvider factory for the given provider type.
+   *
+   * This is the single place in the codebase that knows provider package names.
+   * Each supported provider package exports a `*RunnerFactory` class that
+   * extends {@link AIProvider}; this method instantiates it directly.
    *
-   * @param aiConfig The AI configuration
-   * @param logger Optional logger for logging provider initialization
-   * @param defaultAiProvider Optional default AI provider to use
+   * @param providerType One of the {@link SUPPORTED_AI_PROVIDERS} values.
+   * @param logger Optional logger forwarded to the provider factory.
+   * @returns A configured {@link AIProvider} instance, or `undefined` if the
+   *   package cannot be loaded.
    */
-  static async create(aiConfig, logger, defaultAiProvider) {
-    const providerName = aiConfig.provider?.name?.toLowerCase();
-    const providersToTry = this._getProvidersToTry(defaultAiProvider, providerName);
-    for (const providerType of providersToTry) {
-      logger?.debug(
-        `Attempting to create AIProvider for: ${aiConfig.provider?.name} with provider type: ${providerType}`
-      );
-      const provider = await this._tryCreateProvider(providerType, aiConfig, logger);
-      if (provider) {
-        logger?.debug(`Successfully created AIProvider for: ${aiConfig.provider?.name}`);
-        return provider;
+  static async _getProviderFactory(providerType, logger) {
+    try {
+      let module;
+      switch (providerType) {
+        case "openai": {
+          module = await import("@launchdarkly/server-sdk-ai-openai");
+          return new module.OpenAIRunnerFactory(logger);
+        }
+        case "langchain": {
+          module = await import("@launchdarkly/server-sdk-ai-langchain");
+          return new module.LangChainRunnerFactory(logger);
+        }
+        case "vercel": {
+          module = await import("@launchdarkly/server-sdk-ai-vercel");
+          return new module.VercelRunnerFactory(logger);
+        }
+        default:
+          return void 0;
       }
+    } catch (error) {
+      logger?.warn(
+        `Unable to load provider package. Check that you have installed the correct package. ${error.message}`
+      );
+      return void 0;
     }
-    logger?.warn(
-      `Provider is not supported or failed to initialize: ${aiConfig.provider?.name ?? "unknown"}`
-    );
-    return void 0;
   }
   /**
    * Determine which providers to try based on defaultAiProvider and providerName.
+   *
+   * Mirrors Python's `_get_providers_to_try` helper.
    */
   static _getProvidersToTry(defaultAiProvider, providerName) {
     if (defaultAiProvider) {
@@ -843,36 +915,111 @@ var AIProviderFactory = class {
     return Array.from(providerSet);
   }
   /**
-   * Try to create a provider of the specified type.
+   * Try each provider in order and return the first non-undefined result.
+   *
+   * Mirrors Python's `_with_fallback` helper. Loads each provider factory via
+   * {@link _getProviderFactory} and calls `fn` with it. Returns the first
+   * truthy result, or `undefined` if no provider succeeds.
+   *
+   * @param providers Ordered list of provider types to try.
+   * @param fn Callback that calls the appropriate factory method on the provider.
+   * @param logger Optional logger forwarded to each provider factory.
    */
-  static async _tryCreateProvider(providerType, aiConfig, logger) {
-    try {
-      let module;
-      switch (providerType) {
-        case "openai": {
-          module = await import("@launchdarkly/server-sdk-ai-openai");
-          const provider = await module.OpenAIProvider.create(aiConfig, logger);
-          return provider;
-        }
-        case "langchain": {
-          module = await import("@launchdarkly/server-sdk-ai-langchain");
-          const provider = await module.LangChainProvider.create(aiConfig, logger);
-          return provider;
-        }
-        case "vercel": {
-          module = await import("@launchdarkly/server-sdk-ai-vercel");
-          const provider = await module.VercelProvider.create(aiConfig, logger);
-          return provider;
+  static async _withFallback(providers, fn, logger) {
+    for (const providerType of providers) {
+      logger?.debug(`Attempting to create runner with provider: ${providerType}`);
+      const factory = await _RunnerFactory._getProviderFactory(providerType, logger);
+      if (factory) {
+        const result = await fn(factory);
+        if (result) {
+          logger?.debug(`Successfully created runner with provider: ${providerType}`);
+          return result;
         }
-        default:
-          return void 0;
       }
-    } catch (error) {
+    }
+    return void 0;
+  }
+  /**
+   * Create a Runner for the given AI configuration.
+   *
+   * Suitable for completion, judge, and agent config modes. Dynamically
+   * loads the matching provider package via {@link _getProviderFactory} and
+   * delegates to its {@link AIProvider.createModel} method.
+   *
+   * @param config The AI configuration (completion, agent, or judge).
+   * @param logger Optional logger forwarded to the underlying provider.
+   * @param defaultAiProvider Optional provider override
+   *   ('openai', 'langchain', 'vercel', …). When set, only that provider is
+   *   tried. When omitted, providers are tried in priority order based on the
+   *   provider name in the config.
+   * @returns A configured {@link Runner} ready to invoke the model, or
+   *   `undefined` if no suitable provider could be loaded.
+   */
+  static async createModel(config, logger, defaultAiProvider) {
+    const providerName = config.provider?.name?.toLowerCase();
+    const providers = _RunnerFactory._getProvidersToTry(defaultAiProvider, providerName);
+    const runner = await _RunnerFactory._withFallback(
+      providers,
+      (factory) => factory.createModel(config),
+      logger
+    );
+    if (!runner) {
       logger?.warn(
-        `Unable to create AIProvider. Check that you have installed the correct package. ${error.message}`
+        `Provider is not supported or failed to initialize: ${config.provider?.name ?? "unknown"}`
       );
-      return void 0;
     }
+    return runner;
+  }
+  /**
+   * Create a Runner for an agent AI Config.
+   *
+   * Delegates to the provider factory's {@link AIProvider.createAgent} method.
+   *
+   * @param config The agent AI configuration.
+   * @param tools Optional registry of callable tools.
+   * @param logger Optional logger forwarded to the underlying provider.
+   * @param defaultAiProvider Optional provider override.
+   * @returns A configured {@link Runner}, or `undefined` if no suitable
+   *   provider could be loaded.
+   */
+  static async createAgent(config, tools, logger, defaultAiProvider) {
+    const providerName = config.provider?.name?.toLowerCase();
+    const providers = _RunnerFactory._getProvidersToTry(defaultAiProvider, providerName);
+    const runner = await _RunnerFactory._withFallback(
+      providers,
+      (factory) => factory.createAgent(config, tools),
+      logger
+    );
+    if (!runner) {
+      logger?.warn(
+        `Provider is not supported or failed to initialize: ${config.provider?.name ?? "unknown"}`
+      );
+    }
+    return runner;
+  }
+  /**
+   * Create an AgentGraphRunner for the given agent graph definition.
+   *
+   * Delegates to the provider factory's {@link AIProvider.createAgentGraph} method.
+   *
+   * @param graphDef The agent graph definition.
+   * @param tools Optional registry of callable tools.
+   * @param logger Optional logger forwarded to the underlying provider.
+   * @param defaultAiProvider Optional provider override.
+   * @returns A configured {@link AgentGraphRunner}, or `undefined` if no
+   *   suitable provider could be loaded.
+   */
+  static async createAgentGraph(graphDef, tools, logger, defaultAiProvider) {
+    const providers = _RunnerFactory._getProvidersToTry(defaultAiProvider);
+    const runner = await _RunnerFactory._withFallback(
+      providers,
+      (factory) => factory.createAgentGraph(graphDef, tools),
+      logger
+    );
+    if (!runner) {
+      logger?.warn(`No provider could create an AgentGraphRunner for the given graph definition.`);
+    }
+    return runner;
   }
 };
 
@@ -923,6 +1070,7 @@ var LDAIConfigTrackerImpl = class _LDAIConfigTrackerImpl {
     this._context = _context;
     this._graphKey = _graphKey;
     this._trackedMetrics = {};
+    this._trackedMetrics.resumptionToken = this.resumptionToken;
   }
   getTrackData() {
     return {
@@ -1006,6 +1154,10 @@ var LDAIConfigTrackerImpl = class _LDAIConfigTrackerImpl {
     }
   }
   trackToolCall(toolKey) {
+    if (!this._trackedMetrics.toolCalls) {
+      this._trackedMetrics.toolCalls = [];
+    }
+    this._trackedMetrics.toolCalls.push(toolKey);
     this._ldClient.track("$ld:ai:tool_call", this._context, { ...this.getTrackData(), toolKey }, 1);
   }
   trackToolCalls(toolKeys) {
@@ -1061,8 +1213,11 @@ var LDAIConfigTrackerImpl = class _LDAIConfigTrackerImpl {
     } else {
       this.trackError();
     }
-    if (metrics.usage) {
-      this.trackTokens(metrics.usage);
+    if (metrics.tokens) {
+      this.trackTokens(metrics.tokens);
+    }
+    if (metrics.toolCalls?.length) {
+      this.trackToolCalls(metrics.toolCalls);
     }
     return result;
   }
@@ -1086,8 +1241,11 @@ var LDAIConfigTrackerImpl = class _LDAIConfigTrackerImpl {
       } else {
         this.trackError();
       }
-      if (metrics.usage) {
-        this.trackTokens(metrics.usage);
+      if (metrics.tokens) {
+        this.trackTokens(metrics.tokens);
+      }
+      if (metrics.toolCalls?.length) {
+        this.trackToolCalls(metrics.toolCalls);
       }
     } catch (error) {
       this.trackError();
@@ -1172,6 +1330,7 @@ var LDGraphTrackerImpl = class _LDGraphTrackerImpl {
     this._version = _version;
     this._context = _context;
     this._summary = {};
+    this._summary.resumptionToken = this.resumptionToken;
   }
   /**
    * Reconstructs an {@link LDGraphTrackerImpl} from a resumption token, preserving
@@ -1197,15 +1356,12 @@ var LDGraphTrackerImpl = class _LDGraphTrackerImpl {
     );
   }
   getTrackData() {
-    const data = {
+    return {
       runId: this._runId,
       graphKey: this._graphKey,
-      version: this._version
+      version: this._version,
+      ...this._variationKey !== void 0 ? { variationKey: this._variationKey } : {}
     };
-    if (this._variationKey !== void 0) {
-      data.variationKey = this._variationKey;
-    }
-    return data;
   }
   getSummary() {
     return { ...this._summary };
@@ -1310,13 +1466,14 @@ var LDGraphTrackerImpl = class _LDGraphTrackerImpl {
 
 // src/sdkInfo.ts
 var aiSdkName = "@launchdarkly/server-sdk-ai";
-var aiSdkVersion = "0.19.0";
+var aiSdkVersion = "0.20.0";
 var aiSdkLanguage = "javascript";
 
 // src/LDAIClientImpl.ts
 var TRACK_SDK_INFO = "$ld:ai:sdk:info";
 var TRACK_USAGE_COMPLETION_CONFIG = "$ld:ai:usage:completion-config";
 var TRACK_USAGE_CREATE_CHAT = "$ld:ai:usage:create-chat";
+var TRACK_USAGE_CREATE_AGENT = "$ld:ai:usage:create-agent";
 var TRACK_USAGE_JUDGE_CONFIG = "$ld:ai:usage:judge-config";
 var TRACK_USAGE_CREATE_JUDGE = "$ld:ai:usage:create-judge";
 var TRACK_USAGE_AGENT_CONFIG = "$ld:ai:usage:agent-config";
@@ -1344,18 +1501,11 @@ var LDAIClientImpl = class {
     );
   }
   _interpolateTemplate(template, variables) {
-    return Mustache2.render(template, variables, void 0, { escape: (item) => item });
+    return Mustache.render(template, variables, void 0, { escape: (item) => item });
   }
-  async _evaluate(key, context, defaultValue, mode, variables, graphKey) {
+  async _evaluate(key, context, defaultValue, mode, variables, graphKey, defaultAiProvider) {
     const ldFlagValue = LDAIConfigUtils.toFlagValue(defaultValue, mode);
     const value = await this._ldClient.variation(key, context, ldFlagValue);
-    const flagMode = value._ldMeta?.mode ?? "completion";
-    if (flagMode !== mode) {
-      this._logger?.warn(
-        `AI Config mode mismatch for ${key}: expected ${mode}, got ${flagMode}. Returning disabled config.`
-      );
-      return LDAIConfigUtils.createDisabledConfig(key, mode);
-    }
     const trackerFactory = () => new LDAIConfigTrackerImpl(
       this._ldClient,
       randomUUID(),
@@ -1369,7 +1519,23 @@ var LDAIClientImpl = class {
       context,
       graphKey
     );
-    const config = LDAIConfigUtils.fromFlagValue(key, value, trackerFactory);
+    const flagMode = value._ldMeta?.mode ?? "completion";
+    let evaluator = Evaluator.noop();
+    if (flagMode !== mode) {
+      this._logger?.warn(
+        `AI Config mode mismatch for ${key}: expected ${mode}, got ${flagMode}. Returning disabled config.`
+      );
+      return LDAIConfigUtils.createDisabledConfig(key, mode, trackerFactory, evaluator);
+    }
+    if (flagMode !== "judge") {
+      evaluator = await this._buildEvaluator(
+        value.judgeConfiguration?.judges ?? [],
+        context,
+        variables,
+        defaultAiProvider
+      );
+    }
+    const config = LDAIConfigUtils.fromFlagValue(key, value, trackerFactory, evaluator);
     return this._applyInterpolation(config, context, variables);
   }
   _applyInterpolation(config, context, variables) {
@@ -1391,33 +1557,44 @@ var LDAIClientImpl = class {
     }
     return config;
   }
-  async _initializeJudges(judgeConfigs, context, variables, defaultAiProvider) {
-    const judges = {};
-    const judgePromises = judgeConfigs.map(async (judgeConfig) => {
-      const judge = await this.createJudge(
-        judgeConfig.key,
-        context,
-        void 0,
-        variables,
-        defaultAiProvider
-      );
-      return judge ? { key: judgeConfig.key, judge } : null;
-    });
-    const results = await Promise.all(judgePromises);
-    results.forEach((result) => {
-      if (result) {
-        judges[result.key] = result.judge;
-      }
-    });
-    return judges;
-  }
-  async _completionConfig(key, context, defaultValue, variables) {
-    const config = await this._evaluate(key, context, defaultValue, "completion", variables);
-    return config;
+  async _buildEvaluator(judgeConfigs, context, variables, defaultAiProvider) {
+    if (judgeConfigs.length === 0) {
+      return Evaluator.noop();
+    }
+    const judgeInstances = (await Promise.all(
+      judgeConfigs.map(
+        (jc) => this._createJudgeInstance(
+          jc.key,
+          context,
+          void 0,
+          variables,
+          defaultAiProvider,
+          jc.samplingRate
+        )
+      )
+    )).filter((j) => j !== void 0);
+    return new Evaluator(judgeInstances);
+  }
+  async _completionConfig(key, context, defaultValue, variables, defaultAiProvider) {
+    return await this._evaluate(
+      key,
+      context,
+      defaultValue,
+      "completion",
+      variables,
+      void 0,
+      defaultAiProvider
+    );
   }
-  async completionConfig(key, context, defaultValue, variables) {
+  async completionConfig(key, context, defaultValue, variables, defaultAiProvider) {
     this._ldClient.track(TRACK_USAGE_COMPLETION_CONFIG, context, key, 1);
-    return this._completionConfig(key, context, defaultValue ?? disabledAIConfig, variables);
+    return this._completionConfig(
+      key,
+      context,
+      defaultValue ?? disabledAIConfig,
+      variables,
+      defaultAiProvider
+    );
   }
   /**
    * @deprecated Use `completionConfig` instead. This method will be removed in a future version.
@@ -1426,20 +1603,58 @@ var LDAIClientImpl = class {
     return this.completionConfig(key, context, defaultValue, variables);
   }
   async _judgeConfig(key, context, defaultValue, variables) {
-    const config = await this._evaluate(key, context, defaultValue, "judge", variables);
+    if (variables?.message_history !== void 0) {
+      this._logger?.warn(
+        "The variable 'message_history' is reserved by the judge and will be ignored."
+      );
+    }
+    if (variables?.response_to_evaluate !== void 0) {
+      this._logger?.warn(
+        "The variable 'response_to_evaluate' is reserved by the judge and will be ignored."
+      );
+    }
+    const extendedVariables = {
+      ...variables,
+      message_history: "{{message_history}}",
+      response_to_evaluate: "{{response_to_evaluate}}"
+    };
+    const config = await this._evaluate(
+      key,
+      context,
+      defaultValue,
+      "judge",
+      extendedVariables
+    );
+    if (config.messages) {
+      return { ...config, messages: stripLegacyJudgeMessages(config.messages) };
+    }
     return config;
   }
   async judgeConfig(key, context, defaultValue, variables) {
     this._ldClient.track(TRACK_USAGE_JUDGE_CONFIG, context, key, 1);
     return this._judgeConfig(key, context, defaultValue ?? disabledAIConfig, variables);
   }
-  async _agentConfig(key, context, defaultValue, variables, graphKey) {
-    const config = await this._evaluate(key, context, defaultValue, "agent", variables, graphKey);
-    return config;
+  async _agentConfig(key, context, defaultValue, variables, graphKey, defaultAiProvider) {
+    return await this._evaluate(
+      key,
+      context,
+      defaultValue,
+      "agent",
+      variables,
+      graphKey,
+      defaultAiProvider
+    );
   }
-  async agentConfig(key, context, defaultValue, variables) {
+  async agentConfig(key, context, defaultValue, variables, defaultAiProvider) {
     this._ldClient.track(TRACK_USAGE_AGENT_CONFIG, context, key, 1);
-    return this._agentConfig(key, context, defaultValue ?? disabledAIConfig, variables);
+    return this._agentConfig(
+      key,
+      context,
+      defaultValue ?? disabledAIConfig,
+      variables,
+      void 0,
+      defaultAiProvider
+    );
   }
   /**
    * @deprecated Use `agentConfig` instead. This method will be removed in a future version.
@@ -1474,73 +1689,89 @@ var LDAIClientImpl = class {
   async agents(agentConfigs, context) {
     return this.agentConfigs(agentConfigs, context);
   }
-  async createChat(key, context, defaultValue, variables, defaultAiProvider) {
-    this._ldClient.track(TRACK_USAGE_CREATE_CHAT, context, key, 1);
-    const config = await this._completionConfig(
+  async createJudge(key, context, defaultValue, variables, defaultAiProvider, sampleRate = 1) {
+    this._ldClient.track(TRACK_USAGE_CREATE_JUDGE, context, key, 1);
+    return this._createJudgeInstance(
       key,
       context,
-      defaultValue ?? disabledAIConfig,
-      variables
-    );
-    if (!config.enabled) {
-      this._logger?.info(`Chat configuration is disabled: ${key}`);
-      return void 0;
-    }
-    const provider = await AIProviderFactory.create(config, this._logger, defaultAiProvider);
-    if (!provider) {
-      return void 0;
-    }
-    const judges = await this._initializeJudges(
-      config.judgeConfiguration?.judges ?? [],
-      context,
+      defaultValue,
       variables,
-      defaultAiProvider
+      defaultAiProvider,
+      sampleRate
     );
-    return new TrackedChat(config, provider, judges, this._logger);
   }
-  async createJudge(key, context, defaultValue, variables, defaultAiProvider) {
-    this._ldClient.track(TRACK_USAGE_CREATE_JUDGE, context, key, 1);
+  async _createJudgeInstance(key, context, defaultValue, variables, defaultAiProvider, sampleRate = 1) {
     try {
-      if (variables?.message_history !== void 0) {
-        this._logger?.warn(
-          "The variable 'message_history' is reserved by the judge and will be ignored."
-        );
-      }
-      if (variables?.response_to_evaluate !== void 0) {
-        this._logger?.warn(
-          "The variable 'response_to_evaluate' is reserved by the judge and will be ignored."
-        );
-      }
-      const extendedVariables = {
-        ...variables,
-        message_history: "{{message_history}}",
-        response_to_evaluate: "{{response_to_evaluate}}"
-      };
       const judgeConfig = await this._judgeConfig(
         key,
         context,
         defaultValue ?? disabledAIConfig,
-        extendedVariables
+        variables
       );
       if (!judgeConfig.enabled) {
         this._logger?.info(`Judge configuration is disabled: ${key}`);
         return void 0;
       }
-      const provider = await AIProviderFactory.create(judgeConfig, this._logger, defaultAiProvider);
-      if (!provider) {
+      const runner = await RunnerFactory.createModel(judgeConfig, this._logger, defaultAiProvider);
+      if (!runner) {
         return void 0;
       }
-      return new Judge(judgeConfig, provider, this._logger);
+      return new Judge(judgeConfig, runner, sampleRate, this._logger);
     } catch (error) {
       this._logger?.error(`Failed to initialize judge ${key}:`, error);
       return void 0;
     }
   }
+  async createModel(key, context, defaultValue, variables, defaultAiProvider) {
+    this._ldClient.track(TRACK_USAGE_CREATE_CHAT, context, key, 1);
+    const config = await this._completionConfig(
+      key,
+      context,
+      defaultValue ?? disabledAIConfig,
+      variables,
+      defaultAiProvider
+    );
+    if (!config.enabled) {
+      this._logger?.info(`Completion configuration is disabled: ${key}`);
+      return void 0;
+    }
+    const runner = await RunnerFactory.createModel(config, this._logger, defaultAiProvider);
+    if (!runner) {
+      return void 0;
+    }
+    return new ManagedModel(config, runner, this._logger);
+  }
+  async createAgent(key, context, defaultValue, variables, defaultAiProvider) {
+    this._ldClient.track(TRACK_USAGE_CREATE_AGENT, context, key, 1);
+    const config = await this._agentConfig(
+      key,
+      context,
+      defaultValue ?? disabledAIConfig,
+      variables,
+      void 0,
+      defaultAiProvider
+    );
+    if (!config.enabled) {
+      this._logger?.info(`Agent configuration is disabled: ${key}`);
+      return void 0;
+    }
+    const runner = await RunnerFactory.createAgent(config, void 0, this._logger, defaultAiProvider);
+    if (!runner) {
+      return void 0;
+    }
+    return new ManagedAgent(config, runner, this._logger);
+  }
+  /**
+   * @deprecated Use `createModel` instead. This method will be removed in a future version.
+   */
+  async createChat(key, context, defaultValue, variables, defaultAiProvider) {
+    return this.createModel(key, context, defaultValue, variables, defaultAiProvider);
+  }
   /**
-   * @deprecated Use `createChat` instead. This method will be removed in a future version.
+   * @deprecated Use `createModel` instead. This method will be removed in a future version.
    */
   async initChat(key, context, defaultValue, variables, defaultAiProvider) {
-    return this.createChat(key, context, defaultValue, variables, defaultAiProvider);
+    return this.createModel(key, context, defaultValue, variables, defaultAiProvider);
   }
   createTracker(token, context) {
     return LDAIConfigTrackerImpl.fromResumptionToken(token, this._ldClient, context);
@@ -1625,14 +1856,16 @@ function initAi(ldClient) {
 }
 export {
   AIProvider,
-  AIProviderFactory,
   AgentGraphDefinition,
   AgentGraphNode,
   Judge,
   LDFeedbackKind,
   LDGraphTrackerImpl,
+  ManagedAgent,
+  ManagedAgentGraph,
+  ManagedModel,
+  RunnerFactory,
   SUPPORTED_AI_PROVIDERS,
-  TrackedChat,
   createBedrockTokenUsage,
   createOpenAiUsage,
   createVercelAISDKTokenUsage,