@learning-commons/evaluators 0.4.0 → 0.6.0

package/dist/{base-Ced9oKKa.d.ts → base-DKcAYXfb.d.ts}

@@ -1,3 +1,5 @@
+import { z } from 'zod';
+
 /**
  * Logging interface for the Evaluators SDK
  *
@@ -77,6 +79,82 @@ interface Logger {
     error(message: string, context?: LogContext): void;
 }
 
+/**
+ * Message format for LLM conversations
+ */
+interface Message {
+    role: 'system' | 'user' | 'assistant';
+    content: string;
+}
+/**
+ * Request configuration for structured LLM generation
+ */
+interface LLMRequest<T> {
+    messages: Message[];
+    schema: z.ZodSchema<T>;
+    temperature?: number;
+    maxTokens?: number;
+}
+/**
+ * Response from LLM with usage metadata
+ */
+interface LLMResponse<T> {
+    data: T;
+    model: string;
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+    };
+    latencyMs: number;
+}
+/**
+ * Response from plain text generation
+ */
+interface TextGenerationResponse {
+    text: string;
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+    };
+    latencyMs: number;
+}
+/**
+ * Base interface for LLM provider implementations
+ */
+interface LLMProvider {
+    /** Canonical label for the provider and model in use (e.g. "openai:gpt-4o") */
+    readonly label: string;
+    /**
+     * Generate structured output from LLM using Zod schema
+     */
+    generateStructured<T>(request: LLMRequest<T>): Promise<LLMResponse<T>>;
+    /**
+     * Generate plain text from LLM
+     */
+    generateText(messages: Message[], temperature?: number): Promise<TextGenerationResponse>;
+}
+/**
+ * Named constants for LLM provider types — use instead of raw string literals.
+ */
+declare const Providers: {
+    readonly google: "google";
+    readonly openai: "openai";
+    readonly anthropic: "anthropic";
+    readonly custom: "custom";
+};
+/**
+ * Configuration for LLM provider
+ */
+interface ProviderConfig {
+    type: 'openai' | 'anthropic' | 'google' | 'custom';
+    apiKey?: string;
+    model?: string;
+    temperature?: number;
+    baseURL?: string;
+    customProvider?: LLMProvider;
+    maxRetries?: number;
+}
+
 /**
  * Evaluation status
  */
@@ -130,6 +208,7 @@ interface TelemetryEvent {
     provider: string;
     token_usage?: TokenUsage;
     metadata?: TelemetryMetadata;
+    model_override?: boolean;
     input_text?: string;
 }
 /**
@@ -166,6 +245,14 @@ declare class TelemetryClient {
     send(event: TelemetryEvent): Promise<void>;
 }
 
+/**
+ * Supported LLM providers
+ */
+declare enum Provider {
+    OpenAI = "openai",
+    Google = "google",
+    Anthropic = "anthropic"
+}
 /**
  * Granular telemetry configuration options
  */
@@ -175,6 +262,24 @@ interface TelemetryOptions {
     /** Record input text in telemetry (default: false) */
     recordInputs?: boolean;
 }
+/**
+ * Override the provider and model used by an evaluator.
+ *
+ * When set, all LLM calls use this provider and model instead of the defaults.
+ * The evaluator's normal key requirements are bypassed — provide the key for
+ * the chosen provider via the matching top-level config field
+ * (e.g. `anthropicApiKey` for `Provider.Anthropic`).
+ *
+ * Both `provider` and `model` are required. An empty or missing `model` throws
+ * `ConfigurationError` at construction time. An unrecognised model ID throws
+ * `ConfigurationError` at evaluation time when the provider rejects it.
+ *
+ * Results may vary; evaluators are validated against their recommended models.
+ */
+interface ModelOverride {
+    provider: Provider;
+    model: string;
+}
 /**
  * Base configuration for all evaluators
  */
@@ -183,8 +288,16 @@ interface BaseEvaluatorConfig {
     googleApiKey?: string;
     /** OpenAI API key (for evaluators using GPT) */
     openaiApiKey?: string;
+    /** Anthropic API key (for evaluators using Claude) */
+    anthropicApiKey?: string;
     /** Learning Commons partner key for authenticated telemetry (optional) */
     partnerKey?: string;
+    /**
+     * Override the provider and model used by this evaluator.
+     * When set, all LLM calls use this provider and model instead of the defaults.
+     * See {@link ModelOverride} for details.
+     */
+    modelOverride?: ModelOverride;
     /**
      * Maximum number of retries for failed API calls (default: 2)
      * Set to 0 to disable retries.
@@ -232,10 +345,8 @@ interface EvaluatorMetadata {
     readonly description: string;
     /** Supported grade levels (e.g., ['3', '4', '5', ...]) */
     readonly supportedGrades: readonly string[];
-    /** Whether this evaluator requires a Google API key */
-    readonly requiresGoogleKey: boolean;
-    /** Whether this evaluator requires an OpenAI API key */
-    readonly requiresOpenAIKey: boolean;
+    /** Providers required by this evaluator's default configuration */
+    readonly defaultProviders: readonly Provider[];
 }
 /**
  * Abstract base class for all evaluators
@@ -254,6 +365,10 @@ declare abstract class BaseEvaluator {
     protected logger: Logger;
     protected config: Required<Pick<BaseEvaluatorConfig, 'maxRetries'>> & {
         telemetry: Required<TelemetryOptions>;
+        modelOverride?: ModelOverride;
+        googleApiKey?: string;
+        openaiApiKey?: string;
+        anthropicApiKey?: string;
     };
     /**
      * Static metadata for the evaluator
@@ -268,13 +383,17 @@ declare abstract class BaseEvaluator {
      *     name: 'My Evaluator',
      *     description: 'Does something useful',
      *     supportedGrades: ['3', '4', '5'],
-     *     requiresGoogleKey: true,
-     *     requiresOpenAIKey: false,
+     *     defaultProviders: [Provider.Google],
      *   };
      * }
      * ```
      */
     static readonly metadata: EvaluatorMetadata;
+    /**
+     * @throws {ConfigurationError} If the subclass has not defined static metadata
+     * @throws {ConfigurationError} If modelOverride has an invalid provider or empty model
+     * @throws {ConfigurationError} If a required API key is missing
+     */
     constructor(config: BaseEvaluatorConfig);
     /**
      * Get metadata for this evaluator instance
@@ -282,8 +401,16 @@ declare abstract class BaseEvaluator {
      */
     protected get metadata(): EvaluatorMetadata;
     /**
-     * Validate that required API keys are provided based on metadata
-     * @throws {ConfigurationError} If required API keys are missing
+     * Validate modelOverride shape: provider must be a known Provider value and
+     * model must be a non-empty string.
+     * @throws {ConfigurationError} If the override is malformed
+     */
+    private validateModelOverride;
+    /**
+     * Validate that the required API key is present.
+     * When modelOverride is set, checks the override provider's key.
+     * Otherwise checks the keys required by the evaluator's default providers.
+     * @throws {ConfigurationError} If a required key is missing
      */
     private validateApiKeys;
     /**
@@ -311,6 +438,12 @@ declare abstract class BaseEvaluator {
      * @throws {ValidationError} If grade is invalid
      */
     protected validateGrade(grade: string, validGrades: Set<string>): void;
+    /**
+     * Create an LLM provider, honouring modelOverride if set.
+     * When override is active, the key for the override provider is resolved
+     * from the matching top-level config field (e.g. anthropicApiKey for Anthropic).
+     */
+    protected createConfiguredProvider(defaultType: Provider, defaultModel: string, defaultApiKey: string | undefined): LLMProvider;
     /**
      * Send telemetry event to analytics service
      * Common helper for all evaluators
@@ -328,4 +461,4 @@ declare abstract class BaseEvaluator {
     }): Promise<void>;
 }
 
-export { BaseEvaluator as B, type EvaluatorMetadata as E, type Logger as L, type TelemetryOptions as T, type BaseEvaluatorConfig as a, type LogContext as b, LogLevel as c };
+export { BaseEvaluator as B, type EvaluatorMetadata as E, type LLMProvider as L, type Message as M, Provider as P, type TelemetryOptions as T, type BaseEvaluatorConfig as a, type LLMRequest as b, type LLMResponse as c, type LogContext as d, LogLevel as e, type Logger as f, type ModelOverride as g, type ProviderConfig as h, Providers as i, type TextGenerationResponse as j };