@learning-commons/evaluators 0.3.0 → 0.5.0

package/dist/base-DKcAYXfb.d.cts

@@ -0,0 +1,464 @@
+import { z } from 'zod';
+
+/**
+ * Logging interface for the Evaluators SDK
+ *
+ * Provides structured logging with verbosity levels.
+ * Users can inject custom loggers or use the default console logger.
+ */
+/**
+ * Log levels in order of verbosity
+ */
+declare enum LogLevel {
+    /** Debug messages - very verbose, for development */
+    DEBUG = 0,
+    /** Informational messages - normal operations */
+    INFO = 1,
+    /** Warning messages - potentially problematic situations */
+    WARN = 2,
+    /** Error messages - errors that need attention */
+    ERROR = 3,
+    /** Silent - no logging */
+    SILENT = 4
+}
+/**
+ * Context object for structured logging
+ */
+interface LogContext {
+    /** Evaluator type (vocabulary, sentence-structure, etc.) */
+    evaluator?: string;
+    /** Current operation or stage */
+    operation?: string;
+    /** Error object if applicable */
+    error?: Error;
+    /** Additional metadata */
+    [key: string]: unknown;
+}
+/**
+ * Logger interface
+ *
+ * Implement this interface to provide custom logging behavior.
+ *
+ * @example
+ * ```typescript
+ * const customLogger: Logger = {
+ *   debug: (msg, ctx) => myLogger.debug(msg, ctx),
+ *   info: (msg, ctx) => myLogger.info(msg, ctx),
+ *   warn: (msg, ctx) => myLogger.warn(msg, ctx),
+ *   error: (msg, ctx) => myLogger.error(msg, ctx),
+ * };
+ *
+ * const evaluator = new VocabularyEvaluator({
+ *   googleApiKey: '...',
+ *   openaiApiKey: '...',
+ *   logger: customLogger,
+ *   logLevel: LogLevel.INFO,
+ * });
+ * ```
+ */
+interface Logger {
+    /**
+     * Log debug message
+     * Used for detailed debugging information
+     */
+    debug(message: string, context?: LogContext): void;
+    /**
+     * Log informational message
+     * Used for normal operations
+     */
+    info(message: string, context?: LogContext): void;
+    /**
+     * Log warning message
+     * Used for potentially problematic situations
+     */
+    warn(message: string, context?: LogContext): void;
+    /**
+     * Log error message
+     * Used for errors that need attention
+     */
+    error(message: string, context?: LogContext): void;
+}
+
+/**
+ * Message format for LLM conversations
+ */
+interface Message {
+    role: 'system' | 'user' | 'assistant';
+    content: string;
+}
+/**
+ * Request configuration for structured LLM generation
+ */
+interface LLMRequest<T> {
+    messages: Message[];
+    schema: z.ZodSchema<T>;
+    temperature?: number;
+    maxTokens?: number;
+}
+/**
+ * Response from LLM with usage metadata
+ */
+interface LLMResponse<T> {
+    data: T;
+    model: string;
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+    };
+    latencyMs: number;
+}
+/**
+ * Response from plain text generation
+ */
+interface TextGenerationResponse {
+    text: string;
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+    };
+    latencyMs: number;
+}
+/**
+ * Base interface for LLM provider implementations
+ */
+interface LLMProvider {
+    /** Canonical label for the provider and model in use (e.g. "openai:gpt-4o") */
+    readonly label: string;
+    /**
+     * Generate structured output from LLM using Zod schema
+     */
+    generateStructured<T>(request: LLMRequest<T>): Promise<LLMResponse<T>>;
+    /**
+     * Generate plain text from LLM
+     */
+    generateText(messages: Message[], temperature?: number): Promise<TextGenerationResponse>;
+}
+/**
+ * Named constants for LLM provider types — use instead of raw string literals.
+ */
+declare const Providers: {
+    readonly google: "google";
+    readonly openai: "openai";
+    readonly anthropic: "anthropic";
+    readonly custom: "custom";
+};
+/**
+ * Configuration for LLM provider
+ */
+interface ProviderConfig {
+    type: 'openai' | 'anthropic' | 'google' | 'custom';
+    apiKey?: string;
+    model?: string;
+    temperature?: number;
+    baseURL?: string;
+    customProvider?: LLMProvider;
+    maxRetries?: number;
+}
+
+/**
+ * Evaluation status
+ */
+type EvaluationStatus = 'success' | 'error';
+/**
+ * Token usage metrics from LLM providers
+ */
+interface TokenUsage {
+    input_tokens: number;
+    output_tokens: number;
+}
+/**
+ * Per-stage details for multi-stage evaluations
+ */
+interface StageDetail {
+    /** Stage name (e.g., "background_knowledge", "complexity_evaluation") */
+    stage: string;
+    /** Provider used for this stage (e.g., "openai:gpt-4o") */
+    provider: string;
+    /** Total latency including all retries (ms) */
+    latency_ms: number;
+    /** Token usage aggregated across all attempts */
+    token_usage?: TokenUsage;
+    /**
+     * Whether schema validation failed (indicates prompt needs clearer instructions)
+     *
+     * TODO: Not currently tracked. Vercel AI SDK abstracts validation away.
+     * To implement: Add custom retry wrapper that catches validation errors.
+     */
+    schema_validation_failed?: boolean;
+}
+/**
+ * Extensible metadata for telemetry events
+ */
+interface TelemetryMetadata {
+    /** Detailed breakdown by stage (for multi-stage evaluations) */
+    stage_details?: StageDetail[];
+}
+/**
+ * Telemetry event payload
+ */
+interface TelemetryEvent {
+    timestamp: string;
+    sdk_version: string;
+    evaluator_type: string;
+    grade?: string;
+    status: EvaluationStatus;
+    error_code?: string;
+    latency_ms: number;
+    text_length_chars: number;
+    provider: string;
+    token_usage?: TokenUsage;
+    metadata?: TelemetryMetadata;
+    model_override?: boolean;
+    input_text?: string;
+}
+/**
+ * Configuration for telemetry client
+ */
+interface TelemetryConfig {
+    /** Analytics service endpoint URL */
+    endpoint: string;
+    /** Learning Commons partner key (optional, sent as X-API-Key header) */
+    partnerKey?: string;
+    /** Client ID for anonymous tracking (persistent UUID from ~/.config/learning-commons/config.json) */
+    clientId: string;
+    /** Enable telemetry (default: true) */
+    enabled: boolean;
+    /** Logger instance (respects the SDK's configured log level and custom logger) */
+    logger: Logger;
+}
+
+/**
+ * Telemetry client for sending analytics events
+ *
+ * Fire-and-forget implementation that never blocks SDK operations.
+ * Errors are logged but don't fail evaluations.
+ */
+declare class TelemetryClient {
+    private config;
+    private logger;
+    constructor(config: TelemetryConfig);
+    /**
+     * Send telemetry event to analytics service
+     *
+     * Fire-and-forget: Errors are logged but don't throw.
+     */
+    send(event: TelemetryEvent): Promise<void>;
+}
+
+/**
+ * Supported LLM providers
+ */
+declare enum Provider {
+    OpenAI = "openai",
+    Google = "google",
+    Anthropic = "anthropic"
+}
+/**
+ * Granular telemetry configuration options
+ */
+interface TelemetryOptions {
+    /** Enable telemetry (default: true) */
+    enabled?: boolean;
+    /** Record input text in telemetry (default: false) */
+    recordInputs?: boolean;
+}
+/**
+ * Override the provider and model used by an evaluator.
+ *
+ * When set, all LLM calls use this provider and model instead of the defaults.
+ * The evaluator's normal key requirements are bypassed — provide the key for
+ * the chosen provider via the matching top-level config field
+ * (e.g. `anthropicApiKey` for `Provider.Anthropic`).
+ *
+ * Both `provider` and `model` are required. An empty or missing `model` throws
+ * `ConfigurationError` at construction time. An unrecognised model ID throws
+ * `ConfigurationError` at evaluation time when the provider rejects it.
+ *
+ * Results may vary; evaluators are validated against their recommended models.
+ */
+interface ModelOverride {
+    provider: Provider;
+    model: string;
+}
+/**
+ * Base configuration for all evaluators
+ */
+interface BaseEvaluatorConfig {
+    /** Google API key (for evaluators using Gemini) */
+    googleApiKey?: string;
+    /** OpenAI API key (for evaluators using GPT) */
+    openaiApiKey?: string;
+    /** Anthropic API key (for evaluators using Claude) */
+    anthropicApiKey?: string;
+    /** Learning Commons partner key for authenticated telemetry (optional) */
+    partnerKey?: string;
+    /**
+     * Override the provider and model used by this evaluator.
+     * When set, all LLM calls use this provider and model instead of the defaults.
+     * See {@link ModelOverride} for details.
+     */
+    modelOverride?: ModelOverride;
+    /**
+     * Maximum number of retries for failed API calls (default: 2)
+     * Set to 0 to disable retries.
+     *
+     * Note: With maxRetries=2, a failed call will be attempted up to 3 times total
+     * (1 initial attempt + 2 retries)
+     */
+    maxRetries?: number;
+    /**
+     * Telemetry configuration (default: all enabled)
+     *
+     * Can be:
+     * - `true`: Enable with defaults (recordInputs: false)
+     * - `false`: Disable completely
+     * - `TelemetryOptions`: Granular control
+     */
+    telemetry?: boolean | TelemetryOptions;
+    /**
+     * Custom logger implementation (optional)
+     * If not provided, uses console logger with specified logLevel
+     */
+    logger?: Logger;
+    /**
+     * Log level for default console logger (default: WARN)
+     * Only used if custom logger is not provided
+     *
+     * - DEBUG: Very verbose, shows all operations
+     * - INFO: Normal operations
+     * - WARN: Warnings only (default)
+     * - ERROR: Errors only
+     * - SILENT: No logging
+     */
+    logLevel?: LogLevel;
+}
+/**
+ * Evaluator metadata interface
+ * Each evaluator must provide this metadata as static properties
+ */
+interface EvaluatorMetadata {
+    /** Unique identifier for the evaluator (e.g., 'vocabulary', 'sentence-structure') */
+    readonly id: string;
+    /** Human-readable name (e.g., 'Vocabulary', 'Sentence Structure') */
+    readonly name: string;
+    /** Brief description of what the evaluator does */
+    readonly description: string;
+    /** Supported grade levels (e.g., ['3', '4', '5', ...]) */
+    readonly supportedGrades: readonly string[];
+    /** Providers required by this evaluator's default configuration */
+    readonly defaultProviders: readonly Provider[];
+}
+/**
+ * Abstract base class for all evaluators
+ *
+ * Provides common functionality:
+ * - Telemetry setup and event sending
+ * - Text validation
+ * - Grade validation (with overridable default)
+ * - Metadata creation
+ *
+ * Concrete evaluators must implement:
+ * - static metadata: Provide evaluator metadata (see EvaluatorMetadata interface)
+ */
+declare abstract class BaseEvaluator {
+    protected telemetryClient?: TelemetryClient;
+    protected logger: Logger;
+    protected config: Required<Pick<BaseEvaluatorConfig, 'maxRetries'>> & {
+        telemetry: Required<TelemetryOptions>;
+        modelOverride?: ModelOverride;
+        googleApiKey?: string;
+        openaiApiKey?: string;
+        anthropicApiKey?: string;
+    };
+    /**
+     * Static metadata for the evaluator
+     *
+     * Concrete evaluators MUST define this property.
+     *
+     * @example
+     * ```typescript
+     * class MyEvaluator extends BaseEvaluator {
+     *   static readonly metadata = {
+     *     id: 'my-evaluator',
+     *     name: 'My Evaluator',
+     *     description: 'Does something useful',
+     *     supportedGrades: ['3', '4', '5'],
+     *     defaultProviders: [Provider.Google],
+     *   };
+     * }
+     * ```
+     */
+    static readonly metadata: EvaluatorMetadata;
+    /**
+     * @throws {ConfigurationError} If the subclass has not defined static metadata
+     * @throws {ConfigurationError} If modelOverride has an invalid provider or empty model
+     * @throws {ConfigurationError} If a required API key is missing
+     */
+    constructor(config: BaseEvaluatorConfig);
+    /**
+     * Get metadata for this evaluator instance
+     * @throws {ConfigurationError} If the subclass has not defined static metadata
+     */
+    protected get metadata(): EvaluatorMetadata;
+    /**
+     * Validate modelOverride shape: provider must be a known Provider value and
+     * model must be a non-empty string.
+     * @throws {ConfigurationError} If the override is malformed
+     */
+    private validateModelOverride;
+    /**
+     * Validate that the required API key is present.
+     * When modelOverride is set, checks the override provider's key.
+     * Otherwise checks the keys required by the evaluator's default providers.
+     * @throws {ConfigurationError} If a required key is missing
+     */
+    private validateApiKeys;
+    /**
+     * Normalize telemetry config to standard format
+     */
+    private normalizeTelemetryConfig;
+    /**
+     * Get the evaluator type identifier from metadata
+     * @returns The evaluator type ID (e.g., "vocabulary", "sentence-structure")
+     */
+    protected getEvaluatorType(): string;
+    /**
+     * Validate text meets requirements
+     * Default implementation - can be overridden by concrete evaluators
+     *
+     * @throws {ValidationError} If text is invalid
+     */
+    protected validateText(text: string): void;
+    /**
+     * Validate grade is in supported range
+     * Default implementation - can be overridden by concrete evaluators
+     *
+     * @param grade - Grade level to validate
+     * @param validGrades - Set of valid grades for this evaluator
+     * @throws {ValidationError} If grade is invalid
+     */
+    protected validateGrade(grade: string, validGrades: Set<string>): void;
+    /**
+     * Create an LLM provider, honouring modelOverride if set.
+     * When override is active, the key for the override provider is resolved
+     * from the matching top-level config field (e.g. anthropicApiKey for Anthropic).
+     */
+    protected createConfiguredProvider(defaultType: Provider, defaultModel: string, defaultApiKey: string | undefined): LLMProvider;
+    /**
+     * Send telemetry event to analytics service
+     * Common helper for all evaluators
+     */
+    protected sendTelemetry(params: {
+        status: 'success' | 'error';
+        latencyMs: number;
+        textLength: number;
+        grade?: string;
+        provider: string;
+        errorCode?: string;
+        tokenUsage?: TokenUsage;
+        metadata?: TelemetryMetadata;
+        inputText?: string;
+    }): Promise<void>;
+}
+
+export { BaseEvaluator as B, type EvaluatorMetadata as E, type LLMProvider as L, type Message as M, Provider as P, type TelemetryOptions as T, type BaseEvaluatorConfig as a, type LLMRequest as b, type LLMResponse as c, type LogContext as d, LogLevel as e, type Logger as f, type ModelOverride as g, type ProviderConfig as h, Providers as i, type TextGenerationResponse as j };