@learning-commons/evaluators 0.2.0 → 0.4.0

package/dist/base-Ced9oKKa.d.ts

@@ -0,0 +1,331 @@
+/**
+ * Logging interface for the Evaluators SDK
+ *
+ * Provides structured logging with verbosity levels.
+ * Users can inject custom loggers or use the default console logger.
+ */
+/**
+ * Log levels in order of verbosity
+ */
+declare enum LogLevel {
+    /** Debug messages - very verbose, for development */
+    DEBUG = 0,
+    /** Informational messages - normal operations */
+    INFO = 1,
+    /** Warning messages - potentially problematic situations */
+    WARN = 2,
+    /** Error messages - errors that need attention */
+    ERROR = 3,
+    /** Silent - no logging */
+    SILENT = 4
+}
+/**
+ * Context object for structured logging
+ */
+interface LogContext {
+    /** Evaluator type (vocabulary, sentence-structure, etc.) */
+    evaluator?: string;
+    /** Current operation or stage */
+    operation?: string;
+    /** Error object if applicable */
+    error?: Error;
+    /** Additional metadata */
+    [key: string]: unknown;
+}
+/**
+ * Logger interface
+ *
+ * Implement this interface to provide custom logging behavior.
+ *
+ * @example
+ * ```typescript
+ * const customLogger: Logger = {
+ *   debug: (msg, ctx) => myLogger.debug(msg, ctx),
+ *   info: (msg, ctx) => myLogger.info(msg, ctx),
+ *   warn: (msg, ctx) => myLogger.warn(msg, ctx),
+ *   error: (msg, ctx) => myLogger.error(msg, ctx),
+ * };
+ *
+ * const evaluator = new VocabularyEvaluator({
+ *   googleApiKey: '...',
+ *   openaiApiKey: '...',
+ *   logger: customLogger,
+ *   logLevel: LogLevel.INFO,
+ * });
+ * ```
+ */
+interface Logger {
+    /**
+     * Log debug message
+     * Used for detailed debugging information
+     */
+    debug(message: string, context?: LogContext): void;
+    /**
+     * Log informational message
+     * Used for normal operations
+     */
+    info(message: string, context?: LogContext): void;
+    /**
+     * Log warning message
+     * Used for potentially problematic situations
+     */
+    warn(message: string, context?: LogContext): void;
+    /**
+     * Log error message
+     * Used for errors that need attention
+     */
+    error(message: string, context?: LogContext): void;
+}
+
+/**
+ * Evaluation status
+ */
+type EvaluationStatus = 'success' | 'error';
+/**
+ * Token usage metrics from LLM providers
+ */
+interface TokenUsage {
+    input_tokens: number;
+    output_tokens: number;
+}
+/**
+ * Per-stage details for multi-stage evaluations
+ */
+interface StageDetail {
+    /** Stage name (e.g., "background_knowledge", "complexity_evaluation") */
+    stage: string;
+    /** Provider used for this stage (e.g., "openai:gpt-4o") */
+    provider: string;
+    /** Total latency including all retries (ms) */
+    latency_ms: number;
+    /** Token usage aggregated across all attempts */
+    token_usage?: TokenUsage;
+    /**
+     * Whether schema validation failed (indicates prompt needs clearer instructions)
+     *
+     * TODO: Not currently tracked. Vercel AI SDK abstracts validation away.
+     * To implement: Add custom retry wrapper that catches validation errors.
+     */
+    schema_validation_failed?: boolean;
+}
+/**
+ * Extensible metadata for telemetry events
+ */
+interface TelemetryMetadata {
+    /** Detailed breakdown by stage (for multi-stage evaluations) */
+    stage_details?: StageDetail[];
+}
+/**
+ * Telemetry event payload
+ */
+interface TelemetryEvent {
+    timestamp: string;
+    sdk_version: string;
+    evaluator_type: string;
+    grade?: string;
+    status: EvaluationStatus;
+    error_code?: string;
+    latency_ms: number;
+    text_length_chars: number;
+    provider: string;
+    token_usage?: TokenUsage;
+    metadata?: TelemetryMetadata;
+    input_text?: string;
+}
+/**
+ * Configuration for telemetry client
+ */
+interface TelemetryConfig {
+    /** Analytics service endpoint URL */
+    endpoint: string;
+    /** Learning Commons partner key (optional, sent as X-API-Key header) */
+    partnerKey?: string;
+    /** Client ID for anonymous tracking (persistent UUID from ~/.config/learning-commons/config.json) */
+    clientId: string;
+    /** Enable telemetry (default: true) */
+    enabled: boolean;
+    /** Logger instance (respects the SDK's configured log level and custom logger) */
+    logger: Logger;
+}
+
+/**
+ * Telemetry client for sending analytics events
+ *
+ * Fire-and-forget implementation that never blocks SDK operations.
+ * Errors are logged but don't fail evaluations.
+ */
+declare class TelemetryClient {
+    private config;
+    private logger;
+    constructor(config: TelemetryConfig);
+    /**
+     * Send telemetry event to analytics service
+     *
+     * Fire-and-forget: Errors are logged but don't throw.
+     */
+    send(event: TelemetryEvent): Promise<void>;
+}
+
+/**
+ * Granular telemetry configuration options
+ */
+interface TelemetryOptions {
+    /** Enable telemetry (default: true) */
+    enabled?: boolean;
+    /** Record input text in telemetry (default: false) */
+    recordInputs?: boolean;
+}
+/**
+ * Base configuration for all evaluators
+ */
+interface BaseEvaluatorConfig {
+    /** Google API key (for evaluators using Gemini) */
+    googleApiKey?: string;
+    /** OpenAI API key (for evaluators using GPT) */
+    openaiApiKey?: string;
+    /** Learning Commons partner key for authenticated telemetry (optional) */
+    partnerKey?: string;
+    /**
+     * Maximum number of retries for failed API calls (default: 2)
+     * Set to 0 to disable retries.
+     *
+     * Note: With maxRetries=2, a failed call will be attempted up to 3 times total
+     * (1 initial attempt + 2 retries)
+     */
+    maxRetries?: number;
+    /**
+     * Telemetry configuration (default: all enabled)
+     *
+     * Can be:
+     * - `true`: Enable with defaults (recordInputs: false)
+     * - `false`: Disable completely
+     * - `TelemetryOptions`: Granular control
+     */
+    telemetry?: boolean | TelemetryOptions;
+    /**
+     * Custom logger implementation (optional)
+     * If not provided, uses console logger with specified logLevel
+     */
+    logger?: Logger;
+    /**
+     * Log level for default console logger (default: WARN)
+     * Only used if custom logger is not provided
+     *
+     * - DEBUG: Very verbose, shows all operations
+     * - INFO: Normal operations
+     * - WARN: Warnings only (default)
+     * - ERROR: Errors only
+     * - SILENT: No logging
+     */
+    logLevel?: LogLevel;
+}
+/**
+ * Evaluator metadata interface
+ * Each evaluator must provide this metadata as static properties
+ */
+interface EvaluatorMetadata {
+    /** Unique identifier for the evaluator (e.g., 'vocabulary', 'sentence-structure') */
+    readonly id: string;
+    /** Human-readable name (e.g., 'Vocabulary', 'Sentence Structure') */
+    readonly name: string;
+    /** Brief description of what the evaluator does */
+    readonly description: string;
+    /** Supported grade levels (e.g., ['3', '4', '5', ...]) */
+    readonly supportedGrades: readonly string[];
+    /** Whether this evaluator requires a Google API key */
+    readonly requiresGoogleKey: boolean;
+    /** Whether this evaluator requires an OpenAI API key */
+    readonly requiresOpenAIKey: boolean;
+}
+/**
+ * Abstract base class for all evaluators
+ *
+ * Provides common functionality:
+ * - Telemetry setup and event sending
+ * - Text validation
+ * - Grade validation (with overridable default)
+ * - Metadata creation
+ *
+ * Concrete evaluators must implement:
+ * - static metadata: Provide evaluator metadata (see EvaluatorMetadata interface)
+ */
+declare abstract class BaseEvaluator {
+    protected telemetryClient?: TelemetryClient;
+    protected logger: Logger;
+    protected config: Required<Pick<BaseEvaluatorConfig, 'maxRetries'>> & {
+        telemetry: Required<TelemetryOptions>;
+    };
+    /**
+     * Static metadata for the evaluator
+     *
+     * Concrete evaluators MUST define this property.
+     *
+     * @example
+     * ```typescript
+     * class MyEvaluator extends BaseEvaluator {
+     *   static readonly metadata = {
+     *     id: 'my-evaluator',
+     *     name: 'My Evaluator',
+     *     description: 'Does something useful',
+     *     supportedGrades: ['3', '4', '5'],
+     *     requiresGoogleKey: true,
+     *     requiresOpenAIKey: false,
+     *   };
+     * }
+     * ```
+     */
+    static readonly metadata: EvaluatorMetadata;
+    constructor(config: BaseEvaluatorConfig);
+    /**
+     * Get metadata for this evaluator instance
+     * @throws {ConfigurationError} If the subclass has not defined static metadata
+     */
+    protected get metadata(): EvaluatorMetadata;
+    /**
+     * Validate that required API keys are provided based on metadata
+     * @throws {ConfigurationError} If required API keys are missing
+     */
+    private validateApiKeys;
+    /**
+     * Normalize telemetry config to standard format
+     */
+    private normalizeTelemetryConfig;
+    /**
+     * Get the evaluator type identifier from metadata
+     * @returns The evaluator type ID (e.g., "vocabulary", "sentence-structure")
+     */
+    protected getEvaluatorType(): string;
+    /**
+     * Validate text meets requirements
+     * Default implementation - can be overridden by concrete evaluators
+     *
+     * @throws {ValidationError} If text is invalid
+     */
+    protected validateText(text: string): void;
+    /**
+     * Validate grade is in supported range
+     * Default implementation - can be overridden by concrete evaluators
+     *
+     * @param grade - Grade level to validate
+     * @param validGrades - Set of valid grades for this evaluator
+     * @throws {ValidationError} If grade is invalid
+     */
+    protected validateGrade(grade: string, validGrades: Set<string>): void;
+    /**
+     * Send telemetry event to analytics service
+     * Common helper for all evaluators
+     */
+    protected sendTelemetry(params: {
+        status: 'success' | 'error';
+        latencyMs: number;
+        textLength: number;
+        grade?: string;
+        provider: string;
+        errorCode?: string;
+        tokenUsage?: TokenUsage;
+        metadata?: TelemetryMetadata;
+        inputText?: string;
+    }): Promise<void>;
+}
+
+export { BaseEvaluator as B, type EvaluatorMetadata as E, type Logger as L, type TelemetryOptions as T, type BaseEvaluatorConfig as a, type LogContext as b, LogLevel as c };