@learning-commons/evaluators 0.2.0 → 0.4.0

package/CHANGELOG.md

@@ -0,0 +1,44 @@
+# Changelog
+
+All notable changes to the `@learning-commons/evaluators` TypeScript SDK will be documented in this file.
+
+## [0.4.0] — 2026-03-23
+
+### Added
+
+- **Batch CSV Evaluator** — CLI tool and programmatic API for evaluating multiple texts from a CSV file in parallel. Runs the `text-complexity` group (GLA, SMK, Vocabulary, Sentence Structure, and Conventionality) across up to 50 rows and produces CSV and HTML reports.
+
+---
+
+## [0.3.0] — 2026-03-20
+
+### Added
+
+- **Conventionality Evaluator** — evaluates how explicit, literal, and straightforward a text's meaning is versus how abstract, ironic, figurative, or archaic it is, relative to grades 3–12.
+- **Conventionality added to TextComplexityEvaluator** — composite evaluator now runs vocabulary, sentence structure, SMK, and conventionality in parallel; result includes `conventionality` key.
+
+---
+
+## [0.2.0] — 2026-03-18
+
+### Added
+
+- **Subject Matter Knowledge (SMK) Evaluator** — evaluates background knowledge demands of educational texts relative to grades 3–12.
+- **SMK added to TextComplexityEvaluator** — composite evaluator now runs vocabulary, sentence structure, and SMK in parallel; result includes `subjectMatterKnowledge` key.
+- **Prompt versioning** — prompts updated to v1.3.0 (`evals/prompts/subject-matter-knowledge/`).
+
+---
+
+## [0.1.0] — Early Release
+
+Initial early release of the TypeScript SDK for Learning Commons educational evaluators.
+
+### Added
+
+- **Vocabulary Evaluator** — grades 3–12 vocabulary difficulty assessment.
+- **Sentence Structure Evaluator** — syntactic complexity analysis by grade level.
+- **Grade Level Appropriateness (GLA) Evaluator** — overall grade-level suitability scoring.
+- **Text Complexity Evaluator** — composite evaluation combining Vocabulary, Sentence Structure, and GLA.
+- **Provider abstraction** — model-agnostic via Vercel AI SDK; OpenAI, Google, and Anthropic supported.
+- **Telemetry** — opt-in, with `partnerKey` and `recordInputs` (defaults to `false`).
+- **Prompt versioning** — prompts versioned in `evals/prompts/` (v1.2.0), shared with Python notebooks.

package/README.md

@@ -180,9 +180,71 @@ console.log(result._internal.identified_topics); // ["hydraulics", "propulsion",
 
 ---
 
-### 4. Text Complexity Evaluator
+### 4. Conventionality Evaluator
 
-Composite evaluator that analyzes vocabulary, sentence structure, and subject matter knowledge complexity in parallel.
+Evaluates how explicit, literal, and straightforward a text's meaning is versus how abstract, ironic, figurative, or archaic it is for the target grade level. Based on the Common Core Qualitative Text Complexity Rubric.
+
+**Supported Grades:** 3-12
+
+**Uses:** Google Gemini 3 Flash Preview
+
+**Constructor:**
+```typescript
+const evaluator = new ConventionalityEvaluator({
+  googleApiKey?: string;  // Google API key (required by this evaluator)
+  maxRetries?: number;    // Optional - Max retry attempts (default: 2)
+  telemetry?: boolean | TelemetryOptions; // Optional (default: true)
+  logger?: Logger;        // Optional - Custom logger
+  logLevel?: LogLevel;    // Optional - Logging verbosity (default: WARN)
+});
+```
+
+**API:**
+```typescript
+await evaluator.evaluate(text: string, grade: string)
+```
+
+**Returns:**
+```typescript
+{
+  score: 'Slightly complex' | 'Moderately complex' | 'Very complex' | 'Exceedingly complex';
+  reasoning: string;
+  metadata: {
+    model: string;
+    processingTimeMs: number;
+  };
+  _internal: {
+    conventionality_features: string[];
+    grade_context: string;
+    instructional_insights: string;
+    complexity_score: 'Slightly complex' | 'Moderately complex' | 'Very complex' | 'Exceedingly complex';
+    reasoning: string;
+  };
+}
+```
+
+**Example:**
+```typescript
+import { ConventionalityEvaluator } from '@learning-commons/evaluators';
+
+const evaluator = new ConventionalityEvaluator({
+  googleApiKey: process.env.GOOGLE_API_KEY,
+});
+
+const result = await evaluator.evaluate(
+  "The author uses sustained irony to critique societal norms throughout the passage.",
+  "10"
+);
+console.log(result.score);          // "Very complex"
+console.log(result.reasoning);
+console.log(result._internal.conventionality_features); // ["sustained irony", ...]
+```
+
+---
+
+### 5. Text Complexity Evaluator
+
+Composite evaluator that analyzes vocabulary, sentence structure, subject matter knowledge, and conventionality complexity in parallel.
 
 **Supported Grades:** 3-12
 
@@ -211,10 +273,11 @@ await evaluator.evaluate(text: string, grade: string)
   vocabulary: EvaluationResult<TextComplexityLevel> | { error: Error };
   sentenceStructure: EvaluationResult<TextComplexityLevel> | { error: Error };
   subjectMatterKnowledge: EvaluationResult<TextComplexityLevel> | { error: Error };
+  conventionality: EvaluationResult<TextComplexityLevel> | { error: Error };
 }
 ```
 
-Each sub-evaluator result is either a full `EvaluationResult` or `{ error: Error }` if that evaluator failed. An error is only thrown if all three fail.
+Each sub-evaluator result is either a full `EvaluationResult` or `{ error: Error }` if that evaluator failed. An error is only thrown if all four fail.
 
 **Example:**
 ```typescript
@@ -236,11 +299,14 @@ if (!('error' in result.sentenceStructure)) {
 if (!('error' in result.subjectMatterKnowledge)) {
   console.log('Subject matter knowledge:', result.subjectMatterKnowledge.score);
 }
+if (!('error' in result.conventionality)) {
+  console.log('Conventionality:', result.conventionality.score);
+}
 ```
 
 ---
 
-### 5. Grade Level Appropriateness Evaluator
+### 6. Grade Level Appropriateness Evaluator
 
 Determines appropriate grade level for text.
 
@@ -284,6 +350,21 @@ await evaluator.evaluate(text: string)
 
 ---
 
+## Batch CSV Evaluation
+
+For evaluating many texts at once, the SDK ships a CLI tool that reads a CSV file, runs all evaluators in a group, and produces CSV and HTML reports.
+
+```bash
+# Run from the directory containing your CSV
+npx evaluators-batch
+```
+
+The CLI will prompt for your CSV path, API keys, and output directory, then process all rows in parallel with real-time progress.
+
+See [`src/batch/README.md`](./src/batch/README.md) for full documentation.
+
+---
+
 ## Error Handling
 
 The SDK provides specific error types to help you handle different scenarios:
@@ -388,6 +469,7 @@ interface BaseEvaluatorConfig {
 - **Vocabulary**: Requires both `googleApiKey` and `openaiApiKey`
 - **Sentence Structure**: Requires `openaiApiKey` only
 - **Subject Matter Knowledge**: Requires `googleApiKey` only
+- **Conventionality**: Requires `googleApiKey` only
 - **Text Complexity**: Requires both `googleApiKey` and `openaiApiKey`
 - **Grade Level Appropriateness**: Requires `googleApiKey` only
 

package/dist/base-Ced9oKKa.d.cts

@@ -0,0 +1,331 @@
+/**
+ * Logging interface for the Evaluators SDK
+ *
+ * Provides structured logging with verbosity levels.
+ * Users can inject custom loggers or use the default console logger.
+ */
+/**
+ * Log levels in order of verbosity
+ */
+declare enum LogLevel {
+    /** Debug messages - very verbose, for development */
+    DEBUG = 0,
+    /** Informational messages - normal operations */
+    INFO = 1,
+    /** Warning messages - potentially problematic situations */
+    WARN = 2,
+    /** Error messages - errors that need attention */
+    ERROR = 3,
+    /** Silent - no logging */
+    SILENT = 4
+}
+/**
+ * Context object for structured logging
+ */
+interface LogContext {
+    /** Evaluator type (vocabulary, sentence-structure, etc.) */
+    evaluator?: string;
+    /** Current operation or stage */
+    operation?: string;
+    /** Error object if applicable */
+    error?: Error;
+    /** Additional metadata */
+    [key: string]: unknown;
+}
+/**
+ * Logger interface
+ *
+ * Implement this interface to provide custom logging behavior.
+ *
+ * @example
+ * ```typescript
+ * const customLogger: Logger = {
+ *   debug: (msg, ctx) => myLogger.debug(msg, ctx),
+ *   info: (msg, ctx) => myLogger.info(msg, ctx),
+ *   warn: (msg, ctx) => myLogger.warn(msg, ctx),
+ *   error: (msg, ctx) => myLogger.error(msg, ctx),
+ * };
+ *
+ * const evaluator = new VocabularyEvaluator({
+ *   googleApiKey: '...',
+ *   openaiApiKey: '...',
+ *   logger: customLogger,
+ *   logLevel: LogLevel.INFO,
+ * });
+ * ```
+ */
+interface Logger {
+    /**
+     * Log debug message
+     * Used for detailed debugging information
+     */
+    debug(message: string, context?: LogContext): void;
+    /**
+     * Log informational message
+     * Used for normal operations
+     */
+    info(message: string, context?: LogContext): void;
+    /**
+     * Log warning message
+     * Used for potentially problematic situations
+     */
+    warn(message: string, context?: LogContext): void;
+    /**
+     * Log error message
+     * Used for errors that need attention
+     */
+    error(message: string, context?: LogContext): void;
+}
+
+/**
+ * Evaluation status
+ */
+type EvaluationStatus = 'success' | 'error';
+/**
+ * Token usage metrics from LLM providers
+ */
+interface TokenUsage {
+    input_tokens: number;
+    output_tokens: number;
+}
+/**
+ * Per-stage details for multi-stage evaluations
+ */
+interface StageDetail {
+    /** Stage name (e.g., "background_knowledge", "complexity_evaluation") */
+    stage: string;
+    /** Provider used for this stage (e.g., "openai:gpt-4o") */
+    provider: string;
+    /** Total latency including all retries (ms) */
+    latency_ms: number;
+    /** Token usage aggregated across all attempts */
+    token_usage?: TokenUsage;
+    /**
+     * Whether schema validation failed (indicates prompt needs clearer instructions)
+     *
+     * TODO: Not currently tracked. Vercel AI SDK abstracts validation away.
+     * To implement: Add custom retry wrapper that catches validation errors.
+     */
+    schema_validation_failed?: boolean;
+}
+/**
+ * Extensible metadata for telemetry events
+ */
+interface TelemetryMetadata {
+    /** Detailed breakdown by stage (for multi-stage evaluations) */
+    stage_details?: StageDetail[];
+}
+/**
+ * Telemetry event payload
+ */
+interface TelemetryEvent {
+    timestamp: string;
+    sdk_version: string;
+    evaluator_type: string;
+    grade?: string;
+    status: EvaluationStatus;
+    error_code?: string;
+    latency_ms: number;
+    text_length_chars: number;
+    provider: string;
+    token_usage?: TokenUsage;
+    metadata?: TelemetryMetadata;
+    input_text?: string;
+}
+/**
+ * Configuration for telemetry client
+ */
+interface TelemetryConfig {
+    /** Analytics service endpoint URL */
+    endpoint: string;
+    /** Learning Commons partner key (optional, sent as X-API-Key header) */
+    partnerKey?: string;
+    /** Client ID for anonymous tracking (persistent UUID from ~/.config/learning-commons/config.json) */
+    clientId: string;
+    /** Enable telemetry (default: true) */
+    enabled: boolean;
+    /** Logger instance (respects the SDK's configured log level and custom logger) */
+    logger: Logger;
+}
+
+/**
+ * Telemetry client for sending analytics events
+ *
+ * Fire-and-forget implementation that never blocks SDK operations.
+ * Errors are logged but don't fail evaluations.
+ */
+declare class TelemetryClient {
+    private config;
+    private logger;
+    constructor(config: TelemetryConfig);
+    /**
+     * Send telemetry event to analytics service
+     *
+     * Fire-and-forget: Errors are logged but don't throw.
+     */
+    send(event: TelemetryEvent): Promise<void>;
+}
+
+/**
+ * Granular telemetry configuration options
+ */
+interface TelemetryOptions {
+    /** Enable telemetry (default: true) */
+    enabled?: boolean;
+    /** Record input text in telemetry (default: false) */
+    recordInputs?: boolean;
+}
+/**
+ * Base configuration for all evaluators
+ */
+interface BaseEvaluatorConfig {
+    /** Google API key (for evaluators using Gemini) */
+    googleApiKey?: string;
+    /** OpenAI API key (for evaluators using GPT) */
+    openaiApiKey?: string;
+    /** Learning Commons partner key for authenticated telemetry (optional) */
+    partnerKey?: string;
+    /**
+     * Maximum number of retries for failed API calls (default: 2)
+     * Set to 0 to disable retries.
+     *
+     * Note: With maxRetries=2, a failed call will be attempted up to 3 times total
+     * (1 initial attempt + 2 retries)
+     */
+    maxRetries?: number;
+    /**
+     * Telemetry configuration (default: all enabled)
+     *
+     * Can be:
+     * - `true`: Enable with defaults (recordInputs: false)
+     * - `false`: Disable completely
+     * - `TelemetryOptions`: Granular control
+     */
+    telemetry?: boolean | TelemetryOptions;
+    /**
+     * Custom logger implementation (optional)
+     * If not provided, uses console logger with specified logLevel
+     */
+    logger?: Logger;
+    /**
+     * Log level for default console logger (default: WARN)
+     * Only used if custom logger is not provided
+     *
+     * - DEBUG: Very verbose, shows all operations
+     * - INFO: Normal operations
+     * - WARN: Warnings only (default)
+     * - ERROR: Errors only
+     * - SILENT: No logging
+     */
+    logLevel?: LogLevel;
+}
+/**
+ * Evaluator metadata interface
+ * Each evaluator must provide this metadata as static properties
+ */
+interface EvaluatorMetadata {
+    /** Unique identifier for the evaluator (e.g., 'vocabulary', 'sentence-structure') */
+    readonly id: string;
+    /** Human-readable name (e.g., 'Vocabulary', 'Sentence Structure') */
+    readonly name: string;
+    /** Brief description of what the evaluator does */
+    readonly description: string;
+    /** Supported grade levels (e.g., ['3', '4', '5', ...]) */
+    readonly supportedGrades: readonly string[];
+    /** Whether this evaluator requires a Google API key */
+    readonly requiresGoogleKey: boolean;
+    /** Whether this evaluator requires an OpenAI API key */
+    readonly requiresOpenAIKey: boolean;
+}
+/**
+ * Abstract base class for all evaluators
+ *
+ * Provides common functionality:
+ * - Telemetry setup and event sending
+ * - Text validation
+ * - Grade validation (with overridable default)
+ * - Metadata creation
+ *
+ * Concrete evaluators must implement:
+ * - static metadata: Provide evaluator metadata (see EvaluatorMetadata interface)
+ */
+declare abstract class BaseEvaluator {
+    protected telemetryClient?: TelemetryClient;
+    protected logger: Logger;
+    protected config: Required<Pick<BaseEvaluatorConfig, 'maxRetries'>> & {
+        telemetry: Required<TelemetryOptions>;
+    };
+    /**
+     * Static metadata for the evaluator
+     *
+     * Concrete evaluators MUST define this property.
+     *
+     * @example
+     * ```typescript
+     * class MyEvaluator extends BaseEvaluator {
+     *   static readonly metadata = {
+     *     id: 'my-evaluator',
+     *     name: 'My Evaluator',
+     *     description: 'Does something useful',
+     *     supportedGrades: ['3', '4', '5'],
+     *     requiresGoogleKey: true,
+     *     requiresOpenAIKey: false,
+     *   };
+     * }
+     * ```
+     */
+    static readonly metadata: EvaluatorMetadata;
+    constructor(config: BaseEvaluatorConfig);
+    /**
+     * Get metadata for this evaluator instance
+     * @throws {ConfigurationError} If the subclass has not defined static metadata
+     */
+    protected get metadata(): EvaluatorMetadata;
+    /**
+     * Validate that required API keys are provided based on metadata
+     * @throws {ConfigurationError} If required API keys are missing
+     */
+    private validateApiKeys;
+    /**
+     * Normalize telemetry config to standard format
+     */
+    private normalizeTelemetryConfig;
+    /**
+     * Get the evaluator type identifier from metadata
+     * @returns The evaluator type ID (e.g., "vocabulary", "sentence-structure")
+     */
+    protected getEvaluatorType(): string;
+    /**
+     * Validate text meets requirements
+     * Default implementation - can be overridden by concrete evaluators
+     *
+     * @throws {ValidationError} If text is invalid
+     */
+    protected validateText(text: string): void;
+    /**
+     * Validate grade is in supported range
+     * Default implementation - can be overridden by concrete evaluators
+     *
+     * @param grade - Grade level to validate
+     * @param validGrades - Set of valid grades for this evaluator
+     * @throws {ValidationError} If grade is invalid
+     */
+    protected validateGrade(grade: string, validGrades: Set<string>): void;
+    /**
+     * Send telemetry event to analytics service
+     * Common helper for all evaluators
+     */
+    protected sendTelemetry(params: {
+        status: 'success' | 'error';
+        latencyMs: number;
+        textLength: number;
+        grade?: string;
+        provider: string;
+        errorCode?: string;
+        tokenUsage?: TokenUsage;
+        metadata?: TelemetryMetadata;
+        inputText?: string;
+    }): Promise<void>;
+}
+
+export { BaseEvaluator as B, type EvaluatorMetadata as E, type Logger as L, type TelemetryOptions as T, type BaseEvaluatorConfig as a, type LogContext as b, LogLevel as c };