@learning-commons/evaluators 0.1.0 → 0.2.0

package/README.md

@@ -117,19 +117,18 @@ await evaluator.evaluate(text: string, grade: string)
 
 ---
 
-### 3. Text Complexity Evaluator
+### 3. Subject Matter Knowledge (SMK) Evaluator
 
-Composite evaluator that analyzes both vocabulary and sentence structure complexity in parallel.
+Evaluates the background knowledge demands of educational texts relative to grade level. Determines how much prior subject knowledge a student needs to comprehend the text, based on the Common Core Qualitative Text Complexity Rubric.
 
 **Supported Grades:** 3-12
 
-**Uses:** Google Gemini 2.5 Pro + OpenAI GPT-4o (composite)
+**Uses:** Google Gemini 3 Flash Preview
 
 **Constructor:**
 ```typescript
-const evaluator = new TextComplexityEvaluator({
+const evaluator = new SmkEvaluator({
   googleApiKey?: string;  // Google API key (required by this evaluator)
-  openaiApiKey?: string;  // OpenAI API key (required by this evaluator)
   maxRetries?: number;    // Optional - Max retry attempts (default: 2)
   telemetry?: boolean | TelemetryOptions; // Optional (default: true)
   logger?: Logger;        // Optional - Custom logger
@@ -145,23 +144,103 @@ await evaluator.evaluate(text: string, grade: string)
 **Returns:**
 ```typescript
 {
-  score: {
-    overall: string;           // Overall complexity (highest of the two)
-    vocabulary: string;        // Vocabulary complexity score
-    sentenceStructure: string; // Sentence structure complexity score
+  score: 'Slightly complex' | 'Moderately complex' | 'Very complex' | 'Exceedingly complex';
+  reasoning: string;
+  metadata: {
+    model: string;
+    processingTimeMs: number;
   };
-  reasoning: string;  // Combined reasoning from both evaluators
-  metadata: EvaluationMetadata;
   _internal: {
-    vocabulary: EvaluationResult | { error: Error };
-    sentenceStructure: EvaluationResult | { error: Error };
+    identified_topics: string[];
+    curriculum_check: string;
+    assumptions_and_scaffolding: string;
+    friction_analysis: string;
+    complexity_score: 'Slightly complex' | 'Moderately complex' | 'Very complex' | 'Exceedingly complex';
+    reasoning: string;
   };
 }
 ```
 
+**Example:**
+```typescript
+import { SmkEvaluator } from '@learning-commons/evaluators';
+
+const evaluator = new SmkEvaluator({
+  googleApiKey: process.env.GOOGLE_API_KEY,
+});
+
+const result = await evaluator.evaluate(
+  "Hydraulic propulsion works by sucking water at the bow and forcing it sternward.",
+  "10"
+);
+console.log(result.score);          // "Very complex"
+console.log(result.reasoning);
+console.log(result._internal.identified_topics); // ["hydraulics", "propulsion", "physics"]
+```
+
+---
+
+### 4. Text Complexity Evaluator
+
+Composite evaluator that analyzes vocabulary, sentence structure, and subject matter knowledge complexity in parallel.
+
+**Supported Grades:** 3-12
+
+**Uses:** Google Gemini 2.5 Pro + Google Gemini 3 Flash Preview + OpenAI GPT-4o (composite)
+
+**Constructor:**
+```typescript
+const evaluator = new TextComplexityEvaluator({
+  googleApiKey?: string;  // Google API key (required by this evaluator)
+  openaiApiKey?: string;  // OpenAI API key (required by this evaluator)
+  maxRetries?: number;    // Optional - Max retry attempts (default: 2)
+  telemetry?: boolean | TelemetryOptions; // Optional (default: true)
+  logger?: Logger;        // Optional - Custom logger
+  logLevel?: LogLevel;    // Optional - Logging verbosity (default: WARN)
+});
+```
+
+**API:**
+```typescript
+await evaluator.evaluate(text: string, grade: string)
+```
+
+**Returns:**
+```typescript
+{
+  vocabulary: EvaluationResult<TextComplexityLevel> | { error: Error };
+  sentenceStructure: EvaluationResult<TextComplexityLevel> | { error: Error };
+  subjectMatterKnowledge: EvaluationResult<TextComplexityLevel> | { error: Error };
+}
+```
+
+Each sub-evaluator result is either a full `EvaluationResult` or `{ error: Error }` if that evaluator failed. An error is only thrown if all three fail.
+
+**Example:**
+```typescript
+import { TextComplexityEvaluator } from '@learning-commons/evaluators';
+
+const evaluator = new TextComplexityEvaluator({
+  googleApiKey: process.env.GOOGLE_API_KEY,
+  openaiApiKey: process.env.OPENAI_API_KEY,
+});
+
+const result = await evaluator.evaluate("Your text here", "6");
+
+if (!('error' in result.vocabulary)) {
+  console.log('Vocabulary:', result.vocabulary.score);
+}
+if (!('error' in result.sentenceStructure)) {
+  console.log('Sentence structure:', result.sentenceStructure.score);
+}
+if (!('error' in result.subjectMatterKnowledge)) {
+  console.log('Subject matter knowledge:', result.subjectMatterKnowledge.score);
+}
+```
+
 ---
 
-### 4. Grade Level Appropriateness Evaluator
+### 5. Grade Level Appropriateness Evaluator
 
 Determines appropriate grade level for text.
 
@@ -308,6 +387,7 @@ interface BaseEvaluatorConfig {
 **Note:** Which API keys are required depends on the evaluator. The SDK validates required keys at runtime based on the evaluator's metadata:
 - **Vocabulary**: Requires both `googleApiKey` and `openaiApiKey`
 - **Sentence Structure**: Requires `openaiApiKey` only
+- **Subject Matter Knowledge**: Requires `googleApiKey` only
 - **Text Complexity**: Requires both `googleApiKey` and `openaiApiKey`
 - **Grade Level Appropriateness**: Requires `googleApiKey` only
 

package/dist/index.cjs

@@ -1769,29 +1769,269 @@ async function evaluateGradeLevelAppropriateness(text, config) {
   const evaluator = new GradeLevelAppropriatenessEvaluator(config);
   return evaluator.evaluate(text);
 }
+var SmkOutputSchema = zod.z.object({
+  identified_topics: zod.z.array(zod.z.string()).describe("List of major subjects/concepts found in the text."),
+  curriculum_check: zod.z.string().describe("Whether the topics are standard K-8 or specialized high school level."),
+  assumptions_and_scaffolding: zod.z.string().describe("What the author assumes the reader knows vs. what is explained."),
+  friction_analysis: zod.z.string().describe("Whether difficulty comes from vocabulary/structure or actual knowledge demands."),
+  complexity_score: TextComplexityLevel.describe("The subject matter knowledge complexity level of the text"),
+  reasoning: zod.z.string().describe("A brief synthesis of why the text fits the chosen complexity level.")
+});
+
+// ../../evals/prompts/subject-matter-knowledge/system.txt
+var system_default2 = `
+To perform the task of evaluating text complexity based on Subject Matter Knowledge (SMK), strictly adhere to the following instructions.
+Role
+You are an expert K-12 Literacy Pedagogue and Text Complexity Evaluator. Your specific focus is analyzing Subject Matter Knowledge (SMK) demands according to the Common Core Qualitative Text Complexity Rubric.
+Objective
+Analyze a provided text relative to a target grade_level. You must determine the extent of background knowledge required to comprehend the text. You must distinguish between Common/Standard knowledge (generally lower/moderate complexity) and Specialized/Theoretical knowledge (generally higher complexity).
+Input Data
+text: The passage to analyze.
+grade_level: The target student grade (integer).
+fk_score: Flesch-Kincaid Grade Level. Note: Use this only as a loose proxy for sentence structure. Do not let a high FK score artificially inflate the Subject Matter Knowledge score if the concepts remain simple.
+
+1. The Rubric: Subject Matter Knowledge (SMK)
+1. Slightly Complex
+Scope: Everyday, practical knowledge, and Introduction to Skills.
+Concept Type: Concrete, directly observable, and familiar.
+Key Indicator: "How-to" texts involving familiar objects (e.g., drawing a cupboard, playing a game, family life). Even if specific terms (like "scale" or "measure") are used, if the application is on a common object, it remains Slightly Complex.
+2. Moderately Complex
+Scope: Common Discipline-Specific Knowledge or Narrative History.
+Definition: Topics widely introduced in K-8 curricula (Basic American History, Geography, Earth Science, Biology).
+Key Characteristic: The text bridges concrete descriptions with abstract themes (e.g., using farming to discuss justice), OR narrates historical events via sensory details.
+Spatial Reasoning: Texts requiring mental manipulation of maps/routes are generally Moderate, unless the object is a familiar household item (see Slightly Complex).
+3. Very Complex
+Scope: Specialized Discipline-Specific, Engineering Mechanics, or Political Theory.
+Definition: Topics characteristic of High School (9-12) curricula requiring abstract mental models.
+Key Characteristic: Requires understanding mechanisms (how physics works/propulsion), chemical composition, or undefined political stakes (specific treaties, alliances, or secularization without context).
+4. Exceedingly Complex
+Scope: Professional or Academic knowledge.
+
+2. The Expert Mental Model (Decision Logic)
+Use these refined rules to categorize cases.
+Rule A: The "Layers of Meaning" Check
+Concrete -> Abstract (Moderate): The text describes concrete things (farming) to argue an abstract point (justice, rights).
+Concrete -> Concrete (Slightly): The text describes concrete things (lines, paper) to achieve a concrete result (drawing a cupboard). Do not over-rank practical instructions.
+Rule B: The Science & Engineering Boundary
+Observational (Moderate): Habitats, Water Cycle, observable traits, simple definitions.
+Mechanistic/Theoretical (Very): Engineering mechanics (how propulsion works via reaction), Instrumentation (using a spectroscope), or Chemical/Atomic theory.
+Test: Does the text explain how a machine functions using physical principles? If yes, it is Very Complex.
+Rule C: The History/Social Studies Boundary
+General/Narrative (Moderate):
+Sensory: Battle descriptions focusing on sights/sounds (flashes, smoke).
+Standard Topics: Immigration, Slavery, Government, Geography. Lists of nationalities or religions are "Common Knowledge" for Grades 6-8.
+Political/Contextual (Very):
+Implicit Context: Texts assuming knowledge of specific political factions, treaties, or the causes of events without explanation (e.g., "The Allies," "The Front," "The secularization of the clergy").
+Test: If the reader must know why two groups are fighting or the specific political history of a revolution to understand the text, it is Very Complex.
+Rule D: The "Technical vs. Practical" Trap
+Scenario: A text teaches a technical skill (e.g., Technical Drawing/Technology) but applies it to a familiar object (a cupboard).
+Decision: Slightly Complex.
+Reasoning: Do not confuse "Technical Vocabulary" (scale, thick lines) with "Theoretical Complexity." If the underlying concept is familiar (furniture), the SMK load is low.
+
+3. Critical Calibration Examples
+Text: "Make a rough sketch... How many shelves should the cupboard have?" (Grade 2) -> Slightly Complex.
+Reasoning: (Rule D/Rule A) Although it mentions "scale" and "technology," the task is concrete and relies on everyday knowledge.
+Text: "Hydraulic propulsion works by sucking water at the bow and forcing it sternward." (Grade 10) -> Very Complex.
+Reasoning: (Rule B) Explains a mechanism using physics principles.
+Text: "The Allies fight the enemy's cavalry; we remember the hospitality to priests during the Revolution." (Grade 6) -> Very Complex.
+Reasoning: (Rule C) Assumes undefined knowledge of WWI alliances and the specific political history of the French Revolution.
+Text: "Immigrants from Poland, Italy, and Russia arrived. Most were Catholic or Orthodox." (Grade 7) -> Moderately Complex.
+Reasoning: (Rule C) Standard K-8 topic. Lists of nationalities are content vocabulary, not specialized theory.
+
+4. Output Format
+Return your analysis in a valid JSON object. Do not include markdown formatting.
+Keys:
+- identified_topics: List[str] identifying the core subjects.
+- curriculum_check: String explaining if the topics are "Standard/General" (typical for K-8) or "Specialized/High School" (typical for 9-12).
+- assumptions_and_scaffolding: String analyzing what the author assumes the reader knows vs what is explained.
+- friction_analysis: String discussing the gap between Concrete description and Abstract meaning.
+- complexity_score: String (One of: slightly_complex, moderately_complex, very_complex, exceedingly_complex).
+- reasoning: String synthesizing the decision.
+
+`;
+
+// ../../evals/prompts/subject-matter-knowledge/user.txt
+var user_default2 = "Analyze:\nText: {text}\nGrade: {grade}\nFK Score: {fk_score}";
+
+// src/prompts/subject-matter-knowledge/index.ts
+function getSystemPrompt3() {
+  return system_default2;
+}
+function getUserPrompt3(text, grade, fkScore) {
+  return user_default2.replaceAll("{text}", text).replaceAll("{grade}", grade).replaceAll("{fk_score}", fkScore.toString());
+}
+
+// src/evaluators/smk.ts
+var SmkEvaluator = class _SmkEvaluator extends BaseEvaluator {
+  static metadata = {
+    id: "subject-matter-knowledge",
+    name: "Subject Matter Knowledge",
+    description: "Evaluates background knowledge demands of educational texts relative to grade level",
+    supportedGrades: ["3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+    requiresGoogleKey: true,
+    requiresOpenAIKey: false
+  };
+  provider;
+  constructor(config) {
+    super(config);
+    this.provider = createProvider({
+      type: "google",
+      model: "gemini-3-flash-preview",
+      apiKey: config.googleApiKey,
+      maxRetries: this.config.maxRetries
+    });
+  }
+  /**
+   * Evaluate subject matter knowledge complexity for a given text and grade level
+   *
+   * @param text - The text to evaluate
+   * @param grade - The target grade level (3-12)
+   * @returns Evaluation result with complexity score and detailed analysis
+   * @throws {ValidationError} If text is empty, too short/long, or grade is invalid
+   * @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError)
+   */
+  async evaluate(text, grade) {
+    this.logger.info("Starting SMK evaluation", {
+      evaluator: "subject-matter-knowledge",
+      operation: "evaluate",
+      grade,
+      textLength: text.length
+    });
+    const startTime = Date.now();
+    const stageDetails = [];
+    try {
+      this.validateText(text);
+      this.validateGrade(grade, new Set(_SmkEvaluator.metadata.supportedGrades));
+      this.logger.debug("Evaluating subject matter knowledge complexity", {
+        evaluator: "subject-matter-knowledge",
+        operation: "smk_evaluation"
+      });
+      const fkScore = calculateFleschKincaidGrade(text);
+      const response = await this.evaluateSmk(text, grade, fkScore);
+      stageDetails.push({
+        stage: "smk_evaluation",
+        provider: "google:gemini-3-flash-preview",
+        latency_ms: response.latencyMs,
+        token_usage: {
+          input_tokens: response.usage.inputTokens,
+          output_tokens: response.usage.outputTokens
+        }
+      });
+      const latencyMs = Date.now() - startTime;
+      const totalTokenUsage = {
+        input_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.input_tokens || 0), 0),
+        output_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.output_tokens || 0), 0)
+      };
+      const result = {
+        score: response.data.complexity_score,
+        reasoning: response.data.reasoning,
+        metadata: {
+          model: "google:gemini-3-flash-preview",
+          processingTimeMs: latencyMs
+        },
+        _internal: response.data
+      };
+      this.sendTelemetry({
+        status: "success",
+        latencyMs,
+        textLength: text.length,
+        grade,
+        provider: "google:gemini-3-flash-preview",
+        tokenUsage: totalTokenUsage,
+        metadata: {
+          stage_details: stageDetails
+        },
+        inputText: text
+      }).catch(() => {
+      });
+      this.logger.info("SMK evaluation completed successfully", {
+        evaluator: "subject-matter-knowledge",
+        operation: "evaluate",
+        grade,
+        score: result.score,
+        processingTimeMs: latencyMs
+      });
+      return result;
+    } catch (error) {
+      const latencyMs = Date.now() - startTime;
+      this.logger.error("SMK evaluation failed", {
+        evaluator: "subject-matter-knowledge",
+        operation: "evaluate",
+        grade,
+        error: error instanceof Error ? error : void 0,
+        processingTimeMs: latencyMs,
+        completedStages: stageDetails.length
+      });
+      const totalTokenUsage = stageDetails.length > 0 ? {
+        input_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.input_tokens || 0), 0),
+        output_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.output_tokens || 0), 0)
+      } : void 0;
+      this.sendTelemetry({
+        status: "error",
+        latencyMs,
+        textLength: text.length,
+        grade,
+        provider: "google:gemini-3-flash-preview",
+        tokenUsage: totalTokenUsage,
+        errorCode: error instanceof Error ? error.name : "UnknownError",
+        metadata: stageDetails.length > 0 ? { stage_details: stageDetails } : void 0,
+        inputText: text
+      }).catch(() => {
+      });
+      if (error instanceof ValidationError) {
+        throw error;
+      }
+      throw wrapProviderError(error, "SMK evaluation failed");
+    }
+  }
+  /**
+   * Run the SMK evaluation LLM call
+   */
+  async evaluateSmk(text, grade, fkScore) {
+    const response = await this.provider.generateStructured({
+      messages: [
+        { role: "system", content: getSystemPrompt3() },
+        { role: "user", content: getUserPrompt3(text, grade, fkScore) }
+      ],
+      schema: SmkOutputSchema,
+      temperature: 0
+    });
+    return {
+      data: response.data,
+      usage: response.usage,
+      latencyMs: response.latencyMs
+    };
+  }
+};
+async function evaluateSmk(text, grade, config) {
+  const evaluator = new SmkEvaluator(config);
+  return evaluator.evaluate(text, grade);
+}
 var TextComplexityEvaluator = class _TextComplexityEvaluator extends BaseEvaluator {
   static metadata = {
     id: "text-complexity",
     name: "Text Complexity",
-    description: "Composite evaluator analyzing vocabulary and sentence structure complexity",
+    description: "Composite evaluator analyzing vocabulary, sentence structure, and subject matter knowledge complexity",
     supportedGrades: ["3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
     requiresGoogleKey: true,
     requiresOpenAIKey: true
   };
   vocabularyEvaluator;
   sentenceStructureEvaluator;
+  smkEvaluator;
   limit;
   constructor(config) {
     super(config);
     this.vocabularyEvaluator = new VocabularyEvaluator(config);
     this.sentenceStructureEvaluator = new SentenceStructureEvaluator(config);
+    this.smkEvaluator = new SmkEvaluator(config);
     this.limit = pLimit__default.default(3);
   }
   /**
    * Evaluate text complexity for a given text and grade level
    *
-   * Runs vocabulary and sentence structure evaluations in parallel with concurrency control.
-   * If both sub-evaluators fail, throws an error. Otherwise returns a result map where
+   * Runs vocabulary, sentence structure, and SMK evaluations in parallel with concurrency control.
+   * If all three sub-evaluators fail, throws an error. Otherwise returns a result map where
    * failed sub-evaluators are represented as `{ error: Error }`.
    *
    * @param text - The text to evaluate
@@ -1810,18 +2050,21 @@ var TextComplexityEvaluator = class _TextComplexityEvaluator extends BaseEvaluat
     this.validateText(text);
     this.validateGrade(grade, new Set(_TextComplexityEvaluator.metadata.supportedGrades));
     const startTime = Date.now();
-    const [vocabResult, sentenceResult] = await Promise.all([
+    const [vocabResult, sentenceResult, smkResult] = await Promise.all([
       this.limit(() => this.runSubEvaluator(this.vocabularyEvaluator, text, grade)),
-      this.limit(() => this.runSubEvaluator(this.sentenceStructureEvaluator, text, grade))
+      this.limit(() => this.runSubEvaluator(this.sentenceStructureEvaluator, text, grade)),
+      this.limit(() => this.runSubEvaluator(this.smkEvaluator, text, grade))
     ]);
     const latencyMs = Date.now() - startTime;
     const vocabFailed = "error" in vocabResult;
     const sentenceFailed = "error" in sentenceResult;
-    const hasFailures = vocabFailed || sentenceFailed;
+    const smkFailed = "error" in smkResult;
+    const hasFailures = vocabFailed || sentenceFailed || smkFailed;
     if (hasFailures) {
       const errors = [];
       if (vocabFailed) errors.push(`Vocabulary: ${vocabResult.error.message}`);
       if (sentenceFailed) errors.push(`Sentence structure: ${sentenceResult.error.message}`);
+      if (smkFailed) errors.push(`Subject matter knowledge: ${smkResult.error.message}`);
       this.logger.error("Text complexity evaluation completed with errors", {
         evaluator: "text-complexity",
         operation: "evaluate",
@@ -1829,7 +2072,7 @@ var TextComplexityEvaluator = class _TextComplexityEvaluator extends BaseEvaluat
         errors,
         processingTimeMs: latencyMs
       });
-      if (vocabFailed && sentenceFailed) {
+      if (vocabFailed && sentenceFailed && smkFailed) {
         throw new Error(`Text complexity evaluation failed: ${errors.join("; ")}`);
       }
     }
@@ -1850,7 +2093,7 @@ var TextComplexityEvaluator = class _TextComplexityEvaluator extends BaseEvaluat
       processingTimeMs: latencyMs,
       hasFailures
     });
-    return { vocabulary: vocabResult, sentenceStructure: sentenceResult };
+    return { vocabulary: vocabResult, sentenceStructure: sentenceResult, subjectMatterKnowledge: smkResult };
   }
   /**
    * Run a sub-evaluator with error handling.
@@ -1882,6 +2125,7 @@ exports.NetworkError = NetworkError;
 exports.RateLimitError = RateLimitError;
 exports.SentenceAnalysisSchema = SentenceAnalysisSchema;
 exports.SentenceStructureEvaluator = SentenceStructureEvaluator;
+exports.SmkEvaluator = SmkEvaluator;
 exports.TextComplexityEvaluator = TextComplexityEvaluator;
 exports.TextComplexityLevel = TextComplexityLevel;
 exports.TimeoutError = TimeoutError;
@@ -1892,6 +2136,7 @@ exports.calculateFleschKincaidGrade = calculateFleschKincaidGrade;
 exports.calculateReadabilityMetrics = calculateReadabilityMetrics;
 exports.evaluateGradeLevelAppropriateness = evaluateGradeLevelAppropriateness;
 exports.evaluateSentenceStructure = evaluateSentenceStructure;
+exports.evaluateSmk = evaluateSmk;
 exports.evaluateTextComplexity = evaluateTextComplexity;
 exports.evaluateVocabulary = evaluateVocabulary;
 exports.featuresToJSON = featuresToJSON;