@stephen-lord/docs 1.0.0 → 1.0.1

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/tests/evaluation.test.ts

@@ -0,0 +1,233 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import { 
+  executeDirectScore, 
+  executePairwiseCompare, 
+  executeGenerateRubric,
+  EvaluatorAgent
+} from '../src/index.js';
+import { validateConfig } from '../src/config/index.js';
+
+// Test fixtures
+const TEST_PROMPT = 'Explain quantum entanglement to a high school student';
+
+const GOOD_RESPONSE = `Quantum entanglement is like having two magical coins that are connected in a special way. 
+When you flip one coin and it lands on heads, the other coin will instantly show tails, 
+no matter how far apart they are - even if one is on Earth and one is on Mars.
+
+Here's what makes it special:
+1. The connection is instantaneous - faster than light
+2. You can't predict which side either coin will land on
+3. But once you see one, you know exactly what the other shows
+
+Scientists like Einstein called this "spooky action at a distance" because it seems impossible, 
+but experiments have proven it's real. This phenomenon is now being used to develop 
+super-secure communication systems and quantum computers.`;
+
+const POOR_RESPONSE = `Quantum entanglement is when particles are connected. 
+It's complicated physics stuff. Scientists study it.`;
+
+const MEDIUM_RESPONSE = `Quantum entanglement happens when two particles become linked together. 
+When you measure one particle, you instantly know something about the other particle, 
+even if they're far apart. It's used in quantum computing research.`;
+
+// Validate config once before all tests
+beforeAll(() => {
+  validateConfig();
+});
+
+describe('Direct Score Tool', () => {
+  it('should score a response against criteria', async () => {
+    const result = await executeDirectScore({
+      response: GOOD_RESPONSE,
+      prompt: TEST_PROMPT,
+      criteria: [
+        {
+          name: 'Accuracy',
+          description: 'Scientific correctness of the explanation',
+          weight: 0.4
+        },
+        {
+          name: 'Clarity',
+          description: 'Understandable for a high school student',
+          weight: 0.3
+        },
+        {
+          name: 'Engagement',
+          description: 'Interesting and memorable',
+          weight: 0.3
+        }
+      ],
+      rubric: {
+        scale: '1-5'
+      }
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.scores).toHaveLength(3);
+    expect(result.overallScore).toBeGreaterThan(0);
+    expect(result.overallScore).toBeLessThanOrEqual(5);
+    expect(result.metadata.criteriaCount).toBe(3);
+    
+    // Good response should score reasonably well
+    expect(result.overallScore).toBeGreaterThanOrEqual(3);
+  }, 60000);
+
+  it('should provide lower scores for poor responses', async () => {
+    const goodResult = await executeDirectScore({
+      response: GOOD_RESPONSE,
+      prompt: TEST_PROMPT,
+      criteria: [
+        { name: 'Quality', description: 'Overall quality', weight: 1 }
+      ]
+    });
+
+    const poorResult = await executeDirectScore({
+      response: POOR_RESPONSE,
+      prompt: TEST_PROMPT,
+      criteria: [
+        { name: 'Quality', description: 'Overall quality', weight: 1 }
+      ]
+    });
+
+    expect(goodResult.success).toBe(true);
+    expect(poorResult.success).toBe(true);
+    expect(goodResult.overallScore).toBeGreaterThan(poorResult.overallScore);
+  }, 120000);
+});
+
+describe('Pairwise Compare Tool', () => {
+  it('should correctly identify the better response', async () => {
+    const result = await executePairwiseCompare({
+      responseA: GOOD_RESPONSE,
+      responseB: POOR_RESPONSE,
+      prompt: TEST_PROMPT,
+      criteria: ['accuracy', 'clarity', 'completeness', 'engagement'],
+      allowTie: true,
+      swapPositions: true
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.winner).toBe('A');
+    expect(result.confidence).toBeGreaterThan(0.5);
+  }, 120000);
+
+  it('should handle similar responses appropriately', async () => {
+    const result = await executePairwiseCompare({
+      responseA: MEDIUM_RESPONSE,
+      responseB: MEDIUM_RESPONSE,
+      prompt: TEST_PROMPT,
+      criteria: ['quality'],
+      allowTie: true,
+      swapPositions: true
+    });
+
+    expect(result.success).toBe(true);
+    // Same response should tie
+    expect(result.winner).toBe('TIE');
+  }, 120000);
+
+  it('should provide comparison details for each criterion', async () => {
+    const result = await executePairwiseCompare({
+      responseA: GOOD_RESPONSE,
+      responseB: MEDIUM_RESPONSE,
+      prompt: TEST_PROMPT,
+      criteria: ['accuracy', 'completeness'],
+      allowTie: true,
+      swapPositions: false
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.comparison).toHaveLength(2);
+    result.comparison.forEach(c => {
+      expect(c.criterion).toBeDefined();
+      expect(['A', 'B', 'TIE']).toContain(c.winner);
+      expect(c.reasoning).toBeDefined();
+    });
+  }, 60000);
+});
+
+describe('Generate Rubric Tool', () => {
+  it('should generate a complete rubric', async () => {
+    const result = await executeGenerateRubric({
+      criterionName: 'Factual Accuracy',
+      criterionDescription: 'How factually correct is the content',
+      scale: '1-5',
+      domain: 'educational content',
+      includeExamples: true,
+      strictness: 'balanced'
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.levels).toHaveLength(5);
+    expect(result.scale.min).toBe(1);
+    expect(result.scale.max).toBe(5);
+    expect(result.scoringGuidelines.length).toBeGreaterThan(0);
+    expect(result.edgeCases.length).toBeGreaterThan(0);
+
+    // Check level structure
+    result.levels.forEach(level => {
+      expect(level.score).toBeGreaterThanOrEqual(1);
+      expect(level.score).toBeLessThanOrEqual(5);
+      expect(level.label).toBeDefined();
+      expect(level.description).toBeDefined();
+      expect(level.characteristics.length).toBeGreaterThan(0);
+    });
+  }, 60000);
+
+  it('should respect strictness setting', async () => {
+    const lenient = await executeGenerateRubric({
+      criterionName: 'Code Quality',
+      criterionDescription: 'Quality of code implementation',
+      scale: '1-5',
+      includeExamples: false,
+      strictness: 'lenient'
+    });
+
+    const strict = await executeGenerateRubric({
+      criterionName: 'Code Quality',
+      criterionDescription: 'Quality of code implementation',
+      scale: '1-5',
+      includeExamples: false,
+      strictness: 'strict'
+    });
+
+    expect(lenient.success).toBe(true);
+    expect(strict.success).toBe(true);
+    expect(lenient.metadata.strictness).toBe('lenient');
+    expect(strict.metadata.strictness).toBe('strict');
+  }, 120000);
+});
+
+describe('Evaluator Agent', () => {
+  let agent: EvaluatorAgent;
+
+  beforeAll(() => {
+    agent = new EvaluatorAgent();
+  });
+
+  it('should provide integrated evaluation workflow', async () => {
+    const result = await agent.evaluateWithGeneratedRubric(
+      GOOD_RESPONSE,
+      TEST_PROMPT,
+      [
+        { name: 'Accuracy', description: 'Scientific correctness' },
+        { name: 'Accessibility', description: 'Appropriate for audience' }
+      ]
+    );
+
+    expect(result.success).toBe(true);
+    expect(result.scores.length).toBeGreaterThan(0);
+  }, 120000);
+
+  it('should support chat-based evaluation', async () => {
+    const result = await agent.chat(`
+      Please evaluate this response for accuracy:
+      
+      Question: What is photosynthesis?
+      Response: Photosynthesis is how plants make food using sunlight, water, and carbon dioxide.
+    `);
+
+    expect(result.text).toBeDefined();
+    expect(result.text.length).toBeGreaterThan(50);
+  }, 60000);
+});

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/tests/setup.ts

@@ -0,0 +1,27 @@
+/**
+ * Test setup file
+ * 
+ * This file runs before all tests to configure the test environment.
+ */
+
+import { beforeAll, afterAll } from 'vitest';
+
+// Load environment variables for testing
+import 'dotenv/config';
+
+// Global test configuration
+beforeAll(() => {
+  // Suppress console output during tests unless DEBUG is set
+  if (!process.env.DEBUG) {
+    console.log = () => {};
+    console.info = () => {};
+  }
+});
+
+afterAll(() => {
+  // Cleanup
+});
+
+// Increase timeout for LLM API calls
+// Note: Individual tests can override this with their own timeout
+

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/tests/skills.test.ts

@@ -0,0 +1,213 @@
+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';
+import { EvaluatorAgent } from '../src/agents/evaluator.js';
+import { validateConfig } from '../src/config/index.js';
+
+/**
+ * Tests for skills implementation based on LLM-as-a-Judge research
+ */
+
+describe('LLM Evaluator Skill Tests', () => {
+  let agent: EvaluatorAgent;
+
+  beforeAll(() => {
+    validateConfig();
+  });
+
+  beforeEach(() => {
+    agent = new EvaluatorAgent();
+  });
+
+  describe('Direct Scoring Skill', () => {
+    it('should use chain-of-thought in scoring', async () => {
+      const result = await agent.score({
+        response: 'Machine learning is a type of artificial intelligence that allows computers to learn from data.',
+        prompt: 'Define machine learning',
+        criteria: [
+          { name: 'Accuracy', description: 'Factual correctness', weight: 1 }
+        ]
+      });
+
+      expect(result.success).toBe(true);
+      // Should have justification (evidence of CoT)
+      if (result.scores.length > 0) {
+        expect(result.scores[0].justification.length).toBeGreaterThan(20);
+      }
+    }, 60000);
+
+    it('should handle multiple weighted criteria', async () => {
+      const result = await agent.score({
+        response: 'The mitochondria is the powerhouse of the cell. It produces ATP.',
+        prompt: 'Explain the function of mitochondria',
+        criteria: [
+          { name: 'Accuracy', description: 'Scientific correctness', weight: 0.5 },
+          { name: 'Completeness', description: 'Covers key points', weight: 0.3 },
+          { name: 'Clarity', description: 'Easy to understand', weight: 0.2 }
+        ]
+      });
+
+      expect(result.success).toBe(true);
+      expect(result.scores).toHaveLength(3);
+      expect(result.weightedScore).toBeDefined();
+    }, 60000);
+  });
+
+  describe('Pairwise Comparison Skill', () => {
+    it('should mitigate position bias with swap', async () => {
+      const response1 = 'Water boils at 100 degrees Celsius at sea level.';
+      const response2 = 'Water boils at 100°C (212°F) at standard atmospheric pressure (sea level).';
+
+      const result = await agent.compare({
+        responseA: response1,
+        responseB: response2,
+        prompt: 'At what temperature does water boil?',
+        criteria: ['accuracy', 'completeness'],
+        allowTie: true,
+        swapPositions: true
+      });
+
+      expect(result.success).toBe(true);
+      expect(result.positionConsistency).toBeDefined();
+    }, 120000);
+
+    it('should identify clear winner for quality difference', async () => {
+      const good = `The Earth revolves around the Sun in an elliptical orbit, 
+        taking approximately 365.25 days to complete one revolution. 
+        This is why we have leap years every 4 years.`;
+      
+      const poor = 'The earth goes around the sun.';
+
+      const result = await agent.compare({
+        responseA: good,
+        responseB: poor,
+        prompt: 'How does the Earth orbit the Sun?',
+        criteria: ['completeness', 'accuracy', 'detail'],
+        allowTie: true,
+        swapPositions: true
+      });
+
+      expect(result.success).toBe(true);
+      expect(result.winner).toBe('A');
+      expect(result.confidence).toBeGreaterThan(0.5);
+    }, 120000);
+  });
+
+  describe('Rubric Generation Skill', () => {
+    it('should generate domain-specific rubrics', async () => {
+      const result = await agent.generateRubric({
+        criterionName: 'Code Readability',
+        criterionDescription: 'How easy the code is to understand and maintain',
+        scale: '1-5',
+        domain: 'software engineering',
+        includeExamples: true,
+        strictness: 'balanced'
+      });
+
+      expect(result.success).toBe(true);
+      expect(result.levels.length).toBe(5);
+      expect(result.metadata.domain).toBe('software engineering');
+      
+      // Should have code-specific terminology
+      const allText = result.levels.map(l => l.description + l.characteristics.join(' ')).join(' ');
+      expect(allText.toLowerCase()).toMatch(/variable|function|comment|name|structure|code|read/i);
+    }, 60000);
+
+    it('should provide edge case guidance', async () => {
+      const result = await agent.generateRubric({
+        criterionName: 'Factual Accuracy',
+        criterionDescription: 'Whether claims are factually correct',
+        scale: '1-5',
+        includeExamples: false,
+        strictness: 'strict'
+      });
+
+      expect(result.success).toBe(true);
+      expect(result.edgeCases.length).toBeGreaterThan(0);
+      result.edgeCases.forEach(ec => {
+        expect(ec.situation).toBeDefined();
+        expect(ec.guidance).toBeDefined();
+      });
+    }, 60000);
+  });
+
+  describe('Context Fundamentals Skill Application', () => {
+    it('should utilize provided context in evaluation', async () => {
+      const context = `The user is a medical professional asking about drug interactions.
+        Technical terminology is appropriate.`;
+
+      const result = await agent.score({
+        response: 'Combining SSRIs with MAOIs can lead to serotonin syndrome, a potentially life-threatening condition.',
+        prompt: 'What are the risks of combining antidepressants?',
+        context,
+        criteria: [
+          { name: 'Accuracy', description: 'Medical accuracy', weight: 0.5 },
+          { name: 'Appropriateness', description: 'Appropriate for audience', weight: 0.5 }
+        ]
+      });
+
+      expect(result.success).toBe(true);
+      // Technical response should score well given medical context
+      expect(result.overallScore).toBeGreaterThanOrEqual(2);
+    }, 60000);
+  });
+});
+
+describe('Skill Input/Output Validation', () => {
+  let agent: EvaluatorAgent;
+
+  beforeAll(() => {
+    validateConfig();
+    agent = new EvaluatorAgent();
+  });
+
+  it('should validate DirectScore input schema', async () => {
+    const result = await agent.score({
+      response: 'Test response',
+      prompt: 'Test prompt',
+      criteria: [{ name: 'Test', description: 'Test criterion', weight: 1 }]
+    });
+    
+    expect(result).toHaveProperty('success');
+    expect(result).toHaveProperty('scores');
+    expect(result).toHaveProperty('overallScore');
+    expect(result).toHaveProperty('summary');
+    expect(result).toHaveProperty('metadata');
+  }, 60000);
+
+  it('should validate PairwiseCompare output structure', async () => {
+    const result = await agent.compare({
+      responseA: 'Response A content',
+      responseB: 'Response B content',
+      prompt: 'Test prompt',
+      criteria: ['quality'],
+      allowTie: true,
+      swapPositions: false
+    });
+
+    expect(result).toHaveProperty('success');
+    expect(result).toHaveProperty('winner');
+    expect(['A', 'B', 'TIE']).toContain(result.winner);
+    expect(result).toHaveProperty('confidence');
+    expect(result.confidence).toBeGreaterThanOrEqual(0);
+    expect(result.confidence).toBeLessThanOrEqual(1);
+    expect(result).toHaveProperty('comparison');
+    expect(result).toHaveProperty('metadata');
+  }, 60000);
+
+  it('should validate GenerateRubric output structure', async () => {
+    const result = await agent.generateRubric({
+      criterionName: 'Test',
+      criterionDescription: 'Test criterion',
+      scale: '1-5',
+      includeExamples: false,
+      strictness: 'balanced'
+    });
+
+    expect(result).toHaveProperty('success');
+    expect(result).toHaveProperty('criterion');
+    expect(result).toHaveProperty('scale');
+    expect(result).toHaveProperty('levels');
+    expect(result).toHaveProperty('scoringGuidelines');
+    expect(result).toHaveProperty('edgeCases');
+    expect(result).toHaveProperty('metadata');
+  }, 60000);
+});

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/tools/evaluation/direct-score.md

@@ -0,0 +1,159 @@
+# Direct Score Tool
+
+## Purpose
+
+Evaluate a single LLM response against defined criteria using a scoring rubric.
+
+## Tool Definition
+
+```typescript
+import { tool } from "ai";
+import { z } from "zod";
+
+export const directScore = tool({
+  description: `Evaluate a response by scoring it against specific criteria.
+Use this for objective evaluations where you need to assess quality 
+dimensions like accuracy, completeness, clarity, or task adherence.
+Returns structured scores with justifications.`,
+
+  parameters: z.object({
+    response: z.string()
+      .describe("The LLM response to evaluate"),
+    
+    prompt: z.string()
+      .describe("The original prompt/instruction that generated the response"),
+    
+    context: z.string().optional()
+      .describe("Additional context like retrieved documents or conversation history"),
+    
+    criteria: z.array(z.object({
+      name: z.string().describe("Name of the criterion (e.g., 'Accuracy')"),
+      description: z.string().describe("What this criterion measures"),
+      weight: z.number().min(0).max(1).default(1)
+        .describe("Relative importance, weights should sum to 1")
+    })).min(1).describe("Evaluation criteria to score against"),
+    
+    rubric: z.object({
+      scale: z.enum(["1-3", "1-5", "1-10"]).default("1-5"),
+      levelDescriptions: z.record(z.string(), z.string()).optional()
+        .describe("Optional descriptions for each score level")
+    }).optional().describe("Scoring rubric configuration")
+  }),
+
+  execute: async (input) => {
+    // Implementation delegates to evaluator LLM
+    return evaluateWithLLM(input);
+  }
+});
+```
+
+## Input Schema
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| response | string | Yes | The response being evaluated |
+| prompt | string | Yes | Original prompt that generated the response |
+| context | string | No | Additional context (RAG docs, history) |
+| criteria | Criterion[] | Yes | List of evaluation criteria |
+| rubric | Rubric | No | Scoring scale and level descriptions |
+
+### Criterion Object
+```typescript
+{
+  name: string;        // e.g., "Factual Accuracy"
+  description: string; // e.g., "Response contains no factual errors"
+  weight: number;      // 0-1, relative importance
+}
+```
+
+### Rubric Object
+```typescript
+{
+  scale: "1-3" | "1-5" | "1-10";
+  levelDescriptions?: {
+    "1": "Poor - Major issues",
+    "2": "Below Average - Several issues",
+    "3": "Average - Some issues",
+    "4": "Good - Minor issues",
+    "5": "Excellent - No issues"
+  }
+}
+```
+
+## Output Schema
+
+```typescript
+interface DirectScoreResult {
+  success: boolean;
+  
+  scores: {
+    criterion: string;
+    score: number;
+    maxScore: number;
+    justification: string;
+    examples: string[];  // Specific examples from response
+  }[];
+  
+  overallScore: number;
+  weightedScore: number;
+  
+  summary: {
+    strengths: string[];
+    weaknesses: string[];
+    suggestions: string[];
+  };
+  
+  metadata: {
+    evaluationTimeMs: number;
+    criteriaCount: number;
+    rubricScale: string;
+  };
+}
+```
+
+## Usage Example
+
+```typescript
+const result = await directScore.execute({
+  response: "Machine learning is a subset of AI that enables systems to learn from data...",
+  
+  prompt: "Explain machine learning to a beginner",
+  
+  criteria: [
+    {
+      name: "Accuracy",
+      description: "Technical correctness of explanations",
+      weight: 0.4
+    },
+    {
+      name: "Clarity",
+      description: "Understandable for a beginner",
+      weight: 0.3
+    },
+    {
+      name: "Completeness",
+      description: "Covers key concepts adequately",
+      weight: 0.3
+    }
+  ],
+  
+  rubric: {
+    scale: "1-5",
+    levelDescriptions: {
+      "1": "Completely fails criterion",
+      "2": "Major deficiencies",
+      "3": "Acceptable but improvable",
+      "4": "Good with minor issues",
+      "5": "Excellent, no issues"
+    }
+  }
+});
+```
+
+## Implementation Notes
+
+1. **Chain-of-Thought**: Implementation should use CoT prompting for more reliable scoring
+2. **Calibration**: Include few-shot examples of scores at each level
+3. **Justification First**: Ask for justification before score to reduce bias
+4. **Length Normalization**: Consider response length when appropriate
+