@stephen-lord/docs 1.0.0 → 1.0.1

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/examples/basic-evaluation.ts

@@ -0,0 +1,89 @@
+/**
+ * Basic Evaluation Example
+ * 
+ * Demonstrates how to use the EvaluatorAgent to score responses.
+ * 
+ * Run: npx tsx examples/basic-evaluation.ts
+ */
+
+import 'dotenv/config';
+import { EvaluatorAgent } from '../src/agents/evaluator.js';
+import { validateConfig } from '../src/config/index.js';
+
+async function main() {
+  // Validate API key is configured
+  validateConfig();
+
+  const agent = new EvaluatorAgent();
+
+  console.log('=== Direct Scoring Example ===\n');
+
+  const response = `
+    Machine learning is a subset of artificial intelligence that enables systems 
+    to learn and improve from experience without being explicitly programmed. 
+    It focuses on developing algorithms that can access data and use it to learn for themselves.
+    
+    There are three main types of machine learning:
+    1. Supervised learning - learns from labeled data
+    2. Unsupervised learning - finds patterns in unlabeled data  
+    3. Reinforcement learning - learns through trial and error
+  `;
+
+  const result = await agent.score({
+    response,
+    prompt: 'Explain what machine learning is to a beginner',
+    criteria: [
+      {
+        name: 'Accuracy',
+        description: 'Factual correctness of the explanation',
+        weight: 0.4
+      },
+      {
+        name: 'Clarity',
+        description: 'Easy to understand for a beginner',
+        weight: 0.3
+      },
+      {
+        name: 'Completeness',
+        description: 'Covers the key concepts adequately',
+        weight: 0.3
+      }
+    ],
+    rubric: {
+      scale: '1-5',
+      levelDescriptions: {
+        '1': 'Poor - Major issues',
+        '2': 'Below Average - Several issues',
+        '3': 'Average - Some issues',
+        '4': 'Good - Minor issues only',
+        '5': 'Excellent - No issues'
+      }
+    }
+  });
+
+  if (result.success) {
+    console.log('Evaluation Results:');
+    console.log('-------------------');
+    
+    result.scores.forEach(score => {
+      console.log(`\n${score.criterion}: ${score.score}/${score.maxScore}`);
+      console.log(`Justification: ${score.justification}`);
+      console.log(`Improvement: ${score.improvement}`);
+    });
+
+    console.log('\n-------------------');
+    console.log(`Overall Score: ${result.overallScore}`);
+    console.log(`Weighted Score: ${result.weightedScore}`);
+    console.log(`\nAssessment: ${result.summary.assessment}`);
+    console.log(`\nStrengths:`);
+    result.summary.strengths.forEach(s => console.log(`  - ${s}`));
+    console.log(`\nWeaknesses:`);
+    result.summary.weaknesses.forEach(w => console.log(`  - ${w}`));
+    console.log(`\nEvaluation Time: ${result.metadata.evaluationTimeMs}ms`);
+  } else {
+    console.error('Evaluation failed:', result.summary.assessment);
+  }
+}
+
+main().catch(console.error);
+

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/examples/full-evaluation-workflow.ts

@@ -0,0 +1,136 @@
+/**
+ * Full Evaluation Workflow Example
+ * 
+ * Demonstrates a complete evaluation workflow:
+ * 1. Generate rubrics for criteria
+ * 2. Score a response using generated rubrics
+ * 3. Compare with an alternative response
+ * 
+ * Run: npx tsx examples/full-evaluation-workflow.ts
+ */
+
+import 'dotenv/config';
+import { EvaluatorAgent } from '../src/agents/evaluator.js';
+import { validateConfig } from '../src/config/index.js';
+
+async function main() {
+  validateConfig();
+
+  const agent = new EvaluatorAgent();
+  const startTime = Date.now();
+
+  console.log('=== Full Evaluation Workflow ===\n');
+
+  const prompt = 'Explain how vaccines work to prevent disease';
+
+  const response = `
+    Vaccines work by training your immune system to recognize and fight specific pathogens
+    without causing the disease itself.
+    
+    Here's the process:
+    
+    1. **Introduction**: The vaccine introduces a weakened, killed, or partial version of
+       the pathogen (or instructions to make a piece of it, like mRNA vaccines).
+    
+    2. **Immune Response**: Your immune system detects these foreign substances (antigens)
+       and mounts a response. This includes producing antibodies and training T-cells.
+    
+    3. **Memory Formation**: Some immune cells become "memory cells" that remember
+       how to fight this specific pathogen.
+    
+    4. **Future Protection**: If you're exposed to the real pathogen later, your immune
+       system recognizes it immediately and can fight it off before you get sick.
+    
+    This is why vaccines are so effective - they give your immune system a "practice run"
+    without the risks of actual infection.
+  `;
+
+  // Step 1: Generate rubrics
+  console.log('Step 1: Generating rubrics...\n');
+
+  const criteria = [
+    { name: 'Scientific Accuracy', description: 'Correctness of biological/medical information' },
+    { name: 'Completeness', description: 'Covers the key steps and concepts' },
+    { name: 'Accessibility', description: 'Understandable by general audience' }
+  ];
+
+  const rubrics = await Promise.all(
+    criteria.map(c => agent.generateRubric({
+      criterionName: c.name,
+      criterionDescription: c.description,
+      scale: '1-5',
+      domain: 'health education',
+      includeExamples: false,
+      strictness: 'balanced'
+    }))
+  );
+
+  console.log('Generated rubrics for:');
+  rubrics.forEach(r => {
+    if (r.success) {
+      console.log(`  - ${r.criterion.name} (${r.levels.length} levels)`);
+    }
+  });
+
+  // Step 2: Score the response
+  console.log('\nStep 2: Scoring the response...\n');
+
+  const scoreResult = await agent.score({
+    response,
+    prompt,
+    criteria: criteria.map((c, i) => ({
+      name: c.name,
+      description: c.description,
+      weight: i === 0 ? 0.4 : 0.3 // Weight accuracy higher
+    })),
+    rubric: {
+      scale: '1-5',
+      levelDescriptions: rubrics[0].success 
+        ? Object.fromEntries(rubrics[0].levels.map(l => [String(l.score), l.label]))
+        : undefined
+    }
+  });
+
+  if (scoreResult.success) {
+    console.log('Scores:');
+    scoreResult.scores.forEach(s => {
+      console.log(`  ${s.criterion}: ${s.score}/${s.maxScore}`);
+    });
+    console.log(`\nOverall: ${scoreResult.overallScore} | Weighted: ${scoreResult.weightedScore}`);
+  }
+
+  // Step 3: Compare with an alternative
+  console.log('\nStep 3: Comparing with alternative response...\n');
+
+  const alternativeResponse = `
+    Vaccines prevent disease by helping your body build immunity. When you get
+    vaccinated, your body learns to fight the germ. Then if you're exposed to
+    the real disease, your body already knows how to protect itself.
+  `;
+
+  const comparisonResult = await agent.compare({
+    responseA: response,
+    responseB: alternativeResponse,
+    prompt,
+    criteria: ['accuracy', 'depth', 'clarity'],
+    swapPositions: true
+  });
+
+  if (comparisonResult.success) {
+    console.log(`Comparison Result: Response ${comparisonResult.winner} is better`);
+    console.log(`Confidence: ${(comparisonResult.confidence * 100).toFixed(0)}%`);
+    console.log('\nKey differences:');
+    comparisonResult.differentiators.slice(0, 3).forEach(d => console.log(`  - ${d}`));
+  }
+
+  // Summary
+  const totalTime = Date.now() - startTime;
+  console.log('\n=== Workflow Complete ===');
+  console.log(`Total time: ${totalTime}ms`);
+  console.log(`Rubrics generated: ${rubrics.filter(r => r.success).length}`);
+  console.log(`Final score: ${scoreResult.success ? scoreResult.overallScore : 'N/A'}`);
+  console.log(`Better response: ${comparisonResult.success ? comparisonResult.winner : 'N/A'}`);
+}
+
+main().catch(console.error);
+

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/examples/generate-rubric.ts

@@ -0,0 +1,67 @@
+/**
+ * Rubric Generation Example
+ * 
+ * Demonstrates how to generate evaluation rubrics for custom criteria.
+ * 
+ * Run: npx tsx examples/generate-rubric.ts
+ */
+
+import 'dotenv/config';
+import { EvaluatorAgent } from '../src/agents/evaluator.js';
+import { validateConfig } from '../src/config/index.js';
+
+async function main() {
+  validateConfig();
+
+  const agent = new EvaluatorAgent();
+
+  console.log('=== Rubric Generation Example ===\n');
+
+  // Generate a rubric for code review
+  const result = await agent.generateRubric({
+    criterionName: 'Code Readability',
+    criterionDescription: 'How easy the code is to read, understand, and maintain',
+    scale: '1-5',
+    domain: 'software engineering',
+    includeExamples: true,
+    strictness: 'balanced'
+  });
+
+  if (result.success) {
+    console.log(`Criterion: ${result.criterion.name}`);
+    console.log(`Description: ${result.criterion.description}`);
+    console.log(`Scale: ${result.scale.min}-${result.scale.max}`);
+    console.log(`Domain: ${result.metadata.domain || 'General'}`);
+    console.log(`Strictness: ${result.metadata.strictness}`);
+
+    console.log('\n--- Score Levels ---\n');
+    result.levels.forEach(level => {
+      console.log(`[${level.score}] ${level.label}`);
+      console.log(`    ${level.description}`);
+      console.log(`    Characteristics:`);
+      level.characteristics.forEach(c => console.log(`      - ${c}`));
+      if (level.example) {
+        console.log(`    Example: ${level.example.slice(0, 100)}...`);
+      }
+      console.log();
+    });
+
+    console.log('--- Scoring Guidelines ---');
+    result.scoringGuidelines.forEach((g, i) => {
+      console.log(`${i + 1}. ${g}`);
+    });
+
+    console.log('\n--- Edge Cases ---');
+    result.edgeCases.forEach(ec => {
+      console.log(`\nSituation: ${ec.situation}`);
+      console.log(`Guidance: ${ec.guidance}`);
+    });
+
+    console.log(`\nGeneration Time: ${result.metadata.generationTimeMs}ms`);
+  } else {
+    console.error('Rubric generation failed');
+  }
+}
+
+main().catch(console.error);
+

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/examples/pairwise-comparison.ts

@@ -0,0 +1,97 @@
+/**
+ * Pairwise Comparison Example
+ * 
+ * Demonstrates how to compare two responses and pick the better one.
+ * 
+ * Run: npx tsx examples/pairwise-comparison.ts
+ */
+
+import 'dotenv/config';
+import { EvaluatorAgent } from '../src/agents/evaluator.js';
+import { validateConfig } from '../src/config/index.js';
+
+async function main() {
+  validateConfig();
+
+  const agent = new EvaluatorAgent();
+
+  console.log('=== Pairwise Comparison Example ===\n');
+
+  const prompt = 'Explain the benefits of regular exercise';
+
+  const responseA = `
+    Regular exercise offers numerous health benefits that affect both body and mind.
+    
+    Physical benefits include:
+    - Improved cardiovascular health and reduced heart disease risk
+    - Stronger muscles and bones
+    - Better weight management
+    - Enhanced immune function
+    
+    Mental benefits include:
+    - Reduced stress and anxiety
+    - Improved mood through endorphin release
+    - Better sleep quality
+    - Enhanced cognitive function
+    
+    The CDC recommends at least 150 minutes of moderate aerobic activity per week,
+    plus muscle-strengthening activities twice weekly.
+  `;
+
+  const responseB = `
+    Working out is really good for you. It makes you healthier and feel better.
+    You should try to exercise regularly if you can. Many people find that
+    going to the gym or running helps them stay in shape.
+  `;
+
+  console.log('Prompt:', prompt);
+  console.log('\n--- Response A ---');
+  console.log(responseA.trim());
+  console.log('\n--- Response B ---');
+  console.log(responseB.trim());
+  console.log('\n--- Comparison Results ---\n');
+
+  const result = await agent.compare({
+    responseA,
+    responseB,
+    prompt,
+    criteria: ['accuracy', 'completeness', 'actionability', 'clarity'],
+    allowTie: true,
+    swapPositions: true // Mitigate position bias
+  });
+
+  if (result.success) {
+    console.log(`Winner: Response ${result.winner}`);
+    console.log(`Confidence: ${(result.confidence * 100).toFixed(0)}%`);
+    
+    if (result.positionConsistency) {
+      console.log(`Position Consistency: ${result.positionConsistency.consistent ? 'Yes' : 'No'}`);
+    }
+
+    console.log('\nPer-Criterion Results:');
+    result.comparison.forEach(c => {
+      console.log(`\n  ${c.criterion}:`);
+      console.log(`    Winner: ${c.winner}`);
+      console.log(`    A: ${c.aAssessment}`);
+      console.log(`    B: ${c.bAssessment}`);
+    });
+
+    console.log('\nKey Differentiators:');
+    result.differentiators.forEach(d => console.log(`  - ${d}`));
+
+    console.log('\nResponse A Analysis:');
+    console.log('  Strengths:', result.analysis.responseA.strengths.join(', '));
+    console.log('  Weaknesses:', result.analysis.responseA.weaknesses.join(', '));
+
+    console.log('\nResponse B Analysis:');
+    console.log('  Strengths:', result.analysis.responseB.strengths.join(', '));
+    console.log('  Weaknesses:', result.analysis.responseB.weaknesses.join(', '));
+
+    console.log(`\nEvaluation Time: ${result.metadata.evaluationTimeMs}ms`);
+  } else {
+    console.error('Comparison failed');
+  }
+}
+
+main().catch(console.error);
+

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/package.json

@@ -0,0 +1,79 @@
+{
+  "name": "llm-as-judge-skills",
+  "version": "1.0.0",
+  "description": "LLM-as-a-Judge evaluation skills built with AI SDK - Direct scoring, pairwise comparison, and rubric generation tools",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx watch src/index.ts",
+    "start": "node dist/index.js",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "lint": "eslint src tests examples",
+    "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\" \"examples/**/*.ts\"",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "npm run build",
+    "example:basic": "tsx examples/basic-evaluation.ts",
+    "example:compare": "tsx examples/pairwise-comparison.ts",
+    "example:rubric": "tsx examples/generate-rubric.ts",
+    "example:full": "tsx examples/full-evaluation-workflow.ts"
+  },
+  "keywords": [
+    "ai",
+    "agents",
+    "llm",
+    "evaluation",
+    "llm-as-judge",
+    "ai-sdk",
+    "openai",
+    "gpt",
+    "machine-learning",
+    "natural-language-processing"
+  ],
+  "author": "Muratcan Koylan",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/muratcankoylan/llm-as-judge-skills.git"
+  },
+  "bugs": {
+    "url": "https://github.com/muratcankoylan/llm-as-judge-skills/issues"
+  },
+  "homepage": "https://github.com/muratcankoylan/llm-as-judge-skills#readme",
+  "dependencies": {
+    "ai": "^4.0.0",
+    "@ai-sdk/openai": "^1.0.0",
+    "@ai-sdk/anthropic": "^1.0.0",
+    "zod": "^3.23.0",
+    "dotenv": "^16.4.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^5.6.0",
+    "typescript-eslint": "^8.0.0",
+    "tsx": "^4.19.0",
+    "vitest": "^2.1.0",
+    "@vitest/coverage-v8": "^2.1.0",
+    "@eslint/js": "^9.0.0",
+    "eslint": "^9.0.0",
+    "prettier": "^3.4.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}
+

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/prompts/agent-system/orchestrator-prompt.md

@@ -0,0 +1,197 @@
+# Orchestrator System Prompt
+
+## Purpose
+
+System prompt for the Orchestrator Agent that manages multi-agent workflows.
+
+## Prompt Template
+
+```markdown
+# Workflow Orchestrator
+
+You are a workflow orchestration expert managing a team of specialized AI agents.
+
+## Your Role
+
+- Analyze complex tasks and decompose them into subtasks
+- Assign subtasks to the most appropriate agents
+- Coordinate outputs and manage dependencies
+- Synthesize results into coherent deliverables
+- Handle errors and ensure workflow completion
+
+## Available Agents
+
+### Evaluator Agent
+**Capabilities**: Quality assessment, scoring, pairwise comparison
+**Use when**: Need to assess response quality, compare outputs, validate content
+**Input requirements**: Response to evaluate, criteria, optional rubric
+
+### Researcher Agent
+**Capabilities**: Web search, content extraction, fact synthesis
+**Use when**: Need current information, verification, comprehensive research
+**Input requirements**: Research question, scope constraints
+
+### Writer Agent
+**Capabilities**: Content generation, editing, style adaptation
+**Use when**: Need to produce or refine written content
+**Input requirements**: Writing task, context, style guidelines
+
+### Analyst Agent
+**Capabilities**: Data analysis, pattern identification, insights
+**Use when**: Need to analyze data or identify trends
+**Input requirements**: Data or information to analyze, analysis focus
+
+## Orchestration Principles
+
+1. **Right Agent, Right Task**: Match agent capabilities to task requirements
+2. **Complete Context**: Provide agents with all information they need
+3. **Clear Success Criteria**: Define what "done" looks like for each subtask
+4. **Dependency Awareness**: Sequence dependent tasks appropriately
+5. **Parallel When Possible**: Run independent tasks concurrently
+6. **Fail Gracefully**: Handle errors without abandoning the workflow
+
+## Workflow Execution
+
+When given a complex task:
+
+### Step 1: Task Analysis
+- What is the end goal?
+- What are the component tasks?
+- Which tasks depend on others?
+- Which can run in parallel?
+
+### Step 2: Agent Assignment
+- Which agent is best suited for each task?
+- What context does each agent need?
+- What output format is expected?
+
+### Step 3: Execution Planning
+```
+[Task Dependency Graph]
+  ├── Task 1 (Agent A) ─────────────────────┐
+  ├── Task 2 (Agent B) ───────┐             │
+  └── Task 3 (Agent B) ───────┴─→ Task 4 (Agent C) ─→ Final
+```
+
+### Step 4: Execution & Monitoring
+- Execute tasks according to plan
+- Monitor for failures
+- Retry or adapt as needed
+
+### Step 5: Synthesis
+- Collect all outputs
+- Synthesize into final deliverable
+- Validate against original requirements
+
+## Task Template
+
+When delegating to an agent, provide:
+
+```
+Agent: [agent_name]
+Task: [clear description of what to do]
+Context:
+  - [relevant context item 1]
+  - [relevant context item 2]
+  - [output from prior task if dependency]
+Expected Output:
+  - Format: [text/json/markdown/structured]
+  - Requirements: [specific requirements]
+Success Criteria:
+  - [criterion 1]
+  - [criterion 2]
+```
+
+## Error Handling
+
+When an agent fails:
+
+1. **Assess the error**
+   - Is it transient (retry may help)?
+   - Is it a context issue (can we provide better input)?
+   - Is it a capability issue (wrong agent)?
+
+2. **Decide on action**
+   - Retry with same parameters
+   - Retry with adjusted context
+   - Delegate to different agent
+   - Simplify the task
+   - Escalate if unrecoverable
+
+3. **Document and continue**
+   - Note what failed and why
+   - Adjust remaining workflow if needed
+   - Continue with best effort
+
+## Output Format
+
+Provide workflow status and results:
+
+```json
+{
+  "status": "completed" | "partial" | "failed",
+  "workflow": [
+    {
+      "task": "Task description",
+      "agent": "agent_name",
+      "status": "success" | "failed" | "skipped",
+      "output": "...",
+      "duration_ms": 1234
+    }
+  ],
+  "finalOutput": "Synthesized result",
+  "errors": [],
+  "notes": []
+}
+```
+```
+
+## Variables
+
+| Variable | Description |
+|----------|-------------|
+| task | The complex task to orchestrate |
+| constraints | Time, cost, or quality constraints |
+| preferredAgents | Any agent preferences |
+
+## Example Workflow
+
+### Input
+```
+Task: Create a comprehensive report on LLM evaluation best practices.
+
+Requirements:
+1. Research current methods and tools
+2. Analyze trade-offs between approaches
+3. Write an executive summary
+4. Evaluate the quality of the final report
+```
+
+### Execution Plan
+```
+Phase 1 (Parallel):
+  ├── Researcher: "Research LLM evaluation methods, tools, and recent papers"
+  └── Researcher: "Research case studies and practical implementations"
+
+Phase 2:
+  └── Analyst: "Analyze trade-offs between evaluation approaches"
+      Input: Research outputs from Phase 1
+
+Phase 3:
+  └── Writer: "Write executive summary of evaluation best practices"
+      Input: Research and analysis from Phase 1-2
+
+Phase 4:
+  └── Evaluator: "Evaluate report quality"
+      Input: Written report from Phase 3
+      Criteria: Accuracy, Completeness, Clarity, Actionability
+```
+
+## Best Practices
+
+1. **Start Simple**: Begin with minimal viable workflow, add complexity as needed.
+2. **Monitor Progress**: Provide status updates for long-running workflows,
+3. **Preserve Context**: Pass relevant context between agent handoffs,
+4. **Quality Gates**: Validate intermediate outputs before proceeding,
+5. **Document Decisions**: Log why tasks were assigned to specific agents.
+