@stephen-lord/docs 1.0.0 → 1.0.1

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/src/tools/evaluation/direct-score.ts

@@ -0,0 +1,164 @@
+import { tool } from 'ai';
+import { z } from 'zod';
+import { openai } from '@ai-sdk/openai';
+import { generateText } from 'ai';
+import { config } from '../../config/index.js';
+
+const CriterionSchema = z.object({
+  name: z.string().describe('Name of the criterion'),
+  description: z.string().describe('What this criterion measures'),
+  weight: z.number().min(0).max(1).default(1).describe('Relative importance')
+});
+
+const RubricSchema = z.object({
+  scale: z.enum(['1-3', '1-5', '1-10']).default('1-5'),
+  levelDescriptions: z.record(z.string(), z.string()).optional()
+});
+
+export const DirectScoreInputSchema = z.object({
+  response: z.string().describe('The LLM response to evaluate'),
+  prompt: z.string().describe('The original prompt that generated the response'),
+  context: z.string().optional().describe('Additional context'),
+  criteria: z.array(CriterionSchema).min(1).describe('Evaluation criteria'),
+  rubric: RubricSchema.optional()
+});
+
+export type DirectScoreInput = z.infer<typeof DirectScoreInputSchema>;
+
+export const DirectScoreOutputSchema = z.object({
+  success: z.boolean(),
+  scores: z.array(z.object({
+    criterion: z.string(),
+    score: z.number(),
+    maxScore: z.number(),
+    justification: z.string(),
+    evidence: z.array(z.string()),
+    improvement: z.string()
+  })),
+  overallScore: z.number(),
+  weightedScore: z.number(),
+  summary: z.object({
+    assessment: z.string(),
+    strengths: z.array(z.string()),
+    weaknesses: z.array(z.string()),
+    priorities: z.array(z.string())
+  }),
+  metadata: z.object({
+    evaluationTimeMs: z.number(),
+    model: z.string(),
+    criteriaCount: z.number()
+  })
+});
+
+export type DirectScoreOutput = z.infer<typeof DirectScoreOutputSchema>;
+
+export async function executeDirectScore(input: DirectScoreInput): Promise<DirectScoreOutput> {
+  const startTime = Date.now();
+  const scale = input.rubric?.scale || '1-5';
+  const maxScore = parseInt(scale.split('-')[1]);
+
+  const systemPrompt = `You are an expert evaluator. Assess the response against each criterion.
+For each criterion:
+1. Find specific evidence in the response
+2. Score according to the rubric (1-${maxScore} scale)
+3. Justify your score
+4. Suggest one improvement
+
+Be objective and consistent. Base scores on explicit evidence.`;
+
+  const userPrompt = `## Original Prompt
+${input.prompt}
+
+${input.context ? `## Context\n${input.context}\n` : ''}
+## Response to Evaluate
+${input.response}
+
+## Criteria
+${input.criteria.map((c, i) => `${i + 1}. **${c.name}** (weight: ${c.weight}): ${c.description}`).join('\n')}
+
+${input.rubric?.levelDescriptions ? `## Rubric\n${Object.entries(input.rubric.levelDescriptions).map(([k, v]) => `- ${k}: ${v}`).join('\n')}` : ''}
+
+Respond with valid JSON matching this structure:
+{
+  "scores": [
+    {
+      "criterion": "criterion name",
+      "score": number,
+      "evidence": ["quote or observation 1", "quote 2"],
+      "justification": "why this score",
+      "improvement": "specific suggestion"
+    }
+  ],
+  "summary": {
+    "assessment": "overall quality summary",
+    "strengths": ["strength 1", "strength 2"],
+    "weaknesses": ["weakness 1"],
+    "priorities": ["most important improvement"]
+  }
+}`;
+
+  try {
+    const result = await generateText({
+      model: openai(config.openai.model),
+      system: systemPrompt,
+      prompt: userPrompt,
+      temperature: 0.3
+    });
+
+    const parsed = JSON.parse(result.text);
+    
+    // Calculate scores
+    const totalWeight = input.criteria.reduce((sum, c) => sum + c.weight, 0);
+    const weightedSum = parsed.scores.reduce((sum: number, s: { criterion: string; score: number }) => {
+      const criterion = input.criteria.find(c => c.name === s.criterion);
+      return sum + (s.score * (criterion?.weight || 1));
+    }, 0);
+    
+    const overallScore = parsed.scores.reduce((sum: number, s: { score: number }) => sum + s.score, 0) / parsed.scores.length;
+    const weightedScore = weightedSum / totalWeight;
+
+    return {
+      success: true,
+      scores: parsed.scores.map((s: { criterion: string; score: number; evidence?: string[]; justification: string; improvement: string }) => ({
+        ...s,
+        maxScore,
+        evidence: s.evidence || []
+      })),
+      overallScore: Math.round(overallScore * 100) / 100,
+      weightedScore: Math.round(weightedScore * 100) / 100,
+      summary: parsed.summary,
+      metadata: {
+        evaluationTimeMs: Date.now() - startTime,
+        model: config.openai.model,
+        criteriaCount: input.criteria.length
+      }
+    };
+  } catch (error) {
+    return {
+      success: false,
+      scores: [],
+      overallScore: 0,
+      weightedScore: 0,
+      summary: {
+        assessment: `Evaluation failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        strengths: [],
+        weaknesses: [],
+        priorities: []
+      },
+      metadata: {
+        evaluationTimeMs: Date.now() - startTime,
+        model: config.openai.model,
+        criteriaCount: input.criteria.length
+      }
+    };
+  }
+}
+
+export const directScoreTool = tool({
+  description: `Evaluate a response by scoring it against specific criteria.
+Use for objective evaluations like accuracy, completeness, clarity.
+Returns structured scores with justifications.`,
+  parameters: DirectScoreInputSchema,
+  execute: executeDirectScore
+});
+

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/src/tools/evaluation/generate-rubric.ts

@@ -0,0 +1,161 @@
+import { tool } from 'ai';
+import { z } from 'zod';
+import { openai } from '@ai-sdk/openai';
+import { generateText } from 'ai';
+import { config } from '../../config/index.js';
+
+export const GenerateRubricInputSchema = z.object({
+  criterionName: z.string().describe('Name of the criterion'),
+  criterionDescription: z.string().describe('What this criterion measures'),
+  scale: z.enum(['1-3', '1-5', '1-10']).optional().default('1-5'),
+  domain: z.string().optional().describe('Domain context'),
+  includeExamples: z.boolean().optional().default(true),
+  strictness: z.enum(['lenient', 'balanced', 'strict']).optional().default('balanced')
+});
+
+export type GenerateRubricInput = z.infer<typeof GenerateRubricInputSchema>;
+
+export const GenerateRubricOutputSchema = z.object({
+  success: z.boolean(),
+  criterion: z.object({
+    name: z.string(),
+    description: z.string()
+  }),
+  scale: z.object({
+    min: z.number(),
+    max: z.number(),
+    type: z.string()
+  }),
+  levels: z.array(z.object({
+    score: z.number(),
+    label: z.string(),
+    description: z.string(),
+    characteristics: z.array(z.string()),
+    example: z.string().optional()
+  })),
+  scoringGuidelines: z.array(z.string()),
+  edgeCases: z.array(z.object({
+    situation: z.string(),
+    guidance: z.string()
+  })),
+  metadata: z.object({
+    domain: z.string().nullable(),
+    strictness: z.string(),
+    generationTimeMs: z.number()
+  })
+});
+
+export type GenerateRubricOutput = z.infer<typeof GenerateRubricOutputSchema>;
+
+export async function executeGenerateRubric(input: GenerateRubricInput): Promise<GenerateRubricOutput> {
+  const startTime = Date.now();
+  const [minScore, maxScore] = input.scale.split('-').map(Number);
+
+  const systemPrompt = `You are an expert in creating evaluation rubrics.
+Create clear, actionable rubrics with distinct boundaries between levels.
+Strictness: ${input.strictness}
+- lenient: Lower bar for passing scores
+- balanced: Fair, typical expectations
+- strict: High standards, critical evaluation`;
+
+  const userPrompt = `Create a scoring rubric for:
+
+**Criterion**: ${input.criterionName}
+**Description**: ${input.criterionDescription}
+**Scale**: ${input.scale} (${minScore} = lowest, ${maxScore} = highest)
+${input.domain ? `**Domain**: ${input.domain}` : ''}
+**Include Examples**: ${input.includeExamples}
+
+Generate a rubric with:
+1. Clear descriptions for each score level
+2. Specific characteristics that define each level
+3. ${input.includeExamples ? 'Brief example text for each level' : 'No examples needed'}
+4. General scoring guidelines
+5. Edge cases with guidance
+
+Respond with valid JSON:
+{
+  "levels": [
+    {
+      "score": ${minScore},
+      "label": "Label (e.g., Poor)",
+      "description": "Detailed description of this level",
+      "characteristics": ["characteristic 1", "characteristic 2"],
+      "example": ${input.includeExamples ? '"Brief example text"' : 'null'}
+    }
+    // ... all levels from ${minScore} to ${maxScore}
+  ],
+  "scoringGuidelines": [
+    "General guideline 1",
+    "General guideline 2"
+  ],
+  "edgeCases": [
+    {
+      "situation": "Edge case description",
+      "guidance": "How to handle it"
+    }
+  ]
+}`;
+
+  try {
+    const result = await generateText({
+      model: openai(config.openai.model),
+      system: systemPrompt,
+      prompt: userPrompt,
+      temperature: 0.4
+    });
+
+    const parsed = JSON.parse(result.text);
+
+    return {
+      success: true,
+      criterion: {
+        name: input.criterionName,
+        description: input.criterionDescription
+      },
+      scale: {
+        min: minScore,
+        max: maxScore,
+        type: input.scale
+      },
+      levels: parsed.levels,
+      scoringGuidelines: parsed.scoringGuidelines,
+      edgeCases: parsed.edgeCases,
+      metadata: {
+        domain: input.domain || null,
+        strictness: input.strictness,
+        generationTimeMs: Date.now() - startTime
+      }
+    };
+  } catch (error) {
+    return {
+      success: false,
+      criterion: {
+        name: input.criterionName,
+        description: input.criterionDescription
+      },
+      scale: {
+        min: minScore,
+        max: maxScore,
+        type: input.scale
+      },
+      levels: [],
+      scoringGuidelines: [],
+      edgeCases: [],
+      metadata: {
+        domain: input.domain || null,
+        strictness: input.strictness,
+        generationTimeMs: Date.now() - startTime
+      }
+    };
+  }
+}
+
+export const generateRubricTool = tool({
+  description: `Generate a scoring rubric for an evaluation criterion.
+Creates detailed descriptions for each score level.
+Use to establish consistent evaluation standards.`,
+  parameters: GenerateRubricInputSchema,
+  execute: executeGenerateRubric
+});
+

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/src/tools/evaluation/index.ts

@@ -0,0 +1,9 @@
+export { directScoreTool, executeDirectScore, DirectScoreInputSchema, DirectScoreOutputSchema } from './direct-score.js';
+export type { DirectScoreInput, DirectScoreOutput } from './direct-score.js';
+
+export { pairwiseCompareTool, executePairwiseCompare, PairwiseCompareInputSchema, PairwiseCompareOutputSchema } from './pairwise-compare.js';
+export type { PairwiseCompareInput, PairwiseCompareOutput } from './pairwise-compare.js';
+
+export { generateRubricTool, executeGenerateRubric, GenerateRubricInputSchema, GenerateRubricOutputSchema } from './generate-rubric.js';
+export type { GenerateRubricInput, GenerateRubricOutput } from './generate-rubric.js';
+

package/dist-docs/Agent-Skills-for-Context-Engineering/examples/llm-as-judge-skills/src/tools/evaluation/pairwise-compare.ts

@@ -0,0 +1,255 @@
+import { tool } from 'ai';
+import { z } from 'zod';
+import { openai } from '@ai-sdk/openai';
+import { generateText } from 'ai';
+import { config } from '../../config/index.js';
+
+export const PairwiseCompareInputSchema = z.object({
+  responseA: z.string().describe('First response to compare'),
+  responseB: z.string().describe('Second response to compare'),
+  prompt: z.string().describe('The original prompt both responses address'),
+  context: z.string().optional().describe('Additional context'),
+  criteria: z.array(z.string()).min(1).describe('Comparison criteria'),
+  allowTie: z.boolean().optional().default(true).describe('Allow tie verdict'),
+  swapPositions: z.boolean().optional().default(true).describe('Swap positions to reduce bias')
+});
+
+export type PairwiseCompareInput = z.infer<typeof PairwiseCompareInputSchema>;
+
+export const PairwiseCompareOutputSchema = z.object({
+  success: z.boolean(),
+  winner: z.enum(['A', 'B', 'TIE']),
+  confidence: z.number().min(0).max(1),
+  comparison: z.array(z.object({
+    criterion: z.string(),
+    winner: z.enum(['A', 'B', 'TIE']),
+    aAssessment: z.string(),
+    bAssessment: z.string(),
+    reasoning: z.string()
+  })),
+  analysis: z.object({
+    responseA: z.object({
+      strengths: z.array(z.string()),
+      weaknesses: z.array(z.string())
+    }),
+    responseB: z.object({
+      strengths: z.array(z.string()),
+      weaknesses: z.array(z.string())
+    })
+  }),
+  differentiators: z.array(z.string()),
+  positionConsistency: z.object({
+    consistent: z.boolean(),
+    firstPassWinner: z.enum(['A', 'B', 'TIE']).optional(),
+    secondPassWinner: z.enum(['A', 'B', 'TIE']).optional()
+  }).optional(),
+  metadata: z.object({
+    evaluationTimeMs: z.number(),
+    model: z.string(),
+    positionsSwapped: z.boolean()
+  })
+});
+
+export type PairwiseCompareOutput = z.infer<typeof PairwiseCompareOutputSchema>;
+
+async function evaluatePair(
+  first: string,
+  second: string,
+  prompt: string,
+  criteria: string[],
+  context?: string,
+  allowTie: boolean = true
+): Promise<{ winner: 'A' | 'B' | 'TIE'; confidence: number; comparison: PairwiseCompareOutput['comparison']; analysis: PairwiseCompareOutput['analysis'] }> {
+  const systemPrompt = `You are an expert evaluator comparing two AI responses.
+
+IMPORTANT:
+- Do NOT prefer responses because they are longer
+- Do NOT prefer responses based on position (first vs second)
+- Focus only on quality according to the criteria
+- ${allowTie ? 'Ties are acceptable when responses are genuinely equivalent' : 'You must choose a winner'}`;
+
+  const userPrompt = `## Original Prompt
+${prompt}
+
+${context ? `## Context\n${context}\n` : ''}
+## Response A
+${first}
+
+## Response B
+${second}
+
+## Criteria to Compare
+${criteria.map((c, i) => `${i + 1}. ${c}`).join('\n')}
+
+First analyze each response independently, then compare them.
+Respond with valid JSON:
+{
+  "analysis": {
+    "responseA": { "strengths": [...], "weaknesses": [...] },
+    "responseB": { "strengths": [...], "weaknesses": [...] }
+  },
+  "comparison": [
+    {
+      "criterion": "criterion name",
+      "winner": "A" | "B" | "TIE",
+      "aAssessment": "brief assessment of A",
+      "bAssessment": "brief assessment of B",
+      "reasoning": "why this winner"
+    }
+  ],
+  "result": {
+    "winner": "A" | "B" | "TIE",
+    "confidence": 0.0-1.0,
+    "reasoning": "overall reasoning"
+  }
+}`;
+
+  const result = await generateText({
+    model: openai(config.openai.model),
+    system: systemPrompt,
+    prompt: userPrompt,
+    temperature: 0.3
+  });
+
+  const parsed = JSON.parse(result.text);
+  
+  return {
+    winner: parsed.result.winner,
+    confidence: parsed.result.confidence,
+    comparison: parsed.comparison,
+    analysis: parsed.analysis
+  };
+}
+
+export async function executePairwiseCompare(input: PairwiseCompareInput): Promise<PairwiseCompareOutput> {
+  const startTime = Date.now();
+
+  try {
+    if (input.swapPositions) {
+      // First pass: A first, B second
+      const pass1 = await evaluatePair(
+        input.responseA,
+        input.responseB,
+        input.prompt,
+        input.criteria,
+        input.context,
+        input.allowTie
+      );
+
+      // Second pass: B first, A second
+      const pass2 = await evaluatePair(
+        input.responseB,
+        input.responseA,
+        input.prompt,
+        input.criteria,
+        input.context,
+        input.allowTie
+      );
+
+      // Map pass2 result back
+      const pass2WinnerMapped = pass2.winner === 'A' ? 'B' : pass2.winner === 'B' ? 'A' : 'TIE';
+      const consistent = pass1.winner === pass2WinnerMapped;
+
+      // Determine final winner
+      let finalWinner: 'A' | 'B' | 'TIE';
+      let finalConfidence: number;
+
+      if (consistent) {
+        finalWinner = pass1.winner;
+        finalConfidence = (pass1.confidence + pass2.confidence) / 2;
+      } else {
+        // Inconsistent - return tie with lower confidence
+        finalWinner = 'TIE';
+        finalConfidence = 0.5;
+      }
+
+      // Merge comparisons
+      const mergedComparison = pass1.comparison.map((c, i) => {
+        const c2 = pass2.comparison[i];
+        const c2WinnerMapped = c2.winner === 'A' ? 'B' : c2.winner === 'B' ? 'A' : 'TIE';
+        return {
+          ...c,
+          winner: c.winner === c2WinnerMapped ? c.winner : 'TIE' as const
+        };
+      });
+
+      // Find differentiators
+      const differentiators = mergedComparison
+        .filter(c => c.winner !== 'TIE')
+        .map(c => `${c.criterion}: ${c.winner === 'A' ? 'Response A' : 'Response B'} wins - ${c.reasoning}`);
+
+      return {
+        success: true,
+        winner: finalWinner,
+        confidence: Math.round(finalConfidence * 100) / 100,
+        comparison: mergedComparison,
+        analysis: pass1.analysis,
+        differentiators,
+        positionConsistency: {
+          consistent,
+          firstPassWinner: pass1.winner,
+          secondPassWinner: pass2WinnerMapped
+        },
+        metadata: {
+          evaluationTimeMs: Date.now() - startTime,
+          model: config.openai.model,
+          positionsSwapped: true
+        }
+      };
+    } else {
+      // Single pass without swap
+      const result = await evaluatePair(
+        input.responseA,
+        input.responseB,
+        input.prompt,
+        input.criteria,
+        input.context,
+        input.allowTie
+      );
+
+      const differentiators = result.comparison
+        .filter(c => c.winner !== 'TIE')
+        .map(c => `${c.criterion}: ${c.winner === 'A' ? 'Response A' : 'Response B'} wins - ${c.reasoning}`);
+
+      return {
+        success: true,
+        winner: result.winner,
+        confidence: result.confidence,
+        comparison: result.comparison,
+        analysis: result.analysis,
+        differentiators,
+        metadata: {
+          evaluationTimeMs: Date.now() - startTime,
+          model: config.openai.model,
+          positionsSwapped: false
+        }
+      };
+    }
+  } catch (error) {
+    return {
+      success: false,
+      winner: 'TIE',
+      confidence: 0,
+      comparison: [],
+      analysis: {
+        responseA: { strengths: [], weaknesses: [] },
+        responseB: { strengths: [], weaknesses: [] }
+      },
+      differentiators: [],
+      metadata: {
+        evaluationTimeMs: Date.now() - startTime,
+        model: config.openai.model,
+        positionsSwapped: input.swapPositions
+      }
+    };
+  }
+}
+
+export const pairwiseCompareTool = tool({
+  description: `Compare two responses and select the better one.
+Use for subjective evaluations like tone, persuasiveness, style.
+More reliable than direct scoring for preferences.`,
+  parameters: PairwiseCompareInputSchema,
+  execute: executePairwiseCompare
+});
+