mcp-rubber-duck 1.2.4 → 1.3.0

package/src/tools/duck-judge.ts

@@ -0,0 +1,301 @@
+import { ProviderManager } from '../providers/manager.js';
+import { DuckResponse, JudgeEvaluation } from '../config/types.js';
+import { logger } from '../utils/logger.js';
+
+export interface DuckJudgeArgs {
+  responses: DuckResponse[];
+  judge?: string;
+  criteria?: string[];
+  persona?: string;
+}
+
+interface ParsedJudgment {
+  rankings: Array<{
+    provider: string;
+    score: number;
+    justification: string;
+  }>;
+  criteria_scores?: Record<string, Record<string, number>>;
+  summary: string;
+}
+
+const DEFAULT_CRITERIA = ['accuracy', 'completeness', 'clarity'];
+
+export async function duckJudgeTool(
+  providerManager: ProviderManager,
+  args: Record<string, unknown>
+) {
+  const {
+    responses,
+    judge,
+    criteria = DEFAULT_CRITERIA,
+    persona,
+  } = args as unknown as DuckJudgeArgs;
+
+  // Validate inputs
+  if (!responses || !Array.isArray(responses) || responses.length === 0) {
+    throw new Error('At least one response is required to judge');
+  }
+
+  if (responses.length === 1) {
+    throw new Error('At least two responses are required for comparison');
+  }
+
+  // Determine judge provider
+  const judgeProvider = judge || providerManager.getProviderNames()[0];
+  if (!judgeProvider) {
+    throw new Error('No judge provider available');
+  }
+
+  logger.info(`Starting judgment with ${judgeProvider} on ${responses.length} responses`);
+
+  // Build the judgment prompt
+  const prompt = buildJudgePrompt(responses, criteria, persona);
+
+  // Get judgment from the judge duck
+  const judgeResponse = await providerManager.askDuck(judgeProvider, prompt);
+
+  // Parse the judgment
+  const evaluation = parseJudgment(
+    judgeResponse.content,
+    judgeResponse.provider,
+    judgeResponse.nickname,
+    responses,
+    criteria
+  );
+
+  // Format output
+  const formattedOutput = formatJudgeResult(evaluation);
+
+  logger.info(
+    `Judgment completed by ${judgeProvider}: #1 is ${evaluation.rankings[0]?.provider || 'unknown'}`
+  );
+
+  return {
+    content: [
+      {
+        type: 'text',
+        text: formattedOutput,
+      },
+    ],
+  };
+}
+
+function buildJudgePrompt(
+  responses: DuckResponse[],
+  criteria: string[],
+  persona?: string
+): string {
+  const criteriaList = criteria.map((c, i) => `${i + 1}. ${c}`).join('\n');
+
+  const responsesText = responses.map((r, i) =>
+    `--- Response ${i + 1} (${r.nickname} / ${r.provider}) ---\n${r.content}\n`
+  ).join('\n');
+
+  const personaText = persona
+    ? `You are a ${persona} evaluating these responses.\n\n`
+    : '';
+
+  return `${personaText}You are a judge evaluating ${responses.length} responses to the same prompt.
+
+RESPONSES TO EVALUATE:
+${responsesText}
+
+EVALUATION CRITERIA:
+${criteriaList}
+
+INSTRUCTIONS:
+1. Evaluate each response against ALL criteria
+2. Assign a score from 0-100 for each response
+3. Rank responses from best to worst
+4. Provide a brief justification for each ranking
+5. Give a final summary
+
+Respond with ONLY a JSON object in this exact format:
+{
+  "rankings": [
+    {"provider": "<provider name>", "score": <0-100>, "justification": "<brief explanation>"},
+    {"provider": "<provider name>", "score": <0-100>, "justification": "<brief explanation>"}
+  ],
+  "criteria_scores": {
+    "<provider>": {${criteria.map(c => `"${c}": <0-100>`).join(', ')}}
+  },
+  "summary": "<overall assessment and recommendation>"
+}
+
+IMPORTANT:
+- Rankings must be ordered from highest score to lowest
+- Use the exact provider names from the responses
+- Do NOT include any text before or after the JSON
+- Do NOT use markdown code blocks`;
+}
+
+function matchProvider(
+  judgeProviderName: string,
+  originalResponses: DuckResponse[]
+): DuckResponse | undefined {
+  const nameLower = judgeProviderName.toLowerCase();
+
+  // Try exact match first
+  const exactMatch = originalResponses.find(r => r.provider.toLowerCase() === nameLower);
+  if (exactMatch) return exactMatch;
+
+  // Try matching by provider name contained in judge's response
+  const containsMatch = originalResponses.find(r =>
+    nameLower.includes(r.provider.toLowerCase()) ||
+    nameLower.includes(r.nickname.toLowerCase())
+  );
+  if (containsMatch) return containsMatch;
+
+  // Try matching by nickname
+  const nicknameMatch = originalResponses.find(r =>
+    r.nickname.toLowerCase() === nameLower
+  );
+  if (nicknameMatch) return nicknameMatch;
+
+  return undefined;
+}
+
+function parseJudgment(
+  response: string,
+  judgeProvider: string,
+  judgeNickname: string,
+  originalResponses: DuckResponse[],
+  criteria: string[]
+): JudgeEvaluation {
+  const evaluation: JudgeEvaluation = {
+    judge: judgeProvider,
+    judgeNickname: judgeNickname,
+    prompt: '', // Will be filled by caller if needed
+    criteria,
+    rankings: [],
+    criteriaScores: {},
+    summary: '',
+    rawResponse: response,
+  };
+
+  try {
+    // Try to extract JSON from the response
+    const jsonMatch = response.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) {
+      logger.warn(`No JSON found in judge response from ${judgeProvider}`);
+      return createFallbackEvaluation(evaluation, originalResponses, response);
+    }
+
+    const parsed = JSON.parse(jsonMatch[0]) as ParsedJudgment;
+    const matchedProviders = new Set<string>();
+
+    // Parse rankings
+    if (Array.isArray(parsed.rankings)) {
+      for (const [index, r] of parsed.rankings.entries()) {
+        const matched = matchProvider(r.provider, originalResponses);
+        if (matched && !matchedProviders.has(matched.provider)) {
+          matchedProviders.add(matched.provider);
+          evaluation.rankings.push({
+            provider: matched.provider,
+            nickname: matched.nickname,
+            rank: index + 1,
+            score: typeof r.score === 'number' ? Math.max(0, Math.min(100, r.score)) : 0,
+            justification: r.justification?.toString() || '',
+          });
+        }
+      }
+    }
+
+    // Parse criteria scores
+    if (parsed.criteria_scores && typeof parsed.criteria_scores === 'object') {
+      evaluation.criteriaScores = parsed.criteria_scores;
+    }
+
+    // Parse summary
+    if (parsed.summary) {
+      evaluation.summary = parsed.summary.toString();
+    }
+
+  } catch (error) {
+    logger.warn(`Failed to parse JSON judgment from ${judgeProvider}:`, error);
+    return createFallbackEvaluation(evaluation, originalResponses, response);
+  }
+
+  // Ensure all original responses are represented
+  const rankedProviders = new Set(evaluation.rankings.map(r => r.provider));
+  for (const resp of originalResponses) {
+    if (!rankedProviders.has(resp.provider)) {
+      evaluation.rankings.push({
+        provider: resp.provider,
+        nickname: resp.nickname,
+        rank: evaluation.rankings.length + 1,
+        score: 0,
+        justification: 'Not evaluated by judge',
+      });
+    }
+  }
+
+  return evaluation;
+}
+
+function createFallbackEvaluation(
+  evaluation: JudgeEvaluation,
+  originalResponses: DuckResponse[],
+  rawResponse: string
+): JudgeEvaluation {
+  // Create a basic evaluation when parsing fails
+  evaluation.rankings = originalResponses.map((r, index) => ({
+    provider: r.provider,
+    nickname: r.nickname,
+    rank: index + 1,
+    score: 50,
+    justification: 'Unable to parse judge response',
+  }));
+  evaluation.summary = `Judge evaluation parsing failed. Raw response available for review.`;
+  evaluation.rawResponse = rawResponse;
+  return evaluation;
+}
+
+function formatJudgeResult(evaluation: JudgeEvaluation): string {
+  let output = `⚖️ **Judge Evaluation**\n`;
+  output += `═══════════════════════════════════════\n\n`;
+  output += `**Judge:** ${evaluation.judgeNickname} (${evaluation.judge})\n`;
+  output += `**Criteria:** ${evaluation.criteria.join(', ')}\n\n`;
+
+  // Rankings
+  output += `**Rankings:**\n`;
+  output += `─────────────────────────────────────\n`;
+
+  for (const ranking of evaluation.rankings) {
+    const medal = ranking.rank === 1 ? '🥇' : ranking.rank === 2 ? '🥈' : ranking.rank === 3 ? '🥉' : '  ';
+    const bar = '█'.repeat(Math.floor(ranking.score / 10));
+    const emptyBar = '░'.repeat(10 - Math.floor(ranking.score / 10));
+
+    output += `${medal} **#${ranking.rank} ${ranking.nickname}** (${ranking.provider})\n`;
+    output += `   Score: ${bar}${emptyBar} ${ranking.score}/100\n`;
+    output += `   💭 "${ranking.justification}"\n\n`;
+  }
+
+  // Criteria breakdown if available
+  if (Object.keys(evaluation.criteriaScores).length > 0) {
+    output += `**Criteria Breakdown:**\n`;
+    output += `─────────────────────────────────────\n`;
+
+    for (const [provider, scores] of Object.entries(evaluation.criteriaScores)) {
+      output += `📊 **${provider}:**\n`;
+      for (const [criterion, score] of Object.entries(scores)) {
+        const criterionScore = typeof score === 'number' ? score : 0;
+        output += `   • ${criterion}: ${criterionScore}/100\n`;
+      }
+      output += `\n`;
+    }
+  }
+
+  // Summary
+  if (evaluation.summary) {
+    output += `**Summary:**\n`;
+    output += `─────────────────────────────────────\n`;
+    output += `${evaluation.summary}\n\n`;
+  }
+
+  output += `═══════════════════════════════════════\n`;
+  output += `📋 Evaluated ${evaluation.rankings.length} responses\n`;
+
+  return output;
+}

package/src/tools/duck-vote.ts

@@ -0,0 +1,87 @@
+import { ProviderManager } from '../providers/manager.js';
+import { ConsensusService } from '../services/consensus.js';
+import { VoteResult } from '../config/types.js';
+import { logger } from '../utils/logger.js';
+
+export interface DuckVoteArgs {
+  question: string;
+  options: string[];
+  voters?: string[];
+  require_reasoning?: boolean;
+}
+
+export async function duckVoteTool(
+  providerManager: ProviderManager,
+  args: Record<string, unknown>
+) {
+  const {
+    question,
+    options,
+    voters,
+    require_reasoning = true,
+  } = args as unknown as DuckVoteArgs;
+
+  // Validate inputs
+  if (!question || typeof question !== 'string') {
+    throw new Error('Question is required');
+  }
+
+  if (!options || !Array.isArray(options) || options.length < 2) {
+    throw new Error('At least 2 options are required');
+  }
+
+  if (options.length > 10) {
+    throw new Error('Maximum 10 options allowed');
+  }
+
+  // Get voters (all providers if not specified)
+  const voterNames = voters && voters.length > 0
+    ? voters
+    : providerManager.getProviderNames();
+
+  if (voterNames.length === 0) {
+    throw new Error('No voters available');
+  }
+
+  logger.info(`Starting vote with ${voterNames.length} voters on: "${question}"`);
+
+  const consensusService = new ConsensusService();
+  const votePrompt = consensusService.buildVotePrompt(
+    question,
+    options,
+    require_reasoning
+  );
+
+  // Get votes from all ducks in parallel
+  const responses = await providerManager.compareDucks(votePrompt, voterNames);
+
+  // Parse votes
+  const votes: VoteResult[] = responses.map(response => {
+    return consensusService.parseVote(
+      response.content,
+      response.provider,
+      response.nickname,
+      options
+    );
+  });
+
+  // Aggregate results
+  const aggregatedResult = consensusService.aggregateVotes(question, options, votes);
+
+  // Format output
+  const formattedOutput = consensusService.formatVoteResult(aggregatedResult);
+
+  logger.info(
+    `Vote completed: ${aggregatedResult.consensusLevel} consensus, ` +
+    `winner: ${aggregatedResult.winner || 'none'}`
+  );
+
+  return {
+    content: [
+      {
+        type: 'text',
+        text: formattedOutput,
+      },
+    ],
+  };
+}

package/tests/consensus.test.ts

@@ -0,0 +1,282 @@
+import { describe, it, expect, beforeEach } from '@jest/globals';
+import { ConsensusService } from '../src/services/consensus';
+import { VoteResult } from '../src/config/types';
+
+describe('ConsensusService', () => {
+  let service: ConsensusService;
+
+  beforeEach(() => {
+    service = new ConsensusService();
+  });
+
+  describe('buildVotePrompt', () => {
+    it('should build a vote prompt with reasoning', () => {
+      const prompt = service.buildVotePrompt(
+        'Best programming language?',
+        ['Python', 'JavaScript', 'Rust'],
+        true
+      );
+
+      expect(prompt).toContain('Best programming language?');
+      expect(prompt).toContain('1. Python');
+      expect(prompt).toContain('2. JavaScript');
+      expect(prompt).toContain('3. Rust');
+      expect(prompt).toContain('"reasoning"');
+    });
+
+    it('should build a vote prompt without reasoning', () => {
+      const prompt = service.buildVotePrompt(
+        'Best color?',
+        ['Red', 'Blue'],
+        false
+      );
+
+      expect(prompt).toContain('Best color?');
+      expect(prompt).toContain('1. Red');
+      expect(prompt).toContain('2. Blue');
+      expect(prompt).not.toContain('"reasoning"');
+    });
+  });
+
+  describe('parseVote', () => {
+    const options = ['Option A', 'Option B', 'Option C'];
+
+    it('should parse valid JSON vote with all fields', () => {
+      const response = JSON.stringify({
+        choice: 'Option A',
+        confidence: 85,
+        reasoning: 'It is the best option because...',
+      });
+
+      const result = service.parseVote(response, 'test-provider', 'Test Duck', options);
+
+      expect(result.choice).toBe('Option A');
+      expect(result.confidence).toBe(85);
+      expect(result.reasoning).toBe('It is the best option because...');
+      expect(result.voter).toBe('test-provider');
+      expect(result.nickname).toBe('Test Duck');
+    });
+
+    it('should parse vote with extra text around JSON', () => {
+      const response = `Here is my vote:
+      {"choice": "Option B", "confidence": 70, "reasoning": "Good choice"}
+      Thank you!`;
+
+      const result = service.parseVote(response, 'provider1', 'Duck 1', options);
+
+      expect(result.choice).toBe('Option B');
+      expect(result.confidence).toBe(70);
+    });
+
+    it('should clamp confidence to 0-100 range', () => {
+      const response = JSON.stringify({
+        choice: 'Option A',
+        confidence: 150,
+      });
+
+      const result = service.parseVote(response, 'provider1', 'Duck 1', options);
+      expect(result.confidence).toBe(100);
+
+      const response2 = JSON.stringify({
+        choice: 'Option A',
+        confidence: -20,
+      });
+
+      const result2 = service.parseVote(response2, 'provider1', 'Duck 1', options);
+      expect(result2.confidence).toBe(0);
+    });
+
+    it('should handle case-insensitive choice matching', () => {
+      const response = JSON.stringify({
+        choice: 'option a',
+        confidence: 80,
+      });
+
+      const result = service.parseVote(response, 'provider1', 'Duck 1', options);
+      expect(result.choice).toBe('Option A');
+    });
+
+    it('should handle partial/fuzzy choice matching', () => {
+      const response = JSON.stringify({
+        choice: 'A',
+        confidence: 80,
+      });
+
+      const result = service.parseVote(response, 'provider1', 'Duck 1', options);
+      expect(result.choice).toBe('Option A');
+    });
+
+    it('should fallback parse when JSON is invalid', () => {
+      const response = 'I think Option C is the best because it has great features.';
+
+      const result = service.parseVote(response, 'provider1', 'Duck 1', options);
+      expect(result.choice).toBe('Option C');
+      expect(result.confidence).toBe(50); // Default fallback confidence
+    });
+
+    it('should return empty choice when no match found', () => {
+      const response = 'I cannot decide on any of these options.';
+
+      const result = service.parseVote(response, 'provider1', 'Duck 1', options);
+      expect(result.choice).toBe('');
+    });
+
+    it('should parse string confidence values', () => {
+      const response = JSON.stringify({
+        choice: 'Option A',
+        confidence: '75',
+      });
+
+      const result = service.parseVote(response, 'provider1', 'Duck 1', options);
+      expect(result.confidence).toBe(75);
+    });
+  });
+
+  describe('aggregateVotes', () => {
+    const question = 'Best framework?';
+    const options = ['React', 'Vue', 'Angular'];
+
+    it('should aggregate votes and determine winner', () => {
+      const votes: VoteResult[] = [
+        { voter: 'p1', nickname: 'D1', choice: 'React', confidence: 80, reasoning: 'R1', rawResponse: '' },
+        { voter: 'p2', nickname: 'D2', choice: 'React', confidence: 90, reasoning: 'R2', rawResponse: '' },
+        { voter: 'p3', nickname: 'D3', choice: 'Vue', confidence: 70, reasoning: 'R3', rawResponse: '' },
+      ];
+
+      const result = service.aggregateVotes(question, options, votes);
+
+      expect(result.winner).toBe('React');
+      expect(result.tally['React']).toBe(2);
+      expect(result.tally['Vue']).toBe(1);
+      expect(result.tally['Angular']).toBe(0);
+      expect(result.isTie).toBe(false);
+      expect(result.consensusLevel).toBe('majority');
+      expect(result.validVotes).toBe(3);
+    });
+
+    it('should detect unanimous consensus', () => {
+      const votes: VoteResult[] = [
+        { voter: 'p1', nickname: 'D1', choice: 'React', confidence: 80, reasoning: '', rawResponse: '' },
+        { voter: 'p2', nickname: 'D2', choice: 'React', confidence: 90, reasoning: '', rawResponse: '' },
+        { voter: 'p3', nickname: 'D3', choice: 'React', confidence: 85, reasoning: '', rawResponse: '' },
+      ];
+
+      const result = service.aggregateVotes(question, options, votes);
+
+      expect(result.winner).toBe('React');
+      expect(result.consensusLevel).toBe('unanimous');
+    });
+
+    it('should break tie by confidence', () => {
+      const votes: VoteResult[] = [
+        { voter: 'p1', nickname: 'D1', choice: 'React', confidence: 60, reasoning: '', rawResponse: '' },
+        { voter: 'p2', nickname: 'D2', choice: 'Vue', confidence: 90, reasoning: '', rawResponse: '' },
+      ];
+
+      const result = service.aggregateVotes(question, options, votes);
+
+      expect(result.isTie).toBe(true);
+      expect(result.winner).toBe('Vue'); // Higher confidence
+      expect(result.consensusLevel).toBe('split');
+    });
+
+    it('should handle no valid votes', () => {
+      const votes: VoteResult[] = [
+        { voter: 'p1', nickname: 'D1', choice: '', confidence: 0, reasoning: '', rawResponse: '' },
+        { voter: 'p2', nickname: 'D2', choice: 'InvalidOption', confidence: 50, reasoning: '', rawResponse: '' },
+      ];
+
+      const result = service.aggregateVotes(question, options, votes);
+
+      expect(result.winner).toBeNull();
+      expect(result.validVotes).toBe(0);
+      expect(result.consensusLevel).toBe('none');
+    });
+
+    it('should calculate average confidence per option', () => {
+      const votes: VoteResult[] = [
+        { voter: 'p1', nickname: 'D1', choice: 'React', confidence: 80, reasoning: '', rawResponse: '' },
+        { voter: 'p2', nickname: 'D2', choice: 'React', confidence: 60, reasoning: '', rawResponse: '' },
+        { voter: 'p3', nickname: 'D3', choice: 'Vue', confidence: 90, reasoning: '', rawResponse: '' },
+      ];
+
+      const result = service.aggregateVotes(question, options, votes);
+
+      expect(result.confidenceByOption['React']).toBe(70); // (80+60)/2
+      expect(result.confidenceByOption['Vue']).toBe(90);
+      expect(result.confidenceByOption['Angular']).toBe(0);
+    });
+
+    it('should detect plurality consensus', () => {
+      const votes: VoteResult[] = [
+        { voter: 'p1', nickname: 'D1', choice: 'React', confidence: 80, reasoning: '', rawResponse: '' },
+        { voter: 'p2', nickname: 'D2', choice: 'React', confidence: 70, reasoning: '', rawResponse: '' },
+        { voter: 'p3', nickname: 'D3', choice: 'Vue', confidence: 90, reasoning: '', rawResponse: '' },
+        { voter: 'p4', nickname: 'D4', choice: 'Angular', confidence: 60, reasoning: '', rawResponse: '' },
+        { voter: 'p5', nickname: 'D5', choice: 'Angular', confidence: 50, reasoning: '', rawResponse: '' },
+      ];
+
+      const result = service.aggregateVotes(question, options, votes);
+
+      // React has 2, Angular has 2, Vue has 1 - it's a tie
+      // But if we modify to have clear plurality:
+      const votes2: VoteResult[] = [
+        { voter: 'p1', nickname: 'D1', choice: 'React', confidence: 80, reasoning: '', rawResponse: '' },
+        { voter: 'p2', nickname: 'D2', choice: 'React', confidence: 70, reasoning: '', rawResponse: '' },
+        { voter: 'p3', nickname: 'D3', choice: 'Vue', confidence: 90, reasoning: '', rawResponse: '' },
+        { voter: 'p4', nickname: 'D4', choice: 'Angular', confidence: 60, reasoning: '', rawResponse: '' },
+      ];
+
+      const result2 = service.aggregateVotes(question, options, votes2);
+      expect(result2.winner).toBe('React');
+      expect(result2.consensusLevel).toBe('plurality'); // 2/4 = 50%, not majority
+    });
+  });
+
+  describe('formatVoteResult', () => {
+    it('should format vote results with winner', () => {
+      const votes: VoteResult[] = [
+        { voter: 'p1', nickname: 'Duck 1', choice: 'Option A', confidence: 85, reasoning: 'Best choice', rawResponse: '' },
+        { voter: 'p2', nickname: 'Duck 2', choice: 'Option A', confidence: 75, reasoning: 'Agreed', rawResponse: '' },
+      ];
+
+      const aggregated = service.aggregateVotes('Test question?', ['Option A', 'Option B'], votes);
+      const formatted = service.formatVoteResult(aggregated);
+
+      expect(formatted).toContain('Vote Results');
+      expect(formatted).toContain('Test question?');
+      expect(formatted).toContain('Winner');
+      expect(formatted).toContain('Option A');
+      expect(formatted).toContain('unanimous');
+      expect(formatted).toContain('Duck 1');
+      expect(formatted).toContain('Duck 2');
+      expect(formatted).toContain('Best choice');
+      expect(formatted).toContain('2/2 valid votes');
+    });
+
+    it('should format results with invalid votes', () => {
+      const votes: VoteResult[] = [
+        { voter: 'p1', nickname: 'Duck 1', choice: '', confidence: 0, reasoning: '', rawResponse: '' },
+      ];
+
+      const aggregated = service.aggregateVotes('Test?', ['A', 'B'], votes);
+      const formatted = service.formatVoteResult(aggregated);
+
+      expect(formatted).toContain('No valid votes');
+      expect(formatted).toContain('Invalid vote');
+      expect(formatted).toContain('0/1 valid votes');
+    });
+
+    it('should indicate tie-breaker when applicable', () => {
+      const votes: VoteResult[] = [
+        { voter: 'p1', nickname: 'D1', choice: 'A', confidence: 60, reasoning: '', rawResponse: '' },
+        { voter: 'p2', nickname: 'D2', choice: 'B', confidence: 90, reasoning: '', rawResponse: '' },
+      ];
+
+      const aggregated = service.aggregateVotes('Test?', ['A', 'B'], votes);
+      const formatted = service.formatVoteResult(aggregated);
+
+      expect(formatted).toContain('tie-breaker');
+    });
+  });
+});