mcp-rubber-duck 1.2.4 → 1.3.0

package/tests/duck-judge.test.ts

@@ -0,0 +1,296 @@
+import { describe, it, expect, jest, beforeEach } from '@jest/globals';
+
+// Mock OpenAI BEFORE importing the provider
+const mockCreate = jest.fn();
+jest.mock('openai', () => {
+  const MockOpenAI = jest.fn().mockImplementation(() => ({
+    chat: {
+      completions: {
+        create: mockCreate,
+      },
+    },
+  }));
+  return {
+    __esModule: true,
+    default: MockOpenAI,
+  };
+});
+
+// Mock config manager and logger
+jest.mock('../src/config/config');
+jest.mock('../src/utils/logger');
+
+import { duckJudgeTool } from '../src/tools/duck-judge';
+import { ProviderManager } from '../src/providers/manager';
+import { ConfigManager } from '../src/config/config';
+import { DuckResponse } from '../src/config/types';
+
+describe('duckJudgeTool', () => {
+  let mockProviderManager: ProviderManager;
+  let mockConfigManager: jest.Mocked<ConfigManager>;
+
+  const mockResponses: DuckResponse[] = [
+    {
+      provider: 'openai',
+      nickname: 'GPT-4',
+      model: 'gpt-4',
+      content: 'Response from GPT-4 about error handling using try-catch blocks.',
+      latency: 1000,
+      cached: false,
+    },
+    {
+      provider: 'gemini',
+      nickname: 'Gemini',
+      model: 'gemini-pro',
+      content: 'Response from Gemini about error handling using Result types.',
+      latency: 1500,
+      cached: false,
+    },
+  ];
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+
+    mockConfigManager = {
+      getConfig: jest.fn().mockReturnValue({
+        providers: {
+          openai: {
+            api_key: 'key1',
+            base_url: 'https://api.openai.com/v1',
+            default_model: 'gpt-4',
+            nickname: 'GPT-4',
+            models: ['gpt-4'],
+          },
+          gemini: {
+            api_key: 'key2',
+            base_url: 'https://api.gemini.com/v1',
+            default_model: 'gemini-pro',
+            nickname: 'Gemini',
+            models: ['gemini-pro'],
+          },
+        },
+        default_provider: 'openai',
+        cache_ttl: 300,
+        enable_failover: true,
+        default_temperature: 0.7,
+      }),
+    } as any;
+
+    mockProviderManager = new ProviderManager(mockConfigManager);
+
+    // Override the client method on all providers
+    const provider1 = mockProviderManager.getProvider('openai');
+    const provider2 = mockProviderManager.getProvider('gemini');
+    provider1['client'].chat.completions.create = mockCreate;
+    provider2['client'].chat.completions.create = mockCreate;
+  });
+
+  it('should throw error when responses are missing', async () => {
+    await expect(
+      duckJudgeTool(mockProviderManager, {})
+    ).rejects.toThrow('At least one response is required');
+  });
+
+  it('should throw error when responses is empty array', async () => {
+    await expect(
+      duckJudgeTool(mockProviderManager, { responses: [] })
+    ).rejects.toThrow('At least one response is required');
+  });
+
+  it('should throw error when only one response provided', async () => {
+    await expect(
+      duckJudgeTool(mockProviderManager, { responses: [mockResponses[0]] })
+    ).rejects.toThrow('At least two responses are required');
+  });
+
+  it('should evaluate responses and return rankings', async () => {
+    const judgeResponse = JSON.stringify({
+      rankings: [
+        { provider: 'gemini', score: 85, justification: 'Better type safety explanation' },
+        { provider: 'openai', score: 75, justification: 'Good but less comprehensive' },
+      ],
+      criteria_scores: {
+        gemini: { accuracy: 85, completeness: 90, clarity: 80 },
+        openai: { accuracy: 75, completeness: 70, clarity: 80 },
+      },
+      summary: 'Gemini provided a more comprehensive response with better type safety coverage.',
+    });
+
+    mockCreate.mockResolvedValueOnce({
+      choices: [{
+        message: { content: judgeResponse },
+        finish_reason: 'stop',
+      }],
+      usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 },
+      model: 'gpt-4',
+    });
+
+    const result = await duckJudgeTool(mockProviderManager, {
+      responses: mockResponses,
+    });
+
+    expect(result.content).toHaveLength(1);
+    expect(result.content[0].type).toBe('text');
+
+    const text = result.content[0].text;
+    expect(text).toContain('Judge Evaluation');
+    expect(text).toContain('#1');
+    expect(text).toContain('#2');
+    expect(text).toContain('gemini');
+    expect(text).toContain('85/100');
+  });
+
+  it('should use specified judge provider', async () => {
+    const judgeResponse = JSON.stringify({
+      rankings: [
+        { provider: 'openai', score: 80, justification: 'Good response' },
+        { provider: 'gemini', score: 70, justification: 'Okay response' },
+      ],
+      summary: 'OpenAI wins.',
+    });
+
+    mockCreate.mockResolvedValueOnce({
+      choices: [{
+        message: { content: judgeResponse },
+        finish_reason: 'stop',
+      }],
+      usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 },
+      model: 'gemini-pro',
+    });
+
+    const result = await duckJudgeTool(mockProviderManager, {
+      responses: mockResponses,
+      judge: 'gemini',
+    });
+
+    const text = result.content[0].text;
+    expect(text).toContain('Gemini');
+  });
+
+  it('should use custom criteria', async () => {
+    const judgeResponse = JSON.stringify({
+      rankings: [
+        { provider: 'openai', score: 90, justification: 'Most secure' },
+        { provider: 'gemini', score: 85, justification: 'Good security' },
+      ],
+      summary: 'Security focused evaluation.',
+    });
+
+    mockCreate.mockResolvedValueOnce({
+      choices: [{
+        message: { content: judgeResponse },
+        finish_reason: 'stop',
+      }],
+      usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 },
+      model: 'gpt-4',
+    });
+
+    const result = await duckJudgeTool(mockProviderManager, {
+      responses: mockResponses,
+      criteria: ['security', 'performance', 'maintainability'],
+    });
+
+    const text = result.content[0].text;
+    expect(text).toContain('security');
+    expect(text).toContain('performance');
+    expect(text).toContain('maintainability');
+  });
+
+  it('should handle persona parameter', async () => {
+    const judgeResponse = JSON.stringify({
+      rankings: [
+        { provider: 'openai', score: 85, justification: 'Senior approved' },
+        { provider: 'gemini', score: 80, justification: 'Good for juniors' },
+      ],
+      summary: 'From a senior perspective.',
+    });
+
+    mockCreate.mockResolvedValueOnce({
+      choices: [{
+        message: { content: judgeResponse },
+        finish_reason: 'stop',
+      }],
+      usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 },
+      model: 'gpt-4',
+    });
+
+    const result = await duckJudgeTool(mockProviderManager, {
+      responses: mockResponses,
+      persona: 'senior engineer',
+    });
+
+    expect(result.content[0].text).toContain('Judge Evaluation');
+    expect(mockCreate).toHaveBeenCalledTimes(1);
+  });
+
+  it('should handle invalid JSON gracefully with fallback', async () => {
+    mockCreate.mockResolvedValueOnce({
+      choices: [{
+        message: { content: 'This is not valid JSON at all, just some random text.' },
+        finish_reason: 'stop',
+      }],
+      usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 },
+      model: 'gpt-4',
+    });
+
+    const result = await duckJudgeTool(mockProviderManager, {
+      responses: mockResponses,
+    });
+
+    const text = result.content[0].text;
+    expect(text).toContain('Judge Evaluation');
+    expect(text).toContain('Unable to parse');
+  });
+
+  it('should handle JSON with extra text around it', async () => {
+    const judgeResponse = `Here is my evaluation:
+    {"rankings": [{"provider": "openai", "score": 90, "justification": "Best"}], "summary": "Done"}
+    Hope this helps!`;
+
+    mockCreate.mockResolvedValueOnce({
+      choices: [{
+        message: { content: judgeResponse },
+        finish_reason: 'stop',
+      }],
+      usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 },
+      model: 'gpt-4',
+    });
+
+    const result = await duckJudgeTool(mockProviderManager, {
+      responses: mockResponses,
+    });
+
+    const text = result.content[0].text;
+    expect(text).toContain('90/100');
+    expect(text).toContain('openai');
+  });
+
+  it('should include missing providers in rankings', async () => {
+    // Judge only ranks one provider
+    const judgeResponse = JSON.stringify({
+      rankings: [
+        { provider: 'openai', score: 85, justification: 'Good' },
+      ],
+      summary: 'Only evaluated one.',
+    });
+
+    mockCreate.mockResolvedValueOnce({
+      choices: [{
+        message: { content: judgeResponse },
+        finish_reason: 'stop',
+      }],
+      usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 },
+      model: 'gpt-4',
+    });
+
+    const result = await duckJudgeTool(mockProviderManager, {
+      responses: mockResponses,
+    });
+
+    const text = result.content[0].text;
+    // Should include both providers even though only one was ranked
+    expect(text).toContain('openai');
+    expect(text).toContain('gemini');
+    expect(text).toContain('Not evaluated');
+  });
+});

package/tests/duck-vote.test.ts

@@ -0,0 +1,250 @@
+import { describe, it, expect, jest, beforeEach } from '@jest/globals';
+
+// Mock OpenAI BEFORE importing the provider
+const mockCreate = jest.fn();
+jest.mock('openai', () => {
+  const MockOpenAI = jest.fn().mockImplementation(() => ({
+    chat: {
+      completions: {
+        create: mockCreate,
+      },
+    },
+  }));
+  return {
+    __esModule: true,
+    default: MockOpenAI,
+  };
+});
+
+// Mock config manager and logger
+jest.mock('../src/config/config');
+jest.mock('../src/utils/logger');
+
+import { duckVoteTool } from '../src/tools/duck-vote';
+import { ProviderManager } from '../src/providers/manager';
+import { ConfigManager } from '../src/config/config';
+
+describe('duckVoteTool', () => {
+  let mockProviderManager: ProviderManager;
+  let mockConfigManager: jest.Mocked<ConfigManager>;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+
+    mockConfigManager = {
+      getConfig: jest.fn().mockReturnValue({
+        providers: {
+          openai: {
+            api_key: 'key1',
+            base_url: 'https://api.openai.com/v1',
+            default_model: 'gpt-4',
+            nickname: 'GPT-4',
+            models: ['gpt-4'],
+          },
+          gemini: {
+            api_key: 'key2',
+            base_url: 'https://api.gemini.com/v1',
+            default_model: 'gemini-pro',
+            nickname: 'Gemini',
+            models: ['gemini-pro'],
+          },
+        },
+        default_provider: 'openai',
+        cache_ttl: 300,
+        enable_failover: true,
+        default_temperature: 0.7,
+      }),
+    } as any;
+
+    mockProviderManager = new ProviderManager(mockConfigManager);
+
+    // Override the client method on all providers
+    const provider1 = mockProviderManager.getProvider('openai');
+    const provider2 = mockProviderManager.getProvider('gemini');
+    provider1['client'].chat.completions.create = mockCreate;
+    provider2['client'].chat.completions.create = mockCreate;
+  });
+
+  it('should throw error when question is missing', async () => {
+    await expect(
+      duckVoteTool(mockProviderManager, { options: ['A', 'B'] })
+    ).rejects.toThrow('Question is required');
+  });
+
+  it('should throw error when options are missing', async () => {
+    await expect(
+      duckVoteTool(mockProviderManager, { question: 'Test?' })
+    ).rejects.toThrow('At least 2 options are required');
+  });
+
+  it('should throw error when less than 2 options', async () => {
+    await expect(
+      duckVoteTool(mockProviderManager, { question: 'Test?', options: ['A'] })
+    ).rejects.toThrow('At least 2 options are required');
+  });
+
+  it('should throw error when more than 10 options', async () => {
+    const options = Array.from({ length: 11 }, (_, i) => `Option ${i + 1}`);
+    await expect(
+      duckVoteTool(mockProviderManager, { question: 'Test?', options })
+    ).rejects.toThrow('Maximum 10 options allowed');
+  });
+
+  it('should conduct vote with all providers', async () => {
+    // Mock responses with valid JSON votes
+    mockCreate
+      .mockResolvedValueOnce({
+        choices: [{
+          message: { content: '{"choice": "Option A", "confidence": 85, "reasoning": "Best for performance"}' },
+          finish_reason: 'stop',
+        }],
+        usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+        model: 'gpt-4',
+      })
+      .mockResolvedValueOnce({
+        choices: [{
+          message: { content: '{"choice": "Option A", "confidence": 75, "reasoning": "Scalable solution"}' },
+          finish_reason: 'stop',
+        }],
+        usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+        model: 'gemini-pro',
+      });
+
+    const result = await duckVoteTool(mockProviderManager, {
+      question: 'Best approach?',
+      options: ['Option A', 'Option B'],
+    });
+
+    expect(result.content).toHaveLength(1);
+    expect(result.content[0].type).toBe('text');
+
+    const text = result.content[0].text;
+    expect(text).toContain('Vote Results');
+    expect(text).toContain('Best approach?');
+    expect(text).toContain('Option A');
+    expect(text).toContain('Winner');
+    expect(text).toContain('unanimous');
+  });
+
+  it('should handle split votes', async () => {
+    mockCreate
+      .mockResolvedValueOnce({
+        choices: [{
+          message: { content: '{"choice": "Option A", "confidence": 60, "reasoning": "Good choice"}' },
+          finish_reason: 'stop',
+        }],
+        usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+        model: 'gpt-4',
+      })
+      .mockResolvedValueOnce({
+        choices: [{
+          message: { content: '{"choice": "Option B", "confidence": 90, "reasoning": "Better choice"}' },
+          finish_reason: 'stop',
+        }],
+        usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+        model: 'gemini-pro',
+      });
+
+    const result = await duckVoteTool(mockProviderManager, {
+      question: 'Which option?',
+      options: ['Option A', 'Option B'],
+    });
+
+    const text = result.content[0].text;
+    expect(text).toContain('split');
+    expect(text).toContain('tie-breaker');
+    expect(text).toContain('Option B'); // Higher confidence wins
+  });
+
+  it('should use specific voters when provided', async () => {
+    mockCreate.mockResolvedValueOnce({
+      choices: [{
+        message: { content: '{"choice": "Option A", "confidence": 80, "reasoning": "Only choice"}' },
+        finish_reason: 'stop',
+      }],
+      usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+      model: 'gpt-4',
+    });
+
+    const result = await duckVoteTool(mockProviderManager, {
+      question: 'Test?',
+      options: ['Option A', 'Option B'],
+      voters: ['openai'],
+    });
+
+    expect(mockCreate).toHaveBeenCalledTimes(1);
+    expect(result.content[0].text).toContain('1/1 valid votes');
+  });
+
+  it('should handle invalid JSON responses gracefully', async () => {
+    mockCreate
+      .mockResolvedValueOnce({
+        choices: [{
+          message: { content: 'I think Option A is clearly the best because of its simplicity.' },
+          finish_reason: 'stop',
+        }],
+        usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+        model: 'gpt-4',
+      })
+      .mockResolvedValueOnce({
+        choices: [{
+          message: { content: '{"choice": "Option B", "confidence": 70}' },
+          finish_reason: 'stop',
+        }],
+        usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+        model: 'gemini-pro',
+      });
+
+    const result = await duckVoteTool(mockProviderManager, {
+      question: 'Test?',
+      options: ['Option A', 'Option B'],
+    });
+
+    // Should still work - fallback parsing should find "Option A"
+    const text = result.content[0].text;
+    expect(text).toContain('2/2 valid votes');
+  });
+
+  it('should handle provider errors gracefully', async () => {
+    mockCreate
+      .mockResolvedValueOnce({
+        choices: [{
+          message: { content: '{"choice": "Option A", "confidence": 85}' },
+          finish_reason: 'stop',
+        }],
+        usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+        model: 'gpt-4',
+      })
+      .mockRejectedValueOnce(new Error('API Error'));
+
+    const result = await duckVoteTool(mockProviderManager, {
+      question: 'Test?',
+      options: ['Option A', 'Option B'],
+    });
+
+    // One valid vote, one error
+    const text = result.content[0].text;
+    expect(text).toContain('Option A');
+    expect(text).toContain('Invalid vote'); // Error response should be marked invalid
+  });
+
+  it('should work without reasoning requirement', async () => {
+    mockCreate.mockResolvedValueOnce({
+      choices: [{
+        message: { content: '{"choice": "Option A", "confidence": 80}' },
+        finish_reason: 'stop',
+      }],
+      usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+      model: 'gpt-4',
+    });
+
+    const result = await duckVoteTool(mockProviderManager, {
+      question: 'Test?',
+      options: ['Option A', 'Option B'],
+      voters: ['openai'],
+      require_reasoning: false,
+    });
+
+    expect(result.content[0].text).toContain('Option A');
+  });
+});