cadr-cli 2.0.0 → 2.0.2

package/src/llm/llm.test.ts

@@ -0,0 +1,315 @@
+jest.mock('../providers', () => ({
+  getProvider: jest.fn(),
+}));
+
+jest.mock('../logger', () => ({
+  loggerInstance: {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn(),
+    debug: jest.fn(),
+  },
+}));
+
+import { getProvider } from '../providers';
+import { loggerInstance as logger } from '../logger';
+import { analyzeChanges, generateADRContent } from './llm';
+import type { AnalysisConfig } from '../config';
+import type { LLMProvider } from '../providers/types';
+
+const mockedGetProvider = getProvider as jest.MockedFunction<typeof getProvider>;
+const mockedLogger = logger as jest.Mocked<typeof logger>;
+
+const config: AnalysisConfig = {
+  provider: 'openai',
+  analysis_model: 'gpt-4',
+  api_key_env: 'TEST_API_KEY',
+  timeout_seconds: 15,
+};
+
+const baseAnalysisRequest = {
+  file_paths: ['src/index.ts'],
+  diff_content: 'diff --git a/src/index.ts',
+  repository_context: 'test repo',
+  analysis_prompt: 'Analyze these changes',
+};
+
+const baseGenerationRequest = {
+  file_paths: ['src/index.ts'],
+  diff_content: 'diff --git a/src/index.ts',
+  reason: 'significant architectural change',
+  generation_prompt: 'Generate ADR for these changes',
+};
+
+function createMockProvider(analyzeFn: jest.Mock): LLMProvider {
+  return {
+    name: 'openai',
+    analyze: analyzeFn,
+  };
+}
+
+beforeEach(() => {
+  jest.clearAllMocks();
+  process.env.TEST_API_KEY = 'test-key';
+});
+
+afterEach(() => {
+  delete process.env.TEST_API_KEY;
+  jest.restoreAllMocks();
+});
+
+describe('analyzeChanges', () => {
+  it('should return error when API key is not set', async () => {
+    delete process.env.TEST_API_KEY;
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('TEST_API_KEY');
+  });
+
+  it('should return successful result when provider returns clean JSON', async () => {
+    const mockAnalyze = jest.fn().mockResolvedValue(
+      JSON.stringify({ is_significant: true, reason: 'big' })
+    );
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).not.toBeNull();
+    expect(response.result!.is_significant).toBe(true);
+    expect(response.result!.reason).toBe('big');
+    expect(response.result!.timestamp).toBeDefined();
+    expect(response.error).toBeUndefined();
+  });
+
+  it('should parse JSON wrapped in a code block', async () => {
+    const mockAnalyze = jest.fn().mockResolvedValue(
+      '```json\n{"is_significant": true, "reason": "refactored"}\n```'
+    );
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).not.toBeNull();
+    expect(response.result!.is_significant).toBe(true);
+    expect(response.result!.reason).toBe('refactored');
+  });
+
+  it('should return error when provider returns undefined', async () => {
+    const mockAnalyze = jest.fn().mockResolvedValue(undefined);
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toBe('No response content from LLM');
+  });
+
+  it('should return error when provider returns invalid JSON', async () => {
+    const mockAnalyze = jest.fn().mockResolvedValue('not valid json at all');
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('Failed to parse LLM response as JSON');
+  });
+
+  it('should return error when is_significant is not a boolean', async () => {
+    const mockAnalyze = jest.fn().mockResolvedValue(
+      JSON.stringify({ is_significant: 'yes', reason: 'something' })
+    );
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('Invalid response format');
+  });
+
+  it('should return error when is_significant is true but reason is empty', async () => {
+    const mockAnalyze = jest.fn().mockResolvedValue(
+      JSON.stringify({ is_significant: true, reason: '' })
+    );
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('no reason');
+  });
+
+  it('should include confidence when value is within valid range', async () => {
+    const mockAnalyze = jest.fn().mockResolvedValue(
+      JSON.stringify({ is_significant: true, reason: 'big change', confidence: 0.85 })
+    );
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).not.toBeNull();
+    expect(response.result!.confidence).toBe(0.85);
+  });
+
+  it('should not include confidence when value is out of range', async () => {
+    const mockAnalyze = jest.fn().mockResolvedValue(
+      JSON.stringify({ is_significant: true, reason: 'big change', confidence: 1.5 })
+    );
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).not.toBeNull();
+    expect(response.result!.confidence).toBeUndefined();
+  });
+
+  it('should log warning when estimated tokens exceed 7000', async () => {
+    const longPrompt = 'x'.repeat(28004);
+    const mockAnalyze = jest.fn().mockResolvedValue(
+      JSON.stringify({ is_significant: false, reason: 'minor' })
+    );
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const request = { ...baseAnalysisRequest, analysis_prompt: longPrompt };
+    await analyzeChanges(config, request);
+
+    expect(mockedLogger.warn).toHaveBeenCalledWith(
+      'High token count detected',
+      expect.objectContaining({ estimated_tokens: expect.any(Number) })
+    );
+  });
+
+  it('should return auth error when provider throws with status 401', async () => {
+    const mockAnalyze = jest.fn().mockRejectedValue({ status: 401, message: 'Unauthorized' });
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('Invalid API key');
+  });
+
+  it('should return diff-too-large error when provider throws status 400 with context length message', async () => {
+    const mockAnalyze = jest.fn().mockRejectedValue({
+      status: 400,
+      message: 'maximum context length exceeded with 50000 tokens',
+    });
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('Diff too large');
+  });
+
+  it('should return rate limit error when provider throws with status 429', async () => {
+    const mockAnalyze = jest.fn().mockRejectedValue({ status: 429, message: 'Too many requests' });
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('Rate limit exceeded');
+  });
+
+  it('should return timeout error when provider throws with code ETIMEDOUT', async () => {
+    const mockAnalyze = jest.fn().mockRejectedValue({ code: 'ETIMEDOUT', message: 'connection timed out' });
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('timeout');
+    expect(response.error).toContain('15s');
+  });
+
+  it('should return timeout error when provider throws with timeout in message', async () => {
+    const mockAnalyze = jest.fn().mockRejectedValue({ message: 'Request timeout after 15000ms' });
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('timeout');
+  });
+
+  it('should return network error when provider throws with code ENOTFOUND', async () => {
+    const mockAnalyze = jest.fn().mockRejectedValue({ code: 'ENOTFOUND', message: 'getaddrinfo ENOTFOUND' });
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('Network error');
+  });
+
+  it('should return generic API error for unknown errors', async () => {
+    const mockAnalyze = jest.fn().mockRejectedValue({ message: 'Something unexpected' });
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await analyzeChanges(config, baseAnalysisRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toBe('API error: Something unexpected');
+  });
+});
+
+describe('generateADRContent', () => {
+  it('should return error when API key is not set', async () => {
+    delete process.env.TEST_API_KEY;
+
+    const response = await generateADRContent(config, baseGenerationRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('TEST_API_KEY');
+  });
+
+  it('should extract title from markdown h1 and return cleaned content', async () => {
+    const markdown = '# Adopt New Database\n\n## Context\nWe need a new DB.';
+    const mockAnalyze = jest.fn().mockResolvedValue(markdown);
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await generateADRContent(config, baseGenerationRequest);
+
+    expect(response.result).not.toBeNull();
+    expect(response.result!.title).toBe('Adopt New Database');
+    expect(response.result!.content).toBe(markdown);
+    expect(response.result!.timestamp).toBeDefined();
+    expect(response.error).toBeUndefined();
+  });
+
+  it('should strip code block wrapper from markdown response', async () => {
+    const inner = '# My Decision\n\n## Context\nSome context.';
+    const wrapped = '```markdown\n' + inner + '\n```';
+    const mockAnalyze = jest.fn().mockResolvedValue(wrapped);
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await generateADRContent(config, baseGenerationRequest);
+
+    expect(response.result).not.toBeNull();
+    expect(response.result!.title).toBe('My Decision');
+    expect(response.result!.content).toBe(inner);
+  });
+
+  it('should use Untitled Decision when markdown has no h1', async () => {
+    const markdown = '## Context\nSome context without a title.';
+    const mockAnalyze = jest.fn().mockResolvedValue(markdown);
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await generateADRContent(config, baseGenerationRequest);
+
+    expect(response.result).not.toBeNull();
+    expect(response.result!.title).toBe('Untitled Decision');
+  });
+
+  it('should return auth error when provider throws with status 401', async () => {
+    const mockAnalyze = jest.fn().mockRejectedValue({ status: 401, message: 'Unauthorized' });
+    mockedGetProvider.mockReturnValue(createMockProvider(mockAnalyze));
+
+    const response = await generateADRContent(config, baseGenerationRequest);
+
+    expect(response.result).toBeNull();
+    expect(response.error).toContain('Invalid API key');
+  });
+});

package/src/{llm.ts → llm/llm.ts}

@@ -1,17 +1,7 @@
-/**
- * LLM Client Module
- *
- * Provider-based wrapper for analyzing code changes.
- * Implements fail-open error handling per constitution requirements.
- */
-
-import { getProvider } from './providers';
-import { AnalysisConfig } from './config';
-import { loggerInstance as logger } from './logger';
-
-/**
- * Analysis request data structure
- */
+import { getProvider } from '../providers';
+import { AnalysisConfig } from '../config';
+import { loggerInstance as logger } from '../logger';
+
 export interface AnalysisRequest {
   file_paths: string[];
   diff_content: string;
@@ -19,17 +9,6 @@ export interface AnalysisRequest {
   analysis_prompt: string;
 }
 
-/**
- * Rough token estimation (1 token ≈ 4 characters for English text)
- * This is a conservative estimate
- */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
-/**
- * Analysis result from LLM
- */
 export interface AnalysisResult {
   is_significant: boolean;
   reason: string;
@@ -37,27 +16,38 @@ export interface AnalysisResult {
   timestamp: string;
 }
 
-/**
- * Analysis response including potential errors
- */
 export interface AnalysisResponse {
   result: AnalysisResult | null;
   error?: string;
 }
 
-/**
- * Analyze staged changes using OpenAI LLM
- *
- * @param config - Analysis configuration with API settings
- * @param request - Analysis request with code changes
- * @returns Promise resolving to analysis response with result or error
- */
+export interface GenerationRequest {
+  file_paths: string[];
+  diff_content: string;
+  reason: string;
+  generation_prompt: string;
+}
+
+export interface GenerationResult {
+  content: string;
+  title: string;
+  timestamp: string;
+}
+
+export interface GenerationResponse {
+  result: GenerationResult | null;
+  error?: string;
+}
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
 export async function analyzeChanges(
   config: AnalysisConfig,
   request: AnalysisRequest
 ): Promise<AnalysisResponse> {
   try {
-    // Check if API key is available
     const apiKey = process.env[config.api_key_env];
     if (!apiKey) {
       logger.warn('API key not found in environment', {
@@ -65,21 +55,19 @@ export async function analyzeChanges(
       });
       return {
         result: null,
-        error: `API key not found: ${config.api_key_env} environment variable is not set`
+        error: `API key not found: ${config.api_key_env} environment variable is not set`,
       };
     }
 
-    // Estimate tokens for logging and validation
     const estimatedTokens = estimateTokens(request.analysis_prompt);
-    
+
     logger.info('Sending analysis request to LLM', {
       provider: config.provider,
       model: config.analysis_model,
       file_count: request.file_paths.length,
       estimated_tokens: estimatedTokens,
     });
-    
-    // Warn if token estimate is high (most models have 8k-32k limits)
+
     if (estimatedTokens > 7000) {
       logger.warn('High token count detected', {
         estimated_tokens: estimatedTokens,
@@ -99,28 +87,24 @@ export async function analyzeChanges(
       logger.warn('No response content from LLM', { provider: config.provider });
       return {
         result: null,
-        error: 'No response content from LLM'
+        error: 'No response content from LLM',
       };
     }
 
-    // Parse JSON response - handle markdown-wrapped JSON
     let parsedResponse: { is_significant: boolean; reason: string; confidence?: number };
     try {
-      // Try to extract JSON from markdown code blocks if present
       let jsonContent = responseContent.trim();
-      
-      // Remove markdown code block if present: ```json ... ``` or ``` ... ```
+
       const codeBlockMatch = jsonContent.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
       if (codeBlockMatch) {
         jsonContent = codeBlockMatch[1].trim();
       }
-      
-      // Try to find JSON object if there's surrounding text
+
       const jsonMatch = jsonContent.match(/\{[\s\S]*\}/);
       if (jsonMatch) {
         jsonContent = jsonMatch[0];
       }
-      
+
       parsedResponse = JSON.parse(jsonContent);
     } catch (parseError) {
       logger.warn('Failed to parse LLM response as JSON', {
@@ -129,11 +113,10 @@ export async function analyzeChanges(
       });
       return {
         result: null,
-        error: `Failed to parse LLM response as JSON. Response was:\n${responseContent.substring(0, 200)}...`
+        error: `Failed to parse LLM response as JSON. Response was:\n${responseContent.substring(0, 200)}...`,
       };
     }
 
-    // Validate response format
     if (
       typeof parsedResponse.is_significant !== 'boolean' ||
       typeof parsedResponse.reason !== 'string'
@@ -143,29 +126,26 @@ export async function analyzeChanges(
       });
       return {
         result: null,
-        error: `Invalid response format from LLM. Expected {is_significant: boolean, reason: string}, got: ${JSON.stringify(parsedResponse).substring(0, 150)}...`
+        error: `Invalid response format from LLM. Expected {is_significant: boolean, reason: string}, got: ${JSON.stringify(parsedResponse).substring(0, 150)}...`,
       };
     }
-    
-    // Reason is required when is_significant is true, but can be empty when false
+
     if (parsedResponse.is_significant && !parsedResponse.reason) {
       logger.warn('Missing reason for significant change', {
         response: parsedResponse,
       });
       return {
         result: null,
-        error: 'LLM indicated significant change but provided no reason'
+        error: 'LLM indicated significant change but provided no reason',
       };
     }
 
-    // Build result with timestamp
     const result: AnalysisResult = {
       is_significant: parsedResponse.is_significant,
       reason: parsedResponse.reason,
       timestamp: new Date().toISOString(),
     };
 
-    // Include confidence if provided
     if (
       typeof parsedResponse.confidence === 'number' &&
       parsedResponse.confidence >= 0 &&
@@ -181,22 +161,20 @@ export async function analyzeChanges(
 
     return { result, error: undefined };
   } catch (error) {
-    // Fail-open: log error and return descriptive error message
     const errorObj = error as { status?: number; code?: string; message?: string };
     let errorMessage: string;
 
-    // Check for specific error types and provide helpful messages
     if (errorObj.status === 401) {
       errorMessage = 'Invalid API key - please check your API key configuration';
       logger.warn('LLM API authentication failed', { error: errorObj });
     } else if (errorObj.status === 400 && errorObj.message?.includes('maximum context length')) {
-      // Extract token counts from error message if available
       const tokenMatch = errorObj.message.match(/(\d+)\s+tokens/g);
-      errorMessage = 'Diff too large for model context window. Try:\n' +
+      errorMessage =
+        'Diff too large for model context window. Try:\n' +
         '  • Stage fewer files at once\n' +
         '  • Use a model with larger context window in cadr.yaml\n' +
         '  • Add ignore patterns to filter large files';
-      logger.warn('LLM context length exceeded', { 
+      logger.warn('LLM context length exceeded', {
         error: errorObj,
         tokens: tokenMatch,
       });
@@ -218,46 +196,11 @@ export async function analyzeChanges(
   }
 }
 
-/**
- * Generation request data structure
- */
-export interface GenerationRequest {
-  file_paths: string[];
-  diff_content: string;
-  reason: string;
-  generation_prompt: string;
-}
-
-/**
- * Generation result from LLM
- */
-export interface GenerationResult {
-  content: string;
-  title: string;
-  timestamp: string;
-}
-
-/**
- * Generation response including potential errors
- */
-export interface GenerationResponse {
-  result: GenerationResult | null;
-  error?: string;
-}
-
-/**
- * Generate ADR content using LLM
- *
- * @param config - Analysis configuration with API settings
- * @param request - Generation request with code changes
- * @returns Promise resolving to generation response with result or error
- */
 export async function generateADRContent(
   config: AnalysisConfig,
   request: GenerationRequest
 ): Promise<GenerationResponse> {
   try {
-    // Check if API key is available
     const apiKey = process.env[config.api_key_env];
     if (!apiKey) {
       logger.warn('API key not found in environment for generation', {
@@ -265,7 +208,7 @@ export async function generateADRContent(
       });
       return {
         result: null,
-        error: `API key not found: ${config.api_key_env} environment variable is not set`
+        error: `API key not found: ${config.api_key_env} environment variable is not set`,
       };
     }
 
@@ -278,30 +221,27 @@ export async function generateADRContent(
     const provider = getProvider(config.provider);
     const responseContent = await provider.analyze(request.generation_prompt, {
       apiKey,
-      model: config.analysis_model, // Using same model per user request
+      model: config.analysis_model,
       timeoutMs: config.timeout_seconds * 1000,
     });
 
     if (!responseContent) {
-      logger.warn('No response content from LLM for generation', { 
-        provider: config.provider 
+      logger.warn('No response content from LLM for generation', {
+        provider: config.provider,
       });
       return {
         result: null,
-        error: 'No response content from LLM'
+        error: 'No response content from LLM',
       };
     }
 
-    // Clean up the response - remove markdown code fences if LLM added them
     let cleanedContent = responseContent.trim();
-    
-    // Remove markdown code block if present
+
     const codeBlockMatch = cleanedContent.match(/```(?:markdown|md)?\s*\n?([\s\S]*?)\n?```/);
     if (codeBlockMatch) {
       cleanedContent = codeBlockMatch[1].trim();
     }
 
-    // Extract title from first line (should be # Title)
     const titleMatch = cleanedContent.match(/^#\s+(.+)$/m);
     const title = titleMatch ? titleMatch[1].trim() : 'Untitled Decision';
 
@@ -318,7 +258,6 @@ export async function generateADRContent(
 
     return { result, error: undefined };
   } catch (error) {
-    // Fail-open: log error and return descriptive error message
     const errorObj = error as { status?: number; code?: string; message?: string };
     let errorMessage: string;