@crownpeak/dqm-react-component-dev-mcp 1.2.0

package/data/probe/npm/tests/integration/retryFallback.test.js

@@ -0,0 +1,368 @@
+/**
+ * Integration tests for Retry + Fallback together
+ */
+
+import { describe, test, expect, jest, beforeEach } from '@jest/globals';
+import { RetryManager } from '../../src/agent/RetryManager.js';
+import { FallbackManager } from '../../src/agent/FallbackManager.js';
+
+describe('Retry and Fallback Integration', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('Combined retry and fallback flow', () => {
+    test('should retry on each provider before falling back', async () => {
+      const retry = new RetryManager({
+        maxRetries: 2,
+        initialDelay: 10,
+        debug: false
+      });
+
+      const fallback = new FallbackManager({
+        strategy: 'custom',
+        providers: [
+          { provider: 'anthropic', apiKey: 'key1' },
+          { provider: 'openai', apiKey: 'key2' },
+          { provider: 'google', apiKey: 'key3' }
+        ],
+        debug: false
+      });
+
+      const callLog = [];
+
+      // Mock function that tracks which provider is being used
+      const mockFn = jest.fn().mockImplementation((provider, model, config) => {
+        callLog.push(config.provider);
+
+        // First provider (anthropic) fails 3 times (1 initial + 2 retries)
+        if (config.provider === 'anthropic') {
+          return retry.executeWithRetry(() => {
+            throw new Error('Overloaded');
+          }, { provider: 'anthropic' });
+        }
+
+        // Second provider (openai) fails 3 times
+        if (config.provider === 'openai') {
+          return retry.executeWithRetry(() => {
+            throw new Error('Overloaded');
+          }, { provider: 'openai' });
+        }
+
+        // Third provider (google) succeeds on second retry
+        if (config.provider === 'google') {
+          let attempts = 0;
+          return retry.executeWithRetry(() => {
+            attempts++;
+            if (attempts < 2) {
+              throw new Error('Overloaded');
+            }
+            return 'success from google';
+          }, { provider: 'google' });
+        }
+      });
+
+      const result = await fallback.executeWithFallback(mockFn);
+
+      expect(result).toBe('success from google');
+      expect(callLog).toEqual(['anthropic', 'openai', 'google']);
+
+      // Verify fallback stats
+      const fallbackStats = fallback.getStats();
+      expect(fallbackStats.totalAttempts).toBe(3);
+      expect(fallbackStats.successfulProvider).toContain('google');
+      expect(fallbackStats.failedProviders.length).toBe(2);
+    });
+
+    test('should handle all providers exhausting retries', async () => {
+      const retry = new RetryManager({
+        maxRetries: 1,
+        initialDelay: 5,
+        debug: false
+      });
+
+      const fallback = new FallbackManager({
+        strategy: 'custom',
+        providers: [
+          { provider: 'anthropic', apiKey: 'key1' },
+          { provider: 'openai', apiKey: 'key2' }
+        ],
+        debug: false
+      });
+
+      const mockFn = jest.fn().mockImplementation((provider, model, config) => {
+        return retry.executeWithRetry(() => {
+          throw new Error('Overloaded');
+        }, { provider: config.provider });
+      });
+
+      await expect(fallback.executeWithFallback(mockFn)).rejects.toThrow();
+
+      // All providers should have been attempted
+      expect(mockFn).toHaveBeenCalledTimes(2);
+
+      const fallbackStats = fallback.getStats();
+      expect(fallbackStats.totalAttempts).toBe(2);
+      expect(fallbackStats.successfulProvider).toBeNull();
+      expect(fallbackStats.failedProviders.length).toBe(2);
+    });
+  });
+
+  describe('Real-world scenarios', () => {
+    test('should handle Azure Claude -> Bedrock fallback scenario', async () => {
+      const retry = new RetryManager({
+        maxRetries: 3,
+        initialDelay: 10,
+        debug: false
+      });
+
+      const fallback = new FallbackManager({
+        strategy: 'custom',
+        providers: [
+          {
+            provider: 'anthropic',
+            apiKey: 'azure-key',
+            baseURL: 'https://azure-endpoint.com',
+            model: 'claude-3-7-sonnet-20250219'
+          },
+          {
+            provider: 'bedrock',
+            region: 'us-west-2',
+            apiKey: 'bedrock-key',
+            model: 'anthropic.claude-sonnet-4-20250514-v1:0'
+          }
+        ],
+        debug: false
+      });
+
+      let azureAttempts = 0;
+      const mockFn = jest.fn().mockImplementation((provider, model, config) => {
+        return retry.executeWithRetry(() => {
+          // Azure fails with overloaded
+          if (config.baseURL?.includes('azure')) {
+            azureAttempts++;
+            throw new Error('Overloaded');
+          }
+
+          // Bedrock succeeds
+          return `success with ${model}`;
+        }, { provider: config.provider, model });
+      });
+
+      const result = await fallback.executeWithFallback(mockFn);
+
+      expect(result).toContain('anthropic.claude-sonnet');
+      expect(azureAttempts).toBe(4); // 1 initial + 3 retries
+
+      const stats = fallback.getStats();
+      expect(stats.successfulProvider).toContain('bedrock');
+    });
+
+    test('should handle mixed retryable and non-retryable errors', async () => {
+      const retry = new RetryManager({
+        maxRetries: 2,
+        initialDelay: 10,
+        debug: false
+      });
+
+      const fallback = new FallbackManager({
+        strategy: 'custom',
+        providers: [
+          { provider: 'anthropic', apiKey: 'key1' },
+          { provider: 'openai', apiKey: 'key2' }
+        ],
+        debug: false
+      });
+
+      const mockFn = jest.fn().mockImplementation((provider, model, config) => {
+        return retry.executeWithRetry(() => {
+          // First provider: non-retryable error (invalid API key)
+          if (config.provider === 'anthropic') {
+            throw new Error('Invalid API key');
+          }
+
+          // Second provider: succeeds
+          return 'success';
+        }, { provider: config.provider });
+      });
+
+      const result = await fallback.executeWithFallback(mockFn);
+
+      expect(result).toBe('success');
+
+      // Anthropic should only be attempted once (non-retryable)
+      // OpenAI should succeed on first attempt
+      expect(mockFn).toHaveBeenCalledTimes(2);
+    });
+
+    test('should respect maxTotalAttempts across providers', async () => {
+      const retry = new RetryManager({
+        maxRetries: 5,
+        initialDelay: 5,
+        debug: false
+      });
+
+      const fallback = new FallbackManager({
+        strategy: 'custom',
+        providers: [
+          { provider: 'anthropic', apiKey: 'key1' },
+          { provider: 'openai', apiKey: 'key2' },
+          { provider: 'google', apiKey: 'key3' },
+          { provider: 'bedrock', apiKey: 'key4' }
+        ],
+        maxTotalAttempts: 2,  // Only allow 2 provider attempts total
+        debug: false
+      });
+
+      const mockFn = jest.fn().mockImplementation((provider, model, config) => {
+        return retry.executeWithRetry(() => {
+          throw new Error('Overloaded');
+        }, { provider: config.provider });
+      });
+
+      await expect(fallback.executeWithFallback(mockFn)).rejects.toThrow();
+
+      // Should only attempt 2 providers
+      expect(mockFn).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('Statistics and monitoring', () => {
+    test('should track detailed statistics across retry and fallback', async () => {
+      const retry = new RetryManager({
+        maxRetries: 2,
+        initialDelay: 5,
+        debug: false
+      });
+
+      const fallback = new FallbackManager({
+        strategy: 'custom',
+        providers: [
+          { provider: 'anthropic', apiKey: 'key1' },
+          { provider: 'openai', apiKey: 'key2' }
+        ],
+        debug: false
+      });
+
+      let anthropicAttempts = 0;
+      const mockFn = jest.fn().mockImplementation((provider, model, config) => {
+        if (config.provider === 'anthropic') {
+          return retry.executeWithRetry(() => {
+            anthropicAttempts++;
+            throw new Error('Overloaded');
+          }, { provider: 'anthropic' });
+        }
+
+        return retry.executeWithRetry(() => 'success', { provider: 'openai' });
+      });
+
+      await fallback.executeWithFallback(mockFn);
+
+      // Check fallback stats
+      const fallbackStats = fallback.getStats();
+      expect(fallbackStats.totalAttempts).toBe(2);
+      expect(fallbackStats.failedProviders.length).toBe(1);
+      expect(fallbackStats.successfulProvider).toContain('openai');
+
+      // Anthropic should have 3 total attempts (1 + 2 retries)
+      expect(anthropicAttempts).toBe(3);
+    });
+  });
+
+  describe('Edge cases', () => {
+    test('should handle empty provider list gracefully', async () => {
+      const fallback = new FallbackManager({
+        strategy: 'any',
+        providers: []
+      });
+
+      await expect(fallback.executeWithFallback(() => {})).rejects.toThrow('No providers configured');
+    });
+
+    test('should handle provider creation failures', async () => {
+      const fallback = new FallbackManager({
+        strategy: 'custom',
+        providers: [
+          { provider: 'anthropic', apiKey: 'test-key-1' },
+          { provider: 'openai', apiKey: 'valid-key' }
+        ],
+        debug: false
+      });
+
+      const mockFn = jest.fn().mockImplementation((provider, model, config) => {
+        // First provider throws error during execution
+        if (config.provider === 'anthropic') {
+          throw new Error('Provider creation failed');
+        }
+        // Second provider succeeds
+        return 'success';
+      });
+
+      // Should fallback to second provider
+      const result = await fallback.executeWithFallback(mockFn);
+      expect(result).toBe('success');
+      expect(mockFn).toHaveBeenCalledTimes(2);
+    });
+
+    test('should handle AbortSignal in retry manager', async () => {
+      const retry = new RetryManager({
+        maxRetries: 10,
+        initialDelay: 100,
+        debug: false
+      });
+
+      const controller = new AbortController();
+
+      // Abort after first attempt
+      setTimeout(() => controller.abort(), 50);
+
+      const mockFn = jest.fn().mockImplementation(() => {
+        throw new Error('Overloaded');
+      });
+
+      await expect(
+        retry.executeWithRetry(mockFn, { signal: controller.signal })
+      ).rejects.toThrow('Operation aborted');
+
+      // Should not exhaust all retries
+      expect(mockFn.mock.calls.length).toBeLessThan(10);
+    });
+  });
+
+  describe('Performance', () => {
+    test('should complete fallback reasonably quickly with fast retries', async () => {
+      const retry = new RetryManager({
+        maxRetries: 2,
+        initialDelay: 1,  // 1ms
+        maxDelay: 10,
+        debug: false
+      });
+
+      const fallback = new FallbackManager({
+        strategy: 'custom',
+        providers: [
+          { provider: 'anthropic', apiKey: 'key1' },
+          { provider: 'openai', apiKey: 'key2' }
+        ],
+        debug: false
+      });
+
+      const mockFn = jest.fn().mockImplementation((provider, model, config) => {
+        if (config.provider === 'anthropic') {
+          return retry.executeWithRetry(() => {
+            throw new Error('Overloaded');
+          }, { provider: 'anthropic' });
+        }
+        return retry.executeWithRetry(() => 'success', { provider: 'openai' });
+      });
+
+      const startTime = Date.now();
+      await fallback.executeWithFallback(mockFn);
+      const duration = Date.now() - startTime;
+
+      // Should complete in less than 500ms (with some margin for test execution)
+      // 3 retries * ~2ms delay = ~6ms for anthropic, then openai succeeds immediately
+      expect(duration).toBeLessThan(500);
+    });
+  });
+});

package/data/probe/npm/tests/integration/schema-in-initial-message.test.js

@@ -0,0 +1,318 @@
+/**
+ * Integration test for the JSON schema validation loop bug fix
+ *
+ * This test validates that schema instructions are added to the INITIAL user message,
+ * preventing the validation loop that occurred when the AI responded with plain text
+ * and then received correction prompts.
+ *
+ * Bug: https://github.com/probelabs/probe/issues/XXX
+ * Fix: Schema instructions prepended to user message before AI sees it
+ */
+
+import { describe, test, expect } from '@jest/globals';
+import { generateExampleFromSchema } from '../../src/agent/schemaUtils.js';
+
+describe('Schema in Initial Message - Bug Fix Integration Test', () => {
+
+  describe('generateExampleFromSchema - Used in Initial Message', () => {
+    test('should generate valid JSON example for Visor refine schema', () => {
+      // This is the exact schema from the bug report that caused the validation loop
+      const visorRefineSchema = {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          refined: {
+            type: 'boolean',
+            description: 'true if the task description is clear and actionable, false if clarification is needed'
+          },
+          text: {
+            type: 'string',
+            description: 'If refined=true, confirmation message. If refined=false, specific questions to ask.'
+          }
+        },
+        required: ['refined', 'text']
+      };
+
+      const example = generateExampleFromSchema(visorRefineSchema);
+
+      // Should generate a valid example
+      expect(example).toBeDefined();
+      expect(example).not.toBeNull();
+
+      // Should have the correct structure
+      expect(example).toHaveProperty('refined');
+      expect(example).toHaveProperty('text');
+
+      // Should use correct types
+      expect(typeof example.refined).toBe('boolean');
+      expect(typeof example.text).toBe('string');
+
+      // Should be valid JSON when stringified
+      const jsonString = JSON.stringify(example);
+      expect(() => JSON.parse(jsonString)).not.toThrow();
+
+      // The example should match what the AI is expected to generate
+      expect(example).toEqual({
+        refined: false,
+        text: 'If refined=true, confirmation message. If refined=false, specific questions to ask.'
+      });
+    });
+
+    test('should create a complete schema instruction message', () => {
+      const schema = {
+        type: 'object',
+        properties: {
+          status: { type: 'string', description: 'Operation status' },
+          count: { type: 'number' }
+        }
+      };
+
+      const example = generateExampleFromSchema(schema);
+
+      // Simulate how the message would be constructed in ProbeAgent
+      const userMessage = 'Please analyze this code';
+      const schemaInstructions = `\n\nIMPORTANT: When you provide your final answer using attempt_completion, you MUST format it as valid JSON matching this schema:\n\n${JSON.stringify(schema, null, 2)}\n\nExample:\n<attempt_completion>\n${JSON.stringify(example, null, 2)}\n</attempt_completion>\n\nYour response inside attempt_completion must be ONLY valid JSON - no plain text, no explanations, no markdown.`;
+
+      const fullMessage = userMessage + schemaInstructions;
+
+      // Should contain the original message
+      expect(fullMessage).toContain('Please analyze this code');
+
+      // Should contain clear JSON requirement
+      expect(fullMessage).toContain('IMPORTANT');
+      expect(fullMessage).toContain('you MUST format it as valid JSON');
+
+      // Should contain the schema
+      expect(fullMessage).toContain('"type": "object"');
+      expect(fullMessage).toContain('"status"');
+      expect(fullMessage).toContain('"count"');
+
+      // Should contain a concrete example
+      expect(fullMessage).toContain('Example:');
+      expect(fullMessage).toContain('<attempt_completion>');
+      expect(fullMessage).toContain('"status": "Operation status"');
+      expect(fullMessage).toContain('"count": 0');
+
+      // Should contain explicit restrictions
+      expect(fullMessage).toContain('ONLY valid JSON');
+      expect(fullMessage).toContain('no plain text');
+      expect(fullMessage).toContain('no explanations');
+      expect(fullMessage).toContain('no markdown');
+    });
+
+    test('should handle schema as JSON string (common in Visor)', () => {
+      const schemaString = JSON.stringify({
+        type: 'object',
+        properties: {
+          result: { type: 'string' }
+        }
+      });
+
+      const example = generateExampleFromSchema(schemaString);
+
+      expect(example).toBeDefined();
+      expect(example).toHaveProperty('result');
+      expect(typeof example.result).toBe('string');
+    });
+
+    test('should gracefully handle invalid schema without crashing', () => {
+      const invalidSchema = 'not valid json {';
+
+      const example = generateExampleFromSchema(invalidSchema);
+
+      // Should return null for invalid schema
+      expect(example).toBeNull();
+
+      // This ensures the user message construction doesn't crash
+      // even if schema parsing fails
+    });
+  });
+
+  describe('Bug Prevention - Validation Loop Scenario', () => {
+    test('should document the bug scenario that is now prevented', () => {
+      // BEFORE THE FIX:
+      // 1. User message: "Hi! Just checking"
+      // 2. Schema: {refined: boolean, text: string}
+      // 3. AI receives ONLY: "Hi! Just checking" (no schema info)
+      // 4. AI responds: "Hello! I'm ready to help..." (plain text)
+      // 5. System tries to parse as JSON -> FAILS
+      // 6. System sends correction: "CRITICAL JSON ERROR..."
+      // 7. AI still responds with plain text -> LOOP continues for 3 attempts
+      // 8. Total time: 100+ seconds, 30+ API calls
+
+      // AFTER THE FIX:
+      // 1. User message: "Hi! Just checking"
+      // 2. Schema: {refined: boolean, text: string}
+      const schema = {
+        type: 'object',
+        properties: {
+          refined: { type: 'boolean' },
+          text: { type: 'string' }
+        }
+      };
+
+      // 3. System generates example
+      const example = generateExampleFromSchema(schema);
+
+      // 4. AI receives enriched message with schema instructions
+      const userMessage = 'Hi! Just checking';
+      const enrichedMessage = userMessage + `\n\nIMPORTANT: When you provide your final answer using attempt_completion, you MUST format it as valid JSON matching this schema:\n\n${JSON.stringify(schema, null, 2)}\n\nExample:\n<attempt_completion>\n${JSON.stringify(example, null, 2)}\n</attempt_completion>\n\nYour response inside attempt_completion must be ONLY valid JSON - no plain text, no explanations, no markdown.`;
+
+      // 5. AI now knows to respond with JSON from the start
+      // 6. AI responds: <attempt_completion>{"refined": false, "text": "..."}</attempt_completion>
+      // 7. System parses JSON -> SUCCESS on first try
+      // 8. Total time: 4 seconds, 1 API call
+
+      // Verify the fix is in place
+      expect(enrichedMessage).toContain('you MUST format it as valid JSON');
+      expect(enrichedMessage).toContain(JSON.stringify(example, null, 2));
+      expect(enrichedMessage).not.toContain('CRITICAL JSON ERROR'); // No correction needed!
+    });
+
+    test('should prevent the specific Gemini 2.5 Pro issue', () => {
+      // The bug was specifically observed with Google Gemini 2.5 Pro
+      // which would respond with plain text or <task> XML tags
+
+      const visorSchema = {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          refined: { type: 'boolean' },
+          text: { type: 'string' }
+        },
+        required: ['refined', 'text']
+      };
+
+      const example = generateExampleFromSchema(visorSchema);
+
+      // The example should be immediately usable by the AI
+      expect(JSON.stringify(example)).toBe(
+        '{"refined":false,"text":"your answer here"}'
+      );
+
+      // This is what the AI should see upfront, preventing confusion
+      const instructions = `IMPORTANT: When you provide your final answer using attempt_completion, you MUST format it as valid JSON matching this schema:
+
+${JSON.stringify(visorSchema, null, 2)}
+
+Example:
+<attempt_completion>
+${JSON.stringify(example, null, 2)}
+</attempt_completion>
+
+Your response inside attempt_completion must be ONLY valid JSON - no plain text, no explanations, no markdown.`;
+
+      // Should be crystal clear to the AI
+      expect(instructions).toContain('MUST format it as valid JSON');
+      expect(instructions).toContain('Example:');
+      expect(instructions).toContain('ONLY valid JSON');
+      expect(instructions).toContain('no plain text');
+    });
+  });
+
+  describe('Edge Cases', () => {
+    test('should handle schema with deeply nested objects', () => {
+      const nestedSchema = {
+        type: 'object',
+        properties: {
+          user: {
+            type: 'object',
+            properties: {
+              profile: {
+                type: 'object',
+                properties: {
+                  name: { type: 'string' }
+                }
+              }
+            }
+          }
+        }
+      };
+
+      const example = generateExampleFromSchema(nestedSchema);
+
+      // Should handle nesting
+      expect(example).toHaveProperty('user');
+      expect(typeof example.user).toBe('object');
+
+      // Inner objects default to empty
+      expect(example.user).toEqual({});
+    });
+
+    test('should handle schema with all primitive types', () => {
+      const allTypesSchema = {
+        type: 'object',
+        properties: {
+          str: { type: 'string', description: 'A string' },
+          num: { type: 'number' },
+          bool: { type: 'boolean' },
+          arr: { type: 'array' },
+          obj: { type: 'object' }
+        }
+      };
+
+      const example = generateExampleFromSchema(allTypesSchema);
+
+      expect(example).toEqual({
+        str: 'A string',
+        num: 0,
+        bool: false,
+        arr: [],
+        obj: {}
+      });
+
+      // Should be valid JSON
+      expect(() => JSON.parse(JSON.stringify(example))).not.toThrow();
+    });
+
+    test('should use description as example value for strings', () => {
+      const schema = {
+        type: 'object',
+        properties: {
+          greeting: { type: 'string', description: 'Hello, World!' },
+          noDesc: { type: 'string' }
+        }
+      };
+
+      const example = generateExampleFromSchema(schema);
+
+      // Description should be used as the example value
+      expect(example.greeting).toBe('Hello, World!');
+
+      // Without description, default placeholder
+      expect(example.noDesc).toBe('your answer here');
+    });
+  });
+
+  describe('Performance - Bug Fix Impact', () => {
+    test('should prevent wasted iterations (regression test)', () => {
+      // Before fix: 30+ iterations (tool loop + 3 correction attempts each)
+      // After fix: 1 iteration (AI gets it right on first try)
+
+      const schema = {
+        type: 'object',
+        properties: {
+          refined: { type: 'boolean' },
+          text: { type: 'string' }
+        }
+      };
+
+      const startTime = Date.now();
+
+      // Generate example (happens once per request)
+      const example = generateExampleFromSchema(schema);
+
+      const endTime = Date.now();
+      const generationTime = endTime - startTime;
+
+      // Should be very fast (< 10ms)
+      expect(generationTime).toBeLessThan(10);
+
+      // Should produce valid output
+      expect(example).toBeDefined();
+      expect(() => JSON.parse(JSON.stringify(example))).not.toThrow();
+
+      // This small cost (< 10ms) saves 100+ seconds of validation loops
+    });
+  });
+});