@crownpeak/dqm-react-component-dev-mcp 1.2.0

package/data/probe/npm/tests/integration/schema-validation-loop-prevention.test.js

@@ -0,0 +1,244 @@
+/**
+ * Integration test to verify that schema-aware reminders prevent JSON validation loops
+ * This test validates that our fix solves the original issue
+ */
+import { jest, describe, test, expect, beforeEach, afterEach } from '@jest/globals';
+
+describe('Schema Validation Loop Prevention', () => {
+  let mockProbeAgent;
+  let mockMessages;
+
+  beforeEach(() => {
+    // Mock the current message flow that would happen in ProbeAgent
+    mockMessages = [];
+
+    mockProbeAgent = {
+      debug: true,
+      currentMessages: mockMessages,
+
+      // Mock the logic that sends reminder when no tool call is detected
+      sendReminderMessage(options) {
+        let reminderContent;
+        if (options.schema) {
+          // Schema-aware reminder (our new implementation)
+          reminderContent = `Please use one of the available tools to help answer the question, or use attempt_completion if you have enough information to provide a final answer.
+
+Remember: Use proper XML format with BOTH opening and closing tags:
+
+<tool_name>
+<parameter>value</parameter>
+</tool_name>
+
+IMPORTANT: A schema was provided. You MUST respond with data that matches this schema.
+Use attempt_completion with your response directly inside the tags:
+
+<attempt_completion>
+{"key": "value", "field": "your actual data here matching the schema"}
+</attempt_completion>
+
+Your response must conform to this schema:
+${options.schema}`;
+        } else {
+          // Standard reminder
+          reminderContent = `Please use one of the available tools to help answer the question, or use attempt_completion if you have enough information to provide a final answer.
+
+Remember: Use proper XML format with BOTH opening and closing tags:
+
+<tool_name>
+<parameter>value</parameter>
+</tool_name>
+
+Or for quick completion if your previous response was already correct:
+<attempt_complete>`;
+        }
+
+        this.currentMessages.push({
+          role: 'user',
+          content: reminderContent
+        });
+
+        return reminderContent;
+      },
+
+      // Mock what would happen when AI responds with JSON in attempt_completion
+      simulateJsonResponse(jsonContent) {
+        this.currentMessages.push({
+          role: 'assistant',
+          content: `<attempt_completion>${jsonContent}</attempt_completion>`
+        });
+        return jsonContent;
+      }
+    };
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('With Schema - Loop Prevention', () => {
+    test('should provide clear JSON instructions upfront to prevent validation loops', () => {
+      const jsonSchema = '{"analysis": "string", "confidence": "number", "recommendations": "array"}';
+      const options = { schema: jsonSchema };
+
+      const reminder = mockProbeAgent.sendReminderMessage(options);
+
+      // Should clearly indicate JSON is required
+      expect(reminder).toContain('A schema was provided');
+      expect(reminder).toContain('You MUST respond with data that matches this schema');
+
+      // Should show exact format expected
+      expect(reminder).toContain('<attempt_completion>');
+      expect(reminder).toContain('{"key": "value"');
+      expect(reminder).toContain('</attempt_completion>');
+
+      // Should include the schema for reference
+      expect(reminder).toContain(jsonSchema);
+
+      // Should NOT include the shorthand that doesn't work with schema
+      expect(reminder).not.toContain('<attempt_complete>');
+    });
+
+    test('should simulate successful JSON response after clear instructions', () => {
+      const jsonSchema = '{"status": "string", "message": "string"}';
+      const options = { schema: jsonSchema };
+
+      // Step 1: Send schema-aware reminder
+      mockProbeAgent.sendReminderMessage(options);
+
+      // Step 2: Simulate AI responding with proper JSON format
+      const jsonResponse = '{"status": "complete", "message": "Analysis finished successfully"}';
+      const response = mockProbeAgent.simulateJsonResponse(jsonResponse);
+
+      // Verify the response is valid JSON
+      expect(() => JSON.parse(response)).not.toThrow();
+
+      // Verify it matches the expected schema structure
+      const parsed = JSON.parse(response);
+      expect(typeof parsed.status).toBe('string');
+      expect(typeof parsed.message).toBe('string');
+
+      // Verify message flow shows proper instruction -> proper response
+      expect(mockProbeAgent.currentMessages).toHaveLength(2);
+      expect(mockProbeAgent.currentMessages[0].role).toBe('user');
+      expect(mockProbeAgent.currentMessages[0].content).toContain('A schema was provided');
+      expect(mockProbeAgent.currentMessages[1].role).toBe('assistant');
+      expect(mockProbeAgent.currentMessages[1].content).toContain('<attempt_completion>');
+    });
+
+    test('should handle complex JSON schemas with clear instructions', () => {
+      const complexSchema = `{
+        "analysis": "string",
+        "findings": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "category": "string",
+              "severity": "string",
+              "description": "string"
+            }
+          }
+        },
+        "metrics": {
+          "type": "object",
+          "properties": {
+            "score": "number",
+            "coverage": "number"
+          }
+        }
+      }`;
+
+      const options = { schema: complexSchema };
+
+      const reminder = mockProbeAgent.sendReminderMessage(options);
+
+      // Should include the full complex schema
+      expect(reminder).toContain('"findings"');
+      expect(reminder).toContain('"metrics"');
+      expect(reminder).toContain('"severity"');
+      expect(reminder).toContain('"coverage"');
+
+      // Should still provide clear instructions
+      expect(reminder).toContain('You MUST respond with data that matches this schema');
+    });
+  });
+
+  describe('Without Schema - Existing Behavior', () => {
+    test('should maintain backward compatibility for non-schema cases', () => {
+      const options = {}; // No schema
+
+      const reminder = mockProbeAgent.sendReminderMessage(options);
+
+      // Should use standard reminder
+      expect(reminder).toContain('Please use one of the available tools');
+      expect(reminder).toContain('<attempt_complete>');
+
+      // Should NOT contain schema-specific instructions
+      expect(reminder).not.toContain('A schema was provided');
+      expect(reminder).not.toContain('matches this schema');
+    });
+
+    test('should allow shorthand completion for non-schema cases', () => {
+      const options = {}; // No schema
+
+      const reminder = mockProbeAgent.sendReminderMessage(options);
+
+      // Should include the shorthand option
+      expect(reminder).toContain('for quick completion if your previous response was already correct');
+      expect(reminder).toContain('<attempt_complete>');
+    });
+  });
+
+  describe('Comparison - Before vs After Fix', () => {
+    test('should demonstrate how the fix prevents validation loops', () => {
+      // BEFORE: No schema-specific instructions led to validation loops
+      const beforeReminder = `Please use one of the available tools to help answer the question, or use attempt_completion if you have enough information to provide a final answer.
+
+Remember: Use proper XML format with BOTH opening and closing tags:
+
+<tool_name>
+<parameter>value</parameter>
+</tool_name>
+
+Or for quick completion if your previous response was already correct:
+<attempt_complete>`;
+
+      // AFTER: Schema-aware instructions prevent loops
+      const jsonSchema = '{"result": "string"}';
+      const options = { schema: jsonSchema };
+      const afterReminder = mockProbeAgent.sendReminderMessage(options);
+
+      // Before: No indication that JSON was required
+      expect(beforeReminder).not.toContain('schema');
+      expect(beforeReminder).not.toContain('JSON');
+
+      // After: Clear indication that JSON is required
+      expect(afterReminder).toContain('A schema was provided');
+      expect(afterReminder).toContain('You MUST respond with data that matches this schema');
+      expect(afterReminder).toContain(jsonSchema);
+    });
+
+    test('should show the validation loop scenario that is now prevented', () => {
+      // This test documents the problem that our fix solves
+      const jsonSchema = '{"analysis": "string", "score": "number"}';
+
+      // OLD BEHAVIOR (what would cause loops):
+      // 1. Agent gets no schema info in reminder
+      // 2. Agent responds with plain text in attempt_completion
+      // 3. System detects JSON schema and tries to correct
+      // 4. Agent still responds with plain text -> LOOP
+
+      // NEW BEHAVIOR (our fix):
+      // 1. Agent gets clear schema info in reminder
+      const options = { schema: jsonSchema };
+      const reminder = mockProbeAgent.sendReminderMessage(options);
+
+      // 2. Agent knows to respond with JSON from the start
+      expect(reminder).toContain('You MUST respond with data that matches this schema');
+      expect(reminder).toContain('{"key": "value", "field": "your actual data here matching the schema"}');
+
+      // 3. No validation loop needed - prevention at the source
+      expect(reminder).toContain(jsonSchema);
+    });
+  });
+});

package/data/probe/npm/tests/integration/schemaRetryLogic.test.js

@@ -0,0 +1,94 @@
+/**
+ * Integration tests for schema validation retry logic
+ * Tests the new 3-attempt retry mechanism with escalating prompts
+ */
+
+import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
+import * as schemaUtils from '../../src/agent/schemaUtils.js';
+
+describe('Schema Validation Retry Logic Integration Tests', () => {
+  let mockAnswer;
+  
+  beforeEach(() => {
+    // Create a mock answer function that simulates the ProbeAgent.answer method
+    mockAnswer = jest.fn();
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('should create escalating correction prompts for retry attempts', () => {
+    const invalidResponse = 'Hello, this is not JSON';
+    const schema = '{"result": "string"}';
+    const error = 'Unexpected token H in JSON at position 0';
+    
+    // Test the first retry (retryCount 0)
+    const prompt0 = schemaUtils.createJsonCorrectionPrompt(invalidResponse, schema, error, 0);
+    expect(prompt0).toContain('CRITICAL JSON ERROR:');
+    expect(prompt0).toContain('Return ONLY the corrected JSON');
+    
+    // Test the second retry (retryCount 1)  
+    const prompt1 = schemaUtils.createJsonCorrectionPrompt(invalidResponse, schema, error, 1);
+    expect(prompt1).toContain('URGENT - JSON PARSING FAILED:');
+    expect(prompt1).toContain('second chance');
+    expect(prompt1).toContain('ABSOLUTELY NO explanatory text');
+    
+    // Test the third retry (retryCount 2)
+    const prompt2 = schemaUtils.createJsonCorrectionPrompt(invalidResponse, schema, error, 2);
+    expect(prompt2).toContain('FINAL ATTEMPT - CRITICAL JSON ERROR:');
+    expect(prompt2).toContain('final retry');
+    expect(prompt2).toContain('EXAMPLE:');
+  });
+
+  test('should simulate retry logic behavior with real functions', () => {
+    const invalidResponses = [
+      'Hello, this is not JSON',
+      '```json\n{"incomplete":',
+      'Another invalid response'
+    ];
+    const schema = '{"result": "string"}';
+    
+    // Test that each retry creates the appropriate correction prompt
+    invalidResponses.forEach((response, index) => {
+      const prompt = schemaUtils.createJsonCorrectionPrompt(response, schema, 'Invalid JSON', index);
+      
+      if (index === 0) {
+        expect(prompt).toContain('CRITICAL JSON ERROR:');
+      } else if (index === 1) {
+        expect(prompt).toContain('URGENT - JSON PARSING FAILED:');
+      } else if (index === 2) {
+        expect(prompt).toContain('FINAL ATTEMPT - CRITICAL JSON ERROR:');
+      }
+      
+      expect(prompt).toContain(response.substring(0, 50)); // Should contain truncated response
+      expect(prompt).toContain(schema);
+      expect(prompt).toContain('Invalid JSON');
+    });
+  });
+
+  test('should test enhanced JSON extraction patterns', () => {
+    const testCases = [
+      {
+        input: 'AI: Here is your JSON: ```json\n{"result": "success"}\n```',
+        expected: '{"result": "success"}',
+        description: 'should extract from code blocks with text before'
+      },
+      {
+        input: 'Response:\n{"data": "value"}',
+        expected: 'Response:\n{"data": "value"}', // Returns original - text precedes JSON
+        description: 'should not extract when text precedes JSON'
+      },
+      {
+        input: '```\n[{"item": 1}, {"item": 2}]\n```',
+        expected: '[{"item": 1}, {"item": 2}]',
+        description: 'should extract array from unmarked code block'
+      }
+    ];
+
+    testCases.forEach(({ input, expected, description }) => {
+      const result = schemaUtils.cleanSchemaResponse(input);
+      expect(result).toBe(expected);
+    });
+  });
+});

package/data/probe/npm/tests/integration/validationFlow.test.js

@@ -0,0 +1,329 @@
+/**
+ * Integration tests for the complete validation flow
+ * Tests how JSON and Mermaid validation work together
+ */
+
+import { describe, test, expect, jest } from '@jest/globals';
+import {
+  isJsonSchema,
+  isMermaidSchema,
+  validateJsonResponse,
+  validateMermaidDiagram,
+  validateMermaidResponse,
+  cleanSchemaResponse,
+  processSchemaResponse,
+  createJsonCorrectionPrompt,
+  createMermaidCorrectionPrompt,
+  extractMermaidFromMarkdown
+} from '../../src/agent/schemaUtils.js';
+
+describe('Validation Flow Integration', () => {
+  describe('Schema Detection Priority', () => {
+    test('should detect both JSON and Mermaid schemas correctly', () => {
+      const jsonOnlySchema = '{"users": [{"name": "string"}]}';
+      const mermaidOnlySchema = 'Create a mermaid flowchart diagram';
+      const mixedSchema = 'Create a JSON response with a mermaid diagram showing the flow';
+      
+      expect(isJsonSchema(jsonOnlySchema)).toBe(true);
+      expect(isMermaidSchema(jsonOnlySchema)).toBe(false);
+      
+      expect(isJsonSchema(mermaidOnlySchema)).toBe(false);
+      expect(isMermaidSchema(mermaidOnlySchema)).toBe(true);
+      
+      expect(isJsonSchema(mixedSchema)).toBe(true);
+      expect(isMermaidSchema(mixedSchema)).toBe(true);
+    });
+
+    test('should prioritize validation order (Mermaid first, then JSON)', async () => {
+      const response = `{
+        "description": "Here's the process flow",
+        "diagram": "\`\`\`mermaid\\ngraph TD\\n    A[Start] --> B[Process]\\n    B --> C[End]\\n\`\`\`"
+      }`;
+
+      const schema = 'Return JSON with a mermaid diagram field';
+
+      // Test that we can detect both types
+      expect(isMermaidSchema(schema)).toBe(true);
+      expect(isJsonSchema(schema)).toBe(true);
+
+      // Test Mermaid validation first - with JSON-aware extraction, diagrams in JSON strings are now found
+      const mermaidValidation = await validateMermaidResponse(response);
+      expect(mermaidValidation.isValid).toBe(true); // Mermaid diagrams in JSON strings are now detected and validated
+      expect(mermaidValidation.diagrams).toHaveLength(1);
+      expect(mermaidValidation.diagrams[0].isInJson).toBe(true);
+      expect(mermaidValidation.diagrams[0].diagramType).toBe('flowchart');
+
+      // Test JSON validation second
+      const cleanedResponse = cleanSchemaResponse(response);
+      const jsonValidation = validateJsonResponse(cleanedResponse);
+      expect(jsonValidation.isValid).toBe(true);
+    });
+  });
+
+  describe('Complex Response Validation', () => {
+    test('should handle response with valid JSON and valid Mermaid', async () => {
+      const response = `Here's your analysis:
+
+\`\`\`json
+{
+  "status": "completed",
+  "diagram_count": 1
+}
+\`\`\`
+
+\`\`\`mermaid
+graph TD
+    A[Analysis] --> B[Results]
+    B --> C[Report]
+\`\`\``;
+
+      // Clean and validate JSON part
+      const cleanedJson = '{\n  "status": "completed",\n  "diagram_count": 1\n}';
+      const jsonValidation = validateJsonResponse(cleanedJson);
+      expect(jsonValidation.isValid).toBe(true);
+
+      // Validate Mermaid part
+      const mermaidValidation = await validateMermaidResponse(response);
+      expect(mermaidValidation.isValid).toBe(true);
+    });
+
+    test('should handle response with invalid JSON and valid Mermaid', async () => {
+      const response = `Here's your analysis:
+
+\`\`\`json
+{
+  "status": completed,
+  "diagram_count": 1
+}
+\`\`\`
+
+\`\`\`mermaid
+graph TD
+    A[Analysis] --> B[Results]
+    B --> C[Report]
+\`\`\``;
+
+      // JSON should be invalid (missing quotes)
+      const cleanedJson = '{\n  "status": completed,\n  "diagram_count": 1\n}';
+      const jsonValidation = validateJsonResponse(cleanedJson);
+      expect(jsonValidation.isValid).toBe(false);
+
+      // Mermaid should be valid
+      const mermaidValidation = await validateMermaidResponse(response);
+      expect(mermaidValidation.isValid).toBe(true);
+    });
+
+    test('should handle response with valid JSON and invalid Mermaid', async () => {
+      const response = `Here's your analysis:
+
+\`\`\`json
+{
+  "status": "completed",
+  "diagram_count": 1
+}
+\`\`\`
+
+\`\`\`mermaid
+invalid diagram syntax
+\`\`\``;
+
+      // JSON should be valid
+      const cleanedJson = '{\n  "status": "completed",\n  "diagram_count": 1\n}';
+      const jsonValidation = validateJsonResponse(cleanedJson);
+      expect(jsonValidation.isValid).toBe(true);
+
+      // Mermaid should be invalid
+      const mermaidValidation = await validateMermaidResponse(response);
+      expect(mermaidValidation.isValid).toBe(false);
+    });
+
+    test('should handle multiple Mermaid diagrams with mixed validity', async () => {
+      const response = `\`\`\`mermaid
+graph TD
+    A --> B
+\`\`\`
+
+\`\`\`mermaid
+invalid syntax
+\`\`\`
+
+\`\`\`mermaid
+sequenceDiagram
+    Alice->>Bob: Hello
+\`\`\``;
+
+      const validation = await validateMermaidResponse(response);
+      expect(validation.isValid).toBe(false);
+      expect(validation.diagrams).toHaveLength(3);
+      expect(validation.diagrams[0].isValid).toBe(true);
+      expect(validation.diagrams[1].isValid).toBe(false);
+      expect(validation.diagrams[2].isValid).toBe(true);
+    });
+  });
+
+  describe('Error Message Quality', () => {
+    test('should provide detailed JSON correction prompts', () => {
+      const invalidJson = '{"name": John, "age": 25}';
+      const schema = '{"name": "string", "age": "number"}';
+      const error = 'Unexpected token J in JSON at position 9';
+      const detailedError = 'Unexpected token J in JSON at position 9';
+
+      const prompt = createJsonCorrectionPrompt(invalidJson, schema, error, 0);
+
+      expect(prompt).toContain('CRITICAL JSON ERROR:');
+      expect(prompt).toContain('not valid JSON');
+      expect(prompt).toContain(invalidJson);
+      expect(prompt).toContain('Unexpected token J');
+      expect(prompt).toContain('corrected JSON');
+    });
+
+    test('should provide detailed Mermaid correction prompts', () => {
+      const invalidResponse = `\`\`\`mermaid
+graph TD
+    A[Start --> B[Missing bracket
+\`\`\``;
+
+      const schema = 'Create a mermaid flowchart';
+      const errors = ['Diagram 1: Unclosed bracket on line 2'];
+      const diagrams = [{
+        diagram: 'graph TD\n    A[Start --> B[Missing bracket',
+        isValid: false,
+        error: 'Unclosed bracket on line 2',
+        detailedError: 'Line "A[Start --> B[Missing bracket" contains an unclosed bracket'
+      }];
+
+      const prompt = createMermaidCorrectionPrompt(invalidResponse, schema, errors, diagrams);
+
+      expect(prompt).toContain('invalid Mermaid diagrams');
+      expect(prompt).toContain('Unclosed bracket');
+      expect(prompt).toContain('mermaid code blocks');
+      expect(prompt).toContain('correct Mermaid syntax');
+    });
+
+    test('should handle complex error scenarios', async () => {
+      const response = `Here are the results:
+
+\`\`\`mermaid
+sequenceDiagram
+    Alice->>Bob Hello world
+    Bob-->>Alice: Response
+\`\`\`
+
+\`\`\`json
+{
+  "results": [
+    {"id": 1, "status": completed}
+  ]
+}
+\`\`\``;
+
+      // Both should be invalid
+      const mermaidValidation = await validateMermaidResponse(response);
+      expect(mermaidValidation.isValid).toBe(false);
+      expect(mermaidValidation.errors[0]).toContain('Missing colon');
+
+      const jsonPart = '{\n  "results": [\n    {"id": 1, "status": completed}\n  ]\n}';
+      const jsonValidation = validateJsonResponse(jsonPart);
+      expect(jsonValidation.isValid).toBe(false);
+    });
+  });
+
+  describe('Edge Cases', () => {
+    test('should handle empty responses', async () => {
+      const emptyInputs = ['', '   ', '\n\n', null, undefined];
+
+      for (const input of emptyInputs) {
+        const mermaidValidation = await validateMermaidResponse(input);
+        expect(mermaidValidation.isValid).toBe(false);
+
+        if (input) {
+          const jsonValidation = validateJsonResponse(input);
+          expect(jsonValidation.isValid).toBe(false);
+        }
+      }
+    });
+
+    test('should handle malformed markdown blocks', async () => {
+      const malformedResponse = `\`\`\`mermaid
+graph TD
+    A --> B
+\`\`\`mermaid (extra text)
+sequenceDiagram
+    Alice->>Bob: Test
+\`\`\``;
+
+      const { diagrams } = extractMermaidFromMarkdown(malformedResponse);
+      expect(diagrams).toHaveLength(1);
+      expect(diagrams[0].content).toContain('graph TD');
+    });
+
+    test('should handle nested code blocks', async () => {
+      const response = `Here's an example:
+
+\`\`\`markdown
+This is how you create a mermaid diagram:
+
+\`\`\`mermaid
+graph TD
+    A --> B
+\`\`\`
+\`\`\`
+
+And here's a real diagram:
+
+\`\`\`mermaid
+sequenceDiagram
+    Alice->>Bob: Hello
+\`\`\``;
+
+      const { diagrams } = extractMermaidFromMarkdown(response);
+      expect(diagrams).toHaveLength(2); // Both embedded and real diagrams are extracted
+      expect(diagrams[0].content).toContain('graph TD');
+      expect(diagrams[1].content).toContain('sequenceDiagram');
+    });
+
+    test('should handle very large diagrams', async () => {
+      const largeDiagram = 'graph TD\n' + Array(1000).fill(0).map((_, i) => 
+        `    A${i}[Node ${i}] --> A${i + 1}[Node ${i + 1}]`
+      ).join('\n');
+
+      const result = await validateMermaidDiagram(largeDiagram);
+      expect(result.isValid).toBe(true);
+      expect(result.diagramType).toBe('flowchart');
+    });
+
+    test('should handle Unicode characters in diagrams', async () => {
+      const unicodeDiagram = `graph TD
+    A[开始] --> B[处理]
+    B --> C[结束]
+    D[🎯 Goal] --> E[✅ Complete]`;
+
+      const result = await validateMermaidDiagram(unicodeDiagram);
+      expect(result.isValid).toBe(true);
+    });
+
+    test('should handle process schema response with all options', () => {
+      const response = `\`\`\`json
+{
+  "test": "value"
+}
+\`\`\`
+
+\`\`\`mermaid
+graph TD
+    A --> B
+\`\`\``;
+
+      const result = processSchemaResponse(response, 'mixed schema', {
+        validateJson: true,
+        validateXml: false,
+        debug: true
+      });
+
+      expect(result.cleaned).toBeDefined();
+      expect(result.debug).toBeDefined();
+      expect(result.jsonValidation).toBeDefined();
+      expect(result.debug.wasModified).toBe(true);
+    });
+  });
+});