devforgeai 1.0.5 → 1.0.7

package/src/claude/scripts/devforgeai_cli/tests/feedback/test_question_routing.py

@@ -0,0 +1,262 @@
+"""
+Unit tests for context-aware question routing (AC5)
+
+Tests cover:
+- AC5: Context-aware question routing by workflow type
+- Question adaptation based on success/failure
+- Cultural appropriateness validation
+"""
+
+import pytest
+from devforgeai_cli.feedback.question_router import (
+    get_context_aware_questions,
+    load_question_bank,
+)
+from devforgeai_cli.feedback.models import Question
+
+
+class TestContextAwareRouting:
+    """AC5: Context-Aware Question Routing"""
+
+    def test_get_questions_for_dev_success(self):
+        """
+        GIVEN a successful /dev workflow completion
+        WHEN get_context_aware_questions is called with workflow_type='dev' and success_status='success'
+        THEN it returns dev-specific success questions
+        """
+        # Arrange
+        workflow_type = 'dev'
+        success_status = 'success'
+
+        # Act
+        questions = get_context_aware_questions(workflow_type, success_status)
+
+        # Assert
+        assert questions is not None
+        assert len(questions) >= 4
+        assert all(isinstance(q, Question) for q in questions)
+
+        # Check for dev-specific content
+        question_texts = [q.question_text.lower() for q in questions]
+        assert any('tdd' in text or 'test' in text or 'development' in text for text in question_texts)
+
+    def test_get_questions_for_qa_success(self):
+        """
+        GIVEN a successful /qa workflow completion
+        WHEN get_context_aware_questions is called with workflow_type='qa'
+        THEN it returns qa-specific questions (coverage, validation)
+        """
+        # Arrange
+        workflow_type = 'qa'
+        success_status = 'success'
+
+        # Act
+        questions = get_context_aware_questions(workflow_type, success_status)
+
+        # Assert
+        assert questions is not None
+        question_texts = [q.question_text.lower() for q in questions]
+        assert any('coverage' in text or 'quality' in text or 'validation' in text for text in question_texts)
+
+    def test_get_questions_for_orchestrate_success(self):
+        """
+        GIVEN a successful /orchestrate workflow completion
+        WHEN get_context_aware_questions is called
+        THEN it returns orchestration-specific questions
+        """
+        # Arrange
+        workflow_type = 'orchestrate'
+        success_status = 'success'
+
+        # Act
+        questions = get_context_aware_questions(workflow_type, success_status)
+
+        # Assert
+        assert questions is not None
+        question_texts = [q.question_text.lower() for q in questions]
+        assert any('workflow' in text or 'phases' in text or 'integration' in text for text in question_texts)
+
+    def test_get_questions_for_failure_differ_from_success(self):
+        """
+        GIVEN failure vs success status
+        WHEN get_context_aware_questions is called
+        THEN failure questions differ from success questions
+        """
+        # Arrange
+        workflow_type = 'dev'
+
+        # Act
+        success_questions = get_context_aware_questions(workflow_type, 'success')
+        failure_questions = get_context_aware_questions(workflow_type, 'failed')
+
+        # Assert
+        success_ids = [q.question_id for q in success_questions]
+        failure_ids = [q.question_id for q in failure_questions]
+
+        # At least some questions should be different
+        assert success_ids != failure_ids
+        assert len(set(success_ids) & set(failure_ids)) < len(success_ids)  # Some overlap OK, but not all
+
+    def test_questions_have_appropriate_response_types(self):
+        """
+        GIVEN context-aware questions
+        WHEN examining response types
+        THEN questions use appropriate types (rating, multiple_choice, open_text)
+        """
+        # Arrange
+        workflow_type = 'dev'
+        success_status = 'success'
+
+        # Act
+        questions = get_context_aware_questions(workflow_type, success_status)
+
+        # Assert
+        response_types = [q.response_type for q in questions]
+        valid_types = ['rating', 'multiple_choice', 'open_text']
+
+        assert all(rt in valid_types for rt in response_types)
+        # Should have a mix of types
+        assert len(set(response_types)) >= 2
+
+
+class TestCulturalAppropriateness:
+    """AC5: Cultural appropriateness validation"""
+
+    def test_questions_never_blame_user(self):
+        """
+        GIVEN all workflow questions
+        WHEN examining question text
+        THEN questions never blame user (no "You failed", "You missed", etc.)
+        """
+        # Arrange
+        workflows = ['dev', 'qa', 'orchestrate', 'release']
+        statuses = ['success', 'failed', 'partial']
+
+        blame_phrases = ['you failed', 'you missed', 'your fault', 'your mistake', 'you should have']
+
+        # Act & Assert
+        for workflow in workflows:
+            for status in statuses:
+                questions = get_context_aware_questions(workflow, status)
+                for question in questions:
+                    text_lower = question.question_text.lower()
+                    for blame_phrase in blame_phrases:
+                        assert blame_phrase not in text_lower, \
+                            f"Question '{question.question_text}' contains blame phrase '{blame_phrase}'"
+
+    def test_questions_focus_on_framework_improvement(self):
+        """
+        GIVEN failure questions
+        WHEN examining question text
+        THEN questions frame issues as framework improvements ("How can we make this clearer?")
+        """
+        # Arrange
+        workflow_type = 'dev'
+        success_status = 'failed'
+
+        # Act
+        questions = get_context_aware_questions(workflow_type, success_status)
+
+        # Assert
+        improvement_indicators = ['how can we', 'what would help', 'what could', 'suggestions', 'improvements']
+        question_texts = [q.question_text.lower() for q in questions]
+
+        # At least one question should focus on improvements
+        assert any(
+            any(indicator in text for indicator in improvement_indicators)
+            for text in question_texts
+        ), "No questions focus on framework improvements"
+
+    def test_questions_avoid_jargon(self):
+        """
+        GIVEN all questions
+        WHEN examining question text
+        THEN questions use clear English (not overly technical jargon)
+        """
+        # Arrange
+        workflow_type = 'dev'
+        success_status = 'success'
+
+        # Act
+        questions = get_context_aware_questions(workflow_type, success_status)
+
+        # Assert - Questions should be understandable
+        # Check that question text is reasonably short (not overly verbose)
+        for question in questions:
+            assert len(question.question_text) < 200, \
+                f"Question too long (may be too complex): '{question.question_text}'"
+
+            # Should not use ALL CAPS for entire question (aggressive tone)
+            assert not question.question_text.isupper(), \
+                f"Question uses ALL CAPS: '{question.question_text}'"
+
+
+class TestQuestionBankLoading:
+    """Test question bank YAML loading"""
+
+    def test_load_question_bank_returns_structured_data(self):
+        """
+        GIVEN question bank YAML file
+        WHEN load_question_bank is called
+        THEN it returns structured dictionary with workflows
+        """
+        # Act
+        question_bank = load_question_bank()
+
+        # Assert
+        assert question_bank is not None
+        assert 'workflows' in question_bank
+        assert isinstance(question_bank['workflows'], dict)
+
+        # Check required workflows exist
+        required_workflows = ['dev', 'qa', 'orchestrate', 'release']
+        for workflow in required_workflows:
+            assert workflow in question_bank['workflows']
+
+    def test_question_bank_has_success_and_failure_questions(self):
+        """
+        GIVEN question bank
+        WHEN examining workflow questions
+        THEN each workflow has both success_questions and failure_questions
+        """
+        # Act
+        question_bank = load_question_bank()
+
+        # Assert
+        workflows = question_bank['workflows']
+        for workflow_name, workflow_config in workflows.items():
+            assert 'success_questions' in workflow_config, \
+                f"Workflow '{workflow_name}' missing success_questions"
+            assert 'failure_questions' in workflow_config, \
+                f"Workflow '{workflow_name}' missing failure_questions"
+            assert len(workflow_config['success_questions']) >= 3, \
+                f"Workflow '{workflow_name}' needs at least 3 success questions"
+            assert len(workflow_config['failure_questions']) >= 3, \
+                f"Workflow '{workflow_name}' needs at least 3 failure questions"
+
+    def test_question_bank_questions_have_required_fields(self):
+        """
+        GIVEN question bank
+        WHEN examining individual questions
+        THEN each question has id, text, type (and scale/options if applicable)
+        """
+        # Act
+        question_bank = load_question_bank()
+
+        # Assert
+        workflows = question_bank['workflows']
+        for workflow_name, workflow_config in workflows.items():
+            for question_type in ['success_questions', 'failure_questions']:
+                for question in workflow_config[question_type]:
+                    assert 'id' in question, f"Question missing 'id' in {workflow_name}/{question_type}"
+                    assert 'text' in question, f"Question missing 'text' in {workflow_name}/{question_type}"
+                    assert 'type' in question, f"Question missing 'type' in {workflow_name}/{question_type}"
+
+                    # If rating, must have scale
+                    if question['type'] == 'rating':
+                        assert 'scale' in question, f"Rating question missing 'scale' in {workflow_name}"
+
+                    # If multiple_choice, must have options
+                    if question['type'] == 'multiple_choice':
+                        assert 'options' in question, f"Multiple choice question missing 'options' in {workflow_name}"
+                        assert len(question['options']) >= 2, f"Multiple choice needs at least 2 options"

package/src/claude/scripts/devforgeai_cli/tests/feedback/test_retrospective.py

@@ -0,0 +1,333 @@
+"""
+Unit tests for retrospective conversation functionality (AC1, AC2, AC3)
+
+Tests cover:
+- AC1: Retrospective triggered at operation completion
+- AC2: Failed command with root cause feedback
+- AC3: User opts out of feedback
+"""
+
+import pytest
+import json
+import uuid
+from datetime import datetime
+from pathlib import Path
+import tempfile
+import shutil
+
+from devforgeai_cli.feedback.retrospective import (
+    trigger_retrospective,
+    capture_feedback,
+    is_skip_selected,
+)
+from devforgeai_cli.feedback.models import FeedbackSession, Question
+
+
+class TestRetrospectiveTrigger:
+    """AC1: Retrospective Triggered at Operation Completion"""
+
+    def test_trigger_retrospective_success_returns_questions(self):
+        """
+        GIVEN a successful /dev command completion
+        WHEN trigger_retrospective is called with workflow_type='dev' and success_status='success'
+        THEN it returns 4-6 targeted questions
+        """
+        # Arrange
+        workflow_type = 'dev'
+        operation_result = {'status': 'success', 'story_id': 'STORY-001'}
+
+        # Act
+        questions = trigger_retrospective(workflow_type, operation_result)
+
+        # Assert
+        assert questions is not None
+        assert 4 <= len(questions) <= 6
+        assert all(isinstance(q, Question) for q in questions)
+        assert all(hasattr(q, 'question_id') for q in questions)
+        assert all(hasattr(q, 'question_text') for q in questions)
+        assert all(hasattr(q, 'response_type') for q in questions)
+
+    def test_trigger_retrospective_failure_returns_failure_questions(self):
+        """
+        GIVEN a failed /qa command (AC2)
+        WHEN trigger_retrospective is called with success_status='failed'
+        THEN it returns failure-specific questions
+        """
+        # Arrange
+        workflow_type = 'qa'
+        operation_result = {'status': 'failed', 'story_id': 'STORY-002', 'failure_reason': 'Coverage below threshold'}
+
+        # Act
+        questions = trigger_retrospective(workflow_type, operation_result)
+
+        # Assert
+        assert questions is not None
+        assert len(questions) >= 3  # At least 3 failure-specific questions
+        assert any('block' in q.question_text.lower() or 'fail' in q.question_text.lower() for q in questions)
+        assert any('help' in q.question_text.lower() or 'improve' in q.question_text.lower() for q in questions)
+
+
+class TestFeedbackCapture:
+    """AC1: Feedback capture and storage"""
+
+    @pytest.fixture
+    def temp_feedback_dir(self):
+        """Create temporary feedback directory for tests"""
+        temp_dir = tempfile.mkdtemp()
+        yield Path(temp_dir)
+        shutil.rmtree(temp_dir)
+
+    def test_capture_feedback_stores_json_correctly(self, temp_feedback_dir):
+        """
+        GIVEN user responses to retrospective questions
+        WHEN capture_feedback is called
+        THEN feedback is stored in devforgeai/feedback/{STORY-ID}/{timestamp}-retrospective.json
+        AND returns confirmation message
+        """
+        # Arrange
+        story_id = 'STORY-001'
+        responses = [
+            {'question_id': 'dev_success_01', 'response': 4, 'skip': False},
+            {'question_id': 'dev_success_02', 'response': 'Green', 'skip': False},
+        ]
+
+        # Act
+        result = capture_feedback(
+            responses=responses,
+            story_id=story_id,
+            workflow_type='dev',
+            success_status='success',
+            feedback_dir=temp_feedback_dir
+        )
+
+        # Assert
+        assert result['status'] == 'recorded'
+        assert 'feedback_id' in result
+        assert 'file_path' in result
+
+        # Verify file was created
+        story_feedback_dir = temp_feedback_dir / story_id
+        assert story_feedback_dir.exists()
+
+        json_files = list(story_feedback_dir.glob('*-retrospective.json'))
+        assert len(json_files) == 1
+
+        # Verify JSON structure
+        with open(json_files[0], 'r') as f:
+            feedback_data = json.load(f)
+
+        assert feedback_data['story_id'] == story_id
+        assert feedback_data['workflow_type'] == 'dev'
+        assert feedback_data['success_status'] == 'success'
+        assert 'feedback_id' in feedback_data
+        assert 'timestamp' in feedback_data
+        assert len(feedback_data['questions']) == 2
+        assert feedback_data['metadata']['total_questions'] == 2
+        assert feedback_data['metadata']['answered'] == 2
+        assert feedback_data['metadata']['skipped'] == 0
+
+    def test_capture_feedback_validates_required_fields(self, temp_feedback_dir):
+        """
+        GIVEN incomplete feedback (AC2: at least 2 of 5 required)
+        WHEN capture_feedback is called with only 1 substantive response
+        THEN it raises ValueError
+        """
+        # Arrange - Only 1 substantive response, not enough
+        story_id = 'STORY-002'
+        responses = [
+            {'question_id': 'dev_failure_01', 'response': 'Git issue', 'skip': False},
+            {'question_id': 'dev_failure_02', 'response': '', 'skip': True},
+            {'question_id': 'dev_failure_03', 'response': '', 'skip': True},
+            {'question_id': 'dev_failure_04', 'response': '', 'skip': True},
+            {'question_id': 'dev_failure_05', 'response': '', 'skip': True},
+        ]
+
+        # Act & Assert
+        with pytest.raises(ValueError, match="At least 2 of 5 questions must have substantive responses"):
+            capture_feedback(
+                responses=responses,
+                story_id=story_id,
+                workflow_type='dev',
+                success_status='failed',
+                feedback_dir=temp_feedback_dir
+            )
+
+    def test_capture_feedback_accepts_valid_partial_completion(self, temp_feedback_dir):
+        """
+        GIVEN partial feedback with 2 substantive responses
+        WHEN capture_feedback is called
+        THEN it accepts and stores the feedback
+        """
+        # Arrange - 2 substantive responses (minimum required)
+        story_id = 'STORY-003'
+        responses = [
+            {'question_id': 'qa_failure_01', 'response': 'Coverage metrics unclear', 'skip': False},
+            {'question_id': 'qa_failure_02', 'response': 'Better documentation would help', 'skip': False},
+            {'question_id': 'qa_failure_03', 'response': '', 'skip': True},
+        ]
+
+        # Act
+        result = capture_feedback(
+            responses=responses,
+            story_id=story_id,
+            workflow_type='qa',
+            success_status='failed',
+            feedback_dir=temp_feedback_dir
+        )
+
+        # Assert
+        assert result['status'] == 'recorded'
+        assert 'feedback_id' in result
+
+
+class TestUserOptOut:
+    """AC3: User Opts Out of Feedback"""
+
+    def test_is_skip_selected_returns_true_for_skip_option(self):
+        """
+        GIVEN user selects "Skip feedback" option
+        WHEN is_skip_selected is called
+        THEN it returns True
+        """
+        # Arrange
+        user_response = 'Skip feedback'
+
+        # Act
+        result = is_skip_selected(user_response)
+
+        # Assert
+        assert result is True
+
+    def test_is_skip_selected_returns_true_for_decline_variations(self):
+        """
+        GIVEN user declines feedback with various phrases
+        WHEN is_skip_selected is called
+        THEN it returns True
+        """
+        # Arrange
+        decline_phrases = [
+            'Skip',
+            'No thanks',
+            'Not now',
+            'Later',
+            'Decline',
+        ]
+
+        # Act & Assert
+        for phrase in decline_phrases:
+            assert is_skip_selected(phrase) is True
+
+    def test_is_skip_selected_returns_false_for_normal_responses(self):
+        """
+        GIVEN user provides normal feedback response
+        WHEN is_skip_selected is called
+        THEN it returns False
+        """
+        # Arrange
+        normal_responses = [
+            'The TDD workflow was helpful',
+            '4',
+            'Green phase',
+        ]
+
+        # Act & Assert
+        for response in normal_responses:
+            assert is_skip_selected(response) is False
+
+    @pytest.fixture
+    def temp_feedback_dir(self):
+        """Create temporary feedback directory for tests"""
+        temp_dir = tempfile.mkdtemp()
+        yield Path(temp_dir)
+        shutil.rmtree(temp_dir)
+
+    def test_capture_feedback_respects_skip_without_storing(self, temp_feedback_dir):
+        """
+        GIVEN user skips all questions
+        WHEN capture_feedback is called
+        THEN it does NOT create feedback file
+        AND returns skip status
+        """
+        # Arrange
+        story_id = 'STORY-004'
+        responses = [
+            {'question_id': 'dev_success_01', 'response': '', 'skip': True},
+            {'question_id': 'dev_success_02', 'response': '', 'skip': True},
+            {'question_id': 'dev_success_03', 'response': '', 'skip': True},
+            {'question_id': 'dev_success_04', 'response': '', 'skip': True},
+        ]
+
+        # Act
+        result = capture_feedback(
+            responses=responses,
+            story_id=story_id,
+            workflow_type='dev',
+            success_status='success',
+            feedback_dir=temp_feedback_dir,
+            allow_skip=True
+        )
+
+        # Assert
+        assert result['status'] == 'skipped'
+        assert 'message' in result
+        assert 'thanks' in result['message'].lower()
+
+        # Verify no file was created
+        story_feedback_dir = temp_feedback_dir / story_id
+        if story_feedback_dir.exists():
+            json_files = list(story_feedback_dir.glob('*-retrospective.json'))
+            assert len(json_files) == 0
+
+
+class TestTimestampAndMetadata:
+    """Test ISO 8601 timestamps and metadata generation"""
+
+    @pytest.fixture
+    def temp_feedback_dir(self):
+        temp_dir = tempfile.mkdtemp()
+        yield Path(temp_dir)
+        shutil.rmtree(temp_dir)
+
+    def test_capture_feedback_generates_iso8601_timestamp(self, temp_feedback_dir):
+        """
+        GIVEN feedback capture
+        WHEN feedback is stored
+        THEN timestamp is in ISO 8601 format
+        """
+        # Arrange
+        story_id = 'STORY-005'
+        responses = [
+            {'question_id': 'dev_success_01', 'response': 4, 'skip': False},
+            {'question_id': 'dev_success_02', 'response': 'Refactor', 'skip': False},
+        ]
+
+        # Act
+        result = capture_feedback(
+            responses=responses,
+            story_id=story_id,
+            workflow_type='dev',
+            success_status='success',
+            feedback_dir=temp_feedback_dir
+        )
+
+        # Assert
+        story_feedback_dir = temp_feedback_dir / story_id
+        json_files = list(story_feedback_dir.glob('*-retrospective.json'))
+
+        with open(json_files[0], 'r') as f:
+            feedback_data = json.load(f)
+
+        # Verify ISO 8601 format
+        timestamp = feedback_data['timestamp']
+        try:
+            datetime.fromisoformat(timestamp.replace('Z', '+00:00'))
+        except ValueError:
+            pytest.fail(f"Timestamp '{timestamp}' is not valid ISO 8601")
+
+        # Verify metadata completeness
+        metadata = feedback_data['metadata']
+        assert 'duration_seconds' in metadata
+        assert 'total_questions' in metadata
+        assert 'answered' in metadata
+        assert 'skipped' in metadata
+        assert metadata['total_questions'] == metadata['answered'] + metadata['skipped']