devforgeai 1.0.5 → 1.0.7

package/src/claude/scripts/devforgeai_cli/tests/feedback/test_validation_comprehensive.py

@@ -0,0 +1,210 @@
+"""Comprehensive unit tests for validation.py module.
+
+Tests all validation functions for complete coverage.
+"""
+
+import pytest
+from devforgeai_cli.feedback.validation import (
+    validate_response_length,
+    detect_spam,
+    is_coherent_text,
+    check_sensitive_content,
+    validate_story_id,
+    validate_workflow_type,
+)
+
+
+class TestValidateResponseLength:
+    """Test validate_response_length function."""
+
+    def test_valid_length_no_warning(self):
+        """Test response within valid range with no warning."""
+        response = "This is a normal response"
+        is_valid, warning = validate_response_length(response)
+        assert is_valid is True
+        assert warning is None
+
+    def test_too_short_response(self):
+        """Test response below minimum length."""
+        response = "Hi"
+        is_valid, warning = validate_response_length(response, min_length=5)
+        assert is_valid is False
+        assert "too short" in warning
+
+    def test_too_long_response(self):
+        """Test response exceeding maximum length."""
+        response = "x" * 11000
+        is_valid, warning = validate_response_length(response, max_length=10000)
+        assert is_valid is False
+        assert "too long" in warning
+
+    def test_warning_threshold_triggered(self):
+        """Test response triggering warning threshold."""
+        response = "x" * 2500
+        is_valid, warning = validate_response_length(response, warn_threshold=2000)
+        assert is_valid is True
+        assert warning is not None
+        assert "long" in warning
+
+
+class TestDetectSpam:
+    """Test detect_spam function."""
+
+    def test_empty_text_is_spam(self):
+        """Test that empty text is detected as spam (line 47)."""
+        assert detect_spam("") is True
+
+    def test_character_repetition_detected(self):
+        """Test detection of character repetition."""
+        # Need more than 10 chars with 3 or fewer unique chars
+        assert detect_spam("aaaaaaaaaaa") is True  # 11 a's (only 1 unique char)
+        assert detect_spam("abababababab") is True  # 12 chars (only 2 unique chars)
+        assert detect_spam("111111111111") is True  # 12 1's (only 1 unique char)
+
+    def test_pattern_repetition_detected(self):
+        """Test detection of pattern repetition (lines 61-70)."""
+        # Pattern "123" repeated
+        assert detect_spam("123123123123123123") is True
+        # Pattern "abc" repeated
+        assert detect_spam("abcabcabcabcabcabcabcabc") is True
+
+    def test_low_word_count_spam(self):
+        """Test detection of low word count with long text."""
+        # More than 50 chars but less than 5 words
+        # Need to have 4 or fewer words with more than 50 chars
+        assert detect_spam("aaa bbb ccc ddd" + "x" * 40) is True  # 4 words, 55 chars total
+
+    def test_high_non_alphanumeric_ratio(self):
+        """Test detection of high non-alphanumeric ratio."""
+        # Mostly special characters
+        assert detect_spam("!@#$%^&*()_+{}|:<>?[]\\;',./~`") is True
+
+    def test_valid_text_not_spam(self):
+        """Test that valid text is not flagged as spam."""
+        assert detect_spam("This is a coherent piece of feedback text") is False
+        assert detect_spam("I found the documentation unclear and would like more examples") is False
+
+
+class TestIsCoherentText:
+    """Test is_coherent_text function."""
+
+    def test_short_text_always_coherent(self):
+        """Test that short text (<5 chars) is always considered coherent (line 84)."""
+        assert is_coherent_text("hi") is True
+        assert is_coherent_text("ok") is True
+        assert is_coherent_text("yes") is True
+
+    def test_single_character_repetition(self):
+        """Test detection of single character repetition."""
+        assert is_coherent_text("aaaaaaaaaa") is False
+        assert is_coherent_text("1111111111") is False
+
+    def test_pattern_repetition_incoherent(self):
+        """Test detection of repeated patterns."""
+        # Pattern "123" repeated 3+ times
+        assert is_coherent_text("123123123") is False
+        # Pattern "ab" repeated 3+ times
+        assert is_coherent_text("ababababab") is False
+
+    def test_partial_pattern_match_at_end(self):
+        """Test detection of pattern with partial match at end."""
+        # Pattern "abc" repeated 3 times plus partial "ab"
+        assert is_coherent_text("abcabcabcab") is False
+
+    def test_coherent_text_accepted(self):
+        """Test that truly coherent text is accepted."""
+        assert is_coherent_text("This is coherent text") is True
+        assert is_coherent_text("I would like to provide feedback on the TDD workflow") is True
+
+
+class TestCheckSensitiveContent:
+    """Test check_sensitive_content function."""
+
+    def test_api_key_pattern_detected(self):
+        """Test detection of API key pattern (secret)."""
+        feedback = "The command exposed my key: sk-proj1234567890abcdefghij1234567890"
+        is_sensitive, types = check_sensitive_content(feedback)
+        assert is_sensitive is True
+        assert 'secret' in types
+
+    def test_api_key_word_detected(self):
+        """Test detection of 'api key' mention."""
+        feedback = "My API_KEY was visible in the output"
+        is_sensitive, types = check_sensitive_content(feedback)
+        assert is_sensitive is True
+        assert 'api_key' in types
+
+    def test_data_loss_concern_detected(self):
+        """Test detection of data loss concerns."""
+        feedback = "I experienced data loss during the migration"
+        is_sensitive, types = check_sensitive_content(feedback)
+        assert is_sensitive is True
+        assert 'data_loss' in types
+
+    def test_security_breach_detected(self):
+        """Test detection of security breach (lines 129-130)."""
+        feedback = "There was a security breach in the system"
+        is_sensitive, types = check_sensitive_content(feedback)
+        assert is_sensitive is True
+        assert 'critical_issue' in types
+
+    def test_vulnerability_without_api_key(self):
+        """Test vulnerability detection when no API key mentioned."""
+        feedback = "I found a vulnerability in the input validation"
+        is_sensitive, types = check_sensitive_content(feedback)
+        assert is_sensitive is True
+        assert 'critical_issue' in types
+
+    def test_non_sensitive_feedback(self):
+        """Test that normal feedback is not flagged as sensitive."""
+        feedback = "The documentation could be clearer"
+        is_sensitive, types = check_sensitive_content(feedback)
+        assert is_sensitive is False
+        assert len(types) == 0
+
+    def test_multiple_sensitive_types(self):
+        """Test detection of multiple sensitive types."""
+        feedback = "API key exposed and data loss occurred in production database"
+        is_sensitive, types = check_sensitive_content(feedback)
+        assert is_sensitive is True
+        assert 'api_key' in types
+        assert 'data_loss' in types
+
+
+class TestValidateStoryId:
+    """Test validate_story_id function."""
+
+    def test_valid_story_ids(self):
+        """Test that valid story IDs are accepted."""
+        assert validate_story_id("STORY-001") is True
+        assert validate_story_id("STORY-123") is True
+        assert validate_story_id("STORY-999") is True
+        assert validate_story_id("STORY-1") is True
+
+    def test_invalid_story_ids(self):
+        """Test that invalid story IDs are rejected."""
+        assert validate_story_id("story-001") is False  # lowercase
+        assert validate_story_id("STORY-abc") is False  # non-numeric
+        assert validate_story_id("TASK-001") is False   # wrong prefix
+        assert validate_story_id("STORY001") is False   # missing hyphen
+        assert validate_story_id("STORY-") is False     # missing number
+
+
+class TestValidateWorkflowType:
+    """Test validate_workflow_type function."""
+
+    def test_valid_workflow_types(self):
+        """Test that all valid workflow types are accepted."""
+        valid_types = [
+            'dev', 'qa', 'orchestrate', 'release', 'ideate',
+            'create-story', 'create-epic', 'create-sprint'
+        ]
+        for workflow_type in valid_types:
+            assert validate_workflow_type(workflow_type) is True
+
+    def test_invalid_workflow_types(self):
+        """Test that invalid workflow types are rejected."""
+        assert validate_workflow_type('invalid') is False
+        assert validate_workflow_type('deploy') is False
+        assert validate_workflow_type('build') is False
+        assert validate_workflow_type('') is False

package/src/claude/scripts/devforgeai_cli/tests/fixtures/autonomous-deferral-story.md

@@ -0,0 +1,46 @@
+---
+id: STORY-TEST-002
+title: Test Story - Autonomous Deferral (VIOLATION)
+epic: EPIC-001
+status: Dev Complete
+priority: HIGH
+points: 8
+---
+
+# Test Story - Autonomous Deferral
+
+This story demonstrates the autonomous deferral violation.
+DoD marked [x] but Implementation Notes shows [ ] WITHOUT user approval.
+
+## Definition of Done
+
+- [x] Unit tests written and passing
+- [x] Integration tests created
+- [x] Deadlock retry with exponential backoff
+- [x] Code follows coding-standards.md
+
+## Acceptance Criteria
+
+- Given a test scenario
+- When executed
+- Then should pass
+
+## Implementation Notes
+
+**Developer:** DevForgeAI AI Agent
+**Implemented:** 2025-11-04
+
+- [x] Unit tests written and passing - Completed: Created tests
+- [x] Integration tests created - Completed: Created integration tests
+- [ ] Deadlock retry with exponential backoff - Deferred to STORY-XXX: Complex retry logic
+- [x] Code follows coding-standards.md - Completed: Applied standards
+
+**VIOLATION:** "Deadlock retry" is marked [x] in DoD but [ ] in Implementation Notes
+WITHOUT "User approved:" marker or valid reference.
+
+This is AUTONOMOUS DEFERRAL - should FAIL validation.
+
+## Workflow History
+
+### 2025-11-04 - Status: Dev Complete
+- Implementation complete (partial)

package/src/claude/scripts/devforgeai_cli/tests/fixtures/missing-impl-notes.md

@@ -0,0 +1,31 @@
+---
+id: STORY-TEST-004
+title: Test Story - Missing Implementation Notes (VIOLATION)
+epic: EPIC-001
+status: Dev Complete
+priority: HIGH
+points: 3
+---
+
+# Test Story - Missing Implementation Notes
+
+This story demonstrates missing Implementation Notes section.
+Should FAIL validation.
+
+## Definition of Done
+
+- [x] Unit tests written and passing
+- [x] Code follows coding-standards.md
+
+## Acceptance Criteria
+
+- Given a test scenario
+- When executed
+- Then should pass
+
+## Workflow History
+
+### 2025-11-04 - Status: Dev Complete
+- Implementation complete
+
+**VIOLATION:** No Implementation Notes section - should FAIL validation.

package/src/claude/scripts/devforgeai_cli/tests/fixtures/valid-deferral-story.md

@@ -0,0 +1,46 @@
+---
+id: STORY-TEST-003
+title: Test Story - Valid Deferral with User Approval
+epic: EPIC-001
+status: Dev Complete
+priority: HIGH
+points: 8
+---
+
+# Test Story - Valid Deferral
+
+This story demonstrates a VALID deferral with proper user approval marker.
+
+## Definition of Done
+
+- [x] Unit tests written and passing
+- [x] Integration tests created
+- [x] Performance benchmarks created
+- [x] Code follows coding-standards.md
+
+## Acceptance Criteria
+
+- Given a test scenario
+- When executed
+- Then should pass
+
+## Implementation Notes
+
+**Developer:** DevForgeAI AI Agent
+**Implemented:** 2025-11-04
+
+- [x] Unit tests written and passing - Completed: Created 10 unit tests
+- [x] Integration tests created - Completed: Created integration suite
+- [ ] Performance benchmarks created - Deferred to STORY-042
+  **User Approved:** YES (via AskUserQuestion on 2025-11-04)
+  **Approval Method:** AskUserQuestion presented 4 options
+  **Selected Option:** "Defer to follow-up story"
+  **Rationale:** Performance testing requires load testing environment (STORY-041 prerequisite)
+- [x] Code follows coding-standards.md - Completed: Applied standards
+
+**VALID:** Deferral has "User Approved: YES" marker - should PASS validation.
+
+## Workflow History
+
+### 2025-11-04 - Status: Dev Complete
+- Implementation complete with justified deferral

package/src/claude/scripts/devforgeai_cli/tests/fixtures/valid-story-complete.md

@@ -0,0 +1,48 @@
+---
+id: STORY-TEST-001
+title: Test Story - All DoD Complete
+epic: EPIC-001
+status: Dev Complete
+priority: HIGH
+points: 5
+---
+
+# Test Story - All DoD Complete
+
+This is a test story with all DoD items completed.
+
+## Definition of Done
+
+- [x] Unit tests written and passing
+- [x] Integration tests created
+- [x] Code follows coding-standards.md
+- [x] Documentation updated
+- [x] All acceptance criteria met
+
+## Acceptance Criteria
+
+- Given a user
+- When action performed
+- Then expected result
+
+## Implementation Notes
+
+**Developer:** DevForgeAI AI Agent
+**Implemented:** 2025-11-04
+
+- [x] Unit tests written and passing - Completed: Created 10 unit tests, all passing
+- [x] Integration tests created - Completed: Created 3 integration tests with fixtures
+- [x] Code follows coding-standards.md - Completed: Applied naming conventions, error handling
+- [x] Documentation updated - Completed: Updated API docs and README
+- [x] All acceptance criteria met - Completed: Verified via test suite
+
+### Files Created
+- src/module.py
+- tests/test_module.py
+
+## Workflow History
+
+### 2025-11-04 - Status: Dev Complete
+- All DoD items completed
+- Tests passing
+- Ready for QA

package/src/claude/scripts/devforgeai_cli/tests/manual_test_invoke_hooks.sh

@@ -0,0 +1,200 @@
+#!/bin/bash
+# Manual Test Script for STORY-022: invoke-hooks CLI command
+# This script tests the invoke-hooks command manually to verify DoD items
+
+set -e
+
+echo "=========================================="
+echo "Manual Test Suite for invoke-hooks Command"
+echo "=========================================="
+echo ""
+
+# Colors for output
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Test counter
+PASS=0
+FAIL=0
+
+# Helper function to run test
+run_test() {
+    local test_name="$1"
+    local command="$2"
+    local expected_result="$3"
+
+    echo -e "${YELLOW}[TEST]${NC} $test_name"
+    echo "Command: $command"
+
+    if eval "$command"; then
+        if [ "$expected_result" = "success" ]; then
+            echo -e "${GREEN}✓ PASSED${NC}"
+            ((PASS++))
+        else
+            echo -e "${RED}✗ FAILED${NC} (Expected failure but got success)"
+            ((FAIL++))
+        fi
+    else
+        if [ "$expected_result" = "failure" ]; then
+            echo -e "${GREEN}✓ PASSED${NC}"
+            ((PASS++))
+        else
+            echo -e "${RED}✗ FAILED${NC} (Expected success but got failure)"
+            ((FAIL++))
+        fi
+    fi
+    echo ""
+}
+
+# Test 1: Manual test - invoke-hooks triggers feedback conversation
+echo "=========================================="
+echo "Test 1: invoke-hooks triggers feedback conversation"
+echo "=========================================="
+echo ""
+echo "This test verifies that running invoke-hooks attempts to invoke the feedback skill."
+echo "Note: This is a mock test since devforgeai-feedback skill integration is not yet complete."
+echo ""
+
+# Create a mock test that verifies the command runs
+run_test \
+    "Invoke hooks with operation=dev and story=STORY-001" \
+    "python3 -c \"
+import sys
+sys.path.insert(0, '.claude/scripts')
+from devforgeai_cli.hooks import invoke_hooks
+
+# Mock invocation (will fail gracefully since feedback skill isn't integrated)
+result = invoke_hooks('dev', 'STORY-001')
+print(f'invoke_hooks returned: {result}')
+# We expect False since feedback skill isn't actually callable yet
+sys.exit(0 if result == False else 1)
+\"" \
+    "success"
+
+echo "Manual verification needed: The invoke_hooks() function was called."
+echo "Actual feedback skill integration will be completed in STORY-023."
+echo ""
+
+# Test 2: Manual test - Context includes todos, errors, timing
+echo "=========================================="
+echo "Test 2: Context includes todos, errors, timing"
+echo "=========================================="
+echo ""
+
+run_test \
+    "Extract context and verify it contains todos, errors, timing" \
+    "python3 -c \"
+import sys
+sys.path.insert(0, '.claude/scripts')
+from devforgeai_cli.context_extraction import extract_context
+
+# Extract context with test operation
+context = extract_context('dev', 'STORY-001')
+
+# Verify structure
+assert 'operation_id' in context, 'Missing operation_id'
+assert 'operation' in context, 'Missing operation'
+assert 'story_id' in context, 'Missing story_id'
+assert 'start_time' in context, 'Missing start_time'
+assert 'end_time' in context, 'Missing end_time'
+assert 'duration' in context, 'Missing duration'
+assert 'status' in context, 'Missing status'
+assert 'todos' in context, 'Missing todos'
+assert 'errors' in context, 'Missing errors'
+assert 'context_size_bytes' in context, 'Missing context_size_bytes'
+
+print('✓ Context structure validated')
+print(f'  - operation_id: {context[\"operation_id\"]}')
+print(f'  - operation: {context[\"operation\"]}')
+print(f'  - story_id: {context[\"story_id\"]}')
+print(f'  - duration: {context[\"duration\"]}s')
+print(f'  - todos: {len(context[\"todos\"])} items')
+print(f'  - errors: {len(context[\"errors\"])} items')
+print(f'  - context_size: {context[\"context_size_bytes\"]} bytes')
+\"" \
+    "success"
+
+# Test 3: Manual test - Timeout triggers after 30s
+echo "=========================================="
+echo "Test 3: Timeout triggers after 30s (LONG TEST)"
+echo "=========================================="
+echo ""
+echo "WARNING: This test will take 30+ seconds to complete."
+echo "Press Ctrl+C to skip, or wait for timeout verification."
+echo ""
+
+# Give user 5 seconds to cancel
+sleep 3
+
+run_test \
+    "Verify 30-second timeout triggers and aborts" \
+    "python3 -c \"
+import sys
+import time
+import threading
+sys.path.insert(0, '.claude/scripts')
+from devforgeai_cli.hooks import HookInvocationService
+
+# Create service
+service = HookInvocationService()
+
+# Mock a slow skill invocation (35 seconds)
+def slow_skill():
+    time.sleep(35)
+    return True
+
+# Override invoke_feedback_skill to use slow_skill
+original_method = service.invoke_feedback_skill
+service.invoke_feedback_skill = lambda context: slow_skill()
+
+# Call invoke (should timeout after 30s)
+start = time.time()
+result = service.invoke('dev', 'STORY-001')
+elapsed = time.time() - start
+
+print(f'Elapsed time: {elapsed:.1f}s')
+print(f'Result: {result}')
+
+# Verify timeout occurred (~30s, allow ±2s margin)
+if 28 <= elapsed <= 32 and result == False:
+    print('✓ Timeout triggered correctly after ~30s')
+    sys.exit(0)
+else:
+    print(f'✗ Timeout failed: elapsed={elapsed:.1f}s, result={result}')
+    sys.exit(1)
+\"" \
+    "success"
+
+# Test 4: Integration test - Called from /dev command
+echo "=========================================="
+echo "Test 4: Integration test - Called from /dev command"
+echo "=========================================="
+echo ""
+echo "This test verifies the CLI command can be invoked from command line."
+echo ""
+
+run_test \
+    "Run invoke-hooks CLI command" \
+    "cd .claude/scripts && python3 -m devforgeai_cli.cli invoke-hooks --operation dev --story STORY-001 --verbose 2>&1 | head -20" \
+    "failure"  # Expected to fail since feedback skill isn't integrated
+
+echo "Note: Command executed but failed gracefully (expected until STORY-023)."
+echo ""
+
+# Summary
+echo "=========================================="
+echo "Test Summary"
+echo "=========================================="
+echo -e "${GREEN}Passed: $PASS${NC}"
+echo -e "${RED}Failed: $FAIL${NC}"
+echo ""
+
+if [ $FAIL -eq 0 ]; then
+    echo -e "${GREEN}All manual tests PASSED ✓${NC}"
+    exit 0
+else
+    echo -e "${RED}Some tests FAILED ✗${NC}"
+    exit 1
+fi