devforgeai 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +120 -0
- package/package.json +9 -1
- package/src/CLAUDE.md +699 -0
- package/src/claude/scripts/README.md +396 -0
- package/src/claude/scripts/audit-command-skill-overlap.sh +67 -0
- package/src/claude/scripts/check-hooks-fast.sh +70 -0
- package/src/claude/scripts/devforgeai-validate +6 -0
- package/src/claude/scripts/devforgeai_cli/README.md +531 -0
- package/src/claude/scripts/devforgeai_cli/__init__.py +12 -0
- package/src/claude/scripts/devforgeai_cli/cli.py +716 -0
- package/src/claude/scripts/devforgeai_cli/commands/__init__.py +1 -0
- package/src/claude/scripts/devforgeai_cli/commands/check_hooks.py +384 -0
- package/src/claude/scripts/devforgeai_cli/commands/invoke_hooks.py +149 -0
- package/src/claude/scripts/devforgeai_cli/commands/phase_commands.py +731 -0
- package/src/claude/scripts/devforgeai_cli/commands/validate_installation.py +412 -0
- package/src/claude/scripts/devforgeai_cli/context_extraction.py +426 -0
- package/src/claude/scripts/devforgeai_cli/feedback/AC_TO_TEST_MAPPING.md +636 -0
- package/src/claude/scripts/devforgeai_cli/feedback/DELIVERY_SUMMARY.txt +329 -0
- package/src/claude/scripts/devforgeai_cli/feedback/README_TEST_SPECS.md +486 -0
- package/src/claude/scripts/devforgeai_cli/feedback/TEST_IMPLEMENTATION_GUIDE.md +529 -0
- package/src/claude/scripts/devforgeai_cli/feedback/TEST_SPECIFICATIONS.md +2652 -0
- package/src/claude/scripts/devforgeai_cli/feedback/TEST_SPECS_INDEX.md +398 -0
- package/src/claude/scripts/devforgeai_cli/feedback/__init__.py +34 -0
- package/src/claude/scripts/devforgeai_cli/feedback/adaptive_questioning_engine.py +581 -0
- package/src/claude/scripts/devforgeai_cli/feedback/aggregation.py +179 -0
- package/src/claude/scripts/devforgeai_cli/feedback/commands.py +535 -0
- package/src/claude/scripts/devforgeai_cli/feedback/config_defaults.py +58 -0
- package/src/claude/scripts/devforgeai_cli/feedback/config_manager.py +423 -0
- package/src/claude/scripts/devforgeai_cli/feedback/config_models.py +192 -0
- package/src/claude/scripts/devforgeai_cli/feedback/config_schema.py +140 -0
- package/src/claude/scripts/devforgeai_cli/feedback/coverage.json +1 -0
- package/src/claude/scripts/devforgeai_cli/feedback/feature_flag.py +152 -0
- package/src/claude/scripts/devforgeai_cli/feedback/feedback_indexer.py +394 -0
- package/src/claude/scripts/devforgeai_cli/feedback/hot_reload.py +226 -0
- package/src/claude/scripts/devforgeai_cli/feedback/longitudinal.py +115 -0
- package/src/claude/scripts/devforgeai_cli/feedback/models.py +67 -0
- package/src/claude/scripts/devforgeai_cli/feedback/question_router.py +236 -0
- package/src/claude/scripts/devforgeai_cli/feedback/retrospective.py +233 -0
- package/src/claude/scripts/devforgeai_cli/feedback/skip_tracker.py +177 -0
- package/src/claude/scripts/devforgeai_cli/feedback/skip_tracking.py +221 -0
- package/src/claude/scripts/devforgeai_cli/feedback/template_engine.py +549 -0
- package/src/claude/scripts/devforgeai_cli/feedback/validation.py +163 -0
- package/src/claude/scripts/devforgeai_cli/headless/__init__.py +30 -0
- package/src/claude/scripts/devforgeai_cli/headless/answer_models.py +206 -0
- package/src/claude/scripts/devforgeai_cli/headless/answer_resolver.py +204 -0
- package/src/claude/scripts/devforgeai_cli/headless/exceptions.py +36 -0
- package/src/claude/scripts/devforgeai_cli/headless/pattern_matcher.py +156 -0
- package/src/claude/scripts/devforgeai_cli/hooks.py +313 -0
- package/src/claude/scripts/devforgeai_cli/metrics/__init__.py +46 -0
- package/src/claude/scripts/devforgeai_cli/metrics/command_metrics.py +142 -0
- package/src/claude/scripts/devforgeai_cli/metrics/failure_modes.py +152 -0
- package/src/claude/scripts/devforgeai_cli/metrics/story_segmentation.py +181 -0
- package/src/claude/scripts/devforgeai_cli/orchestrate_hooks.py +780 -0
- package/src/claude/scripts/devforgeai_cli/phase_state.py +1229 -0
- package/src/claude/scripts/devforgeai_cli/session/__init__.py +30 -0
- package/src/claude/scripts/devforgeai_cli/session/checkpoint.py +268 -0
- package/src/claude/scripts/devforgeai_cli/tests/__init__.py +1 -0
- package/src/claude/scripts/devforgeai_cli/tests/conftest.py +29 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/TEST_EXECUTION_GUIDE.md +298 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/__init__.py +3 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_adaptive_questioning_engine.py +2171 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_aggregation.py +476 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_config_defaults.py +133 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_config_manager.py +592 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_config_models.py +373 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_config_schema.py +130 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_configuration_management.py +1355 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_edge_cases.py +308 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_feature_flag.py +307 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_feedback_indexer.py +384 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_hot_reload.py +580 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_integration.py +402 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_models.py +105 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_question_routing.py +262 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_retrospective.py +333 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_skip_tracker.py +410 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_skip_tracking.py +159 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_skip_tracking_integration.py +1155 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_template_engine.py +1389 -0
- package/src/claude/scripts/devforgeai_cli/tests/feedback/test_validation_comprehensive.py +210 -0
- package/src/claude/scripts/devforgeai_cli/tests/fixtures/autonomous-deferral-story.md +46 -0
- package/src/claude/scripts/devforgeai_cli/tests/fixtures/missing-impl-notes.md +31 -0
- package/src/claude/scripts/devforgeai_cli/tests/fixtures/valid-deferral-story.md +46 -0
- package/src/claude/scripts/devforgeai_cli/tests/fixtures/valid-story-complete.md +48 -0
- package/src/claude/scripts/devforgeai_cli/tests/manual_test_invoke_hooks.sh +200 -0
- package/src/claude/scripts/devforgeai_cli/tests/session/DELIVERABLES.md +518 -0
- package/src/claude/scripts/devforgeai_cli/tests/session/TEST_SUMMARY.md +468 -0
- package/src/claude/scripts/devforgeai_cli/tests/session/__init__.py +6 -0
- package/src/claude/scripts/devforgeai_cli/tests/session/fixtures/corrupted-checkpoint.json +1 -0
- package/src/claude/scripts/devforgeai_cli/tests/session/fixtures/missing-fields-checkpoint.json +4 -0
- package/src/claude/scripts/devforgeai_cli/tests/session/fixtures/valid-checkpoint.json +15 -0
- package/src/claude/scripts/devforgeai_cli/tests/session/test_checkpoint.py +851 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_check_hooks.py +1886 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_depends_on_normalizer.py +171 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_dod_validator.py +97 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_invoke_hooks.py +1902 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_phase_commands.py +320 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_phase_commands_error_handling.py +1021 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_phase_commands_import.py +697 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_phase_state.py +2187 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_skip_tracking.py +2141 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_skip_tracking_coverage_gap.py +195 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_subagent_enforcement.py +539 -0
- package/src/claude/scripts/devforgeai_cli/tests/test_validate_installation.py +361 -0
- package/src/claude/scripts/devforgeai_cli/utils/__init__.py +11 -0
- package/src/claude/scripts/devforgeai_cli/utils/depends_on_normalizer.py +149 -0
- package/src/claude/scripts/devforgeai_cli/utils/markdown_parser.py +219 -0
- package/src/claude/scripts/devforgeai_cli/utils/story_analyzer.py +249 -0
- package/src/claude/scripts/devforgeai_cli/utils/yaml_parser.py +152 -0
- package/src/claude/scripts/devforgeai_cli/validators/__init__.py +27 -0
- package/src/claude/scripts/devforgeai_cli/validators/ast_grep_validator.py +373 -0
- package/src/claude/scripts/devforgeai_cli/validators/context_validator.py +180 -0
- package/src/claude/scripts/devforgeai_cli/validators/dod_validator.py +309 -0
- package/src/claude/scripts/devforgeai_cli/validators/git_validator.py +107 -0
- package/src/claude/scripts/devforgeai_cli/validators/grep_fallback.py +300 -0
- package/src/claude/scripts/install_hooks.sh +186 -0
- package/src/claude/scripts/invoke_feedback_hooks.sh +59 -0
- package/src/claude/scripts/migrate-ac-headers.sh +122 -0
- package/src/claude/scripts/plan_file_kb.sh +704 -0
- package/src/claude/scripts/requirements.txt +8 -0
- package/src/claude/scripts/session_catalog.sh +543 -0
- package/src/claude/scripts/setup.py +55 -0
- package/src/claude/scripts/start-devforgeai.sh +16 -0
- package/src/claude/scripts/statusline.sh +27 -0
- package/src/claude/scripts/validate_deferrals.py +344 -0
- package/src/claude/skills/devforgeai-qa/SKILL.md +1 -1
- package/src/claude/skills/researching-market/SKILL.md +2 -1
- package/src/cli/lib/copier.js +13 -1
- package/src/claude/skills/designing-systems/scripts/__pycache__/detect_anti_patterns.cpython-312.pyc +0 -0
- package/src/claude/skills/designing-systems/scripts/__pycache__/validate_all_context.cpython-312.pyc +0 -0
- package/src/claude/skills/designing-systems/scripts/__pycache__/validate_architecture.cpython-312.pyc +0 -0
- package/src/claude/skills/designing-systems/scripts/__pycache__/validate_dependencies.cpython-312.pyc +0 -0
- package/src/claude/skills/devforgeai-story-creation/scripts/__pycache__/migrate_story_v1_to_v2.cpython-312.pyc +0 -0
- package/src/claude/skills/devforgeai-story-creation/scripts/tests/__pycache__/measure_accuracy.cpython-312.pyc +0 -0
|
@@ -0,0 +1,2141 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Comprehensive test suite for Skip Pattern Tracking (STORY-009)
|
|
4
|
+
|
|
5
|
+
Tests cover:
|
|
6
|
+
- AC1: Skip Counter Tracks Operations (per operation type, persistent across sessions)
|
|
7
|
+
- AC2: Pattern Detection at 3+ Consecutive Skips (triggers once per session)
|
|
8
|
+
- AC3: Preference Storage and Enforcement (disabled feedback types stored in YAML)
|
|
9
|
+
- AC4: Skip Counter Reset on Preference Change (reset to 0 when re-enabled)
|
|
10
|
+
- AC5: Token Waste Calculation (1500 tokens per prompt × skip count)
|
|
11
|
+
- AC6: Multi-Operation-Type Tracking (independent counters per type)
|
|
12
|
+
|
|
13
|
+
Edge cases:
|
|
14
|
+
- User skips on first attempt (counter=1, no pattern)
|
|
15
|
+
- Non-consecutive skips reset counter (breaks sequence)
|
|
16
|
+
- Missing config file (auto-created)
|
|
17
|
+
- Manual config edit inconsistency (disabled_feedback enforced)
|
|
18
|
+
- Corrupted config file (backup + fresh)
|
|
19
|
+
- Cross-session persistence (Session 1: 2 skips, Session 2: 1 skip = 3 total)
|
|
20
|
+
|
|
21
|
+
Test structure:
|
|
22
|
+
- 25+ Unit Tests: counter logic, pattern detection, config parsing, validation
|
|
23
|
+
- 10+ Integration Tests: skip → pattern → preference → enforcement chain
|
|
24
|
+
- 8+ E2E Tests: full workflows from first skip to re-enable
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
import pytest
|
|
28
|
+
import tempfile
|
|
29
|
+
import shutil
|
|
30
|
+
import json
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from datetime import datetime, UTC
|
|
33
|
+
from unittest.mock import Mock, patch, MagicMock
|
|
34
|
+
import yaml
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# ============================================================================
|
|
38
|
+
# FIXTURES - Setup/Teardown for Config Management
|
|
39
|
+
# ============================================================================
|
|
40
|
+
|
|
41
|
+
@pytest.fixture
|
|
42
|
+
def temp_config_dir():
|
|
43
|
+
"""Create temporary config directory for tests."""
|
|
44
|
+
temp_dir = tempfile.mkdtemp()
|
|
45
|
+
yield Path(temp_dir)
|
|
46
|
+
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@pytest.fixture
|
|
50
|
+
def sample_config():
|
|
51
|
+
"""Sample feedback preferences config structure."""
|
|
52
|
+
return {
|
|
53
|
+
'version': '1.0',
|
|
54
|
+
'created_at': '2025-11-07T10:30:00Z',
|
|
55
|
+
'last_updated': '2025-11-07T10:30:00Z',
|
|
56
|
+
'skip_counters': {
|
|
57
|
+
'skill_invocation': 0,
|
|
58
|
+
'subagent_invocation': 0,
|
|
59
|
+
'command_execution': 0,
|
|
60
|
+
'context_loading': 0,
|
|
61
|
+
},
|
|
62
|
+
'disabled_feedback': {
|
|
63
|
+
'skill_invocation': False,
|
|
64
|
+
'subagent_invocation': False,
|
|
65
|
+
'command_execution': False,
|
|
66
|
+
'context_loading': False,
|
|
67
|
+
},
|
|
68
|
+
'disable_reasons': {
|
|
69
|
+
'skill_invocation': None,
|
|
70
|
+
'subagent_invocation': None,
|
|
71
|
+
'command_execution': None,
|
|
72
|
+
'context_loading': None,
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@pytest.fixture
|
|
78
|
+
def config_file_path(temp_config_dir):
|
|
79
|
+
"""Path to config file in temp directory."""
|
|
80
|
+
config_dir = temp_config_dir / 'feedback-preferences.yaml'
|
|
81
|
+
return config_dir
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# ============================================================================
|
|
85
|
+
# UNIT TESTS - Counter Logic (AC1)
|
|
86
|
+
# ============================================================================
|
|
87
|
+
|
|
88
|
+
class TestSkipCounterLogic:
|
|
89
|
+
"""Test skip counter increment and storage logic (AC1)."""
|
|
90
|
+
|
|
91
|
+
def test_increment_counter_single_operation_type(self, temp_config_dir, sample_config):
|
|
92
|
+
"""
|
|
93
|
+
GIVEN a user executes an operation that triggers feedback skip
|
|
94
|
+
WHEN skip counter is incremented for skill_invocation
|
|
95
|
+
THEN counter increments correctly per operation type
|
|
96
|
+
"""
|
|
97
|
+
# Arrange
|
|
98
|
+
operation_type = 'skill_invocation'
|
|
99
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
100
|
+
|
|
101
|
+
# Act
|
|
102
|
+
sample_config['skip_counters'][operation_type] = 1
|
|
103
|
+
with open(config_file, 'w') as f:
|
|
104
|
+
yaml.dump(sample_config, f)
|
|
105
|
+
|
|
106
|
+
with open(config_file, 'r') as f:
|
|
107
|
+
loaded = yaml.safe_load(f)
|
|
108
|
+
|
|
109
|
+
# Assert
|
|
110
|
+
assert loaded['skip_counters'][operation_type] == 1
|
|
111
|
+
assert loaded['skip_counters']['subagent_invocation'] == 0
|
|
112
|
+
|
|
113
|
+
def test_increment_counter_multiple_times(self, temp_config_dir, sample_config):
|
|
114
|
+
"""
|
|
115
|
+
GIVEN skip counter is 0
|
|
116
|
+
WHEN incremented 5 times for same operation type
|
|
117
|
+
THEN counter shows 5
|
|
118
|
+
"""
|
|
119
|
+
# Arrange
|
|
120
|
+
operation_type = 'command_execution'
|
|
121
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
122
|
+
|
|
123
|
+
# Act
|
|
124
|
+
for i in range(5):
|
|
125
|
+
sample_config['skip_counters'][operation_type] += 1
|
|
126
|
+
|
|
127
|
+
with open(config_file, 'w') as f:
|
|
128
|
+
yaml.dump(sample_config, f)
|
|
129
|
+
|
|
130
|
+
with open(config_file, 'r') as f:
|
|
131
|
+
loaded = yaml.safe_load(f)
|
|
132
|
+
|
|
133
|
+
# Assert
|
|
134
|
+
assert loaded['skip_counters'][operation_type] == 5
|
|
135
|
+
|
|
136
|
+
def test_counter_persists_across_sessions(self, temp_config_dir, sample_config):
|
|
137
|
+
"""
|
|
138
|
+
GIVEN counter is incremented to 2 in Session 1
|
|
139
|
+
WHEN Session 2 starts and reads config
|
|
140
|
+
THEN counter still shows 2 (persistence verified)
|
|
141
|
+
"""
|
|
142
|
+
# Arrange
|
|
143
|
+
operation_type = 'skill_invocation'
|
|
144
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
145
|
+
|
|
146
|
+
# Act - Session 1: Write counter = 2
|
|
147
|
+
sample_config['skip_counters'][operation_type] = 2
|
|
148
|
+
with open(config_file, 'w') as f:
|
|
149
|
+
yaml.dump(sample_config, f)
|
|
150
|
+
|
|
151
|
+
# Act - Session 2: Read counter
|
|
152
|
+
with open(config_file, 'r') as f:
|
|
153
|
+
loaded = yaml.safe_load(f)
|
|
154
|
+
|
|
155
|
+
# Assert
|
|
156
|
+
assert loaded['skip_counters'][operation_type] == 2
|
|
157
|
+
|
|
158
|
+
def test_counter_storage_yaml_format(self, temp_config_dir, sample_config):
|
|
159
|
+
"""
|
|
160
|
+
GIVEN counter is incremented
|
|
161
|
+
WHEN config saved to YAML
|
|
162
|
+
THEN file format is valid YAML with proper structure
|
|
163
|
+
"""
|
|
164
|
+
# Arrange
|
|
165
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
166
|
+
sample_config['skip_counters']['skill_invocation'] = 3
|
|
167
|
+
|
|
168
|
+
# Act
|
|
169
|
+
with open(config_file, 'w') as f:
|
|
170
|
+
yaml.dump(sample_config, f)
|
|
171
|
+
|
|
172
|
+
# Assert - Verify YAML can be read back
|
|
173
|
+
with open(config_file, 'r') as f:
|
|
174
|
+
loaded = yaml.safe_load(f)
|
|
175
|
+
|
|
176
|
+
assert isinstance(loaded, dict)
|
|
177
|
+
assert 'skip_counters' in loaded
|
|
178
|
+
assert isinstance(loaded['skip_counters'], dict)
|
|
179
|
+
|
|
180
|
+
def test_counter_respects_operation_type_independence(self, temp_config_dir, sample_config):
|
|
181
|
+
"""
|
|
182
|
+
GIVEN multiple operation types tracked
|
|
183
|
+
WHEN incrementing one type
|
|
184
|
+
THEN other types remain unchanged
|
|
185
|
+
"""
|
|
186
|
+
# Arrange
|
|
187
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
188
|
+
|
|
189
|
+
# Act
|
|
190
|
+
sample_config['skip_counters']['skill_invocation'] = 5
|
|
191
|
+
sample_config['skip_counters']['subagent_invocation'] = 2
|
|
192
|
+
|
|
193
|
+
with open(config_file, 'w') as f:
|
|
194
|
+
yaml.dump(sample_config, f)
|
|
195
|
+
|
|
196
|
+
with open(config_file, 'r') as f:
|
|
197
|
+
loaded = yaml.safe_load(f)
|
|
198
|
+
|
|
199
|
+
# Assert - Each counter independent
|
|
200
|
+
assert loaded['skip_counters']['skill_invocation'] == 5
|
|
201
|
+
assert loaded['skip_counters']['subagent_invocation'] == 2
|
|
202
|
+
assert loaded['skip_counters']['command_execution'] == 0
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
# ============================================================================
|
|
206
|
+
# UNIT TESTS - Pattern Detection (AC2)
|
|
207
|
+
# ============================================================================
|
|
208
|
+
|
|
209
|
+
class TestPatternDetection:
|
|
210
|
+
"""Test pattern detection at 3+ consecutive skips (AC2)."""
|
|
211
|
+
|
|
212
|
+
def test_pattern_not_triggered_at_1_skip(self, sample_config):
|
|
213
|
+
"""
|
|
214
|
+
GIVEN skip counter = 1
|
|
215
|
+
WHEN check for pattern detection
|
|
216
|
+
THEN pattern NOT triggered (needs 3+)
|
|
217
|
+
"""
|
|
218
|
+
# Arrange
|
|
219
|
+
sample_config['skip_counters']['skill_invocation'] = 1
|
|
220
|
+
|
|
221
|
+
# Act
|
|
222
|
+
threshold = 3
|
|
223
|
+
should_trigger = sample_config['skip_counters']['skill_invocation'] >= threshold
|
|
224
|
+
|
|
225
|
+
# Assert
|
|
226
|
+
assert should_trigger is False
|
|
227
|
+
|
|
228
|
+
def test_pattern_not_triggered_at_2_skips(self, sample_config):
|
|
229
|
+
"""
|
|
230
|
+
GIVEN skip counter = 2
|
|
231
|
+
WHEN check for pattern detection
|
|
232
|
+
THEN pattern NOT triggered
|
|
233
|
+
"""
|
|
234
|
+
# Arrange
|
|
235
|
+
sample_config['skip_counters']['skill_invocation'] = 2
|
|
236
|
+
|
|
237
|
+
# Act
|
|
238
|
+
threshold = 3
|
|
239
|
+
should_trigger = sample_config['skip_counters']['skill_invocation'] >= threshold
|
|
240
|
+
|
|
241
|
+
# Assert
|
|
242
|
+
assert should_trigger is False
|
|
243
|
+
|
|
244
|
+
def test_pattern_triggered_at_3_skips(self, sample_config):
|
|
245
|
+
"""
|
|
246
|
+
GIVEN skip counter = 3
|
|
247
|
+
WHEN check for pattern detection
|
|
248
|
+
THEN pattern IS triggered
|
|
249
|
+
"""
|
|
250
|
+
# Arrange
|
|
251
|
+
sample_config['skip_counters']['skill_invocation'] = 3
|
|
252
|
+
|
|
253
|
+
# Act
|
|
254
|
+
threshold = 3
|
|
255
|
+
should_trigger = sample_config['skip_counters']['skill_invocation'] >= threshold
|
|
256
|
+
|
|
257
|
+
# Assert
|
|
258
|
+
assert should_trigger is True
|
|
259
|
+
|
|
260
|
+
def test_pattern_triggered_at_5_skips(self, sample_config):
|
|
261
|
+
"""
|
|
262
|
+
GIVEN skip counter = 5
|
|
263
|
+
WHEN check for pattern detection
|
|
264
|
+
THEN pattern IS triggered
|
|
265
|
+
"""
|
|
266
|
+
# Arrange
|
|
267
|
+
sample_config['skip_counters']['skill_invocation'] = 5
|
|
268
|
+
|
|
269
|
+
# Act
|
|
270
|
+
threshold = 3
|
|
271
|
+
should_trigger = sample_config['skip_counters']['skill_invocation'] >= threshold
|
|
272
|
+
|
|
273
|
+
# Assert
|
|
274
|
+
assert should_trigger is True
|
|
275
|
+
|
|
276
|
+
def test_pattern_detection_per_operation_type(self, sample_config):
|
|
277
|
+
"""
|
|
278
|
+
GIVEN different skip counters per operation type
|
|
279
|
+
WHEN checking pattern for each type
|
|
280
|
+
THEN only types with 3+ skips trigger
|
|
281
|
+
"""
|
|
282
|
+
# Arrange
|
|
283
|
+
sample_config['skip_counters']['skill_invocation'] = 3
|
|
284
|
+
sample_config['skip_counters']['subagent_invocation'] = 2
|
|
285
|
+
sample_config['skip_counters']['command_execution'] = 4
|
|
286
|
+
sample_config['skip_counters']['context_loading'] = 1
|
|
287
|
+
|
|
288
|
+
threshold = 3
|
|
289
|
+
|
|
290
|
+
# Act
|
|
291
|
+
pattern_skill = sample_config['skip_counters']['skill_invocation'] >= threshold
|
|
292
|
+
pattern_subagent = sample_config['skip_counters']['subagent_invocation'] >= threshold
|
|
293
|
+
pattern_command = sample_config['skip_counters']['command_execution'] >= threshold
|
|
294
|
+
pattern_context = sample_config['skip_counters']['context_loading'] >= threshold
|
|
295
|
+
|
|
296
|
+
# Assert
|
|
297
|
+
assert pattern_skill is True
|
|
298
|
+
assert pattern_subagent is False
|
|
299
|
+
assert pattern_command is True
|
|
300
|
+
assert pattern_context is False
|
|
301
|
+
|
|
302
|
+
def test_pattern_detection_occurs_once_per_session(self, temp_config_dir, sample_config):
|
|
303
|
+
"""
|
|
304
|
+
GIVEN pattern detected at 3rd skip
|
|
305
|
+
WHEN additional skips occur (4th, 5th)
|
|
306
|
+
THEN pattern detection flag only set once per session
|
|
307
|
+
"""
|
|
308
|
+
# Arrange
|
|
309
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
310
|
+
operation_type = 'skill_invocation'
|
|
311
|
+
|
|
312
|
+
# Add session tracking flag to config
|
|
313
|
+
sample_config['pattern_detection_session'] = None
|
|
314
|
+
|
|
315
|
+
# Act - First pattern detection at skip 3
|
|
316
|
+
sample_config['skip_counters'][operation_type] = 3
|
|
317
|
+
if sample_config['pattern_detection_session'] is None:
|
|
318
|
+
sample_config['pattern_detection_session'] = {'timestamp': datetime.now(UTC).isoformat()}
|
|
319
|
+
|
|
320
|
+
first_pattern_time = sample_config['pattern_detection_session']['timestamp']
|
|
321
|
+
|
|
322
|
+
# Act - Skip 4 occurs, but pattern detection already triggered
|
|
323
|
+
sample_config['skip_counters'][operation_type] = 4
|
|
324
|
+
# Pattern detection flag already set, don't reset
|
|
325
|
+
|
|
326
|
+
# Assert
|
|
327
|
+
assert sample_config['pattern_detection_session']['timestamp'] == first_pattern_time
|
|
328
|
+
assert sample_config['skip_counters'][operation_type] == 4
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
# ============================================================================
|
|
332
|
+
# UNIT TESTS - Preference Storage and Enforcement (AC3)
|
|
333
|
+
# ============================================================================
|
|
334
|
+
|
|
335
|
+
class TestPreferenceStorage:
|
|
336
|
+
"""Test preference storage and enforcement (AC3)."""
|
|
337
|
+
|
|
338
|
+
def test_preference_stored_in_yaml(self, temp_config_dir, sample_config):
|
|
339
|
+
"""
|
|
340
|
+
GIVEN user disables feedback for skill_invocation
|
|
341
|
+
WHEN preference saved to YAML
|
|
342
|
+
THEN disabled_feedback[skill_invocation] = true
|
|
343
|
+
"""
|
|
344
|
+
# Arrange
|
|
345
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
346
|
+
operation_type = 'skill_invocation'
|
|
347
|
+
|
|
348
|
+
# Act
|
|
349
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
350
|
+
sample_config['disable_reasons'][operation_type] = 'User disabled after 3+ skips'
|
|
351
|
+
|
|
352
|
+
with open(config_file, 'w') as f:
|
|
353
|
+
yaml.dump(sample_config, f)
|
|
354
|
+
|
|
355
|
+
with open(config_file, 'r') as f:
|
|
356
|
+
loaded = yaml.safe_load(f)
|
|
357
|
+
|
|
358
|
+
# Assert
|
|
359
|
+
assert loaded['disabled_feedback'][operation_type] is True
|
|
360
|
+
assert 'User disabled' in loaded['disable_reasons'][operation_type]
|
|
361
|
+
|
|
362
|
+
def test_disabled_preference_prevents_prompts(self, sample_config):
|
|
363
|
+
"""
|
|
364
|
+
GIVEN disabled_feedback[operation_type] = true
|
|
365
|
+
WHEN checking if prompt should be shown
|
|
366
|
+
THEN prompt is NOT shown
|
|
367
|
+
"""
|
|
368
|
+
# Arrange
|
|
369
|
+
operation_type = 'subagent_invocation'
|
|
370
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
371
|
+
|
|
372
|
+
# Act
|
|
373
|
+
should_show_prompt = not sample_config['disabled_feedback'][operation_type]
|
|
374
|
+
|
|
375
|
+
# Assert
|
|
376
|
+
assert should_show_prompt is False
|
|
377
|
+
|
|
378
|
+
def test_enabled_preference_allows_prompts(self, sample_config):
|
|
379
|
+
"""
|
|
380
|
+
GIVEN disabled_feedback[operation_type] = false
|
|
381
|
+
WHEN checking if prompt should be shown
|
|
382
|
+
THEN prompt IS shown (if other conditions met)
|
|
383
|
+
"""
|
|
384
|
+
# Arrange
|
|
385
|
+
operation_type = 'command_execution'
|
|
386
|
+
sample_config['disabled_feedback'][operation_type] = False
|
|
387
|
+
|
|
388
|
+
# Act
|
|
389
|
+
should_show_prompt = not sample_config['disabled_feedback'][operation_type]
|
|
390
|
+
|
|
391
|
+
# Assert
|
|
392
|
+
assert should_show_prompt is True
|
|
393
|
+
|
|
394
|
+
def test_disable_reason_documented(self, temp_config_dir, sample_config):
|
|
395
|
+
"""
|
|
396
|
+
GIVEN user disables feedback
|
|
397
|
+
WHEN reason stored
|
|
398
|
+
THEN disable_reasons contains "User disabled after 3+ skips"
|
|
399
|
+
"""
|
|
400
|
+
# Arrange
|
|
401
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
402
|
+
operation_type = 'skill_invocation'
|
|
403
|
+
|
|
404
|
+
# Act
|
|
405
|
+
reason = f'User disabled after 3+ skips on {datetime.now(UTC).isoformat()}'
|
|
406
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
407
|
+
sample_config['disable_reasons'][operation_type] = reason
|
|
408
|
+
|
|
409
|
+
with open(config_file, 'w') as f:
|
|
410
|
+
yaml.dump(sample_config, f)
|
|
411
|
+
|
|
412
|
+
with open(config_file, 'r') as f:
|
|
413
|
+
loaded = yaml.safe_load(f)
|
|
414
|
+
|
|
415
|
+
# Assert
|
|
416
|
+
assert 'User disabled after 3+ skips' in loaded['disable_reasons'][operation_type]
|
|
417
|
+
|
|
418
|
+
def test_multiple_disabled_feedback_types(self, sample_config):
|
|
419
|
+
"""
|
|
420
|
+
GIVEN user disables multiple operation types
|
|
421
|
+
WHEN checking preferences
|
|
422
|
+
THEN each disabled type enforced independently
|
|
423
|
+
"""
|
|
424
|
+
# Arrange
|
|
425
|
+
sample_config['disabled_feedback']['skill_invocation'] = True
|
|
426
|
+
sample_config['disabled_feedback']['subagent_invocation'] = True
|
|
427
|
+
sample_config['disabled_feedback']['command_execution'] = False
|
|
428
|
+
|
|
429
|
+
# Act
|
|
430
|
+
skill_blocked = sample_config['disabled_feedback']['skill_invocation']
|
|
431
|
+
subagent_blocked = sample_config['disabled_feedback']['subagent_invocation']
|
|
432
|
+
command_allowed = not sample_config['disabled_feedback']['command_execution']
|
|
433
|
+
|
|
434
|
+
# Assert
|
|
435
|
+
assert skill_blocked is True
|
|
436
|
+
assert subagent_blocked is True
|
|
437
|
+
assert command_allowed is True
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
# ============================================================================
|
|
441
|
+
# UNIT TESTS - Counter Reset on Preference Change (AC4)
|
|
442
|
+
# ============================================================================
|
|
443
|
+
|
|
444
|
+
class TestCounterReset:
|
|
445
|
+
"""Test skip counter reset on preference change (AC4)."""
|
|
446
|
+
|
|
447
|
+
def test_counter_resets_to_zero_on_re_enable(self, temp_config_dir, sample_config):
|
|
448
|
+
"""
|
|
449
|
+
GIVEN user previously disabled feedback after 3+ skips
|
|
450
|
+
WHEN user re-enables feedback
|
|
451
|
+
THEN skip counter resets to 0
|
|
452
|
+
"""
|
|
453
|
+
# Arrange
|
|
454
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
455
|
+
operation_type = 'skill_invocation'
|
|
456
|
+
sample_config['skip_counters'][operation_type] = 5
|
|
457
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
458
|
+
|
|
459
|
+
# Act - User re-enables
|
|
460
|
+
sample_config['disabled_feedback'][operation_type] = False
|
|
461
|
+
sample_config['skip_counters'][operation_type] = 0
|
|
462
|
+
sample_config['disable_reasons'][operation_type] = None
|
|
463
|
+
|
|
464
|
+
with open(config_file, 'w') as f:
|
|
465
|
+
yaml.dump(sample_config, f)
|
|
466
|
+
|
|
467
|
+
with open(config_file, 'r') as f:
|
|
468
|
+
loaded = yaml.safe_load(f)
|
|
469
|
+
|
|
470
|
+
# Assert
|
|
471
|
+
assert loaded['skip_counters'][operation_type] == 0
|
|
472
|
+
assert loaded['disabled_feedback'][operation_type] is False
|
|
473
|
+
assert loaded['disable_reasons'][operation_type] is None
|
|
474
|
+
|
|
475
|
+
def test_pattern_detection_starts_fresh_after_reset(self, sample_config):
|
|
476
|
+
"""
|
|
477
|
+
GIVEN counter reset to 0
|
|
478
|
+
WHEN new skips begin
|
|
479
|
+
THEN pattern detection requires fresh 3 consecutive skips
|
|
480
|
+
"""
|
|
481
|
+
# Arrange
|
|
482
|
+
sample_config['skip_counters']['subagent_invocation'] = 0
|
|
483
|
+
|
|
484
|
+
# Act - New skips after reset
|
|
485
|
+
sample_config['skip_counters']['subagent_invocation'] = 1
|
|
486
|
+
sample_config['skip_counters']['subagent_invocation'] = 2
|
|
487
|
+
|
|
488
|
+
threshold = 3
|
|
489
|
+
should_trigger = sample_config['skip_counters']['subagent_invocation'] >= threshold
|
|
490
|
+
|
|
491
|
+
# Assert
|
|
492
|
+
assert should_trigger is False
|
|
493
|
+
assert sample_config['skip_counters']['subagent_invocation'] == 2
|
|
494
|
+
|
|
495
|
+
def test_only_disabled_type_counter_resets(self, sample_config):
|
|
496
|
+
"""
|
|
497
|
+
GIVEN multiple operation types with counters > 0
|
|
498
|
+
WHEN one type re-enabled
|
|
499
|
+
THEN only that type's counter resets
|
|
500
|
+
"""
|
|
501
|
+
# Arrange
|
|
502
|
+
sample_config['skip_counters']['skill_invocation'] = 5
|
|
503
|
+
sample_config['skip_counters']['subagent_invocation'] = 3
|
|
504
|
+
sample_config['skip_counters']['command_execution'] = 2
|
|
505
|
+
|
|
506
|
+
# Act - Re-enable only skill_invocation
|
|
507
|
+
sample_config['skip_counters']['skill_invocation'] = 0
|
|
508
|
+
# Others remain unchanged
|
|
509
|
+
|
|
510
|
+
# Assert
|
|
511
|
+
assert sample_config['skip_counters']['skill_invocation'] == 0
|
|
512
|
+
assert sample_config['skip_counters']['subagent_invocation'] == 3
|
|
513
|
+
assert sample_config['skip_counters']['command_execution'] == 2
|
|
514
|
+
|
|
515
|
+
def test_disable_reason_cleared_on_re_enable(self, sample_config):
|
|
516
|
+
"""
|
|
517
|
+
GIVEN disable_reason is set
|
|
518
|
+
WHEN user re-enables feedback
|
|
519
|
+
THEN disable_reason is cleared (set to null)
|
|
520
|
+
"""
|
|
521
|
+
# Arrange
|
|
522
|
+
operation_type = 'context_loading'
|
|
523
|
+
sample_config['disable_reasons'][operation_type] = 'User disabled after 3+ skips'
|
|
524
|
+
|
|
525
|
+
# Act
|
|
526
|
+
sample_config['disable_reasons'][operation_type] = None
|
|
527
|
+
|
|
528
|
+
# Assert
|
|
529
|
+
assert sample_config['disable_reasons'][operation_type] is None
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
# ============================================================================
|
|
533
|
+
# UNIT TESTS - Token Waste Calculation (AC5)
|
|
534
|
+
# ============================================================================
|
|
535
|
+
|
|
536
|
+
class TestTokenWasteCalculation:
|
|
537
|
+
"""Test token waste calculation (AC5)."""
|
|
538
|
+
|
|
539
|
+
def test_token_waste_formula_basic(self):
|
|
540
|
+
"""
|
|
541
|
+
GIVEN skip count = 3
|
|
542
|
+
WHEN calculate token waste
|
|
543
|
+
THEN waste = 1500 × 3 = 4500 tokens
|
|
544
|
+
"""
|
|
545
|
+
# Arrange
|
|
546
|
+
skip_count = 3
|
|
547
|
+
tokens_per_prompt = 1500
|
|
548
|
+
|
|
549
|
+
# Act
|
|
550
|
+
token_waste = skip_count * tokens_per_prompt
|
|
551
|
+
|
|
552
|
+
# Assert
|
|
553
|
+
assert token_waste == 4500
|
|
554
|
+
|
|
555
|
+
def test_token_waste_formula_5_skips(self):
|
|
556
|
+
"""
|
|
557
|
+
GIVEN skip count = 5
|
|
558
|
+
WHEN calculate token waste
|
|
559
|
+
THEN waste = 1500 × 5 = 7500 tokens
|
|
560
|
+
"""
|
|
561
|
+
# Arrange
|
|
562
|
+
skip_count = 5
|
|
563
|
+
tokens_per_prompt = 1500
|
|
564
|
+
|
|
565
|
+
# Act
|
|
566
|
+
token_waste = skip_count * tokens_per_prompt
|
|
567
|
+
|
|
568
|
+
# Assert
|
|
569
|
+
assert token_waste == 7500
|
|
570
|
+
|
|
571
|
+
def test_token_waste_formula_10_skips(self):
|
|
572
|
+
"""
|
|
573
|
+
GIVEN skip count = 10
|
|
574
|
+
WHEN calculate token waste
|
|
575
|
+
THEN waste = 1500 × 10 = 15000 tokens
|
|
576
|
+
"""
|
|
577
|
+
# Arrange
|
|
578
|
+
skip_count = 10
|
|
579
|
+
tokens_per_prompt = 1500
|
|
580
|
+
|
|
581
|
+
# Act
|
|
582
|
+
token_waste = skip_count * tokens_per_prompt
|
|
583
|
+
|
|
584
|
+
# Assert
|
|
585
|
+
assert token_waste == 15000
|
|
586
|
+
|
|
587
|
+
def test_token_waste_zero_when_no_skips(self):
|
|
588
|
+
"""
|
|
589
|
+
GIVEN skip count = 0
|
|
590
|
+
WHEN calculate token waste
|
|
591
|
+
THEN waste = 0 tokens
|
|
592
|
+
"""
|
|
593
|
+
# Arrange
|
|
594
|
+
skip_count = 0
|
|
595
|
+
tokens_per_prompt = 1500
|
|
596
|
+
|
|
597
|
+
# Act
|
|
598
|
+
token_waste = skip_count * tokens_per_prompt
|
|
599
|
+
|
|
600
|
+
# Assert
|
|
601
|
+
assert token_waste == 0
|
|
602
|
+
|
|
603
|
+
def test_token_waste_displayed_in_pattern_detection(self, sample_config):
|
|
604
|
+
"""
|
|
605
|
+
GIVEN pattern detected at 3 skips
|
|
606
|
+
WHEN generating AskUserQuestion context
|
|
607
|
+
THEN token waste included: "~4,500 tokens wasted"
|
|
608
|
+
"""
|
|
609
|
+
# Arrange
|
|
610
|
+
operation_type = 'skill_invocation'
|
|
611
|
+
skip_count = sample_config['skip_counters'][operation_type] = 3
|
|
612
|
+
tokens_per_prompt = 1500
|
|
613
|
+
|
|
614
|
+
# Act
|
|
615
|
+
token_waste = skip_count * tokens_per_prompt
|
|
616
|
+
message = f"~{token_waste:,} tokens wasted"
|
|
617
|
+
|
|
618
|
+
# Assert
|
|
619
|
+
assert message == "~4,500 tokens wasted"
|
|
620
|
+
|
|
621
|
+
def test_token_waste_calculation_per_operation_type(self, sample_config):
|
|
622
|
+
"""
|
|
623
|
+
GIVEN different skip counts per operation type
|
|
624
|
+
WHEN calculate waste for each
|
|
625
|
+
THEN each calculation independent
|
|
626
|
+
"""
|
|
627
|
+
# Arrange
|
|
628
|
+
sample_config['skip_counters']['skill_invocation'] = 3
|
|
629
|
+
sample_config['skip_counters']['subagent_invocation'] = 5
|
|
630
|
+
tokens_per_prompt = 1500
|
|
631
|
+
|
|
632
|
+
# Act
|
|
633
|
+
waste_skill = sample_config['skip_counters']['skill_invocation'] * tokens_per_prompt
|
|
634
|
+
waste_subagent = sample_config['skip_counters']['subagent_invocation'] * tokens_per_prompt
|
|
635
|
+
|
|
636
|
+
# Assert
|
|
637
|
+
assert waste_skill == 4500
|
|
638
|
+
assert waste_subagent == 7500
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
# ============================================================================
|
|
642
|
+
# UNIT TESTS - Multi-Operation-Type Tracking (AC6)
|
|
643
|
+
# ============================================================================
|
|
644
|
+
|
|
645
|
+
class TestMultiOperationTypeTracking:
|
|
646
|
+
"""Test multi-operation-type tracking (AC6)."""
|
|
647
|
+
|
|
648
|
+
def test_four_operation_types_tracked(self, sample_config):
|
|
649
|
+
"""
|
|
650
|
+
GIVEN skip tracking for 4 operation types
|
|
651
|
+
WHEN checking config structure
|
|
652
|
+
THEN all 4 types present with initial 0 count
|
|
653
|
+
"""
|
|
654
|
+
# Arrange
|
|
655
|
+
expected_types = [
|
|
656
|
+
'skill_invocation',
|
|
657
|
+
'subagent_invocation',
|
|
658
|
+
'command_execution',
|
|
659
|
+
'context_loading'
|
|
660
|
+
]
|
|
661
|
+
|
|
662
|
+
# Act
|
|
663
|
+
actual_types = list(sample_config['skip_counters'].keys())
|
|
664
|
+
|
|
665
|
+
# Assert
|
|
666
|
+
assert len(actual_types) == 4
|
|
667
|
+
for op_type in expected_types:
|
|
668
|
+
assert op_type in actual_types
|
|
669
|
+
assert sample_config['skip_counters'][op_type] == 0
|
|
670
|
+
|
|
671
|
+
def test_independent_counters_per_type(self, sample_config):
|
|
672
|
+
"""
|
|
673
|
+
GIVEN incrementing different operation types
|
|
674
|
+
WHEN checking counters
|
|
675
|
+
THEN each counter independent
|
|
676
|
+
"""
|
|
677
|
+
# Arrange
|
|
678
|
+
sample_config['skip_counters']['skill_invocation'] = 1
|
|
679
|
+
sample_config['skip_counters']['subagent_invocation'] = 2
|
|
680
|
+
sample_config['skip_counters']['command_execution'] = 3
|
|
681
|
+
sample_config['skip_counters']['context_loading'] = 4
|
|
682
|
+
|
|
683
|
+
# Act & Assert
|
|
684
|
+
assert sample_config['skip_counters']['skill_invocation'] == 1
|
|
685
|
+
assert sample_config['skip_counters']['subagent_invocation'] == 2
|
|
686
|
+
assert sample_config['skip_counters']['command_execution'] == 3
|
|
687
|
+
assert sample_config['skip_counters']['context_loading'] == 4
|
|
688
|
+
|
|
689
|
+
def test_independent_disabled_preferences_per_type(self, sample_config):
|
|
690
|
+
"""
|
|
691
|
+
GIVEN disabling different operation types
|
|
692
|
+
WHEN checking disabled_feedback
|
|
693
|
+
THEN disabling one doesn't affect others
|
|
694
|
+
"""
|
|
695
|
+
# Arrange
|
|
696
|
+
sample_config['disabled_feedback']['skill_invocation'] = True
|
|
697
|
+
sample_config['disabled_feedback']['subagent_invocation'] = False
|
|
698
|
+
sample_config['disabled_feedback']['command_execution'] = True
|
|
699
|
+
|
|
700
|
+
# Act & Assert
|
|
701
|
+
assert sample_config['disabled_feedback']['skill_invocation'] is True
|
|
702
|
+
assert sample_config['disabled_feedback']['subagent_invocation'] is False
|
|
703
|
+
assert sample_config['disabled_feedback']['command_execution'] is True
|
|
704
|
+
assert sample_config['disabled_feedback']['context_loading'] is False
|
|
705
|
+
|
|
706
|
+
def test_separate_pattern_detection_per_type(self, sample_config):
|
|
707
|
+
"""
|
|
708
|
+
GIVEN different skip counts per type
|
|
709
|
+
WHEN checking pattern detection for each
|
|
710
|
+
THEN patterns detected independently
|
|
711
|
+
"""
|
|
712
|
+
# Arrange
|
|
713
|
+
sample_config['skip_counters']['skill_invocation'] = 3
|
|
714
|
+
sample_config['skip_counters']['subagent_invocation'] = 3
|
|
715
|
+
sample_config['skip_counters']['command_execution'] = 2
|
|
716
|
+
threshold = 3
|
|
717
|
+
|
|
718
|
+
# Act
|
|
719
|
+
pattern_skill = sample_config['skip_counters']['skill_invocation'] >= threshold
|
|
720
|
+
pattern_subagent = sample_config['skip_counters']['subagent_invocation'] >= threshold
|
|
721
|
+
pattern_command = sample_config['skip_counters']['command_execution'] >= threshold
|
|
722
|
+
|
|
723
|
+
# Assert
|
|
724
|
+
assert pattern_skill is True
|
|
725
|
+
assert pattern_subagent is True
|
|
726
|
+
assert pattern_command is False
|
|
727
|
+
|
|
728
|
+
def test_operation_type_validation_whitelist(self, sample_config):
|
|
729
|
+
"""
|
|
730
|
+
GIVEN 4 allowed operation types
|
|
731
|
+
WHEN validating operation type
|
|
732
|
+
THEN only whitelisted types accepted
|
|
733
|
+
"""
|
|
734
|
+
# Arrange
|
|
735
|
+
allowed_types = {
|
|
736
|
+
'skill_invocation',
|
|
737
|
+
'subagent_invocation',
|
|
738
|
+
'command_execution',
|
|
739
|
+
'context_loading'
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
# Act & Assert
|
|
743
|
+
for op_type in allowed_types:
|
|
744
|
+
assert op_type in sample_config['skip_counters']
|
|
745
|
+
assert op_type in sample_config['disabled_feedback']
|
|
746
|
+
|
|
747
|
+
# Test invalid type rejection
|
|
748
|
+
invalid_type = 'invalid_operation'
|
|
749
|
+
assert invalid_type not in sample_config['skip_counters']
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
# ============================================================================
|
|
753
|
+
# UNIT TESTS - Config File Management
|
|
754
|
+
# ============================================================================
|
|
755
|
+
|
|
756
|
+
class TestConfigFileManagement:
|
|
757
|
+
"""Test config file creation, parsing, validation."""
|
|
758
|
+
|
|
759
|
+
def test_config_file_created_if_missing(self, temp_config_dir, sample_config):
|
|
760
|
+
"""
|
|
761
|
+
GIVEN config file doesn't exist
|
|
762
|
+
WHEN first skip occurs
|
|
763
|
+
THEN config file created with proper structure
|
|
764
|
+
"""
|
|
765
|
+
# Arrange
|
|
766
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
767
|
+
|
|
768
|
+
# Assert - File doesn't exist yet
|
|
769
|
+
assert not config_file.exists()
|
|
770
|
+
|
|
771
|
+
# Act - Create config file
|
|
772
|
+
with open(config_file, 'w') as f:
|
|
773
|
+
yaml.dump(sample_config, f)
|
|
774
|
+
|
|
775
|
+
# Assert - File created with structure
|
|
776
|
+
assert config_file.exists()
|
|
777
|
+
with open(config_file, 'r') as f:
|
|
778
|
+
loaded = yaml.safe_load(f)
|
|
779
|
+
assert 'skip_counters' in loaded
|
|
780
|
+
assert 'disabled_feedback' in loaded
|
|
781
|
+
assert 'disable_reasons' in loaded
|
|
782
|
+
|
|
783
|
+
def test_config_file_yaml_format_valid(self, temp_config_dir, sample_config):
|
|
784
|
+
"""
|
|
785
|
+
GIVEN config file written
|
|
786
|
+
WHEN reading back
|
|
787
|
+
THEN YAML format valid and parseable
|
|
788
|
+
"""
|
|
789
|
+
# Arrange
|
|
790
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
791
|
+
|
|
792
|
+
# Act
|
|
793
|
+
with open(config_file, 'w') as f:
|
|
794
|
+
yaml.dump(sample_config, f)
|
|
795
|
+
|
|
796
|
+
# Assert - Can parse without error
|
|
797
|
+
with open(config_file, 'r') as f:
|
|
798
|
+
loaded = yaml.safe_load(f)
|
|
799
|
+
|
|
800
|
+
assert loaded is not None
|
|
801
|
+
assert isinstance(loaded, dict)
|
|
802
|
+
|
|
803
|
+
def test_config_file_corrupted_detected(self, temp_config_dir):
|
|
804
|
+
"""
|
|
805
|
+
GIVEN config file is corrupted (invalid YAML)
|
|
806
|
+
WHEN attempting to read
|
|
807
|
+
THEN error detected
|
|
808
|
+
"""
|
|
809
|
+
# Arrange
|
|
810
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
811
|
+
corrupted_yaml = "skip_counters: [invalid: yaml: structure:"
|
|
812
|
+
|
|
813
|
+
with open(config_file, 'w') as f:
|
|
814
|
+
f.write(corrupted_yaml)
|
|
815
|
+
|
|
816
|
+
# Act & Assert
|
|
817
|
+
with pytest.raises(yaml.YAMLError):
|
|
818
|
+
with open(config_file, 'r') as f:
|
|
819
|
+
yaml.safe_load(f)
|
|
820
|
+
|
|
821
|
+
def test_config_backup_created_before_modification(self, temp_config_dir, sample_config):
|
|
822
|
+
"""
|
|
823
|
+
GIVEN existing config file
|
|
824
|
+
WHEN modification occurs
|
|
825
|
+
THEN backup created with timestamp
|
|
826
|
+
"""
|
|
827
|
+
# Arrange
|
|
828
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
829
|
+
backup_dir = temp_config_dir / 'backups'
|
|
830
|
+
backup_dir.mkdir(parents=True, exist_ok=True)
|
|
831
|
+
|
|
832
|
+
with open(config_file, 'w') as f:
|
|
833
|
+
yaml.dump(sample_config, f)
|
|
834
|
+
|
|
835
|
+
# Act - Create backup before modification
|
|
836
|
+
import shutil
|
|
837
|
+
timestamp = datetime.now(UTC).strftime('%Y%m%d_%H%M%S')
|
|
838
|
+
backup_file = backup_dir / f'feedback-preferences-{timestamp}.yaml.backup'
|
|
839
|
+
shutil.copy2(config_file, backup_file)
|
|
840
|
+
|
|
841
|
+
# Assert
|
|
842
|
+
assert backup_file.exists()
|
|
843
|
+
|
|
844
|
+
def test_config_corrupted_recovery(self, temp_config_dir, sample_config):
|
|
845
|
+
"""
|
|
846
|
+
GIVEN config file is corrupted
|
|
847
|
+
WHEN recovery initiated
|
|
848
|
+
THEN backup created and fresh config generated
|
|
849
|
+
"""
|
|
850
|
+
# Arrange
|
|
851
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
852
|
+
backup_dir = temp_config_dir / 'backups'
|
|
853
|
+
backup_dir.mkdir(parents=True, exist_ok=True)
|
|
854
|
+
|
|
855
|
+
corrupted_yaml = "skip_counters: [invalid"
|
|
856
|
+
with open(config_file, 'w') as f:
|
|
857
|
+
f.write(corrupted_yaml)
|
|
858
|
+
|
|
859
|
+
# Act - Backup corrupted file
|
|
860
|
+
timestamp = datetime.now(UTC).strftime('%Y%m%d_%H%M%S')
|
|
861
|
+
backup_file = backup_dir / f'feedback-preferences-{timestamp}.yaml.backup'
|
|
862
|
+
import shutil
|
|
863
|
+
shutil.copy2(config_file, backup_file)
|
|
864
|
+
|
|
865
|
+
# Act - Create fresh config
|
|
866
|
+
with open(config_file, 'w') as f:
|
|
867
|
+
yaml.dump(sample_config, f)
|
|
868
|
+
|
|
869
|
+
# Assert
|
|
870
|
+
assert backup_file.exists()
|
|
871
|
+
with open(config_file, 'r') as f:
|
|
872
|
+
loaded = yaml.safe_load(f)
|
|
873
|
+
assert loaded is not None
|
|
874
|
+
assert 'skip_counters' in loaded
|
|
875
|
+
|
|
876
|
+
def test_config_version_validated(self, sample_config):
|
|
877
|
+
"""
|
|
878
|
+
GIVEN config file with version field
|
|
879
|
+
WHEN validating version
|
|
880
|
+
THEN version matches expected format
|
|
881
|
+
"""
|
|
882
|
+
# Arrange
|
|
883
|
+
expected_version = '1.0'
|
|
884
|
+
|
|
885
|
+
# Assert
|
|
886
|
+
assert sample_config['version'] == expected_version
|
|
887
|
+
|
|
888
|
+
def test_config_timestamps_iso8601_format(self, sample_config):
|
|
889
|
+
"""
|
|
890
|
+
GIVEN config file with timestamps
|
|
891
|
+
WHEN validating format
|
|
892
|
+
THEN timestamps in ISO 8601 format
|
|
893
|
+
"""
|
|
894
|
+
# Arrange
|
|
895
|
+
created_at = sample_config['created_at']
|
|
896
|
+
|
|
897
|
+
# Act - Try to parse ISO 8601
|
|
898
|
+
try:
|
|
899
|
+
datetime.fromisoformat(created_at.replace('Z', '+00:00'))
|
|
900
|
+
valid = True
|
|
901
|
+
except ValueError:
|
|
902
|
+
valid = False
|
|
903
|
+
|
|
904
|
+
# Assert
|
|
905
|
+
assert valid is True
|
|
906
|
+
|
|
907
|
+
def test_config_required_sections_present(self, sample_config):
|
|
908
|
+
"""
|
|
909
|
+
GIVEN config file
|
|
910
|
+
WHEN checking structure
|
|
911
|
+
THEN all required sections present
|
|
912
|
+
"""
|
|
913
|
+
# Arrange
|
|
914
|
+
required_sections = [
|
|
915
|
+
'version',
|
|
916
|
+
'created_at',
|
|
917
|
+
'last_updated',
|
|
918
|
+
'skip_counters',
|
|
919
|
+
'disabled_feedback',
|
|
920
|
+
'disable_reasons'
|
|
921
|
+
]
|
|
922
|
+
|
|
923
|
+
# Act & Assert
|
|
924
|
+
for section in required_sections:
|
|
925
|
+
assert section in sample_config
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
# ============================================================================
|
|
929
|
+
# EDGE CASE TESTS
|
|
930
|
+
# ============================================================================
|
|
931
|
+
|
|
932
|
+
class TestEdgeCases:
|
|
933
|
+
"""Test edge cases from story specification."""
|
|
934
|
+
|
|
935
|
+
def test_edge_user_skips_on_first_attempt(self, sample_config):
|
|
936
|
+
"""
|
|
937
|
+
EDGE CASE: User skips feedback on operation #1
|
|
938
|
+
EXPECTED: Skip counter increments to 1, no pattern detected
|
|
939
|
+
VALIDATION: Counter shows "1 of 3 for pattern detection"
|
|
940
|
+
"""
|
|
941
|
+
# Arrange
|
|
942
|
+
operation_type = 'skill_invocation'
|
|
943
|
+
|
|
944
|
+
# Act
|
|
945
|
+
sample_config['skip_counters'][operation_type] = 1
|
|
946
|
+
|
|
947
|
+
# Assert
|
|
948
|
+
assert sample_config['skip_counters'][operation_type] == 1
|
|
949
|
+
threshold = 3
|
|
950
|
+
pattern_triggered = sample_config['skip_counters'][operation_type] >= threshold
|
|
951
|
+
assert pattern_triggered is False
|
|
952
|
+
progress = f"{sample_config['skip_counters'][operation_type]} of 3 for pattern detection"
|
|
953
|
+
assert progress == "1 of 3 for pattern detection"
|
|
954
|
+
|
|
955
|
+
def test_edge_non_consecutive_skips_reset_counter(self, sample_config):
|
|
956
|
+
"""
|
|
957
|
+
EDGE CASE: User skips feedback, answers next feedback, then skips 2 more
|
|
958
|
+
EXPECTED: Skip counter resets to 1 (sequence broken)
|
|
959
|
+
VALIDATION: Only consecutive skips count toward 3+
|
|
960
|
+
"""
|
|
961
|
+
# Arrange
|
|
962
|
+
operation_type = 'subagent_invocation'
|
|
963
|
+
sample_config['skip_counters'][operation_type] = 1
|
|
964
|
+
|
|
965
|
+
# Act - User answers feedback (breaks streak)
|
|
966
|
+
sample_config['skip_counters'][operation_type] = 0
|
|
967
|
+
|
|
968
|
+
# Act - User skips again (new sequence)
|
|
969
|
+
sample_config['skip_counters'][operation_type] = 1
|
|
970
|
+
sample_config['skip_counters'][operation_type] = 2
|
|
971
|
+
|
|
972
|
+
# Assert
|
|
973
|
+
assert sample_config['skip_counters'][operation_type] == 2
|
|
974
|
+
threshold = 3
|
|
975
|
+
pattern_triggered = sample_config['skip_counters'][operation_type] >= threshold
|
|
976
|
+
assert pattern_triggered is False
|
|
977
|
+
|
|
978
|
+
def test_edge_missing_config_file_on_first_skip(self, temp_config_dir, sample_config):
|
|
979
|
+
"""
|
|
980
|
+
EDGE CASE: devforgeai/config/feedback-preferences.yaml doesn't exist
|
|
981
|
+
EXPECTED: System creates config file with initial structure and increments counter
|
|
982
|
+
VALIDATION: File created with YAML frontmatter and initial counters
|
|
983
|
+
"""
|
|
984
|
+
# Arrange
|
|
985
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
986
|
+
assert not config_file.exists()
|
|
987
|
+
|
|
988
|
+
# Act
|
|
989
|
+
with open(config_file, 'w') as f:
|
|
990
|
+
yaml.dump(sample_config, f)
|
|
991
|
+
|
|
992
|
+
sample_config['skip_counters']['skill_invocation'] = 1
|
|
993
|
+
|
|
994
|
+
with open(config_file, 'w') as f:
|
|
995
|
+
yaml.dump(sample_config, f)
|
|
996
|
+
|
|
997
|
+
# Assert
|
|
998
|
+
assert config_file.exists()
|
|
999
|
+
with open(config_file, 'r') as f:
|
|
1000
|
+
loaded = yaml.safe_load(f)
|
|
1001
|
+
|
|
1002
|
+
assert 'skip_counters' in loaded
|
|
1003
|
+
assert loaded['skip_counters']['skill_invocation'] == 1
|
|
1004
|
+
|
|
1005
|
+
def test_edge_manual_config_edit_inconsistency(self, sample_config):
|
|
1006
|
+
"""
|
|
1007
|
+
EDGE CASE: User manually edits config: skip_counter=5 while disabled_feedback=true
|
|
1008
|
+
EXPECTED: System prioritizes disabled_feedback flag (no prompts shown)
|
|
1009
|
+
VALIDATION: Disabled status enforced regardless of counter value
|
|
1010
|
+
"""
|
|
1011
|
+
# Arrange
|
|
1012
|
+
operation_type = 'command_execution'
|
|
1013
|
+
sample_config['skip_counters'][operation_type] = 5
|
|
1014
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
1015
|
+
|
|
1016
|
+
# Act - Check if prompt should be shown
|
|
1017
|
+
counter = sample_config['skip_counters'][operation_type]
|
|
1018
|
+
disabled = sample_config['disabled_feedback'][operation_type]
|
|
1019
|
+
should_show_prompt = (not disabled) and (counter < 3)
|
|
1020
|
+
|
|
1021
|
+
# Assert
|
|
1022
|
+
assert should_show_prompt is False # Disabled takes precedence
|
|
1023
|
+
assert counter == 5 # Counter value ignored when disabled
|
|
1024
|
+
|
|
1025
|
+
def test_edge_corrupted_config_file(self, temp_config_dir, sample_config):
|
|
1026
|
+
"""
|
|
1027
|
+
EDGE CASE: devforgeai/config/feedback-preferences.yaml is malformed YAML
|
|
1028
|
+
EXPECTED: System logs error, creates backup, creates fresh config
|
|
1029
|
+
VALIDATION: Fresh config created, operations continue
|
|
1030
|
+
"""
|
|
1031
|
+
# Arrange
|
|
1032
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1033
|
+
backup_dir = temp_config_dir / 'backups'
|
|
1034
|
+
backup_dir.mkdir(parents=True, exist_ok=True)
|
|
1035
|
+
|
|
1036
|
+
corrupted_yaml = "skip_counters: [invalid yaml structure"
|
|
1037
|
+
with open(config_file, 'w') as f:
|
|
1038
|
+
f.write(corrupted_yaml)
|
|
1039
|
+
|
|
1040
|
+
# Act - Detect corruption
|
|
1041
|
+
try:
|
|
1042
|
+
with open(config_file, 'r') as f:
|
|
1043
|
+
yaml.safe_load(f)
|
|
1044
|
+
corruption_detected = False
|
|
1045
|
+
except yaml.YAMLError:
|
|
1046
|
+
corruption_detected = True
|
|
1047
|
+
|
|
1048
|
+
# Act - Backup and recovery
|
|
1049
|
+
if corruption_detected:
|
|
1050
|
+
import shutil
|
|
1051
|
+
timestamp = datetime.now(UTC).strftime('%Y%m%d_%H%M%S')
|
|
1052
|
+
backup_file = backup_dir / f'feedback-preferences-{timestamp}.yaml.backup'
|
|
1053
|
+
shutil.copy2(config_file, backup_file)
|
|
1054
|
+
|
|
1055
|
+
with open(config_file, 'w') as f:
|
|
1056
|
+
yaml.dump(sample_config, f)
|
|
1057
|
+
|
|
1058
|
+
# Assert
|
|
1059
|
+
assert corruption_detected is True
|
|
1060
|
+
assert backup_file.exists()
|
|
1061
|
+
with open(config_file, 'r') as f:
|
|
1062
|
+
loaded = yaml.safe_load(f)
|
|
1063
|
+
assert loaded is not None
|
|
1064
|
+
assert 'skip_counters' in loaded
|
|
1065
|
+
|
|
1066
|
+
def test_edge_cross_session_persistence(self, temp_config_dir, sample_config):
|
|
1067
|
+
"""
|
|
1068
|
+
EDGE CASE: Session 1: 2 skips, Session 2: 1 skip = 3 total consecutive
|
|
1069
|
+
EXPECTED: Consecutive count maintained across sessions (total = 3 consecutive)
|
|
1070
|
+
VALIDATION: Pattern detection triggers at start of Session 2 on 3rd skip
|
|
1071
|
+
"""
|
|
1072
|
+
# Arrange
|
|
1073
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1074
|
+
operation_type = 'skill_invocation'
|
|
1075
|
+
|
|
1076
|
+
# Act - Session 1: Record 2 skips
|
|
1077
|
+
sample_config['skip_counters'][operation_type] = 2
|
|
1078
|
+
with open(config_file, 'w') as f:
|
|
1079
|
+
yaml.dump(sample_config, f)
|
|
1080
|
+
|
|
1081
|
+
# Act - Session 2: Start fresh (simulate restart), increment from previous
|
|
1082
|
+
with open(config_file, 'r') as f:
|
|
1083
|
+
session2_config = yaml.safe_load(f)
|
|
1084
|
+
|
|
1085
|
+
previous_count = session2_config['skip_counters'][operation_type]
|
|
1086
|
+
session2_config['skip_counters'][operation_type] = previous_count + 1
|
|
1087
|
+
|
|
1088
|
+
with open(config_file, 'w') as f:
|
|
1089
|
+
yaml.dump(session2_config, f)
|
|
1090
|
+
|
|
1091
|
+
# Assert - Pattern detected at Session 2 start with 3rd skip
|
|
1092
|
+
with open(config_file, 'r') as f:
|
|
1093
|
+
final_config = yaml.safe_load(f)
|
|
1094
|
+
|
|
1095
|
+
assert final_config['skip_counters'][operation_type] == 3
|
|
1096
|
+
threshold = 3
|
|
1097
|
+
pattern_triggered = final_config['skip_counters'][operation_type] >= threshold
|
|
1098
|
+
assert pattern_triggered is True
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
# ============================================================================
|
|
1102
|
+
# DATA VALIDATION TESTS
|
|
1103
|
+
# ============================================================================
|
|
1104
|
+
|
|
1105
|
+
class TestDataValidation:
|
|
1106
|
+
"""Test data validation rules."""
|
|
1107
|
+
|
|
1108
|
+
def test_skip_counter_type_integer(self, sample_config):
|
|
1109
|
+
"""
|
|
1110
|
+
VALIDATION: Skip counter must be integer type
|
|
1111
|
+
"""
|
|
1112
|
+
# Arrange
|
|
1113
|
+
operation_type = 'skill_invocation'
|
|
1114
|
+
|
|
1115
|
+
# Act
|
|
1116
|
+
sample_config['skip_counters'][operation_type] = 5
|
|
1117
|
+
|
|
1118
|
+
# Assert
|
|
1119
|
+
assert isinstance(sample_config['skip_counters'][operation_type], int)
|
|
1120
|
+
|
|
1121
|
+
def test_skip_counter_range_valid(self, sample_config):
|
|
1122
|
+
"""
|
|
1123
|
+
VALIDATION: Skip counter range 0-100 (prevents overflow)
|
|
1124
|
+
"""
|
|
1125
|
+
# Arrange
|
|
1126
|
+
operation_type = 'skill_invocation'
|
|
1127
|
+
|
|
1128
|
+
# Act & Assert
|
|
1129
|
+
for count in [0, 1, 50, 100]:
|
|
1130
|
+
sample_config['skip_counters'][operation_type] = count
|
|
1131
|
+
assert 0 <= sample_config['skip_counters'][operation_type] <= 100
|
|
1132
|
+
|
|
1133
|
+
def test_skip_counter_range_invalid_negative(self, sample_config):
|
|
1134
|
+
"""
|
|
1135
|
+
VALIDATION: Skip counter cannot be negative
|
|
1136
|
+
"""
|
|
1137
|
+
# Arrange
|
|
1138
|
+
operation_type = 'skill_invocation'
|
|
1139
|
+
|
|
1140
|
+
# Act
|
|
1141
|
+
sample_config['skip_counters'][operation_type] = -1
|
|
1142
|
+
|
|
1143
|
+
# Assert
|
|
1144
|
+
assert sample_config['skip_counters'][operation_type] < 0 # Invalid
|
|
1145
|
+
|
|
1146
|
+
def test_operation_type_lowercase_enforcement(self):
|
|
1147
|
+
"""
|
|
1148
|
+
VALIDATION: Operation types must be lowercase
|
|
1149
|
+
"""
|
|
1150
|
+
# Arrange
|
|
1151
|
+
invalid_types = ['Skill_Invocation', 'SKILL_INVOCATION', 'skill_Invocation']
|
|
1152
|
+
valid_type = 'skill_invocation'
|
|
1153
|
+
|
|
1154
|
+
# Act & Assert
|
|
1155
|
+
for invalid in invalid_types:
|
|
1156
|
+
assert invalid.lower() == valid_type
|
|
1157
|
+
|
|
1158
|
+
def test_operation_type_snake_case_format(self):
|
|
1159
|
+
"""
|
|
1160
|
+
VALIDATION: Operation types must be snake_case (regex: ^[a-z_]+$)
|
|
1161
|
+
"""
|
|
1162
|
+
# Arrange
|
|
1163
|
+
import re
|
|
1164
|
+
pattern = r'^[a-z_]+$'
|
|
1165
|
+
|
|
1166
|
+
valid_types = [
|
|
1167
|
+
'skill_invocation',
|
|
1168
|
+
'subagent_invocation',
|
|
1169
|
+
'command_execution',
|
|
1170
|
+
'context_loading'
|
|
1171
|
+
]
|
|
1172
|
+
|
|
1173
|
+
invalid_types = [
|
|
1174
|
+
'skill-invocation', # Dashes not allowed
|
|
1175
|
+
'skill.invocation', # Dots not allowed
|
|
1176
|
+
'SkillInvocation', # PascalCase not allowed
|
|
1177
|
+
'skill invocation', # Spaces not allowed
|
|
1178
|
+
]
|
|
1179
|
+
|
|
1180
|
+
# Act & Assert
|
|
1181
|
+
for valid in valid_types:
|
|
1182
|
+
assert re.match(pattern, valid)
|
|
1183
|
+
|
|
1184
|
+
for invalid in invalid_types:
|
|
1185
|
+
assert not re.match(pattern, invalid)
|
|
1186
|
+
|
|
1187
|
+
def test_disabled_feedback_boolean_type(self, sample_config):
|
|
1188
|
+
"""
|
|
1189
|
+
VALIDATION: disabled_feedback must be boolean
|
|
1190
|
+
"""
|
|
1191
|
+
# Arrange
|
|
1192
|
+
operation_type = 'skill_invocation'
|
|
1193
|
+
|
|
1194
|
+
# Act
|
|
1195
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
1196
|
+
|
|
1197
|
+
# Assert
|
|
1198
|
+
assert isinstance(sample_config['disabled_feedback'][operation_type], bool)
|
|
1199
|
+
|
|
1200
|
+
def test_disable_reason_max_length_200_chars(self, sample_config):
|
|
1201
|
+
"""
|
|
1202
|
+
VALIDATION: Disable reason max length 200 characters
|
|
1203
|
+
"""
|
|
1204
|
+
# Arrange
|
|
1205
|
+
operation_type = 'skill_invocation'
|
|
1206
|
+
max_length = 200
|
|
1207
|
+
|
|
1208
|
+
# Act
|
|
1209
|
+
reason = 'U' * 150 + ' on 2025-11-07'
|
|
1210
|
+
sample_config['disable_reasons'][operation_type] = reason
|
|
1211
|
+
|
|
1212
|
+
# Assert
|
|
1213
|
+
assert len(sample_config['disable_reasons'][operation_type]) <= max_length
|
|
1214
|
+
|
|
1215
|
+
def test_disable_reason_null_allowed(self, sample_config):
|
|
1216
|
+
"""
|
|
1217
|
+
VALIDATION: Disable reason can be null (null value)
|
|
1218
|
+
"""
|
|
1219
|
+
# Arrange
|
|
1220
|
+
operation_type = 'command_execution'
|
|
1221
|
+
|
|
1222
|
+
# Act
|
|
1223
|
+
sample_config['disable_reasons'][operation_type] = None
|
|
1224
|
+
|
|
1225
|
+
# Assert
|
|
1226
|
+
assert sample_config['disable_reasons'][operation_type] is None
|
|
1227
|
+
|
|
1228
|
+
|
|
1229
|
+
# ============================================================================
|
|
1230
|
+
# INTEGRATION TESTS - Skip → Pattern → Preference → Enforcement Chain
|
|
1231
|
+
# ============================================================================
|
|
1232
|
+
|
|
1233
|
+
class TestIntegrationWorkflow:
|
|
1234
|
+
"""Test complete workflow chain: skip → pattern detection → preference → enforcement."""
|
|
1235
|
+
|
|
1236
|
+
def test_workflow_skip_to_pattern_detection(self, temp_config_dir, sample_config):
|
|
1237
|
+
"""
|
|
1238
|
+
INTEGRATION: Skip flow → Pattern detection
|
|
1239
|
+
Given: User skips 3 times consecutively
|
|
1240
|
+
When: Pattern detection checked
|
|
1241
|
+
Then: Pattern triggered and preference change offered
|
|
1242
|
+
"""
|
|
1243
|
+
# Arrange
|
|
1244
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1245
|
+
operation_type = 'skill_invocation'
|
|
1246
|
+
|
|
1247
|
+
# Act - Step 1: Initialize config
|
|
1248
|
+
with open(config_file, 'w') as f:
|
|
1249
|
+
yaml.dump(sample_config, f)
|
|
1250
|
+
|
|
1251
|
+
# Act - Step 2: Record 3 skips
|
|
1252
|
+
sample_config['skip_counters'][operation_type] = 1
|
|
1253
|
+
sample_config['skip_counters'][operation_type] = 2
|
|
1254
|
+
sample_config['skip_counters'][operation_type] = 3
|
|
1255
|
+
|
|
1256
|
+
with open(config_file, 'w') as f:
|
|
1257
|
+
yaml.dump(sample_config, f)
|
|
1258
|
+
|
|
1259
|
+
# Act - Step 3: Check pattern
|
|
1260
|
+
threshold = 3
|
|
1261
|
+
pattern_triggered = sample_config['skip_counters'][operation_type] >= threshold
|
|
1262
|
+
|
|
1263
|
+
# Assert
|
|
1264
|
+
assert pattern_triggered is True
|
|
1265
|
+
|
|
1266
|
+
def test_workflow_pattern_detection_to_preference_storage(self, temp_config_dir, sample_config):
|
|
1267
|
+
"""
|
|
1268
|
+
INTEGRATION: Pattern detection → Preference storage
|
|
1269
|
+
Given: Pattern detected (3 skips)
|
|
1270
|
+
When: User confirms "Disable feedback"
|
|
1271
|
+
Then: Preference stored in config
|
|
1272
|
+
"""
|
|
1273
|
+
# Arrange
|
|
1274
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1275
|
+
operation_type = 'subagent_invocation'
|
|
1276
|
+
|
|
1277
|
+
# Act - Step 1: Trigger pattern
|
|
1278
|
+
sample_config['skip_counters'][operation_type] = 3
|
|
1279
|
+
with open(config_file, 'w') as f:
|
|
1280
|
+
yaml.dump(sample_config, f)
|
|
1281
|
+
|
|
1282
|
+
# Act - Step 2: User disables (simulation of AskUserQuestion response)
|
|
1283
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
1284
|
+
sample_config['disable_reasons'][operation_type] = f'User disabled after 3+ skips on {datetime.now(UTC).isoformat()}'
|
|
1285
|
+
|
|
1286
|
+
with open(config_file, 'w') as f:
|
|
1287
|
+
yaml.dump(sample_config, f)
|
|
1288
|
+
|
|
1289
|
+
# Assert - Step 3: Verify stored
|
|
1290
|
+
with open(config_file, 'r') as f:
|
|
1291
|
+
loaded = yaml.safe_load(f)
|
|
1292
|
+
|
|
1293
|
+
assert loaded['disabled_feedback'][operation_type] is True
|
|
1294
|
+
assert 'User disabled' in loaded['disable_reasons'][operation_type]
|
|
1295
|
+
|
|
1296
|
+
def test_workflow_preference_to_prompt_enforcement(self, sample_config):
|
|
1297
|
+
"""
|
|
1298
|
+
INTEGRATION: Preference storage → Prompt enforcement
|
|
1299
|
+
Given: Feedback disabled for operation type
|
|
1300
|
+
When: Operation of that type occurs
|
|
1301
|
+
Then: No prompt shown
|
|
1302
|
+
"""
|
|
1303
|
+
# Arrange
|
|
1304
|
+
operation_type = 'command_execution'
|
|
1305
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
1306
|
+
|
|
1307
|
+
# Act - Check if prompt should be shown
|
|
1308
|
+
should_show_prompt = not sample_config['disabled_feedback'][operation_type]
|
|
1309
|
+
|
|
1310
|
+
# Assert
|
|
1311
|
+
assert should_show_prompt is False
|
|
1312
|
+
|
|
1313
|
+
def test_workflow_re_enable_to_counter_reset(self, temp_config_dir, sample_config):
|
|
1314
|
+
"""
|
|
1315
|
+
INTEGRATION: Re-enable preference → Counter reset
|
|
1316
|
+
Given: Feedback disabled and counter at 3
|
|
1317
|
+
When: User re-enables feedback
|
|
1318
|
+
Then: Counter reset to 0
|
|
1319
|
+
"""
|
|
1320
|
+
# Arrange
|
|
1321
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1322
|
+
operation_type = 'context_loading'
|
|
1323
|
+
|
|
1324
|
+
sample_config['skip_counters'][operation_type] = 3
|
|
1325
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
1326
|
+
|
|
1327
|
+
# Act - Re-enable
|
|
1328
|
+
sample_config['disabled_feedback'][operation_type] = False
|
|
1329
|
+
sample_config['skip_counters'][operation_type] = 0
|
|
1330
|
+
sample_config['disable_reasons'][operation_type] = None
|
|
1331
|
+
|
|
1332
|
+
with open(config_file, 'w') as f:
|
|
1333
|
+
yaml.dump(sample_config, f)
|
|
1334
|
+
|
|
1335
|
+
# Assert
|
|
1336
|
+
with open(config_file, 'r') as f:
|
|
1337
|
+
loaded = yaml.safe_load(f)
|
|
1338
|
+
|
|
1339
|
+
assert loaded['skip_counters'][operation_type] == 0
|
|
1340
|
+
assert loaded['disabled_feedback'][operation_type] is False
|
|
1341
|
+
|
|
1342
|
+
def test_workflow_multiple_operation_types_independent(self, temp_config_dir, sample_config):
|
|
1343
|
+
"""
|
|
1344
|
+
INTEGRATION: Multiple operation types tracked independently
|
|
1345
|
+
Given: User skips different operation types
|
|
1346
|
+
When: Checking preferences for each type
|
|
1347
|
+
Then: Each tracked/disabled independently
|
|
1348
|
+
"""
|
|
1349
|
+
# Arrange
|
|
1350
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1351
|
+
|
|
1352
|
+
# Act - Set different states per type
|
|
1353
|
+
sample_config['skip_counters']['skill_invocation'] = 3
|
|
1354
|
+
sample_config['skip_counters']['subagent_invocation'] = 1
|
|
1355
|
+
sample_config['disabled_feedback']['skill_invocation'] = True
|
|
1356
|
+
sample_config['disabled_feedback']['subagent_invocation'] = False
|
|
1357
|
+
|
|
1358
|
+
with open(config_file, 'w') as f:
|
|
1359
|
+
yaml.dump(sample_config, f)
|
|
1360
|
+
|
|
1361
|
+
# Assert - Each independent
|
|
1362
|
+
with open(config_file, 'r') as f:
|
|
1363
|
+
loaded = yaml.safe_load(f)
|
|
1364
|
+
|
|
1365
|
+
# Skill type: disabled, count 3
|
|
1366
|
+
assert loaded['disabled_feedback']['skill_invocation'] is True
|
|
1367
|
+
assert loaded['skip_counters']['skill_invocation'] == 3
|
|
1368
|
+
|
|
1369
|
+
# Subagent type: enabled, count 1
|
|
1370
|
+
assert loaded['disabled_feedback']['subagent_invocation'] is False
|
|
1371
|
+
assert loaded['skip_counters']['subagent_invocation'] == 1
|
|
1372
|
+
|
|
1373
|
+
|
|
1374
|
+
# ============================================================================
|
|
1375
|
+
# E2E TESTS - End-to-End Workflows
|
|
1376
|
+
# ============================================================================
|
|
1377
|
+
|
|
1378
|
+
class TestEndToEndWorkflows:
|
|
1379
|
+
"""Test complete end-to-end workflows."""
|
|
1380
|
+
|
|
1381
|
+
def test_e2e_first_skip_to_tracking(self, temp_config_dir, sample_config):
|
|
1382
|
+
"""
|
|
1383
|
+
E2E: User's first skip triggers counter increment
|
|
1384
|
+
Given: User skips feedback for skill invocation
|
|
1385
|
+
When: Skip recorded
|
|
1386
|
+
Then: Counter increments to 1 (no pattern yet)
|
|
1387
|
+
"""
|
|
1388
|
+
# Arrange
|
|
1389
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1390
|
+
operation_type = 'skill_invocation'
|
|
1391
|
+
|
|
1392
|
+
# Act
|
|
1393
|
+
with open(config_file, 'w') as f:
|
|
1394
|
+
yaml.dump(sample_config, f)
|
|
1395
|
+
|
|
1396
|
+
sample_config['skip_counters'][operation_type] += 1
|
|
1397
|
+
|
|
1398
|
+
with open(config_file, 'w') as f:
|
|
1399
|
+
yaml.dump(sample_config, f)
|
|
1400
|
+
|
|
1401
|
+
# Assert
|
|
1402
|
+
with open(config_file, 'r') as f:
|
|
1403
|
+
loaded = yaml.safe_load(f)
|
|
1404
|
+
|
|
1405
|
+
assert loaded['skip_counters'][operation_type] == 1
|
|
1406
|
+
|
|
1407
|
+
def test_e2e_three_skips_to_pattern_suggestion(self, temp_config_dir, sample_config):
|
|
1408
|
+
"""
|
|
1409
|
+
E2E: Three consecutive skips trigger AskUserQuestion suggestion
|
|
1410
|
+
Given: User skips feedback 3 times
|
|
1411
|
+
When: 3rd skip processed
|
|
1412
|
+
Then: AskUserQuestion appears with disable/keep/ask-later options
|
|
1413
|
+
"""
|
|
1414
|
+
# Arrange
|
|
1415
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1416
|
+
operation_type = 'subagent_invocation'
|
|
1417
|
+
threshold = 3
|
|
1418
|
+
|
|
1419
|
+
# Act
|
|
1420
|
+
with open(config_file, 'w') as f:
|
|
1421
|
+
yaml.dump(sample_config, f)
|
|
1422
|
+
|
|
1423
|
+
# Skip 1, 2, 3
|
|
1424
|
+
for i in range(3):
|
|
1425
|
+
sample_config['skip_counters'][operation_type] += 1
|
|
1426
|
+
|
|
1427
|
+
with open(config_file, 'w') as f:
|
|
1428
|
+
yaml.dump(sample_config, f)
|
|
1429
|
+
|
|
1430
|
+
# Assert
|
|
1431
|
+
with open(config_file, 'r') as f:
|
|
1432
|
+
loaded = yaml.safe_load(f)
|
|
1433
|
+
|
|
1434
|
+
pattern_triggered = loaded['skip_counters'][operation_type] >= threshold
|
|
1435
|
+
assert pattern_triggered is True
|
|
1436
|
+
assert loaded['skip_counters'][operation_type] == 3
|
|
1437
|
+
|
|
1438
|
+
def test_e2e_non_consecutive_skips_reset(self, temp_config_dir, sample_config):
|
|
1439
|
+
"""
|
|
1440
|
+
E2E: Non-consecutive skips reset counter
|
|
1441
|
+
Given: User skips, answers prompt, skips again (2 times total)
|
|
1442
|
+
When: Counter tracked
|
|
1443
|
+
Then: Sequence broken, counter is 1 (not 3)
|
|
1444
|
+
"""
|
|
1445
|
+
# Arrange
|
|
1446
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1447
|
+
operation_type = 'command_execution'
|
|
1448
|
+
|
|
1449
|
+
# Act - Skip 1
|
|
1450
|
+
sample_config['skip_counters'][operation_type] = 1
|
|
1451
|
+
with open(config_file, 'w') as f:
|
|
1452
|
+
yaml.dump(sample_config, f)
|
|
1453
|
+
|
|
1454
|
+
# Act - User answers prompt (sequence broken)
|
|
1455
|
+
sample_config['skip_counters'][operation_type] = 0
|
|
1456
|
+
|
|
1457
|
+
# Act - Skip again (new sequence)
|
|
1458
|
+
sample_config['skip_counters'][operation_type] = 1
|
|
1459
|
+
|
|
1460
|
+
with open(config_file, 'w') as f:
|
|
1461
|
+
yaml.dump(sample_config, f)
|
|
1462
|
+
|
|
1463
|
+
# Assert
|
|
1464
|
+
with open(config_file, 'r') as f:
|
|
1465
|
+
loaded = yaml.safe_load(f)
|
|
1466
|
+
|
|
1467
|
+
assert loaded['skip_counters'][operation_type] == 1 # Not 3
|
|
1468
|
+
|
|
1469
|
+
def test_e2e_disable_preference_prevents_prompts(self, temp_config_dir, sample_config):
|
|
1470
|
+
"""
|
|
1471
|
+
E2E: User disables feedback → No prompts shown
|
|
1472
|
+
Given: User responds "Disable feedback for skill_invocation"
|
|
1473
|
+
When: Preference stored
|
|
1474
|
+
Then: Subsequent prompts not shown for that type
|
|
1475
|
+
"""
|
|
1476
|
+
# Arrange
|
|
1477
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1478
|
+
operation_type = 'skill_invocation'
|
|
1479
|
+
|
|
1480
|
+
# Act - Disable feedback
|
|
1481
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
1482
|
+
sample_config['disable_reasons'][operation_type] = 'User disabled after 3+ skips'
|
|
1483
|
+
|
|
1484
|
+
with open(config_file, 'w') as f:
|
|
1485
|
+
yaml.dump(sample_config, f)
|
|
1486
|
+
|
|
1487
|
+
# Act - Subsequent operation type check
|
|
1488
|
+
should_show_prompt = not sample_config['disabled_feedback'][operation_type]
|
|
1489
|
+
|
|
1490
|
+
# Assert
|
|
1491
|
+
assert should_show_prompt is False
|
|
1492
|
+
|
|
1493
|
+
def test_e2e_re_enable_feedback_resets_counter(self, temp_config_dir, sample_config):
|
|
1494
|
+
"""
|
|
1495
|
+
E2E: User re-enables feedback → Counter resets
|
|
1496
|
+
Given: Feedback disabled with counter=3
|
|
1497
|
+
When: User re-enables via config edit
|
|
1498
|
+
Then: Counter resets to 0, prompts resume
|
|
1499
|
+
"""
|
|
1500
|
+
# Arrange
|
|
1501
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1502
|
+
operation_type = 'context_loading'
|
|
1503
|
+
|
|
1504
|
+
sample_config['skip_counters'][operation_type] = 3
|
|
1505
|
+
sample_config['disabled_feedback'][operation_type] = True
|
|
1506
|
+
|
|
1507
|
+
# Act - Re-enable
|
|
1508
|
+
sample_config['disabled_feedback'][operation_type] = False
|
|
1509
|
+
sample_config['skip_counters'][operation_type] = 0
|
|
1510
|
+
sample_config['disable_reasons'][operation_type] = None
|
|
1511
|
+
|
|
1512
|
+
with open(config_file, 'w') as f:
|
|
1513
|
+
yaml.dump(sample_config, f)
|
|
1514
|
+
|
|
1515
|
+
# Act - Check if prompt should show
|
|
1516
|
+
should_show_prompt = not sample_config['disabled_feedback'][operation_type]
|
|
1517
|
+
|
|
1518
|
+
# Assert
|
|
1519
|
+
with open(config_file, 'r') as f:
|
|
1520
|
+
loaded = yaml.safe_load(f)
|
|
1521
|
+
|
|
1522
|
+
assert loaded['skip_counters'][operation_type] == 0
|
|
1523
|
+
assert should_show_prompt is True
|
|
1524
|
+
|
|
1525
|
+
def test_e2e_missing_config_auto_creation(self, temp_config_dir, sample_config):
|
|
1526
|
+
"""
|
|
1527
|
+
E2E: Missing config file auto-created
|
|
1528
|
+
Given: Config file doesn't exist
|
|
1529
|
+
When: First skip occurs
|
|
1530
|
+
Then: Config file created with structure and counter incremented
|
|
1531
|
+
"""
|
|
1532
|
+
# Arrange
|
|
1533
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1534
|
+
assert not config_file.exists()
|
|
1535
|
+
|
|
1536
|
+
# Act - First skip creates config
|
|
1537
|
+
with open(config_file, 'w') as f:
|
|
1538
|
+
yaml.dump(sample_config, f)
|
|
1539
|
+
|
|
1540
|
+
sample_config['skip_counters']['skill_invocation'] = 1
|
|
1541
|
+
|
|
1542
|
+
with open(config_file, 'w') as f:
|
|
1543
|
+
yaml.dump(sample_config, f)
|
|
1544
|
+
|
|
1545
|
+
# Assert
|
|
1546
|
+
assert config_file.exists()
|
|
1547
|
+
with open(config_file, 'r') as f:
|
|
1548
|
+
loaded = yaml.safe_load(f)
|
|
1549
|
+
|
|
1550
|
+
assert loaded['skip_counters']['skill_invocation'] == 1
|
|
1551
|
+
|
|
1552
|
+
def test_e2e_corrupted_config_recovery(self, temp_config_dir, sample_config):
|
|
1553
|
+
"""
|
|
1554
|
+
E2E: Corrupted config → Backup created → Fresh config
|
|
1555
|
+
Given: Config file is corrupted YAML
|
|
1556
|
+
When: System detects corruption
|
|
1557
|
+
Then: Backup created and fresh config generated
|
|
1558
|
+
"""
|
|
1559
|
+
# Arrange
|
|
1560
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1561
|
+
backup_dir = temp_config_dir / 'backups'
|
|
1562
|
+
backup_dir.mkdir(parents=True, exist_ok=True)
|
|
1563
|
+
|
|
1564
|
+
corrupted = "skip_counters: [invalid"
|
|
1565
|
+
with open(config_file, 'w') as f:
|
|
1566
|
+
f.write(corrupted)
|
|
1567
|
+
|
|
1568
|
+
# Act - Detect and recover
|
|
1569
|
+
try:
|
|
1570
|
+
with open(config_file, 'r') as f:
|
|
1571
|
+
yaml.safe_load(f)
|
|
1572
|
+
corrupted_detected = False
|
|
1573
|
+
except yaml.YAMLError:
|
|
1574
|
+
corrupted_detected = True
|
|
1575
|
+
|
|
1576
|
+
if corrupted_detected:
|
|
1577
|
+
import shutil
|
|
1578
|
+
timestamp = datetime.now(UTC).strftime('%Y%m%d_%H%M%S')
|
|
1579
|
+
backup_file = backup_dir / f'feedback-preferences-{timestamp}.yaml.backup'
|
|
1580
|
+
shutil.copy2(config_file, backup_file)
|
|
1581
|
+
|
|
1582
|
+
with open(config_file, 'w') as f:
|
|
1583
|
+
yaml.dump(sample_config, f)
|
|
1584
|
+
|
|
1585
|
+
# Assert
|
|
1586
|
+
assert corrupted_detected is True
|
|
1587
|
+
assert backup_file.exists()
|
|
1588
|
+
with open(config_file, 'r') as f:
|
|
1589
|
+
loaded = yaml.safe_load(f)
|
|
1590
|
+
assert 'skip_counters' in loaded
|
|
1591
|
+
|
|
1592
|
+
def test_e2e_cross_session_persistence(self, temp_config_dir, sample_config):
|
|
1593
|
+
"""
|
|
1594
|
+
E2E: Cross-session persistence
|
|
1595
|
+
Given: Session 1 records 2 skips
|
|
1596
|
+
When: Session 2 starts and records 1 more skip
|
|
1597
|
+
Then: Pattern detected with 3 consecutive total
|
|
1598
|
+
"""
|
|
1599
|
+
# Arrange
|
|
1600
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1601
|
+
operation_type = 'skill_invocation'
|
|
1602
|
+
|
|
1603
|
+
# Act - Session 1: Write 2 skips
|
|
1604
|
+
sample_config['skip_counters'][operation_type] = 2
|
|
1605
|
+
with open(config_file, 'w') as f:
|
|
1606
|
+
yaml.dump(sample_config, f)
|
|
1607
|
+
|
|
1608
|
+
# Act - Session 2: Read previous count, increment
|
|
1609
|
+
with open(config_file, 'r') as f:
|
|
1610
|
+
session2_config = yaml.safe_load(f)
|
|
1611
|
+
|
|
1612
|
+
previous = session2_config['skip_counters'][operation_type]
|
|
1613
|
+
session2_config['skip_counters'][operation_type] = previous + 1
|
|
1614
|
+
|
|
1615
|
+
with open(config_file, 'w') as f:
|
|
1616
|
+
yaml.dump(session2_config, f)
|
|
1617
|
+
|
|
1618
|
+
# Assert - Pattern detected
|
|
1619
|
+
with open(config_file, 'r') as f:
|
|
1620
|
+
final = yaml.safe_load(f)
|
|
1621
|
+
|
|
1622
|
+
assert final['skip_counters'][operation_type] == 3
|
|
1623
|
+
threshold = 3
|
|
1624
|
+
pattern_triggered = final['skip_counters'][operation_type] >= threshold
|
|
1625
|
+
assert pattern_triggered is True
|
|
1626
|
+
|
|
1627
|
+
|
|
1628
|
+
# ============================================================================
|
|
1629
|
+
# RELEASE READINESS TESTS - Config Permissions & Audit Trail Logging
|
|
1630
|
+
# ============================================================================
|
|
1631
|
+
|
|
1632
|
+
class TestReleaseReadiness:
|
|
1633
|
+
"""Test Release Readiness items (deferred from DoD)."""
|
|
1634
|
+
|
|
1635
|
+
# ========================================================================
|
|
1636
|
+
# CONFIG FILE PERMISSIONS - mode 600 (User read/write only)
|
|
1637
|
+
# ========================================================================
|
|
1638
|
+
|
|
1639
|
+
def test_config_file_created_with_mode_600(self, temp_config_dir, sample_config):
|
|
1640
|
+
"""
|
|
1641
|
+
GIVEN config file is created for first time
|
|
1642
|
+
WHEN file is written with YAML content
|
|
1643
|
+
THEN file has permissions mode 600 (user read/write only)
|
|
1644
|
+
"""
|
|
1645
|
+
# Arrange
|
|
1646
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1647
|
+
import stat
|
|
1648
|
+
import os
|
|
1649
|
+
|
|
1650
|
+
# Act
|
|
1651
|
+
with open(config_file, 'w') as f:
|
|
1652
|
+
yaml.dump(sample_config, f)
|
|
1653
|
+
|
|
1654
|
+
# Set permissions to 600 (user read/write)
|
|
1655
|
+
os.chmod(config_file, stat.S_IRUSR | stat.S_IWUSR)
|
|
1656
|
+
|
|
1657
|
+
# Assert - Verify permissions
|
|
1658
|
+
file_stat = os.stat(config_file)
|
|
1659
|
+
file_mode = stat.S_IMODE(file_stat.st_mode)
|
|
1660
|
+
expected_mode = stat.S_IRUSR | stat.S_IWUSR # 0o600
|
|
1661
|
+
|
|
1662
|
+
assert file_mode == expected_mode, f"Expected mode 600, got {oct(file_mode)}"
|
|
1663
|
+
|
|
1664
|
+
def test_config_file_permissions_mode_600_octal(self, temp_config_dir, sample_config):
|
|
1665
|
+
"""
|
|
1666
|
+
GIVEN config file exists
|
|
1667
|
+
WHEN checking file permissions
|
|
1668
|
+
THEN permissions are 0o600 (octal notation)
|
|
1669
|
+
"""
|
|
1670
|
+
# Arrange
|
|
1671
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1672
|
+
import stat
|
|
1673
|
+
import os
|
|
1674
|
+
|
|
1675
|
+
with open(config_file, 'w') as f:
|
|
1676
|
+
yaml.dump(sample_config, f)
|
|
1677
|
+
|
|
1678
|
+
# Act
|
|
1679
|
+
os.chmod(config_file, 0o600)
|
|
1680
|
+
file_stat = os.stat(config_file)
|
|
1681
|
+
actual_mode = stat.S_IMODE(file_stat.st_mode)
|
|
1682
|
+
|
|
1683
|
+
# Assert
|
|
1684
|
+
assert actual_mode == 0o600
|
|
1685
|
+
|
|
1686
|
+
def test_config_file_permissions_user_read_enabled(self, temp_config_dir, sample_config):
|
|
1687
|
+
"""
|
|
1688
|
+
GIVEN config file with mode 600
|
|
1689
|
+
WHEN checking read permission for user
|
|
1690
|
+
THEN user has read permission
|
|
1691
|
+
"""
|
|
1692
|
+
# Arrange
|
|
1693
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1694
|
+
import stat
|
|
1695
|
+
import os
|
|
1696
|
+
|
|
1697
|
+
with open(config_file, 'w') as f:
|
|
1698
|
+
yaml.dump(sample_config, f)
|
|
1699
|
+
|
|
1700
|
+
os.chmod(config_file, 0o600)
|
|
1701
|
+
file_stat = os.stat(config_file)
|
|
1702
|
+
file_mode = stat.S_IMODE(file_stat.st_mode)
|
|
1703
|
+
|
|
1704
|
+
# Act
|
|
1705
|
+
user_can_read = bool(file_mode & stat.S_IRUSR)
|
|
1706
|
+
|
|
1707
|
+
# Assert
|
|
1708
|
+
assert user_can_read is True
|
|
1709
|
+
|
|
1710
|
+
def test_config_file_permissions_user_write_enabled(self, temp_config_dir, sample_config):
|
|
1711
|
+
"""
|
|
1712
|
+
GIVEN config file with mode 600
|
|
1713
|
+
WHEN checking write permission for user
|
|
1714
|
+
THEN user has write permission
|
|
1715
|
+
"""
|
|
1716
|
+
# Arrange
|
|
1717
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1718
|
+
import stat
|
|
1719
|
+
import os
|
|
1720
|
+
|
|
1721
|
+
with open(config_file, 'w') as f:
|
|
1722
|
+
yaml.dump(sample_config, f)
|
|
1723
|
+
|
|
1724
|
+
os.chmod(config_file, 0o600)
|
|
1725
|
+
file_stat = os.stat(config_file)
|
|
1726
|
+
file_mode = stat.S_IMODE(file_stat.st_mode)
|
|
1727
|
+
|
|
1728
|
+
# Act
|
|
1729
|
+
user_can_write = bool(file_mode & stat.S_IWUSR)
|
|
1730
|
+
|
|
1731
|
+
# Assert
|
|
1732
|
+
assert user_can_write is True
|
|
1733
|
+
|
|
1734
|
+
def test_config_file_permissions_group_disabled(self, temp_config_dir, sample_config):
|
|
1735
|
+
"""
|
|
1736
|
+
GIVEN config file with mode 600
|
|
1737
|
+
WHEN checking group permissions
|
|
1738
|
+
THEN group has NO read/write permission
|
|
1739
|
+
"""
|
|
1740
|
+
# Arrange
|
|
1741
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1742
|
+
import stat
|
|
1743
|
+
import os
|
|
1744
|
+
|
|
1745
|
+
with open(config_file, 'w') as f:
|
|
1746
|
+
yaml.dump(sample_config, f)
|
|
1747
|
+
|
|
1748
|
+
os.chmod(config_file, 0o600)
|
|
1749
|
+
file_stat = os.stat(config_file)
|
|
1750
|
+
file_mode = stat.S_IMODE(file_stat.st_mode)
|
|
1751
|
+
|
|
1752
|
+
# Act
|
|
1753
|
+
group_can_read = bool(file_mode & stat.S_IRGRP)
|
|
1754
|
+
group_can_write = bool(file_mode & stat.S_IWGRP)
|
|
1755
|
+
|
|
1756
|
+
# Assert
|
|
1757
|
+
assert group_can_read is False
|
|
1758
|
+
assert group_can_write is False
|
|
1759
|
+
|
|
1760
|
+
def test_config_file_permissions_others_disabled(self, temp_config_dir, sample_config):
|
|
1761
|
+
"""
|
|
1762
|
+
GIVEN config file with mode 600
|
|
1763
|
+
WHEN checking others permissions
|
|
1764
|
+
THEN others have NO read/write permission
|
|
1765
|
+
"""
|
|
1766
|
+
# Arrange
|
|
1767
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1768
|
+
import stat
|
|
1769
|
+
import os
|
|
1770
|
+
|
|
1771
|
+
with open(config_file, 'w') as f:
|
|
1772
|
+
yaml.dump(sample_config, f)
|
|
1773
|
+
|
|
1774
|
+
os.chmod(config_file, 0o600)
|
|
1775
|
+
file_stat = os.stat(config_file)
|
|
1776
|
+
file_mode = stat.S_IMODE(file_stat.st_mode)
|
|
1777
|
+
|
|
1778
|
+
# Act
|
|
1779
|
+
others_can_read = bool(file_mode & stat.S_IROTH)
|
|
1780
|
+
others_can_write = bool(file_mode & stat.S_IWOTH)
|
|
1781
|
+
|
|
1782
|
+
# Assert
|
|
1783
|
+
assert others_can_read is False
|
|
1784
|
+
assert others_can_write is False
|
|
1785
|
+
|
|
1786
|
+
def test_config_file_permissions_too_permissive_644_detected(self, temp_config_dir, sample_config):
|
|
1787
|
+
"""
|
|
1788
|
+
GIVEN config file has overly permissive permissions (mode 644)
|
|
1789
|
+
WHEN validating permissions
|
|
1790
|
+
THEN warning/error triggered (file readable by others)
|
|
1791
|
+
"""
|
|
1792
|
+
# Arrange
|
|
1793
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1794
|
+
import stat
|
|
1795
|
+
import os
|
|
1796
|
+
|
|
1797
|
+
with open(config_file, 'w') as f:
|
|
1798
|
+
yaml.dump(sample_config, f)
|
|
1799
|
+
|
|
1800
|
+
# Act - Set to 644 (too permissive)
|
|
1801
|
+
os.chmod(config_file, 0o644)
|
|
1802
|
+
file_stat = os.stat(config_file)
|
|
1803
|
+
actual_mode = stat.S_IMODE(file_stat.st_mode)
|
|
1804
|
+
|
|
1805
|
+
# Validate - Check if others can read
|
|
1806
|
+
others_can_read = bool(actual_mode & stat.S_IROTH)
|
|
1807
|
+
|
|
1808
|
+
# Assert
|
|
1809
|
+
assert actual_mode == 0o644
|
|
1810
|
+
assert others_can_read is True # This is a security issue
|
|
1811
|
+
|
|
1812
|
+
def test_config_file_permissions_too_permissive_666_detected(self, temp_config_dir, sample_config):
|
|
1813
|
+
"""
|
|
1814
|
+
GIVEN config file has overly permissive permissions (mode 666)
|
|
1815
|
+
WHEN validating permissions
|
|
1816
|
+
THEN warning/error triggered (file writable by others)
|
|
1817
|
+
"""
|
|
1818
|
+
# Arrange
|
|
1819
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1820
|
+
import stat
|
|
1821
|
+
import os
|
|
1822
|
+
|
|
1823
|
+
with open(config_file, 'w') as f:
|
|
1824
|
+
yaml.dump(sample_config, f)
|
|
1825
|
+
|
|
1826
|
+
# Act - Set to 666 (dangerously permissive)
|
|
1827
|
+
os.chmod(config_file, 0o666)
|
|
1828
|
+
file_stat = os.stat(config_file)
|
|
1829
|
+
actual_mode = stat.S_IMODE(file_stat.st_mode)
|
|
1830
|
+
|
|
1831
|
+
# Validate - Check if others can write
|
|
1832
|
+
others_can_write = bool(actual_mode & stat.S_IWOTH)
|
|
1833
|
+
|
|
1834
|
+
# Assert
|
|
1835
|
+
assert actual_mode == 0o666
|
|
1836
|
+
assert others_can_write is True # This is a critical security issue
|
|
1837
|
+
|
|
1838
|
+
def test_config_file_permissions_validation_strict_enforcement(self, temp_config_dir, sample_config):
|
|
1839
|
+
"""
|
|
1840
|
+
GIVEN config file with various permission modes
|
|
1841
|
+
WHEN validating against required mode 600
|
|
1842
|
+
THEN only mode 600 passes validation, others fail
|
|
1843
|
+
"""
|
|
1844
|
+
# Arrange
|
|
1845
|
+
config_file = temp_config_dir / 'feedback-preferences.yaml'
|
|
1846
|
+
import stat
|
|
1847
|
+
import os
|
|
1848
|
+
|
|
1849
|
+
with open(config_file, 'w') as f:
|
|
1850
|
+
yaml.dump(sample_config, f)
|
|
1851
|
+
|
|
1852
|
+
# Test multiple permission modes
|
|
1853
|
+
test_modes = {
|
|
1854
|
+
0o600: True, # Expected (pass)
|
|
1855
|
+
0o644: False, # Too permissive (fail)
|
|
1856
|
+
0o666: False, # Too permissive (fail)
|
|
1857
|
+
0o400: False, # Read-only (fail)
|
|
1858
|
+
0o700: False, # Executable (fail)
|
|
1859
|
+
}
|
|
1860
|
+
|
|
1861
|
+
# Act & Assert
|
|
1862
|
+
for mode, should_pass in test_modes.items():
|
|
1863
|
+
os.chmod(config_file, mode)
|
|
1864
|
+
file_stat = os.stat(config_file)
|
|
1865
|
+
actual_mode = stat.S_IMODE(file_stat.st_mode)
|
|
1866
|
+
|
|
1867
|
+
is_valid = (actual_mode == 0o600)
|
|
1868
|
+
assert is_valid == should_pass, f"Mode {oct(mode)} validation failed: expected {should_pass}, got {is_valid}"
|
|
1869
|
+
|
|
1870
|
+
# ========================================================================
|
|
1871
|
+
# AUDIT TRAIL LOGGING - Serilog/Python logging integration
|
|
1872
|
+
# ========================================================================
|
|
1873
|
+
|
|
1874
|
+
@patch('logging.getLogger')
|
|
1875
|
+
def test_counter_increment_logged_at_debug_level(self, mock_get_logger):
|
|
1876
|
+
"""
|
|
1877
|
+
GIVEN skip counter is incremented
|
|
1878
|
+
WHEN counter update occurs
|
|
1879
|
+
THEN operation logged at DEBUG level with details
|
|
1880
|
+
"""
|
|
1881
|
+
# Arrange
|
|
1882
|
+
mock_logger = MagicMock()
|
|
1883
|
+
mock_get_logger.return_value = mock_logger
|
|
1884
|
+
|
|
1885
|
+
import logging
|
|
1886
|
+
logger = logging.getLogger('skip_tracking')
|
|
1887
|
+
|
|
1888
|
+
operation_type = 'skill_invocation'
|
|
1889
|
+
skip_count = 3
|
|
1890
|
+
|
|
1891
|
+
# Act
|
|
1892
|
+
logger.debug(f"Skip counter incremented", extra={
|
|
1893
|
+
'operation_type': operation_type,
|
|
1894
|
+
'new_count': skip_count
|
|
1895
|
+
})
|
|
1896
|
+
|
|
1897
|
+
# Assert
|
|
1898
|
+
mock_logger.debug.assert_called()
|
|
1899
|
+
call_args = mock_logger.debug.call_args
|
|
1900
|
+
assert 'Skip counter incremented' in str(call_args)
|
|
1901
|
+
|
|
1902
|
+
@patch('logging.getLogger')
|
|
1903
|
+
def test_pattern_detection_logged_at_info_level(self, mock_get_logger):
|
|
1904
|
+
"""
|
|
1905
|
+
GIVEN pattern is detected at 3+ skips
|
|
1906
|
+
WHEN pattern detection triggered
|
|
1907
|
+
THEN operation logged at INFO level with context
|
|
1908
|
+
"""
|
|
1909
|
+
# Arrange
|
|
1910
|
+
mock_logger = MagicMock()
|
|
1911
|
+
mock_get_logger.return_value = mock_logger
|
|
1912
|
+
|
|
1913
|
+
import logging
|
|
1914
|
+
logger = logging.getLogger('skip_tracking')
|
|
1915
|
+
|
|
1916
|
+
operation_type = 'subagent_invocation'
|
|
1917
|
+
skip_count = 3
|
|
1918
|
+
token_waste = 4500
|
|
1919
|
+
|
|
1920
|
+
# Act
|
|
1921
|
+
logger.info(f"Skip pattern detected", extra={
|
|
1922
|
+
'operation_type': operation_type,
|
|
1923
|
+
'skip_count': skip_count,
|
|
1924
|
+
'token_waste': token_waste
|
|
1925
|
+
})
|
|
1926
|
+
|
|
1927
|
+
# Assert
|
|
1928
|
+
mock_logger.info.assert_called()
|
|
1929
|
+
call_args = mock_logger.info.call_args
|
|
1930
|
+
assert 'Skip pattern detected' in str(call_args)
|
|
1931
|
+
|
|
1932
|
+
@patch('logging.getLogger')
|
|
1933
|
+
def test_config_corruption_logged_at_error_level(self, mock_get_logger):
|
|
1934
|
+
"""
|
|
1935
|
+
GIVEN config file is corrupted YAML
|
|
1936
|
+
WHEN corruption detected
|
|
1937
|
+
THEN operation logged at ERROR level with recovery details
|
|
1938
|
+
"""
|
|
1939
|
+
# Arrange
|
|
1940
|
+
mock_logger = MagicMock()
|
|
1941
|
+
mock_get_logger.return_value = mock_logger
|
|
1942
|
+
|
|
1943
|
+
import logging
|
|
1944
|
+
logger = logging.getLogger('skip_tracking')
|
|
1945
|
+
|
|
1946
|
+
error_msg = "Invalid YAML in feedback-preferences.yaml"
|
|
1947
|
+
backup_file = "/path/to/backup/feedback-preferences-20251109_143022.yaml.backup"
|
|
1948
|
+
|
|
1949
|
+
# Act
|
|
1950
|
+
logger.error(f"Config file corrupted", extra={
|
|
1951
|
+
'error': error_msg,
|
|
1952
|
+
'backup_file': backup_file,
|
|
1953
|
+
'recovery_action': 'created_fresh_config'
|
|
1954
|
+
})
|
|
1955
|
+
|
|
1956
|
+
# Assert
|
|
1957
|
+
mock_logger.error.assert_called()
|
|
1958
|
+
call_args = mock_logger.error.call_args
|
|
1959
|
+
assert 'Config file corrupted' in str(call_args)
|
|
1960
|
+
|
|
1961
|
+
@patch('logging.getLogger')
|
|
1962
|
+
def test_audit_log_entry_includes_operation_type(self, mock_get_logger):
|
|
1963
|
+
"""
|
|
1964
|
+
GIVEN operation logged to audit trail
|
|
1965
|
+
WHEN log entry created
|
|
1966
|
+
THEN entry includes operation_type field
|
|
1967
|
+
"""
|
|
1968
|
+
# Arrange
|
|
1969
|
+
mock_logger = MagicMock()
|
|
1970
|
+
mock_get_logger.return_value = mock_logger
|
|
1971
|
+
|
|
1972
|
+
import logging
|
|
1973
|
+
logger = logging.getLogger('skip_tracking')
|
|
1974
|
+
|
|
1975
|
+
operation_type = 'command_execution'
|
|
1976
|
+
|
|
1977
|
+
# Act
|
|
1978
|
+
logger.info("Skip counter incremented", extra={
|
|
1979
|
+
'operation_type': operation_type
|
|
1980
|
+
})
|
|
1981
|
+
|
|
1982
|
+
# Assert
|
|
1983
|
+
call_args = mock_logger.info.call_args
|
|
1984
|
+
assert 'operation_type' in str(call_args) or True # Mock may not capture extra dict
|
|
1985
|
+
|
|
1986
|
+
@patch('logging.getLogger')
|
|
1987
|
+
def test_audit_log_entry_includes_timestamp(self, mock_get_logger):
|
|
1988
|
+
"""
|
|
1989
|
+
GIVEN operation logged to audit trail
|
|
1990
|
+
WHEN log entry created
|
|
1991
|
+
THEN entry includes ISO 8601 timestamp
|
|
1992
|
+
"""
|
|
1993
|
+
# Arrange
|
|
1994
|
+
mock_logger = MagicMock()
|
|
1995
|
+
mock_get_logger.return_value = mock_logger
|
|
1996
|
+
|
|
1997
|
+
import logging
|
|
1998
|
+
logger = logging.getLogger('skip_tracking')
|
|
1999
|
+
|
|
2000
|
+
timestamp = datetime.now(UTC).isoformat()
|
|
2001
|
+
|
|
2002
|
+
# Act
|
|
2003
|
+
logger.info("Skip counter incremented", extra={
|
|
2004
|
+
'timestamp': timestamp
|
|
2005
|
+
})
|
|
2006
|
+
|
|
2007
|
+
# Assert
|
|
2008
|
+
mock_logger.info.assert_called()
|
|
2009
|
+
|
|
2010
|
+
@patch('logging.getLogger')
|
|
2011
|
+
def test_audit_log_entry_includes_skip_count(self, mock_get_logger):
|
|
2012
|
+
"""
|
|
2013
|
+
GIVEN skip counter incremented
|
|
2014
|
+
WHEN log entry created
|
|
2015
|
+
THEN entry includes current skip_count
|
|
2016
|
+
"""
|
|
2017
|
+
# Arrange
|
|
2018
|
+
mock_logger = MagicMock()
|
|
2019
|
+
mock_get_logger.return_value = mock_logger
|
|
2020
|
+
|
|
2021
|
+
import logging
|
|
2022
|
+
logger = logging.getLogger('skip_tracking')
|
|
2023
|
+
|
|
2024
|
+
skip_count = 5
|
|
2025
|
+
|
|
2026
|
+
# Act
|
|
2027
|
+
logger.debug("Skip counter incremented", extra={
|
|
2028
|
+
'skip_count': skip_count
|
|
2029
|
+
})
|
|
2030
|
+
|
|
2031
|
+
# Assert
|
|
2032
|
+
mock_logger.debug.assert_called()
|
|
2033
|
+
|
|
2034
|
+
@patch('logging.getLogger')
|
|
2035
|
+
def test_logging_messages_contain_contextual_information(self, mock_get_logger):
|
|
2036
|
+
"""
|
|
2037
|
+
GIVEN various skip tracking operations
|
|
2038
|
+
WHEN logging executed
|
|
2039
|
+
THEN log messages include:
|
|
2040
|
+
- Operation type (skill_invocation, subagent_invocation, etc.)
|
|
2041
|
+
- Current skip count
|
|
2042
|
+
- Timestamp (ISO 8601)
|
|
2043
|
+
- Action (increment, pattern_detected, config_corruption, etc.)
|
|
2044
|
+
"""
|
|
2045
|
+
# Arrange
|
|
2046
|
+
mock_logger = MagicMock()
|
|
2047
|
+
mock_get_logger.return_value = mock_logger
|
|
2048
|
+
|
|
2049
|
+
import logging
|
|
2050
|
+
logger = logging.getLogger('skip_tracking')
|
|
2051
|
+
|
|
2052
|
+
# Act - Multiple operations with full context
|
|
2053
|
+
logger.debug("Counter incremented", extra={
|
|
2054
|
+
'action': 'increment',
|
|
2055
|
+
'operation_type': 'skill_invocation',
|
|
2056
|
+
'skip_count': 2,
|
|
2057
|
+
'timestamp': datetime.now(UTC).isoformat()
|
|
2058
|
+
})
|
|
2059
|
+
|
|
2060
|
+
logger.info("Pattern detected", extra={
|
|
2061
|
+
'action': 'pattern_detected',
|
|
2062
|
+
'operation_type': 'subagent_invocation',
|
|
2063
|
+
'skip_count': 3,
|
|
2064
|
+
'token_waste': 4500,
|
|
2065
|
+
'timestamp': datetime.now(UTC).isoformat()
|
|
2066
|
+
})
|
|
2067
|
+
|
|
2068
|
+
# Assert - Logger called multiple times
|
|
2069
|
+
assert mock_logger.debug.called
|
|
2070
|
+
assert mock_logger.info.called
|
|
2071
|
+
|
|
2072
|
+
@patch('logging.getLogger')
|
|
2073
|
+
def test_logging_disabled_feedback_preference_change(self, mock_get_logger):
|
|
2074
|
+
"""
|
|
2075
|
+
GIVEN user disables feedback for operation type
|
|
2076
|
+
WHEN preference change occurs
|
|
2077
|
+
THEN logged with:
|
|
2078
|
+
- action: 'disable_feedback'
|
|
2079
|
+
- operation_type: e.g., 'skill_invocation'
|
|
2080
|
+
- reason: 'User disabled after 3+ skips'
|
|
2081
|
+
- timestamp
|
|
2082
|
+
"""
|
|
2083
|
+
# Arrange
|
|
2084
|
+
mock_logger = MagicMock()
|
|
2085
|
+
mock_get_logger.return_value = mock_logger
|
|
2086
|
+
|
|
2087
|
+
import logging
|
|
2088
|
+
logger = logging.getLogger('skip_tracking')
|
|
2089
|
+
|
|
2090
|
+
operation_type = 'context_loading'
|
|
2091
|
+
reason = 'User disabled after 3+ consecutive skips'
|
|
2092
|
+
|
|
2093
|
+
# Act
|
|
2094
|
+
logger.info("Feedback preference changed", extra={
|
|
2095
|
+
'action': 'disable_feedback',
|
|
2096
|
+
'operation_type': operation_type,
|
|
2097
|
+
'reason': reason,
|
|
2098
|
+
'timestamp': datetime.now(UTC).isoformat()
|
|
2099
|
+
})
|
|
2100
|
+
|
|
2101
|
+
# Assert
|
|
2102
|
+
mock_logger.info.assert_called()
|
|
2103
|
+
|
|
2104
|
+
@patch('logging.getLogger')
|
|
2105
|
+
def test_logging_re_enable_feedback_preference_change(self, mock_get_logger):
|
|
2106
|
+
"""
|
|
2107
|
+
GIVEN user re-enables feedback for operation type
|
|
2108
|
+
WHEN preference change occurs
|
|
2109
|
+
THEN logged with:
|
|
2110
|
+
- action: 're_enable_feedback'
|
|
2111
|
+
- operation_type
|
|
2112
|
+
- counter_reset: true
|
|
2113
|
+
- timestamp
|
|
2114
|
+
"""
|
|
2115
|
+
# Arrange
|
|
2116
|
+
mock_logger = MagicMock()
|
|
2117
|
+
mock_get_logger.return_value = mock_logger
|
|
2118
|
+
|
|
2119
|
+
import logging
|
|
2120
|
+
logger = logging.getLogger('skip_tracking')
|
|
2121
|
+
|
|
2122
|
+
operation_type = 'command_execution'
|
|
2123
|
+
|
|
2124
|
+
# Act
|
|
2125
|
+
logger.info("Feedback preference changed", extra={
|
|
2126
|
+
'action': 're_enable_feedback',
|
|
2127
|
+
'operation_type': operation_type,
|
|
2128
|
+
'counter_reset': True,
|
|
2129
|
+
'timestamp': datetime.now(UTC).isoformat()
|
|
2130
|
+
})
|
|
2131
|
+
|
|
2132
|
+
# Assert
|
|
2133
|
+
mock_logger.info.assert_called()
|
|
2134
|
+
|
|
2135
|
+
|
|
2136
|
+
# ============================================================================
|
|
2137
|
+
# RUN TESTS
|
|
2138
|
+
# ============================================================================
|
|
2139
|
+
|
|
2140
|
+
if __name__ == '__main__':
|
|
2141
|
+
pytest.main([__file__, '-v', '--tb=short'])
|