add-skill-kit 3.2.4 → 3.2.6

package/lib/agent-cli/__tests__/adaptive_engine.test.js

@@ -0,0 +1,190 @@
+/**
+ * Unit Tests for adaptive_engine.js
+ * 
+ * Tests:
+ * - adjustSeverity
+ * - promoteRules
+ * - demoteRules
+ * - analyzeRules
+ * - runAdaptiveCycle
+ */
+
+import { describe, it, expect, vi } from 'vitest';
+
+// Mock fs module
+vi.mock('fs', async () => {
+    const actual = await vi.importActual('fs');
+    return {
+        ...actual,
+        default: {
+            ...actual,
+            existsSync: vi.fn(() => false),
+            readFileSync: vi.fn(() => '{}'),
+            writeFileSync: vi.fn()
+        },
+        existsSync: vi.fn(() => false),
+        readFileSync: vi.fn(() => '{}')
+    };
+});
+
+// Severity levels constant
+const SEVERITY_LEVELS = ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL'];
+
+describe('Adaptive Engine', () => {
+    describe('adjustSeverity', () => {
+        it('should increase severity for 10+ hits', () => {
+            const lesson = {
+                id: 'TEST-001',
+                hitCount: 15,
+                severity: 'MEDIUM'
+            };
+
+            const currentIndex = SEVERITY_LEVELS.indexOf(lesson.severity);
+
+            if (lesson.hitCount >= 10 && currentIndex < SEVERITY_LEVELS.length - 1) {
+                lesson.severity = SEVERITY_LEVELS[currentIndex + 1];
+            }
+
+            expect(lesson.severity).toBe('HIGH');
+        });
+
+        it('should increase to CRITICAL for 25+ hits', () => {
+            const lesson = {
+                id: 'TEST-001',
+                hitCount: 30,
+                severity: 'LOW'
+            };
+
+            const currentIndex = SEVERITY_LEVELS.indexOf(lesson.severity);
+
+            if (lesson.hitCount >= 25 && currentIndex < SEVERITY_LEVELS.length - 2) {
+                lesson.severity = SEVERITY_LEVELS[currentIndex + 2];
+            }
+
+            expect(lesson.severity).toBe('HIGH');
+        });
+
+        it('should decrease severity for 0 hits in 30 days', () => {
+            const thirtyDaysAgo = new Date(Date.now() - 31 * 24 * 60 * 60 * 1000);
+            const lesson = {
+                id: 'TEST-001',
+                hitCount: 0,
+                severity: 'HIGH',
+                lastHit: thirtyDaysAgo.toISOString()
+            };
+
+            const lastHit = new Date(lesson.lastHit);
+            const thresholdDate = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
+            const currentIndex = SEVERITY_LEVELS.indexOf(lesson.severity);
+
+            if (lastHit < thresholdDate && currentIndex > 0) {
+                lesson.severity = SEVERITY_LEVELS[currentIndex - 1];
+            }
+
+            expect(lesson.severity).toBe('MEDIUM');
+        });
+
+        it('should not decrease LOW severity further', () => {
+            const lesson = {
+                id: 'TEST-001',
+                severity: 'LOW'
+            };
+
+            const currentIndex = SEVERITY_LEVELS.indexOf(lesson.severity);
+            expect(currentIndex).toBe(0);
+        });
+    });
+
+    describe('promoteRules', () => {
+        it('should promote rules with 95%+ accuracy', () => {
+            const lesson = {
+                id: 'TEST-001',
+                hitCount: 1,
+                successCount: 19, // 19/20 = 95%
+                severity: 'MEDIUM'
+            };
+
+            const totalAttempts = lesson.hitCount + lesson.successCount;
+            const accuracy = lesson.successCount / totalAttempts;
+
+            if (accuracy >= 0.95 && lesson.severity !== 'CRITICAL') {
+                lesson.promoted = true;
+                lesson.severity = 'CRITICAL';
+            }
+
+            expect(lesson.promoted).toBe(true);
+            expect(lesson.severity).toBe('CRITICAL');
+        });
+
+        it('should not promote rules with <10 attempts', () => {
+            const lesson = {
+                id: 'TEST-001',
+                hitCount: 1,
+                successCount: 5, // Only 6 attempts
+                severity: 'MEDIUM'
+            };
+
+            const totalAttempts = lesson.hitCount + lesson.successCount;
+
+            // Need 10+ attempts to evaluate
+            const shouldEvaluate = totalAttempts >= 10;
+
+            expect(shouldEvaluate).toBe(false);
+        });
+    });
+
+    describe('demoteRules', () => {
+        it('should demote rules with <50% accuracy', () => {
+            const lesson = {
+                id: 'TEST-001',
+                hitCount: 8,
+                successCount: 2, // 2/10 = 20%
+                severity: 'HIGH'
+            };
+
+            const totalAttempts = lesson.hitCount + lesson.successCount;
+            const accuracy = lesson.successCount / totalAttempts;
+
+            if (accuracy < 0.50 && lesson.severity !== 'LOW') {
+                lesson.demoted = true;
+                lesson.severity = 'LOW';
+            }
+
+            expect(lesson.demoted).toBe(true);
+            expect(lesson.severity).toBe('LOW');
+        });
+
+        it('should not demote LOW severity rules', () => {
+            const lesson = {
+                id: 'TEST-001',
+                severity: 'LOW'
+            };
+
+            const shouldDemote = lesson.severity !== 'LOW';
+
+            expect(shouldDemote).toBe(false);
+        });
+    });
+
+    describe('analyzeRules', () => {
+        it('should group lessons by severity', () => {
+            const lessons = [
+                { severity: 'LOW' },
+                { severity: 'MEDIUM' },
+                { severity: 'MEDIUM' },
+                { severity: 'HIGH' },
+                { severity: 'CRITICAL' }
+            ];
+
+            const bySeverity = {};
+            for (const lesson of lessons) {
+                bySeverity[lesson.severity] = (bySeverity[lesson.severity] || 0) + 1;
+            }
+
+            expect(bySeverity.LOW).toBe(1);
+            expect(bySeverity.MEDIUM).toBe(2);
+            expect(bySeverity.HIGH).toBe(1);
+            expect(bySeverity.CRITICAL).toBe(1);
+        });
+    });
+});

package/lib/agent-cli/__tests__/integration/cross_script.test.js

@@ -0,0 +1,222 @@
+/**
+ * E2E Integration Tests - Cross-Script Integration
+ * 
+ * Tests integration between different scripts:
+ * - error_sensor ↔ pattern_analyzer
+ * - pattern_analyzer ↔ pre_execution_check
+ * - adaptive_engine ↔ skill_injector
+ */
+
+import { describe, it, expect, vi } from 'vitest';
+
+describe('E2E: Cross-Script Integration', () => {
+    describe('Error Sensor → Pattern Analyzer', () => {
+        it('should pass error format between scripts', () => {
+            // Error format from error_sensor
+            const sensorOutput = {
+                id: 'ERR-001',
+                type: 'test-failure',
+                source: 'test',
+                message: 'Test failed',
+                severity: 'HIGH',
+                timestamp: new Date().toISOString(),
+                context: {
+                    project: { projectType: 'nextjs' },
+                    file: { fileType: '.tsx' }
+                }
+            };
+
+            // Pattern analyzer input format
+            const analyzerInput = [sensorOutput];
+
+            // Verify format compatibility
+            expect(analyzerInput[0].type).toBeDefined();
+            expect(analyzerInput[0].severity).toBeDefined();
+            expect(analyzerInput[0].context).toBeDefined();
+        });
+
+        it('should aggregate errors correctly', () => {
+            const errors = [
+                { type: 'lint', severity: 'LOW' },
+                { type: 'lint', severity: 'LOW' },
+                { type: 'test', severity: 'HIGH' },
+                { type: 'lint', severity: 'LOW' }
+            ];
+
+            // Pattern analyzer aggregation
+            const byType = {};
+            const bySeverity = {};
+
+            for (const err of errors) {
+                byType[err.type] = (byType[err.type] || 0) + 1;
+                bySeverity[err.severity] = (bySeverity[err.severity] || 0) + 1;
+            }
+
+            expect(byType.lint).toBe(3);
+            expect(byType.test).toBe(1);
+            expect(bySeverity.LOW).toBe(3);
+            expect(bySeverity.HIGH).toBe(1);
+        });
+    });
+
+    describe('Pattern Analyzer → Pre-Execution Check', () => {
+        it('should generate rules compatible with checker', () => {
+            // Pattern analyzer output
+            const highFrequency = [
+                { pattern: 'missing-await', count: 5, type: 'error' },
+                { pattern: 'no-try-catch', count: 4, type: 'error' }
+            ];
+
+            // Generate rules for pre-execution check
+            const rules = highFrequency.map(hf => ({
+                id: `AUTO-${hf.pattern.toUpperCase()}`,
+                pattern: hf.pattern,
+                prevention: `Avoid ${hf.pattern}`,
+                severity: hf.count >= 5 ? 'HIGH' : 'MEDIUM',
+                status: 'proposed'
+            }));
+
+            expect(rules[0].id).toBe('AUTO-MISSING-AWAIT');
+            expect(rules[0].severity).toBe('HIGH');
+            expect(rules[1].severity).toBe('MEDIUM');
+        });
+
+        it('should match rules against intent', () => {
+            const rules = [
+                { pattern: 'delete', severity: 'CRITICAL' },
+                { pattern: 'async', severity: 'MEDIUM' }
+            ];
+
+            const intent = 'create async function';
+            const matches = rules.filter(r =>
+                intent.toLowerCase().includes(r.pattern)
+            );
+
+            expect(matches.length).toBe(1);
+            expect(matches[0].pattern).toBe('async');
+        });
+    });
+
+    describe('Adaptive Engine → Skill Injector', () => {
+        it('should pass promoted lessons for skill generation', () => {
+            // Adaptive engine output
+            const promotedLessons = [
+                { id: 'SAFE-001', pattern: 'backup-first', severity: 'CRITICAL', promoted: true },
+                { id: 'SAFE-002', pattern: 'confirm-delete', severity: 'CRITICAL', promoted: true },
+                { id: 'SAFE-003', pattern: 'check-exists', severity: 'CRITICAL', promoted: true }
+            ];
+
+            // Skill injector categorization
+            const categories = {};
+            for (const lesson of promotedLessons) {
+                const category = lesson.id.split('-')[0];
+                if (!categories[category]) {
+                    categories[category] = [];
+                }
+                categories[category].push(lesson);
+            }
+
+            expect(categories.SAFE.length).toBe(3);
+            expect(categories.SAFE.every(l => l.promoted)).toBe(true);
+        });
+
+        it('should generate skill from 3+ lessons', () => {
+            const lessons = [
+                { id: 'CODE-001', pattern: 'type-annotation' },
+                { id: 'CODE-002', pattern: 'null-check' },
+                { id: 'CODE-003', pattern: 'error-handling' }
+            ];
+
+            const canGenerateSkill = lessons.length >= 3;
+            expect(canGenerateSkill).toBe(true);
+
+            // Mock skill generation
+            const skill = {
+                name: 'code-patterns',
+                lessonCount: lessons.length,
+                generatedAt: new Date().toISOString()
+            };
+
+            expect(skill.name).toBe('code-patterns');
+            expect(skill.lessonCount).toBe(3);
+        });
+    });
+
+    describe('Success Sensor → Pattern Analyzer', () => {
+        it('should pass success format for balance calculation', () => {
+            const successes = [
+                { pattern: 'async-await', type: 'pattern' },
+                { pattern: 'typescript', type: 'pattern' }
+            ];
+
+            const errors = [
+                { type: 'test', severity: 'HIGH' }
+            ];
+
+            // Calculate balance
+            const totalSuccesses = successes.length;
+            const totalFailures = errors.length;
+            const total = totalSuccesses + totalFailures;
+            const ratio = totalSuccesses / total;
+
+            expect(ratio).toBeCloseTo(0.67, 1);
+        });
+    });
+
+    describe('Rule Sharing → Pre-Execution Check', () => {
+        it('should import rules in correct format', () => {
+            // Exported rule format
+            const exportedRule = {
+                id: 'SAFE-001',
+                pattern: 'delete-without-backup',
+                severity: 'CRITICAL',
+                source: 'imported'
+            };
+
+            // Verify format for pre-execution
+            expect(exportedRule.id).toBeDefined();
+            expect(exportedRule.pattern).toBeDefined();
+            expect(exportedRule.severity).toBeDefined();
+            expect(['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']).toContain(exportedRule.severity);
+        });
+    });
+
+    describe('Full Cross-Script Pipeline', () => {
+        it('should maintain data integrity through pipeline', () => {
+            // 1. Error sensor detects
+            const error = {
+                id: 'ERR-001',
+                type: 'missing-return-type',
+                severity: 'MEDIUM',
+                timestamp: new Date().toISOString()
+            };
+
+            // 2. Pattern analyzer creates lesson
+            const lesson = {
+                id: 'CODE-001',
+                pattern: error.type,
+                severity: error.severity,
+                hitCount: 1
+            };
+
+            // 3. After multiple hits, adaptive engine promotes
+            lesson.hitCount = 25;
+            if (lesson.hitCount >= 25) {
+                lesson.severity = 'CRITICAL';
+                lesson.promoted = true;
+            }
+
+            // 4. Pre-execution check uses promoted rule
+            const intent = 'create function missing-return-type';
+            const matches = intent.includes(lesson.pattern);
+
+            // 5. Skill injector can use for skill
+            const canMakeSkill = lesson.promoted;
+
+            expect(error.type).toBe(lesson.pattern);
+            expect(lesson.severity).toBe('CRITICAL');
+            expect(matches).toBe(true);
+            expect(canMakeSkill).toBe(true);
+        });
+    });
+});

package/lib/agent-cli/__tests__/integration/full_cycle.test.js

@@ -0,0 +1,230 @@
+/**
+ * E2E Integration Tests - Full Learning Cycle
+ * 
+ * Tests the complete auto-learn workflow:
+ * 1. Error Detection → Lesson Extraction
+ * 2. Pattern Analysis → Rule Generation
+ * 3. Pre-execution Check → Prevention
+ * 4. Adaptive Engine → Severity Adjustment
+ */
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import fs from 'fs';
+import path from 'path';
+
+// Mock knowledge path
+const mockKnowledgePath = path.join(process.cwd(), '.test-knowledge');
+
+// Mock data structures
+const createMockError = (id, type, severity = 'HIGH') => ({
+    id,
+    type,
+    severity,
+    message: `Test error ${id}`,
+    timestamp: new Date().toISOString()
+});
+
+const createMockLesson = (id, pattern, severity = 'MEDIUM') => ({
+    id,
+    pattern,
+    severity,
+    message: `Lesson from ${pattern}`,
+    hitCount: 0
+});
+
+describe('E2E: Full Learning Cycle', () => {
+    beforeEach(() => {
+        // Setup mock knowledge directory
+        if (!fs.existsSync(mockKnowledgePath)) {
+            fs.mkdirSync(mockKnowledgePath, { recursive: true });
+        }
+    });
+
+    afterEach(() => {
+        // Cleanup
+        if (fs.existsSync(mockKnowledgePath)) {
+            fs.rmSync(mockKnowledgePath, { recursive: true });
+        }
+    });
+
+    describe('Error → Lesson Flow', () => {
+        it('should create lessons from detected errors', () => {
+            // Simulate error detection
+            const errors = [
+                createMockError('ERR-001', 'test-failure'),
+                createMockError('ERR-002', 'test-failure'),
+                createMockError('ERR-003', 'test-failure')
+            ];
+
+            // Group by type (simulating pattern analysis)
+            const errorsByType = {};
+            for (const err of errors) {
+                errorsByType[err.type] = (errorsByType[err.type] || 0) + 1;
+            }
+
+            // Find high-frequency (3+ occurrences)
+            const threshold = 3;
+            const highFrequency = Object.entries(errorsByType)
+                .filter(([, count]) => count >= threshold);
+
+            expect(highFrequency.length).toBe(1);
+            expect(highFrequency[0][0]).toBe('test-failure');
+
+            // Generate lesson from high-frequency pattern
+            const lesson = createMockLesson(
+                'LEARN-001',
+                'test-failure',
+                'HIGH'
+            );
+
+            expect(lesson.pattern).toBe('test-failure');
+        });
+    });
+
+    describe('Lesson → Rule Flow', () => {
+        it('should generate rules from lessons with 10+ hits', () => {
+            const lesson = createMockLesson('LEARN-001', 'async-await-missing');
+            lesson.hitCount = 15; // High hit count
+
+            const SEVERITY_LEVELS = ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL'];
+            const currentIndex = SEVERITY_LEVELS.indexOf(lesson.severity);
+
+            // Adjust severity for 10+ hits
+            if (lesson.hitCount >= 10 && currentIndex < SEVERITY_LEVELS.length - 1) {
+                lesson.severity = SEVERITY_LEVELS[currentIndex + 1];
+            }
+
+            expect(lesson.severity).toBe('HIGH');
+        });
+
+        it('should maintain lesson through multiple cycles', () => {
+            const lesson = createMockLesson('LEARN-001', 'type-error');
+
+            // Simulate 5 learning cycles
+            for (let i = 0; i < 5; i++) {
+                lesson.hitCount++;
+                lesson.lastHit = new Date().toISOString();
+            }
+
+            expect(lesson.hitCount).toBe(5);
+            expect(lesson.lastHit).toBeDefined();
+        });
+    });
+
+    describe('Rule → Prevention Flow', () => {
+        it('should block execution for matching critical patterns', () => {
+            const lessons = [
+                { id: 'SAFE-001', pattern: 'delete', severity: 'CRITICAL' }
+            ];
+
+            const intent = 'delete old files';
+            const violations = [];
+
+            for (const lesson of lessons) {
+                if (intent.toLowerCase().includes(lesson.pattern.toLowerCase())) {
+                    if (lesson.severity === 'CRITICAL') {
+                        violations.push({
+                            id: lesson.id,
+                            prevention: `Violation: ${lesson.pattern}`,
+                            severity: 'CRITICAL'
+                        });
+                    }
+                }
+            }
+
+            expect(violations.length).toBe(1);
+            expect(violations[0].severity).toBe('CRITICAL');
+        });
+
+        it('should allow execution when no violations', () => {
+            const lessons = [
+                { id: 'SAFE-001', pattern: 'delete', severity: 'CRITICAL' }
+            ];
+
+            const intent = 'create new component';
+            const violations = [];
+
+            for (const lesson of lessons) {
+                if (intent.toLowerCase().includes(lesson.pattern.toLowerCase())) {
+                    violations.push(lesson);
+                }
+            }
+
+            expect(violations.length).toBe(0);
+        });
+    });
+
+    describe('Adaptive Cycle', () => {
+        it('should promote rules with 95%+ accuracy', () => {
+            const lesson = createMockLesson('LEARN-001', 'typescript-strict');
+            lesson.hitCount = 1;
+            lesson.successCount = 19; // 95% accuracy
+
+            const totalAttempts = lesson.hitCount + lesson.successCount;
+            const accuracy = lesson.successCount / totalAttempts;
+
+            if (accuracy >= 0.95 && totalAttempts >= 10) {
+                lesson.promoted = true;
+                lesson.severity = 'CRITICAL';
+            }
+
+            expect(lesson.promoted).toBe(true);
+            expect(lesson.severity).toBe('CRITICAL');
+            expect(accuracy).toBeGreaterThanOrEqual(0.95);
+        });
+
+        it('should demote rules with <50% accuracy', () => {
+            const lesson = createMockLesson('LEARN-001', 'outdated-pattern');
+            lesson.severity = 'HIGH';
+            lesson.hitCount = 8;
+            lesson.successCount = 2; // 20% accuracy
+
+            const totalAttempts = lesson.hitCount + lesson.successCount;
+            const accuracy = lesson.successCount / totalAttempts;
+
+            if (accuracy < 0.50 && totalAttempts >= 10) {
+                lesson.demoted = true;
+                lesson.severity = 'LOW';
+            }
+
+            expect(lesson.demoted).toBe(true);
+            expect(lesson.severity).toBe('LOW');
+        });
+    });
+
+    describe('Complete Workflow', () => {
+        it('should handle full cycle: detect → learn → prevent', () => {
+            // Step 1: Detect errors
+            const detectedErrors = [
+                createMockError('E1', 'missing-await'),
+                createMockError('E2', 'missing-await'),
+                createMockError('E3', 'missing-await')
+            ];
+
+            // Step 2: Analyze patterns
+            const patterns = {};
+            for (const err of detectedErrors) {
+                patterns[err.type] = (patterns[err.type] || 0) + 1;
+            }
+
+            // Step 3: Generate lesson from high-frequency
+            const highFreq = Object.entries(patterns)
+                .filter(([, c]) => c >= 3);
+
+            expect(highFreq.length).toBeGreaterThan(0);
+
+            const lessons = highFreq.map(([type]) =>
+                createMockLesson(`LEARN-${type}`, type, 'HIGH')
+            );
+
+            // Step 4: Check new intent against lessons
+            const newIntent = 'create function with missing await';
+            const matches = lessons.filter(l =>
+                newIntent.toLowerCase().includes(l.pattern)
+            );
+
+            expect(matches.length).toBe(1);
+            expect(matches[0].severity).toBe('HIGH');
+        });
+    });
+});