fraim-framework 2.0.43 → 2.0.45

package/dist/src/ai-manager/evidence-validator.js

@@ -0,0 +1,309 @@
+"use strict";
+/**
+ * Evidence Validator - Validates agent claims with concrete evidence
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvidenceValidator = void 0;
+class EvidenceValidator {
+    /**
+     * Validate test evidence
+     */
+    validateTestEvidence(evidence) {
+        const issues = [];
+        const warnings = [];
+        let score = 0;
+        // Check if test output is provided
+        if (!evidence.testOutput) {
+            issues.push("❌ CRITICAL: No test output provided. Cannot verify test claims.");
+            return { isValid: false, issues, warnings, score: 0 };
+        }
+        // Parse test output for validation
+        const testOutput = evidence.testOutput.toLowerCase();
+        // Check for 100% pass rate
+        if (this.hasFailingTests(testOutput)) {
+            issues.push("❌ CRITICAL: Test output shows failing tests. Cannot proceed with failing tests.");
+            score = 0;
+        }
+        else if (this.hasPassingTests(testOutput)) {
+            score += 40; // Base score for passing tests
+            // Check for comprehensive test coverage
+            if (this.hasComprehensiveTestCoverage(testOutput)) {
+                score += 20;
+            }
+            else {
+                warnings.push("⚠️ WARNING: Test coverage appears limited. Consider adding more test scenarios.");
+            }
+        }
+        else {
+            issues.push("❌ CRITICAL: Cannot determine test results from provided output. Please provide complete test output.");
+            score = 0;
+        }
+        // Check for test count (should have meaningful number of tests)
+        const testCount = this.extractTestCount(testOutput);
+        if (testCount === 0) {
+            issues.push("❌ CRITICAL: No tests detected in output. Must have tests to validate functionality.");
+            score = 0;
+        }
+        else if (testCount < 3) {
+            warnings.push(`⚠️ WARNING: Only ${testCount} tests detected. Consider adding more comprehensive tests.`);
+        }
+        else {
+            score += 10; // Bonus for having multiple tests
+        }
+        // Check for timeout or hanging tests
+        if (this.hasTimeoutIssues(testOutput)) {
+            issues.push("❌ CRITICAL: Test output shows timeouts or hanging tests. Must resolve before proceeding.");
+            score = Math.max(0, score - 30);
+        }
+        return {
+            isValid: issues.length === 0,
+            issues,
+            warnings,
+            score: Math.min(100, score)
+        };
+    }
+    /**
+     * Validate manual validation evidence
+     */
+    validateManualEvidence(evidence) {
+        const issues = [];
+        const warnings = [];
+        let score = 0;
+        // Check for manual validation steps
+        if (!evidence.manualValidationSteps || evidence.manualValidationSteps.length === 0) {
+            issues.push("❌ CRITICAL: No manual validation steps provided. Manual testing is mandatory.");
+            return { isValid: false, issues, warnings, score: 0 };
+        }
+        // Check for screenshots (for UI changes)
+        if (!evidence.screenshots || evidence.screenshots.length === 0) {
+            warnings.push("⚠️ WARNING: No screenshots provided. For UI changes, screenshots are required evidence.");
+            score += 20; // Partial credit
+        }
+        else {
+            score += 40; // Good evidence
+            // Check screenshot quality
+            if (this.hasQualityScreenshots(evidence.screenshots)) {
+                score += 20;
+            }
+        }
+        // Check for API responses (for API changes)
+        if (evidence.apiResponses && evidence.apiResponses.length > 0) {
+            score += 20; // Bonus for API testing
+            if (this.hasComprehensiveApiTesting(evidence.apiResponses)) {
+                score += 10;
+            }
+        }
+        // Validate manual validation steps quality
+        const stepsQuality = this.assessManualValidationQuality(evidence.manualValidationSteps);
+        if (stepsQuality.score < 50) {
+            issues.push("❌ CRITICAL: Manual validation steps are insufficient. Must provide detailed step-by-step validation.");
+            score = 0;
+        }
+        else {
+            score += stepsQuality.score * 0.2; // Up to 20 points
+        }
+        return {
+            isValid: issues.length === 0,
+            issues,
+            warnings,
+            score: Math.min(100, score)
+        };
+    }
+    /**
+     * Validate reflection analysis
+     */
+    validateReflectionEvidence(evidence) {
+        const issues = [];
+        const warnings = [];
+        let score = 0;
+        if (!evidence.reflectionAnalysis) {
+            issues.push("❌ CRITICAL: No reflection analysis provided. Mandatory pre-completion reflection is required.");
+            return { isValid: false, issues, warnings, score: 0 };
+        }
+        const reflection = evidence.reflectionAnalysis.toLowerCase();
+        // Check for all 4 phases
+        const phases = ['phase 1', 'phase 2', 'phase 3', 'phase 4'];
+        const missingPhases = phases.filter(phase => !reflection.includes(phase));
+        if (missingPhases.length > 0) {
+            issues.push(`❌ CRITICAL: Missing reflection phases: ${missingPhases.join(', ')}. All 4 phases are mandatory.`);
+            score = 0;
+        }
+        else {
+            score += 40; // Base score for having all phases
+        }
+        // Check for specific reflection elements
+        const requiredElements = [
+            'evidence',
+            'manual testing',
+            'ship to customers',
+            'bet money',
+            'reproducible'
+        ];
+        const missingElements = requiredElements.filter(element => !reflection.includes(element));
+        if (missingElements.length > 0) {
+            warnings.push(`⚠️ WARNING: Reflection may be incomplete. Consider addressing: ${missingElements.join(', ')}`);
+        }
+        else {
+            score += 30; // Bonus for comprehensive reflection
+        }
+        // Check for honesty indicators
+        if (this.hasHonestyIndicators(reflection)) {
+            score += 20;
+        }
+        else {
+            warnings.push("⚠️ WARNING: Reflection may lack sufficient self-assessment depth.");
+        }
+        // Check for any "NO" answers in critical questions
+        if (this.hasCriticalNoAnswers(reflection)) {
+            issues.push("❌ CRITICAL: Reflection contains 'NO' answers to critical questions. Must address issues before proceeding.");
+            score = 0;
+        }
+        return {
+            isValid: issues.length === 0,
+            issues,
+            warnings,
+            score: Math.min(100, score)
+        };
+    }
+    /**
+     * Comprehensive evidence validation
+     */
+    validateAllEvidence(evidence) {
+        const testValidation = this.validateTestEvidence(evidence);
+        const manualValidation = this.validateManualEvidence(evidence);
+        const reflectionValidation = this.validateReflectionEvidence(evidence);
+        const allIssues = [
+            ...testValidation.issues,
+            ...manualValidation.issues,
+            ...reflectionValidation.issues
+        ];
+        const allWarnings = [
+            ...testValidation.warnings,
+            ...manualValidation.warnings,
+            ...reflectionValidation.warnings
+        ];
+        // Calculate weighted score
+        const weightedScore = (testValidation.score * 0.4 + // 40% weight on tests
+            manualValidation.score * 0.4 + // 40% weight on manual validation
+            reflectionValidation.score * 0.2 // 20% weight on reflection
+        );
+        return {
+            isValid: allIssues.length === 0 && weightedScore >= 70, // Minimum 70% score required
+            issues: allIssues,
+            warnings: allWarnings,
+            score: Math.round(weightedScore)
+        };
+    }
+    // Helper methods for parsing evidence
+    hasFailingTests(testOutput) {
+        const failureIndicators = [
+            'failed',
+            'error',
+            'failing',
+            'fail 1',
+            'fail 2',
+            'timeout',
+            'assertion error',
+            'test failed'
+        ];
+        return failureIndicators.some(indicator => testOutput.includes(indicator));
+    }
+    hasPassingTests(testOutput) {
+        const passIndicators = [
+            'pass',
+            'passed',
+            'success',
+            'ok',
+            '✓',
+            '✔',
+            'all tests passed'
+        ];
+        return passIndicators.some(indicator => testOutput.includes(indicator));
+    }
+    hasComprehensiveTestCoverage(testOutput) {
+        const coverageIndicators = [
+            'coverage',
+            'multiple test',
+            'integration',
+            'unit test',
+            'e2e',
+            'smoke test'
+        ];
+        return coverageIndicators.some(indicator => testOutput.includes(indicator));
+    }
+    extractTestCount(testOutput) {
+        // Look for patterns like "5 tests passed", "tests 10", etc.
+        const testCountPatterns = [
+            /(\d+)\s+tests?\s+passed/i,
+            /tests?\s+(\d+)/i,
+            /(\d+)\s+passing/i,
+            /pass\s+(\d+)/i
+        ];
+        for (const pattern of testCountPatterns) {
+            const match = testOutput.match(pattern);
+            if (match) {
+                return parseInt(match[1], 10);
+            }
+        }
+        return 0;
+    }
+    hasTimeoutIssues(testOutput) {
+        const timeoutIndicators = [
+            'timeout',
+            'timed out',
+            'hanging',
+            'stuck',
+            'exceeded timeout'
+        ];
+        return timeoutIndicators.some(indicator => testOutput.includes(indicator));
+    }
+    hasQualityScreenshots(screenshots) {
+        // Check if screenshots have descriptive names/content
+        return screenshots.some(screenshot => screenshot.length > 20 && // Has some description
+            (screenshot.includes('working') || screenshot.includes('success') || screenshot.includes('complete')));
+    }
+    hasComprehensiveApiTesting(apiResponses) {
+        return apiResponses.some(response => response.includes('200') || response.includes('success') || response.includes('json'));
+    }
+    assessManualValidationQuality(steps) {
+        let score = 0;
+        const issues = [];
+        if (steps.length < 3) {
+            issues.push("Insufficient manual validation steps");
+            return { score: 20, issues };
+        }
+        // Check for specific validation elements
+        const stepText = steps.join(' ').toLowerCase();
+        if (stepText.includes('browser') || stepText.includes('ui'))
+            score += 25;
+        if (stepText.includes('api') || stepText.includes('curl'))
+            score += 25;
+        if (stepText.includes('error') || stepText.includes('edge case'))
+            score += 25;
+        if (stepText.includes('user flow') || stepText.includes('end-to-end'))
+            score += 25;
+        return { score, issues };
+    }
+    hasHonestyIndicators(reflection) {
+        const honestyIndicators = [
+            'honestly',
+            'truthfully',
+            'actually tested',
+            'real validation',
+            'concrete evidence'
+        ];
+        return honestyIndicators.some(indicator => reflection.includes(indicator));
+    }
+    hasCriticalNoAnswers(reflection) {
+        // Look for "NO" answers to critical questions
+        const criticalNoPatterns = [
+            /ship to customers:\s*no/i,
+            /bet money:\s*no/i,
+            /reproducible:\s*no/i,
+            /evidence quality:\s*no/i,
+            /honesty check:\s*no/i
+        ];
+        return criticalNoPatterns.some(pattern => pattern.test(reflection));
+    }
+}
+exports.EvidenceValidator = EvidenceValidator;

package/dist/src/ai-manager/phase-flow.js

@@ -0,0 +1,244 @@
+"use strict";
+/**
+ * Phase flow logic for AI Manager workflows
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getNextPhase = getNextPhase;
+exports.isPhaseValidForWorkflow = isPhaseValidForWorkflow;
+exports.getPhaseOnFailure = getPhaseOnFailure;
+exports.getPhasesForWorkflow = getPhasesForWorkflow;
+exports.getNextPhaseForImplement = getNextPhaseForImplement;
+exports.isPhaseValidForIssueType = isPhaseValidForIssueType;
+exports.getPhasesForIssueType = getPhasesForIssueType;
+exports.getPhaseOnImplementFailure = getPhaseOnImplementFailure;
+/**
+ * Phase flows for different workflow types
+ */
+const BUG_PHASE_FLOW = [
+    'implement-scoping',
+    'implement-repro',
+    'implement-code',
+    'implement-validate',
+    'implement-regression',
+    'implement-completeness-review',
+    'submit-pr',
+    'wait-for-pr-review',
+];
+const FEATURE_PHASE_FLOW = [
+    'implement-scoping',
+    'implement-spike',
+    'implement-code',
+    'implement-validate',
+    'implement-smoke',
+    'implement-regression',
+    'implement-completeness-review',
+    'submit-pr',
+    'wait-for-pr-review',
+];
+const SPEC_PHASE_FLOW = [
+    'spec-spec',
+    'spec-completeness-review',
+    'submit-pr',
+    'wait-for-pr-review',
+];
+const DESIGN_PHASE_FLOW = [
+    'design-design',
+    'design-completeness-review',
+    'submit-pr',
+    'wait-for-pr-review',
+];
+const TEST_PHASE_FLOW = [
+    'test-test',
+    'test-validate',
+    'submit-pr',
+    'wait-for-pr-review',
+];
+/**
+ * Get the next phase in the workflow
+ * @param currentPhase - Current phase
+ * @param workflowType - Type of workflow
+ * @param issueType - Type of issue (only used for implement workflow)
+ * @returns Next phase or null if at end of workflow
+ */
+function getNextPhase(currentPhase, workflowType, issueType) {
+    let flow;
+    if (workflowType === 'implement') {
+        if (!issueType) {
+            throw new Error('Issue type is required for implement workflow');
+        }
+        flow = issueType === 'bug' ? BUG_PHASE_FLOW : FEATURE_PHASE_FLOW;
+    }
+    else if (workflowType === 'spec') {
+        flow = SPEC_PHASE_FLOW;
+    }
+    else if (workflowType === 'design') {
+        flow = DESIGN_PHASE_FLOW;
+    }
+    else if (workflowType === 'test') {
+        flow = TEST_PHASE_FLOW;
+    }
+    else {
+        throw new Error(`Unknown workflow type: ${workflowType}`);
+    }
+    const currentIndex = flow.indexOf(currentPhase);
+    if (currentIndex === -1) {
+        throw new Error(`Invalid phase "${currentPhase}" for workflow type "${workflowType}"`);
+    }
+    if (currentIndex >= flow.length - 1) {
+        return null; // At end of workflow
+    }
+    return flow[currentIndex + 1];
+}
+/**
+ * Check if a phase is valid for the given workflow type
+ * @param phase - Phase to check
+ * @param workflowType - Type of workflow
+ * @param issueType - Type of issue (only used for implement workflow)
+ * @returns True if phase is valid for workflow type
+ */
+function isPhaseValidForWorkflow(phase, workflowType, issueType) {
+    let flow;
+    if (workflowType === 'implement') {
+        if (!issueType)
+            return false;
+        flow = issueType === 'bug' ? BUG_PHASE_FLOW : FEATURE_PHASE_FLOW;
+    }
+    else if (workflowType === 'spec') {
+        flow = SPEC_PHASE_FLOW;
+    }
+    else if (workflowType === 'design') {
+        flow = DESIGN_PHASE_FLOW;
+    }
+    else if (workflowType === 'test') {
+        flow = TEST_PHASE_FLOW;
+    }
+    else {
+        return false;
+    }
+    return flow.includes(phase);
+}
+/**
+ * Get the phase to return to when a phase fails
+ * @param failedPhase - The phase that failed
+ * @param workflowType - Type of workflow
+ * @param issueType - Type of issue (only used for implement workflow)
+ * @returns Phase to return to
+ */
+function getPhaseOnFailure(failedPhase, workflowType, issueType) {
+    if (workflowType === 'implement') {
+        // Implement workflow failure handling
+        switch (failedPhase) {
+            case 'implement-scoping':
+                return 'implement-scoping'; // Start over at scoping
+            case 'implement-repro':
+                return 'implement-scoping'; // Go back to understand the issue better
+            case 'implement-spike':
+                return 'implement-scoping'; // Go back to understand requirements better
+            case 'implement-code':
+                return 'implement-scoping'; // Go back to re-understand requirements
+            case 'implement-validate':
+                return 'implement-code'; // Implementation doesn't work, fix the code
+            case 'implement-smoke':
+                return 'implement-code'; // Broke existing functionality, fix the implementation
+            case 'implement-regression':
+                return 'implement-code'; // Test issues likely indicate implementation problems
+            case 'implement-completeness-review':
+                return 'implement-validate'; // Missing validation, go back to validate properly
+            case 'submit-pr':
+                return 'implement-completeness-review'; // PR submission issues, review completeness
+            case 'wait-for-pr-review':
+                // This phase determines where to go based on PR comment analysis
+                // Will be handled by the PR comment analysis logic
+                return 'implement-code'; // Default fallback for PR feedback
+            default:
+                return 'implement-code'; // Default fallback
+        }
+    }
+    else if (workflowType === 'spec') {
+        // Spec workflow failure handling
+        switch (failedPhase) {
+            case 'spec-spec':
+                return 'spec-spec'; // Start over at spec
+            case 'spec-completeness-review':
+                return 'spec-spec'; // Go back to improve spec
+            case 'submit-pr':
+                return 'spec-completeness-review';
+            case 'wait-for-pr-review':
+                return 'spec-spec'; // PR feedback usually means spec changes needed
+            default:
+                return 'spec-spec';
+        }
+    }
+    else if (workflowType === 'design') {
+        // Design workflow failure handling
+        switch (failedPhase) {
+            case 'design-design':
+                return 'design-design'; // Start over at design
+            case 'design-completeness-review':
+                return 'design-design'; // Go back to improve design
+            case 'submit-pr':
+                return 'design-completeness-review';
+            case 'wait-for-pr-review':
+                return 'design-design'; // PR feedback usually means design changes needed
+            default:
+                return 'design-design';
+        }
+    }
+    else if (workflowType === 'test') {
+        // Test workflow failure handling
+        switch (failedPhase) {
+            case 'test-test':
+                return 'test-test'; // Start over at test
+            case 'test-validate':
+                return 'test-test'; // Go back to fix tests
+            case 'submit-pr':
+                return 'test-validate';
+            case 'wait-for-pr-review':
+                return 'test-test'; // PR feedback usually means test changes needed
+            default:
+                return 'test-test';
+        }
+    }
+    // Unknown workflow, return first phase of implement as fallback
+    return 'implement-scoping';
+}
+/**
+ * Get all phases for a workflow type
+ * @param workflowType - Type of workflow
+ * @param issueType - Type of issue (only used for implement workflow)
+ * @returns Array of phases
+ */
+function getPhasesForWorkflow(workflowType, issueType) {
+    if (workflowType === 'implement') {
+        if (!issueType) {
+            throw new Error('Issue type is required for implement workflow');
+        }
+        return issueType === 'bug' ? [...BUG_PHASE_FLOW] : [...FEATURE_PHASE_FLOW];
+    }
+    else if (workflowType === 'spec') {
+        return [...SPEC_PHASE_FLOW];
+    }
+    else if (workflowType === 'design') {
+        return [...DESIGN_PHASE_FLOW];
+    }
+    else if (workflowType === 'test') {
+        return [...TEST_PHASE_FLOW];
+    }
+    else {
+        throw new Error(`Unknown workflow type: ${workflowType}`);
+    }
+}
+// Legacy functions for backward compatibility
+function getNextPhaseForImplement(currentPhase, issueType) {
+    return getNextPhase(currentPhase, 'implement', issueType);
+}
+function isPhaseValidForIssueType(phase, issueType) {
+    return isPhaseValidForWorkflow(phase, 'implement', issueType);
+}
+function getPhasesForIssueType(issueType) {
+    return getPhasesForWorkflow('implement', issueType);
+}
+// Legacy function for backward compatibility
+function getPhaseOnImplementFailure(failedPhase, issueType) {
+    return getPhaseOnFailure(failedPhase, 'implement', issueType);
+}

package/dist/src/ai-manager/types.js

@@ -0,0 +1,5 @@
+"use strict";
+/**
+ * Type definitions for AI Manager phase-based protocol
+ */
+Object.defineProperty(exports, "__esModule", { value: true });