atabey 0.0.13 → 0.0.15

package/framework-mcp/dist/src/modules/engines/evaluation-engine.js

@@ -0,0 +1,169 @@
+import fs from "fs";
+import path from "path";
+import { scanProjectCompliance } from "../../cli/utils/compliance.js";
+import { getFrameworkDir } from "../../cli/utils/memory.js";
+import { appendFile, ensureDir, runCommandQuiet, writeTextFile } from "../../shared/fs.js";
+import { logger } from "../../shared/logger.js";
+import { asTraceID } from "../../shared/types.js";
+export class EvaluationEngine {
+    /**
+     * Evaluates a completed task, calculates a score, and updates Specialty Memory.
+     * Saves lessons from both failures AND successes.
+     */
+    static evaluateTask(traceIdStr, agentName, durationMs, taskDescription) {
+        const traceId = asTraceID(traceIdStr);
+        logger.debug(`Starting evaluation for Trace ${traceId} by ${agentName}`);
+        let score = 100;
+        const metrics = {
+            compilation: true,
+            lint: true,
+            tests: true,
+            compliance: 0
+        };
+        const projectRoot = process.cwd();
+        const lessons = [];
+        // 1. Compliance Check (AST-based discipline check)
+        try {
+            const issues = scanProjectCompliance("src");
+            metrics.compliance = issues.length;
+            if (issues.length > 0) {
+                score -= Math.min(issues.length * 5, 40);
+                lessons.push("Compliance Violations Detected: Ensure strict adherence to zero-any, zero-console.log policies.");
+            }
+            else {
+                lessons.push("Compliance check passed: No any types or console.log violations found.");
+            }
+        }
+        catch (e) {
+            logger.debug("Compliance check failed during evaluation", e);
+        }
+        // 2. Compilation Check
+        try {
+            if (fs.existsSync(path.join(projectRoot, "tsconfig.json"))) {
+                runCommandQuiet("npx", ["tsc", "--noEmit"], projectRoot);
+                lessons.push("TypeScript compilation succeeded. Project is type-safe.");
+            }
+        }
+        catch {
+            metrics.compilation = false;
+            score -= 20;
+            lessons.push("Compilation Error: Always run 'npx tsc --noEmit' to verify type safety before completing a task.");
+        }
+        // 3. Lint Check
+        try {
+            if (fs.existsSync(path.join(projectRoot, "eslint.config.js"))) {
+                runCommandQuiet("npx", ["eslint", ".", "--max-warnings", "0"], projectRoot);
+                lessons.push("Lint check passed. Code follows project style guidelines.");
+            }
+        }
+        catch {
+            metrics.lint = false;
+            score -= 10;
+        }
+        // 4. Test Check
+        try {
+            if (fs.existsSync(path.join(projectRoot, "vitest.config.ts"))) {
+                runCommandQuiet("npx", ["vitest", "run", "--passWithNoTests"], projectRoot);
+                lessons.push("All tests passed. Changes are verified.");
+            }
+        }
+        catch {
+            metrics.tests = false;
+            score -= 20;
+            lessons.push("Test Failure: Ensure all unit tests pass before marking a task as COMPLETED.");
+        }
+        // 5. Efficiency Check
+        if (durationMs > 120000) {
+            score -= 10;
+            lessons.push("Task took longer than 2 minutes. Consider breaking large tasks into smaller steps.");
+        }
+        else if (score >= 80) {
+            lessons.push("Task completed efficiently within expected timeframe.");
+        }
+        // 6. Success Lesson — if score is high and no critical errors
+        if (score >= 80 && taskDescription) {
+            const successLesson = this.extractSuccessLesson(agentName, taskDescription);
+            if (successLesson) {
+                lessons.push(successLesson);
+            }
+        }
+        // Save all lessons to specialty memory
+        for (const lesson of lessons) {
+            this.updateSpecialtyMemory(agentName, lesson);
+        }
+        const result = {
+            traceId,
+            agent: agentName.replace("@", ""),
+            score: Math.max(0, score),
+            metrics,
+            durationMs,
+            timestamp: new Date().toISOString()
+        };
+        logger.debug(`Evaluation completed for ${agentName}: Score ${result.score}/100`);
+        return result;
+    }
+    /**
+     * Extract a meaningful success lesson from a completed task.
+     */
+    static extractSuccessLesson(agentName, taskDescription) {
+        const patterns = [
+            { keywords: ["api", "endpoint", "route", "rest"], lesson: "REST API pattern used successfully for this task." },
+            { keywords: ["auth", "login", "jwt", "token"], lesson: "Authentication/authorization pattern used successfully." },
+            { keywords: ["test", "spec", "vitest"], lesson: "Test-driven approach used successfully." },
+            { keywords: ["migration", "schema", "database", "sql"], lesson: "Database schema/migration pattern used successfully." },
+            { keywords: ["component", "react", "ui", "page"], lesson: "UI component pattern used successfully for this task." },
+            { keywords: ["middleware", "guard", "interceptor"], lesson: "Middleware/guard pattern used successfully." },
+            { keywords: ["service", "repository", "inject"], lesson: "Service/repository pattern used successfully." },
+            { keywords: ["error", "exception", "catch", "try"], lesson: "Error handling pattern used successfully." },
+            { keywords: ["config", "env", "environment"], lesson: "Configuration/environment pattern used successfully." },
+        ];
+        const lower = taskDescription.toLowerCase();
+        for (const { keywords, lesson } of patterns) {
+            if (keywords.some(k => lower.includes(k))) {
+                return lesson;
+            }
+        }
+        // Generic success lesson if no specific pattern matched
+        return "Task completed with score >= 80. Pattern/approach used was effective.";
+    }
+    /**
+     * Reads learned conventions from an agent's specialty memory.
+     * Used by silent-router to inject context into AI calls.
+     */
+    static readLearnedConventions(agentName) {
+        const cleanName = agentName.replace("@", "");
+        const fwDir = getFrameworkDir();
+        const filePath = path.join(fwDir, "memory", "specialties", `${cleanName}.md`);
+        try {
+            if (fs.existsSync(filePath)) {
+                const content = fs.readFileSync(filePath, "utf8").trim();
+                if (content)
+                    return content;
+            }
+        }
+        catch {
+            // File might not exist yet
+        }
+        return "";
+    }
+    /**
+     * Appends learned lessons to the agent's specialty memory.
+     */
+    static updateSpecialtyMemory(agentName, lesson) {
+        const cleanName = agentName.replace("@", "");
+        const fwDir = getFrameworkDir();
+        const specialtiesDir = path.join(fwDir, "memory", "specialties");
+        ensureDir(specialtiesDir);
+        const filePath = path.join(specialtiesDir, `${cleanName}.md`);
+        const timestamp = new Date().toISOString().split("T")[0];
+        const entry = `- **[${timestamp}]**: ${lesson}\n`;
+        if (!fs.existsSync(filePath)) {
+            writeTextFile(filePath, `# Learned Conventions for @${cleanName}\n\n`);
+        }
+        // Avoid duplicate lessons on the same day
+        const currentContent = fs.readFileSync(filePath, "utf8");
+        if (!currentContent.includes(lesson)) {
+            appendFile(filePath, entry);
+        }
+    }
+}

package/framework-mcp/dist/src/modules/engines/quality-gate.js

@@ -0,0 +1,92 @@
+import { execSync } from "child_process";
+import { scanProjectCompliance } from "../../cli/utils/compliance.js";
+import { logger } from "../../shared/logger.js";
+/**
+ * [ENGINE] Quality Gate
+ * Atabey Order v2 — Deterministic Quality Inspection Center.
+ * Inspects agent outputs and codebase against constitutional rules.
+ *
+ * v0.0.14: Selective file-level lint (not full-project), removed test execution
+ * from quality gate (tests are a CI concern), added `any` type detection.
+ */
+export class QualityGate {
+    /**
+     * Puts an agent's work through quality testing.
+     * 1. Compliance (Constitutional compliance via AST scan)
+     * 2. Lint (Targeted file lint if filePath is provided)
+     * 3. Content Validation (Output quality check)
+     *
+     * Note: Test execution is intentionally excluded from the quality gate.
+     * Tests are run separately via 'npm test' in CI. The gate focuses on
+     * the immediate output quality, not full regression coverage.
+     */
+    static async check(agent, output, task, filePath) {
+        logger.info(`[QUALITY_GATE] Starting audit for ${agent}...`);
+        // Skip deep ecosystem commands during framework unit testing to prevent infinite recursion
+        const isTestEnv = process.env.VITEST === "true" || !!process.env.ATABEY_TEST_DIR;
+        if (!isTestEnv) {
+            // 1. Compliance Check (AST Analysis — scans src/ for policy violations)
+            const complianceIssues = scanProjectCompliance("src");
+            if (complianceIssues.length > 0) {
+                const summary = complianceIssues.map(i => `${i.file}:${i.line} (${i.rule})`).join(", ");
+                return {
+                    passed: false,
+                    reason: `Compliance issues found: ${summary}`,
+                    findings: complianceIssues
+                };
+            }
+            // 2. Lint Check
+            if (filePath) {
+                try {
+                    logger.debug("[QUALITY_GATE] Running ESLint on file...");
+                    execSync(`npx eslint --no-eslintrc --rule '{"@typescript-eslint/no-explicit-any": "error"}' ${filePath}`, { stdio: "pipe" });
+                }
+                catch (err) {
+                    const error = err;
+                    return {
+                        passed: false,
+                        reason: "Lint check failed. Zero errors required.",
+                        findings: error.stderr?.toString() || error.message
+                    };
+                }
+            }
+            else {
+                logger.warn("[QUALITY_GATE] No filePath provided — skipping lint check.");
+            }
+        }
+        else {
+            logger.debug("[QUALITY_GATE] Framework unit test environment detected. Skipping ecosystem calls.");
+        }
+        // 3. Content Validation (Basic output quality check)
+        const contentCheck = this.validateOutputContent(output, task);
+        if (!contentCheck.passed) {
+            return contentCheck;
+        }
+        logger.info(`[QUALITY_GATE] Audit PASSED for ${agent}`);
+        return { passed: true, reason: "All quality gates passed." };
+    }
+    /**
+     * Validates the quality of agent output content.
+     * Checks for: empty output, error indicators, `any` type policy violations.
+     */
+    static validateOutputContent(output, _task) {
+        if (!output || output.trim().length < 10) {
+            return { passed: false, reason: "Output is too short or empty." };
+        }
+        // Check for hard error indicators
+        const errorIndicators = ["ERROR", "FAILED", "CRASHED", "TIMEOUT"];
+        for (const indicator of errorIndicators) {
+            if (output.toUpperCase().includes(indicator)) {
+                return { passed: false, reason: `Output contains error indicator: ${indicator}` };
+            }
+        }
+        // Zero Type Hole policy: detect TypeScript `any` type usage in output
+        if (/:\s*any\b/.test(output)) {
+            return {
+                passed: false,
+                reason: "Output contains `any` type — Zero Type Hole policy violation. Use explicit types.",
+            };
+        }
+        return { passed: true, reason: "Content validation passed." };
+    }
+}

package/framework-mcp/dist/src/modules/engines/risk-engine.js

@@ -0,0 +1,105 @@
+/**
+ * [SECURITY] Risk Engine (The Guardian)
+ * Calculates the danger level of a proposed task or operation.
+ */
+export class RiskEngine {
+    static HIGH_RISK_KEYWORDS = [
+        { word: "delete", weight: 40 },
+        { word: "drop", weight: 50 },
+        { word: "truncate", weight: 50 },
+        { word: "rm -rf", weight: 60 },
+        { word: "purge", weight: 40 },
+        { word: "format", weight: 50 },
+        { word: "force", weight: 20 },
+    ];
+    static SENSITIVE_PATHS = [
+        { pattern: /\.env/i, weight: 50 },
+        { pattern: /config/i, weight: 20 },
+        { pattern: /database/i, weight: 30 },
+        { pattern: /auth/i, weight: 30 },
+        { pattern: /security/i, weight: 30 },
+        { pattern: /atabey/i, weight: 40 }, // Framework protection
+    ];
+    /**
+     * Assesses the risk of a natural language task or command string.
+     */
+    static assessTaskRisk(task) {
+        const factors = [];
+        let totalScore = 0;
+        // 1. Keyword Analysis
+        for (const { word, weight } of this.HIGH_RISK_KEYWORDS) {
+            if (new RegExp(`\\b${word}\\b`, "i").test(task)) {
+                factors.push({
+                    factor: `Keyword: ${word}`,
+                    score: weight,
+                    description: `Detected high-risk keyword '${word}' in task description.`
+                });
+                totalScore += weight;
+            }
+        }
+        // 2. Path Sensitivity (if paths are mentioned in the task)
+        for (const { pattern, weight } of this.SENSITIVE_PATHS) {
+            if (pattern.test(task)) {
+                factors.push({
+                    factor: `Sensitive Path: ${pattern.source}`,
+                    score: weight,
+                    description: `Task involves access to sensitive path or pattern: ${pattern.source}`
+                });
+                totalScore += weight;
+            }
+        }
+        // 3. Complexity Risk (Length of task as a proxy)
+        if (task.length > 300) {
+            const score = 10;
+            factors.push({
+                factor: "High Complexity",
+                score,
+                description: "Task description is unusually long, increasing the chance of misunderstanding."
+            });
+            totalScore += score;
+        }
+        return this.finalizeAssessment(totalScore, factors);
+    }
+    /**
+     * Assesses risk based on a proposed file change.
+     */
+    static assessChangeRisk(filePath, operation) {
+        const factors = [];
+        let totalScore = 0;
+        // 1. Operation Weight
+        const opWeights = { write: 30, replace: 5, patch: 10 };
+        totalScore += opWeights[operation];
+        factors.push({
+            factor: `Operation: ${operation}`,
+            score: opWeights[operation],
+            description: `A '${operation}' operation is inherently riskier than a surgical 'replace'.`
+        });
+        // 2. File Path Risk
+        for (const { pattern, weight } of this.SENSITIVE_PATHS) {
+            if (pattern.test(filePath)) {
+                factors.push({
+                    factor: `Sensitive File: ${filePath}`,
+                    score: weight,
+                    description: "Modifying a sensitive file is high risk."
+                });
+                totalScore += weight;
+            }
+        }
+        return this.finalizeAssessment(totalScore, factors);
+    }
+    static finalizeAssessment(totalScore, factors) {
+        let severity = "LOW";
+        if (totalScore >= 80)
+            severity = "CRITICAL";
+        else if (totalScore >= 50)
+            severity = "HIGH";
+        else if (totalScore >= 20)
+            severity = "MEDIUM";
+        return {
+            totalScore: Math.min(totalScore, 100),
+            severity,
+            factors,
+            requiresApproval: totalScore >= 60
+        };
+    }
+}