@martinloop/mcp 0.3.5 → 0.3.7

package/dist/vendor/core/test-integrity.js

@@ -0,0 +1,197 @@
+/**
+ * Test Integrity Validator
+ *
+ * Detects circular test validation — the problem where AI agents generate
+ * tests that only pass because they test the agent's own output, not real
+ * behavior. A test that passes on both the pre-change and post-change
+ * codebase proves nothing about the change.
+ *
+ * Strategy:
+ * 1. Snapshot which test files existed before the agent ran
+ * 2. After the run, identify NEW test files the agent created
+ * 3. For each new test: check if it imports/references pre-existing code
+ *    (real test) or only references agent-generated code (circular test)
+ * 4. Flag tests that don't exercise any pre-existing function
+ *
+ * This module provides the analysis primitives. The verification pipeline
+ * calls these during the post-run verification step.
+ */
+import { readFileSync, readdirSync, existsSync } from "node:fs";
+import { join, relative } from "node:path";
+const TEST_FILE_PATTERNS = [
+    /\.test\.[tj]sx?$/,
+    /\.spec\.[tj]sx?$/,
+    /\/__tests__\//,
+    /\/tests?\//
+];
+const TRIVIAL_ASSERTION_PATTERNS = [
+    /expect\(true\)\.toBe\(true\)/,
+    /expect\(1\)\.toBe\(1\)/,
+    /expect\(\)\.toBeUndefined\(\)/,
+    /assert\.ok\(true\)/,
+    /expect\(.*\)\.toBeDefined\(\)/,
+    /expect\(typeof\s+\w+\)\.toBe\(['"]\w+['"]\)/
+];
+const IMPORT_PATTERN = /(?:import\s+.*from\s+['"]([^'"]+)['"]|require\s*\(\s*['"]([^'"]+)['"]\s*\))/g;
+/**
+ * Detect test files in a directory tree.
+ */
+export function findTestFiles(rootDir) {
+    const results = [];
+    function walk(dir) {
+        if (!existsSync(dir))
+            return;
+        try {
+            const entries = readdirSync(dir, { withFileTypes: true });
+            for (const entry of entries) {
+                const fullPath = join(dir, entry.name);
+                if (entry.name === "node_modules" || entry.name === "dist" || entry.name === ".git")
+                    continue;
+                if (entry.isDirectory()) {
+                    walk(fullPath);
+                }
+                else if (entry.isFile() && isTestFile(entry.name, fullPath)) {
+                    results.push(relative(rootDir, fullPath));
+                }
+            }
+        }
+        catch {
+            // Permission denied or inaccessible — skip silently
+        }
+    }
+    walk(rootDir);
+    return results;
+}
+/**
+ * Check if a filename matches test file patterns.
+ */
+export function isTestFile(filename, fullPath) {
+    const checkPath = fullPath ?? filename;
+    return TEST_FILE_PATTERNS.some((pattern) => pattern.test(checkPath));
+}
+/**
+ * Snapshot existing test files before an agent run.
+ */
+export function snapshotTestFiles(rootDir) {
+    return new Set(findTestFiles(rootDir));
+}
+/**
+ * Identify new test files created during an agent run.
+ */
+export function identifyNewTests(rootDir, preRunSnapshot) {
+    const currentTests = findTestFiles(rootDir);
+    return currentTests.filter((testPath) => !preRunSnapshot.has(testPath));
+}
+/**
+ * Extract import paths from a TypeScript/JavaScript file.
+ */
+export function extractImports(content) {
+    const imports = [];
+    let match;
+    const pattern = new RegExp(IMPORT_PATTERN.source, "g");
+    while ((match = pattern.exec(content)) !== null) {
+        const importPath = match[1] ?? match[2];
+        if (importPath)
+            imports.push(importPath);
+    }
+    return imports;
+}
+/**
+ * Check if a test file contains only trivial assertions.
+ */
+export function hasTrivialAssertionsOnly(content) {
+    const assertionLines = content.split("\n").filter((line) => /expect\(|assert\.|should\./i.test(line));
+    if (assertionLines.length === 0)
+        return true;
+    return assertionLines.every((line) => TRIVIAL_ASSERTION_PATTERNS.some((pattern) => pattern.test(line)));
+}
+/**
+ * Check if a test file imports any pre-existing source files.
+ */
+export function importsPreExistingCode(testContent, preRunSourceFiles) {
+    const imports = extractImports(testContent);
+    return imports.some((importPath) => {
+        // Relative imports that reference pre-existing files
+        if (importPath.startsWith(".") || importPath.startsWith("/")) {
+            // Normalize: strip extension, check if any pre-existing source matches
+            const normalized = importPath.replace(/\.[tj]sx?$/, "");
+            return [...preRunSourceFiles].some((srcFile) => {
+                const normalizedSrc = srcFile.replace(/\.[tj]sx?$/, "");
+                return normalizedSrc.endsWith(normalized) || normalized.endsWith(normalizedSrc);
+            });
+        }
+        // Package imports always count as real (they reference pre-existing deps)
+        return true;
+    });
+}
+/**
+ * Validate the integrity of tests created during a governed run.
+ */
+export function validateTestIntegrity(rootDir, preRunTestSnapshot, preRunSourceFiles) {
+    const newTests = identifyNewTests(rootDir, preRunTestSnapshot);
+    const findings = [];
+    for (const testPath of newTests) {
+        const fullPath = join(rootDir, testPath);
+        let content;
+        try {
+            content = readFileSync(fullPath, "utf8");
+        }
+        catch {
+            findings.push({
+                filePath: testPath,
+                issue: "empty_test",
+                severity: "high",
+                detail: "Test file could not be read."
+            });
+            continue;
+        }
+        if (content.trim().length === 0) {
+            findings.push({
+                filePath: testPath,
+                issue: "empty_test",
+                severity: "high",
+                detail: "Test file is empty."
+            });
+            continue;
+        }
+        if (hasTrivialAssertionsOnly(content)) {
+            findings.push({
+                filePath: testPath,
+                issue: "trivial_assertion",
+                severity: "high",
+                detail: "Test contains only trivial assertions (expect(true).toBe(true) or equivalent)."
+            });
+            continue;
+        }
+        if (!importsPreExistingCode(content, preRunSourceFiles)) {
+            findings.push({
+                filePath: testPath,
+                issue: "no_pre_existing_imports",
+                severity: "medium",
+                detail: "Test does not import any pre-existing source code. May be testing only agent-generated output."
+            });
+        }
+    }
+    const suspiciousTests = findings.filter((f) => f.severity === "high").length;
+    const realTests = newTests.length - suspiciousTests;
+    const verdict = suspiciousTests > 0 && suspiciousTests >= newTests.length * 0.5
+        ? "fail"
+        : suspiciousTests > 0
+            ? "warn"
+            : "pass";
+    const summary = newTests.length === 0
+        ? "No new test files detected."
+        : verdict === "pass"
+            ? `${newTests.length} new test(s) validated — all reference pre-existing code.`
+            : verdict === "warn"
+                ? `${suspiciousTests} of ${newTests.length} new test(s) flagged for review.`
+                : `Test integrity check failed: ${suspiciousTests} of ${newTests.length} new test(s) appear circular or trivial.`;
+    return {
+        totalNewTests: newTests.length,
+        realTests,
+        suspiciousTests,
+        findings,
+        verdict,
+        summary
+    };
+}

package/dist/workflow-state.d.ts

@@ -1,5 +1,5 @@
 import type { LoopBudget, ReceiptScope } from "./vendor/contracts/index.js";
-type McpWorkflowStepName = "doctor" | "plan" | "preflight";
+type McpWorkflowStepName = "doctor" | "plan" | "preflight" | "estimate" | "start";
 interface McpWorkflowReceipt {
     step: McpWorkflowStepName;
     recordedAt: string;

package/dist/workflow-state.js

@@ -4,6 +4,7 @@ import { join, resolve } from "node:path";
 const WORKFLOW_STATE_DIRECTORY = "_martin";
 const WORKFLOW_STATE_FILENAME = "workflow-state.json";
 const DOCTOR_TTL_MS = 24 * 60 * 60 * 1000;
+const ESTIMATE_TTL_MS = 6 * 60 * 60 * 1000; // Estimate expires after 6h — re-estimate if objective changes
 const PLAN_TTL_MS = 24 * 60 * 60 * 1000;
 const PREFLIGHT_TTL_MS = 6 * 60 * 60 * 1000;
 export async function recordMcpWorkflowStep(input) {
@@ -42,6 +43,11 @@ export async function evaluateMcpRunGate(input) {
         receipt.scopeKey === scopeKey)) {
         missingSteps.push("doctor");
     }
+    // Estimate is required before any run — it proves the agent assessed cost first.
+    // The estimate TTL is 6h; it doesn't need to match the exact objective key.
+    if (!isFresh(mcpState["estimate"], ESTIMATE_TTL_MS, (receipt) => receipt.workingDirectory === workingDirectory)) {
+        missingSteps.push("estimate");
+    }
     if (!isFresh(mcpState["plan"], PLAN_TTL_MS, (receipt) => receipt.workingDirectory === workingDirectory &&
         receipt.scopeKey === scopeKey &&
         receipt.objectiveKey === objectiveKey)) {
@@ -65,10 +71,12 @@ export async function evaluateMcpRunGate(input) {
         };
     }
     const nextAction = missingSteps[0] === "doctor"
-        ? "Call martin_doctor for this workingDirectory before any real run."
-        : missingSteps[0] === "plan"
-            ? "Call martin_plan with the exact objective before martin_run."
-            : "Call martin_preflight with the exact objective, verifier plan, and engine before martin_run.";
+        ? "Call martin_doctor to confirm environment before any work."
+        : missingSteps[0] === "estimate"
+            ? "Call martin_estimate with the objective to get cost and route before spending."
+            : missingSteps[0] === "plan"
+                ? "Call martin_plan with the exact objective before martin_run."
+                : "Call martin_preflight with the exact objective, verifier plan, and engine before martin_run.";
     return {
         allowed: false,
         nextAction,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@martinloop/mcp",
-  "version": "0.3.5",
+  "version": "0.3.7",
   "mcpName": "io.github.Keesan12/martin-loop",
   "private": false,
   "type": "module",

package/server.json

@@ -7,12 +7,12 @@
     "url": "https://github.com/Keesan12/martin-loop",
     "source": "github"
   },
-  "version": "0.3.5",
+  "version": "0.3.7",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@martinloop/mcp",
-      "version": "0.3.5",
+      "version": "0.3.7",
       "transport": {
         "type": "stdio"
       }