@surlymochan/evalid 0.0.0-stage → 0.3.1

package/xflow/atoms/i6b_tdd_proof_validate.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""
+I6b.tdd.proof_validate — Validate TDD proof structure, binding, and re-execution.
+
+Required fields: phase, attempt, command, exit_code, passed, recorded_at.
+Red must prove a failing test. Green/refactor must prove passing tests.
+The binding covers the recorded command, exit code, output hash, failure
+class, and snapshot file hashes. This gate recomputes it and re-runs the
+command in a copy of that snapshot. Green also requires snapshotted test
+files to still match the working tree.
+
+tdd_mode refactor does not require a red proof. tdd_mode docs does not
+require a red or green proof. A proof that is present is still checked.
+An agent who rebuilds a self-consistent failing snapshot can still produce
+a binding that verifies; the check raises the bar from "edit the JSON" to
+"possess a tree on which this command really exits as claimed".
+"""
+import sys, os, json, argparse, subprocess
+from pathlib import Path
+
+import tdd_lib
+
+REQUIRED_FIELDS = ['phase', 'attempt', 'command', 'exit_code', 'passed', 'recorded_at']
+
+def expected_result(phase: str):
+    if phase == 'red':
+        return False
+    if phase in {'green', 'refactor', 'run'}:
+        return True
+    return None
+
+def emit(payload, code):
+    print(json.dumps(payload))
+    sys.exit(code)
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument('--project-root', default='.')
+    p.add_argument('--change-id', required=True)
+    p.add_argument('--phase', default='run')
+    p.add_argument('--attempt', type=int, default=0)
+    p.add_argument('--tdd-mode', default='', help='red-green | refactor | docs; empty reads status.json')
+    args = p.parse_args()
+
+    project = Path(args.project_root).resolve()
+    mode = tdd_lib.resolve_tdd_mode(project, args.change_id, args.tdd_mode)
+    if mode not in tdd_lib.MODES:
+        emit({'ok': False, 'verdict': 'invalid_tdd_mode', 'tdd_mode': mode, 'phase': args.phase}, 1)
+
+    proof_file = os.path.join(
+        args.project_root, 'specs', 'changes', args.change_id, 'tdd',
+        f'{args.phase}-{args.attempt}.json'
+    )
+
+    if not os.path.exists(proof_file):
+        if mode == 'docs' or (mode == 'refactor' and args.phase == 'red'):
+            emit({
+                'ok': True,
+                'verdict': 'not_required',
+                'phase': args.phase,
+                'tdd_mode': mode,
+                'file': proof_file,
+            }, 0)
+        emit({'ok': False, 'verdict': 'missing', 'file': proof_file, 'phase': args.phase, 'tdd_mode': mode}, 1)
+
+    try:
+        proof = json.loads(open(proof_file).read())
+    except json.JSONDecodeError as e:
+        emit({'ok': False, 'verdict': 'invalid_json', 'error': str(e), 'phase': args.phase}, 1)
+
+    missing = [f for f in REQUIRED_FIELDS if f not in proof]
+    if missing:
+        emit({'ok': False, 'verdict': 'schema_error', 'missing_fields': missing, 'phase': args.phase}, 1)
+
+    if proof.get('phase') != args.phase:
+        emit({
+            'ok': False,
+            'verdict': 'phase_mismatch',
+            'expected_phase': args.phase,
+            'actual_phase': proof.get('phase'),
+            'phase': args.phase,
+        }, 1)
+
+    if not proof.get('binding') or not proof.get('failure_kind') or not isinstance(proof.get('file_hashes'), dict):
+        emit({'ok': False, 'verdict': 'binding_missing', 'phase': args.phase}, 1)
+
+    if tdd_lib.compute_binding(proof) != proof.get('binding'):
+        emit({'ok': False, 'verdict': 'binding_mismatch', 'phase': args.phase}, 1)
+
+    if bool(proof.get('passed')) != (proof.get('exit_code') == 0):
+        emit({'ok': False, 'verdict': 'proof_inconsistent', 'phase': args.phase}, 1)
+
+    destination = tdd_lib.snapshot_dir(project, args.change_id, args.phase, args.attempt)
+    found = tdd_lib.snapshot_tree_hashes(destination)
+    if found is None or found != proof.get('file_hashes'):
+        emit({'ok': False, 'verdict': 'snapshot_mismatch', 'phase': args.phase}, 1)
+
+    try:
+        actual_exit, actual_text = tdd_lib.reexec_snapshot(destination, proof.get('command') or '', timeout=120)
+    except subprocess.TimeoutExpired:
+        emit({'ok': False, 'verdict': 'reexec_timeout', 'phase': args.phase}, 1)
+
+    if actual_exit != proof.get('exit_code'):
+        emit({
+            'ok': False,
+            'verdict': 'reexec_exit_mismatch',
+            'phase': args.phase,
+            'expected_exit': proof.get('exit_code'),
+            'actual_exit': actual_exit,
+        }, 1)
+
+    actual_kind = 'timeout' if proof.get('failure_kind') == 'timeout' else tdd_lib.classify_output(actual_text, actual_exit == 0)
+    if proof.get('failure_kind') != actual_kind:
+        emit({
+            'ok': False,
+            'verdict': 'reexec_result_mismatch',
+            'phase': args.phase,
+            'expected_failure_kind': proof.get('failure_kind'),
+            'actual_failure_kind': actual_kind,
+        }, 1)
+
+    if args.phase in {'green', 'refactor', 'run'}:
+        mismatched = []
+        for rel, digest in proof.get('file_hashes', {}).items():
+            if not tdd_lib.is_test_file(rel):
+                continue
+            path = project / rel
+            if not path.is_file() or tdd_lib.sha256_bytes(path.read_bytes()) != digest:
+                mismatched.append(rel)
+        if mismatched:
+            emit({
+                'ok': False,
+                'verdict': 'green_tree_mismatch',
+                'phase': args.phase,
+                'files': mismatched,
+            }, 1)
+
+    if proof.get('failure_kind') == 'skipped' or tdd_lib.output_non_result(actual_text):
+        emit({'ok': False, 'verdict': 'tests_skipped_or_todo', 'phase': args.phase}, 1)
+
+    expected_passed = expected_result(args.phase)
+    if expected_passed is not None and proof.get('passed') is not expected_passed:
+        emit({
+            'ok': False,
+            'verdict': 'semantic_error',
+            'phase': args.phase,
+            'expected_passed': expected_passed,
+            'actual_passed': proof.get('passed'),
+        }, 1)
+
+    if proof.get('expectation_met') is False:
+        emit({'ok': False, 'verdict': 'expectation_not_met', 'phase': args.phase}, 1)
+
+    emit({
+        'ok': True,
+        'verdict': 'valid',
+        'proof_file': proof_file,
+        'passed': proof['passed'],
+        'exit_code': proof['exit_code'],
+        'phase': args.phase,
+        'tdd_mode': mode,
+    }, 0)
+
+if __name__ == '__main__':
+    main()

package/xflow/atoms/i6c_tdd_quality_review.py

@@ -0,0 +1,357 @@
+#!/usr/bin/env python3
+"""
+I6c.tdd.quality_review - deterministic review for TDD test quality.
+
+This gate checks the parts that red/green proof cannot prove by itself:
+code changes should be accompanied by test changes, changed tests must not
+rely on empty assertions or snapshot-only coverage, and mock-heavy tests are
+rejected when they lack enough real assertions.
+
+tdd_mode red-green (default) requires a failing red proof and a passing
+green proof. tdd_mode refactor instead requires the existing tests that
+touch the changed code to pass on the pre-change tree and the current tree,
+with no assertion lines removed. tdd_mode docs requires the diff to stay on
+non-code paths. A semantically wrong but plausible expected value is not
+decidable from the diff in general; gate v2 still catches three decidable
+shapes of it: expected values taken from the implementation's own constants
+(the expectation moves when the constant is edited), assertions inside
+named callbacks that are never referenced again (they execute zero times),
+and self-comparison assertions (x == x). Type-only or existence-only
+assertions are flagged only when they are the sole assertions.
+"""
+import argparse
+import json
+import subprocess
+import sys
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+import tdd_lib
+
+
+def run_git(project_root: Path, args: list[str]) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["git", *args],
+        cwd=project_root,
+        text=True,
+        capture_output=True,
+    )
+
+
+def changed_files(project_root: Path, diff_ref: str) -> list[str]:
+    result = run_git(project_root, ["diff", "--name-only", "--diff-filter=ACMR", diff_ref])
+    tracked = [] if result.returncode != 0 else [
+        line.strip() for line in result.stdout.splitlines() if line.strip()
+    ]
+    untracked_result = run_git(project_root, ["ls-files", "--others", "--exclude-standard"])
+    untracked = [] if untracked_result.returncode != 0 else [
+        line.strip() for line in untracked_result.stdout.splitlines() if line.strip()
+    ]
+    return sorted(set(tracked + untracked))
+
+
+def deleted_files(project_root: Path, diff_ref: str) -> list[str]:
+    result = run_git(project_root, ["diff", "--name-only", "--diff-filter=D", diff_ref])
+    if result.returncode != 0:
+        return []
+    return [line.strip() for line in result.stdout.splitlines() if line.strip()]
+
+
+def read_json(path: Path) -> dict:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+
+
+def read_text(path: Path) -> str:
+    try:
+        return path.read_text(encoding="utf-8")
+    except OSError:
+        return ""
+
+
+def review_test_file(project_root: Path, relative_path: str, implementation_sources: list[str] | None = None) -> list[dict]:
+    path = project_root / relative_path
+    try:
+        content = path.read_text(encoding="utf-8")
+    except OSError:
+        return [{"severity": "fail", "code": "test_unreadable", "path": relative_path}]
+    findings = []
+    for finding in tdd_lib.analyze_test(content, implementation_sources=implementation_sources):
+        item = dict(finding)
+        item["path"] = relative_path
+        findings.append(item)
+    return findings
+
+
+def proof_skipped(proof: dict) -> bool:
+    if not proof:
+        return False
+    if proof.get("failure_kind") == "skipped":
+        return True
+    blob = f"{proof.get('stdout_tail') or ''}\n{proof.get('stderr_tail') or ''}"
+    return tdd_lib.output_non_result(blob)
+
+
+def symbols_in(project_root: Path, code_files: list[str]) -> set[str]:
+    symbols = set()
+    for relative in code_files:
+        symbols |= tdd_lib.defined_symbols(read_text(project_root / relative))
+    return symbols
+
+
+def iter_repo_tests(project_root: Path) -> list[str]:
+    found = []
+    skip = tdd_lib.SKIP_DIR_NAMES
+    for dirpath, dirnames, filenames in os_walk(project_root):
+        dirnames[:] = [name for name in dirnames if name not in skip]
+        for name in filenames:
+            full = Path(dirpath) / name
+            rel = full.relative_to(project_root).as_posix()
+            if tdd_lib.is_test_file(rel):
+                found.append(rel)
+    return found
+
+
+def os_walk(project_root: Path):
+    import os
+    return os.walk(project_root)
+
+
+def refactor_findings(project_root: Path, diff_ref: str, code_files: list[str], test_files: list[str], green_proof: dict) -> list[dict]:
+    findings = []
+    command = green_proof.get("command") or ""
+    if not command:
+        return findings
+
+    for relative in code_files:
+        if tdd_lib.file_at_ref(project_root, diff_ref, relative) is None:
+            findings.append({
+                "severity": "fail",
+                "code": "refactor_new_code",
+                "path": relative,
+                "message": "Refactor mode cannot introduce a code file that is not in the baseline.",
+            })
+
+    for relative in deleted_files(project_root, diff_ref):
+        if tdd_lib.is_test_file(relative):
+            findings.append({
+                "severity": "fail",
+                "code": "assertions_removed",
+                "path": relative,
+                "message": "A test file was deleted during a declared refactor.",
+            })
+
+    for relative in test_files:
+        previous = tdd_lib.file_at_ref(project_root, diff_ref, relative)
+        if previous is None:
+            continue
+        old_lines = tdd_lib.assertion_lines(previous.decode("utf-8", errors="replace"))
+        new_lines = tdd_lib.assertion_lines(read_text(project_root / relative))
+        leftover = list(old_lines)
+        for line in new_lines:
+            if line in leftover:
+                leftover.remove(line)
+        if leftover:
+            findings.append({
+                "severity": "fail",
+                "code": "assertions_removed",
+                "path": relative,
+                "message": "Refactor mode removed or rewrote an existing assertion.",
+            })
+
+    symbols = symbols_in(project_root, code_files)
+    covering = []
+    if symbols:
+        for relative in iter_repo_tests(project_root):
+            if tdd_lib.references_symbol(read_text(project_root / relative), symbols):
+                covering.append(relative)
+        if not covering:
+            findings.append({
+                "severity": "fail",
+                "code": "refactor_missing_covering_test",
+                "message": "No existing test references a symbol defined in the changed code.",
+            })
+    if findings:
+        return findings
+
+    temp_root = Path(tempfile.mkdtemp(prefix="evalid-refactor-"))
+    try:
+        before = temp_root / "before"
+        after = temp_root / "after"
+        before.mkdir()
+        after.mkdir()
+        tdd_lib.copy_project(project_root, before)
+        tdd_lib.copy_project(project_root, after)
+        for relative in code_files:
+            baseline = tdd_lib.file_at_ref(project_root, diff_ref, relative)
+            if baseline is None:
+                continue
+            target = before / relative
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(baseline)
+        try:
+            before_run = tdd_lib.run_shell(command, before, timeout=120)
+            after_run = tdd_lib.run_shell(command, after, timeout=120)
+        except subprocess.TimeoutExpired:
+            findings.append({"severity": "fail", "code": "refactor_command_timeout"})
+            return findings
+        if before_run.returncode != 0:
+            findings.append({
+                "severity": "fail",
+                "code": "refactor_tests_failed_before",
+                "message": "Existing tests must pass on the pre-change tree.",
+            })
+        if after_run.returncode != 0:
+            findings.append({
+                "severity": "fail",
+                "code": "refactor_tests_failed_after",
+                "message": "Existing tests must pass after the refactor.",
+            })
+    finally:
+        import shutil
+        shutil.rmtree(temp_root, ignore_errors=True)
+    return findings
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--project-root", default=".")
+    parser.add_argument("--change-id", required=True)
+    parser.add_argument("--diff-ref", default="HEAD")
+    parser.add_argument("--tdd-mode", default="", help="red-green | refactor | docs; empty reads status.json")
+    args = parser.parse_args()
+
+    project_root = Path(args.project_root).resolve()
+    mode = tdd_lib.resolve_tdd_mode(project_root, args.change_id, args.tdd_mode)
+    change_dir = project_root / "specs" / "changes" / args.change_id
+    proof_dir = change_dir / "tdd"
+    red_proof = read_json(proof_dir / "red-0.json")
+    green_proof = read_json(proof_dir / "green-0.json")
+    status = tdd_lib.read_status(project_root, args.change_id)
+
+    files = changed_files(project_root, args.diff_ref)
+    test_files = [path for path in files if tdd_lib.is_test_file(path)]
+    code_files = [path for path in files if tdd_lib.is_code_file(path)]
+
+    findings = []
+    if mode not in tdd_lib.MODES:
+        findings.append({"severity": "fail", "code": "invalid_tdd_mode", "tdd_mode": mode})
+
+    if mode == "docs":
+        if code_files or test_files:
+            findings.append({
+                "severity": "fail",
+                "code": "docs_mode_with_code_change",
+                "message": "Docs mode was declared, but the diff changes code or tests.",
+                "code_files": code_files,
+                "test_files": test_files,
+            })
+        if green_proof and green_proof.get("passed") is not True:
+            findings.append({"severity": "fail", "code": "green_proof_not_passing"})
+    elif mode == "refactor":
+        if status.get("behavior_change") is True:
+            findings.append({
+                "severity": "fail",
+                "code": "refactor_declares_behavior_change",
+                "message": "Refactor mode cannot declare a public behavior change.",
+            })
+        if green_proof.get("passed") is not True:
+            findings.append({"severity": "fail", "code": "green_proof_not_passing"})
+        else:
+            findings.extend(refactor_findings(project_root, args.diff_ref, code_files, test_files, green_proof))
+    else:
+        if code_files and not test_files:
+            findings.append({
+                "severity": "fail",
+                "code": "code_without_test_change",
+                "message": "Code changed without a matching changed test file.",
+                "code_files": code_files,
+            })
+        if red_proof.get("passed") is not False:
+            findings.append({"severity": "fail", "code": "red_proof_not_failing"})
+        if green_proof.get("passed") is not True:
+            findings.append({"severity": "fail", "code": "green_proof_not_passing"})
+        if code_files or test_files:
+            problem = tdd_lib.command_problem(red_proof.get("command") or "", green_proof.get("command") or "")
+            if problem:
+                findings.append({
+                    "severity": "fail",
+                    "code": problem,
+                    "message": "Red and green must run the same test, or green must be a superset of the red test.",
+                    "red_command": red_proof.get("command"),
+                    "green_command": green_proof.get("command"),
+                })
+            kind = red_proof.get("failure_kind")
+            if kind in {"import_or_syntax", "no_test_evidence", "timeout"}:
+                findings.append({
+                    "severity": "fail",
+                    "code": "red_failure_not_assertion",
+                    "message": "The red failure is not an assertion or test error in the changed test.",
+                    "failure_kind": kind,
+                })
+        if proof_skipped(red_proof) or proof_skipped(green_proof):
+            findings.append({
+                "severity": "fail",
+                "code": "tests_skipped_or_todo",
+                "message": "A recorded run skipped or marked todo the tests it claims to have executed.",
+            })
+        symbols = symbols_in(project_root, code_files)
+        if symbols and test_files:
+            for relative in test_files:
+                if not tdd_lib.references_symbol(read_text(project_root / relative), symbols):
+                    findings.append({
+                        "severity": "fail",
+                        "code": "test_misses_changed_symbol",
+                        "path": relative,
+                        "message": "A changed test does not reference a symbol defined in the changed code.",
+                    })
+
+    reviewed = set(test_files)
+    if mode == "refactor":
+        symbols = symbols_in(project_root, code_files)
+        for relative in iter_repo_tests(project_root):
+            if symbols and tdd_lib.references_symbol(read_text(project_root / relative), symbols):
+                reviewed.add(relative)
+    implementation_sources = [read_text(project_root / relative) for relative in code_files]
+    for test_file in sorted(reviewed):
+        findings.extend(review_test_file(project_root, test_file, implementation_sources))
+
+    failed = any(finding.get("severity") == "fail" for finding in findings)
+    verdict = "failed" if failed else "passed"
+    proof = {
+        "ok": not failed,
+        "verdict": verdict,
+        "tdd_mode": mode,
+        "diff_ref": args.diff_ref,
+        "changed_files": files,
+        "code_files": code_files,
+        "test_files": test_files,
+        "findings": findings,
+        "recorded_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+    proof_dir.mkdir(parents=True, exist_ok=True)
+    proof_file = proof_dir / "quality-0.json"
+    proof_file.write_text(json.dumps(proof, indent=2) + "\n", encoding="utf-8")
+
+    print(json.dumps({
+        "ok": proof["ok"],
+        "verdict": verdict,
+        "tdd_mode": mode,
+        "proof_file": str(proof_file),
+        "failures": sum(1 for finding in findings if finding.get("severity") == "fail"),
+        "warnings": sum(1 for finding in findings if finding.get("severity") == "warn"),
+        "failed_codes": [
+            finding.get("code")
+            for finding in findings
+            if finding.get("severity") == "fail"
+        ],
+    }))
+    sys.exit(0 if proof["ok"] else 1)
+
+
+if __name__ == "__main__":
+    main()

package/xflow/atoms/j1_tests_run.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""
+J1.tests.run — Run planned verification commands, capture pass/fail proof.
+Reads commands from plan.md (lines starting with ` ``` bash` blocks or --commands arg).
+"""
+import sys, os, json, argparse, subprocess, re
+from datetime import datetime, timezone
+
+def extract_commands_from_plan(plan_path):
+    """Extract bash commands from plan.md code blocks."""
+    if not os.path.exists(plan_path):
+        return []
+    content = open(plan_path).read()
+    # Find ```bash ... ``` blocks
+    blocks = re.findall(r'```bash\n(.*?)```', content, re.DOTALL)
+    commands = []
+    for block in blocks:
+        for line in block.strip().splitlines():
+            line = line.strip()
+            if line and not line.startswith('#'):
+                commands.append(line)
+    return commands
+
+def normalize_commands(value):
+    if isinstance(value, str):
+        command = value.strip()
+        return [command] if command else []
+    if isinstance(value, list):
+        commands = []
+        for item in value:
+            if isinstance(item, str) and item.strip():
+                commands.append(item.strip())
+            elif isinstance(item, dict) and isinstance(item.get('command'), str) and item.get('command').strip():
+                commands.append(item['command'].strip())
+        return commands
+    return []
+
+def extract_commands_from_change_contract(project_root, change_id):
+    change_dir = os.path.join(project_root, 'specs', 'changes', change_id)
+    candidates = [
+        'plan.json',
+        'corps-input.json',
+        'yolo-input.json',
+        'input.json',
+        'intake.json',
+    ]
+    command_keys = [
+        'verification_commands',
+        'test_commands',
+        'qa_commands',
+        'commands',
+    ]
+    fallback_keys = [
+        'tdd_green_command',
+    ]
+    for filename in candidates:
+        path = os.path.join(change_dir, filename)
+        if not os.path.exists(path):
+            continue
+        try:
+            data = json.load(open(path))
+        except (OSError, json.JSONDecodeError):
+            continue
+        for key in command_keys:
+            commands = normalize_commands(data.get(key))
+            if commands:
+                return commands
+        for key in fallback_keys:
+            commands = normalize_commands(data.get(key))
+            if commands:
+                return commands
+    return []
+
+def verification_command_env():
+    env = os.environ.copy()
+    # J1's own orchestration change id should not alter the behavior of the
+    # verification commands it runs. Commands that need a CHANGE_ID must set it.
+    env.pop('CHANGE_ID', None)
+    return env
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument('--project-root', default='.')
+    p.add_argument('--change-id', required=True)
+    p.add_argument('--commands', nargs='*', help='Explicit commands to run')
+    args = p.parse_args()
+
+    commands = args.commands
+    if not commands:
+        plan_md = os.path.join(args.project_root, 'specs', 'changes', args.change_id, 'plan.md')
+        commands = extract_commands_from_plan(plan_md)
+    if not commands:
+        commands = extract_commands_from_change_contract(args.project_root, args.change_id)
+
+    if not commands:
+        print(json.dumps({'ok': False, 'verdict': 'no_commands', 'message': 'No test commands found in plan.md or change contract'}))
+        sys.exit(1)
+
+    results = []
+    all_passed = True
+    command_env = verification_command_env()
+    for cmd in commands:
+        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, cwd=args.project_root, timeout=120, env=command_env)
+        passed = r.returncode == 0
+        all_passed = all_passed and passed
+        results.append({
+            'command': cmd,
+            'exit_code': r.returncode,
+            'passed': passed,
+            'stdout_tail': r.stdout[-1000:] if r.stdout else '',
+        })
+
+    proof = {
+        'all_passed': all_passed,
+        'results': results,
+        'recorded_at': datetime.now(timezone.utc).isoformat(),
+    }
+
+    proof_dir = os.path.join(args.project_root, 'specs', 'changes', args.change_id)
+    os.makedirs(proof_dir, exist_ok=True)
+    proof_file = os.path.join(proof_dir, 'verify_proof.json')
+    with open(proof_file, 'w') as f:
+        json.dump(proof, f, indent=2)
+
+    print(json.dumps({'ok': all_passed, 'proof_file': proof_file, 'total': len(results), 'passed': sum(1 for r in results if r['passed'])}))
+    sys.exit(0 if all_passed else 1)
+
+if __name__ == '__main__':
+    main()