@timmeck/brain 1.8.0 → 1.8.2

package/tests/e2e/test_full_integration.py

@@ -0,0 +1,534 @@
+#!/usr/bin/env python3
+"""
+Brain v1.8.1 — Full System Integration Test
+Tests EVERY remaining endpoint, CLI, MCP HTTP, SSE, error handling, and performance.
+~80 assertions covering the complete system.
+
+Run AFTER test_error_memory.py and test_code_intelligence.py for richer data.
+"""
+
+import sys
+import time
+import json
+import uuid
+import subprocess
+import threading
+import httpx
+
+BASE = "http://localhost:7777/api/v1"
+MCP_BASE = "http://localhost:7778"
+PASS = 0
+FAIL = 0
+ERRORS: list[str] = []
+PERF: list[tuple[str, float]] = []
+
+
+def check(condition: bool, label: str) -> bool:
+    global PASS, FAIL
+    if condition:
+        PASS += 1
+        print(f"  \033[32mPASS\033[0m {label}")
+    else:
+        FAIL += 1
+        ERRORS.append(label)
+        print(f"  \033[31mFAIL\033[0m {label}")
+    return condition
+
+
+def timed_get(path: str, params: dict | None = None, base: str = BASE) -> httpx.Response:
+    t0 = time.perf_counter()
+    r = httpx.get(f"{base}{path}", params=params, timeout=15)
+    dt = (time.perf_counter() - t0) * 1000
+    PERF.append((f"GET {path}", dt))
+    return r
+
+
+def timed_post(path: str, json_data: dict | list | None = None, base: str = BASE) -> httpx.Response:
+    t0 = time.perf_counter()
+    r = httpx.post(f"{base}{path}", json=json_data or {}, timeout=15)
+    dt = (time.perf_counter() - t0) * 1000
+    PERF.append((f"POST {path}", dt))
+    return r
+
+
+def post(path: str, json_data: dict | list | None = None, base: str = BASE) -> httpx.Response:
+    return httpx.post(f"{base}{path}", json=json_data or {}, timeout=15)
+
+
+def get(path: str, params: dict | None = None, base: str = BASE) -> httpx.Response:
+    return httpx.get(f"{base}{path}", params=params, timeout=15)
+
+
+def section(title: str) -> None:
+    print(f"\n{'-' * 50}")
+    print(f"  {title}")
+    print(f"{'-' * 50}")
+
+
+def main() -> int:
+    print("\n" + "=" * 60)
+    print("  BRAIN E2E TEST: Full System Integration")
+    print("=" * 60)
+
+    # ══════════════════════════════════════════════════════════
+    # Section A: REST Infrastructure
+    # ══════════════════════════════════════════════════════════
+    section("A: REST Infrastructure")
+
+    # A1: Health check
+    r = timed_get("/health")
+    check(r.status_code == 200, "Health endpoint returns 200")
+    health = r.json()
+    check(health.get("status") == "ok", "Health status is 'ok'")
+    check("timestamp" in health, "Health includes timestamp")
+
+    # A2: Methods listing
+    r = timed_get("/methods")
+    check(r.status_code == 200, "Methods endpoint returns 200")
+    methods = r.json().get("methods", [])
+    check(isinstance(methods, list) and len(methods) >= 30, f"Listed {len(methods)} methods (expect 30+)")
+
+    # A3: Single RPC call
+    r = timed_post("/rpc", {"method": "analytics.summary", "params": {}})
+    check(r.status_code == 200, "Single RPC returns 200")
+    check("result" in r.json(), "RPC response has 'result' key")
+
+    # A4: Batch RPC call
+    batch = [
+        {"method": "analytics.summary", "params": {}, "id": 1},
+        {"method": "synapse.stats", "params": {}, "id": 2},
+        {"method": "project.list", "params": {}, "id": 3},
+    ]
+    r = timed_post("/rpc", batch)
+    check(r.status_code == 200, "Batch RPC returns 200")
+    results = r.json()
+    check(isinstance(results, list) and len(results) == 3, f"Batch returned {len(results)} results")
+    check(all("result" in item or "error" in item for item in results), "All batch items have result or error")
+
+    # A5: RPC with unknown method
+    r = post("/rpc", {"method": "nonexistent.method", "params": {}})
+    check(r.status_code == 400, "Unknown RPC method returns 400")
+    check("error" in r.json(), "Unknown method has error message")
+
+    # A6: RPC with missing method field
+    r = post("/rpc", {"params": {}})
+    check(r.status_code == 400, "Missing method field returns 400")
+
+    # A7: RPC with empty body
+    r = httpx.post(f"{BASE}/rpc", content=b"", headers={"Content-Type": "application/json"}, timeout=10)
+    check(r.status_code == 400, "Empty RPC body returns 400")
+
+    # A8: 404 for unknown route
+    r = get("/nonexistent/route")
+    check(r.status_code == 404, "Unknown route returns 404")
+
+    # A9: CORS headers
+    r = httpx.options(f"{BASE}/health", timeout=10)
+    check(r.status_code == 204, "OPTIONS returns 204")
+    check("access-control-allow-origin" in r.headers, "CORS headers present")
+
+    # ══════════════════════════════════════════════════════════
+    # Section B: SSE Events
+    # ══════════════════════════════════════════════════════════
+    section("B: SSE Events")
+
+    sse_events: list[str] = []
+    sse_connected = threading.Event()
+
+    def sse_listener():
+        try:
+            with httpx.stream("GET", f"{BASE}/events", timeout=10) as stream:
+                for line in stream.iter_lines():
+                    if line.startswith("data: "):
+                        data = line[6:]
+                        sse_events.append(data)
+                        parsed = json.loads(data)
+                        if parsed.get("type") == "connected":
+                            sse_connected.set()
+                        if len(sse_events) >= 3:
+                            break
+        except (httpx.ReadTimeout, httpx.RemoteProtocolError):
+            pass
+
+    t = threading.Thread(target=sse_listener, daemon=True)
+    t.start()
+    sse_connected.wait(timeout=5)
+    check(sse_connected.is_set(), "SSE connection established")
+
+    # Trigger an event by reporting an error
+    post("/errors", {
+        "project": "test-sse",
+        "errorOutput": "Error: SSE test trigger\n    at test (/tmp/test.js:1:1)",
+    })
+    time.sleep(1)
+    check(len(sse_events) >= 1, f"SSE received {len(sse_events)} event(s)")
+
+    # ══════════════════════════════════════════════════════════
+    # Section C: MCP HTTP/SSE
+    # ══════════════════════════════════════════════════════════
+    section("C: MCP HTTP/SSE")
+
+    # C1: Root endpoint
+    try:
+        r = get("/", base=MCP_BASE)
+        check(r.status_code == 200, "MCP root endpoint returns 200")
+        mcp_info = r.json()
+        check(mcp_info.get("name") == "brain", f"MCP name: {mcp_info.get('name')}")
+        check(mcp_info.get("protocol") == "MCP", "MCP protocol field present")
+        check("endpoints" in mcp_info, "MCP endpoints listed")
+    except httpx.ConnectError:
+        check(False, "MCP HTTP server reachable on port 7778")
+        check(False, "MCP name check (skipped)")
+        check(False, "MCP protocol check (skipped)")
+        check(False, "MCP endpoints check (skipped)")
+
+    # C2: SSE endpoint (just verify it starts streaming)
+    try:
+        with httpx.stream("GET", f"{MCP_BASE}/sse", timeout=5) as stream:
+            first_chunk = None
+            for line in stream.iter_lines():
+                first_chunk = line
+                break
+            check(first_chunk is not None, f"MCP SSE stream started: {first_chunk[:50] if first_chunk else 'empty'}...")
+    except (httpx.ReadTimeout, httpx.ConnectError):
+        check(False, "MCP SSE connection (timeout or unreachable)")
+
+    # C3: Messages endpoint without sessionId
+    try:
+        r = post("/messages", base=MCP_BASE)
+        check(r.status_code == 400, "MCP /messages without sessionId returns 400")
+    except httpx.ConnectError:
+        check(False, "MCP /messages reachable")
+
+    # ══════════════════════════════════════════════════════════
+    # Section D: All REST Endpoints
+    # ══════════════════════════════════════════════════════════
+    section("D: All REST Endpoints (comprehensive)")
+
+    # D1-D6: Error endpoints (basic shape verification)
+    r = timed_get("/errors")
+    check(r.status_code == 200, "GET /errors returns 200")
+    errors = r.json().get("result", [])
+    check(isinstance(errors, list), f"Errors list: {len(errors)} items")
+
+    if errors:
+        eid = errors[0]["id"] if isinstance(errors[0], dict) else errors[0]
+        r = timed_get(f"/errors/{eid}")
+        check(r.status_code == 200, f"GET /errors/{eid} returns 200")
+
+        r = timed_get(f"/errors/{eid}/match")
+        check(r.status_code == 200, f"GET /errors/{eid}/match returns 200")
+
+        r = timed_get(f"/errors/{eid}/chain")
+        check(r.status_code == 200, f"GET /errors/{eid}/chain returns 200")
+
+    # D7-D10: Solution endpoints
+    r = timed_get("/solutions")
+    check(r.status_code == 200, "GET /solutions returns 200")
+
+    r = timed_get("/solutions/efficiency")
+    check(r.status_code == 200, "GET /solutions/efficiency returns 200")
+
+    # D11: Projects
+    r = timed_get("/projects")
+    check(r.status_code == 200, "GET /projects returns 200")
+
+    # D12-D16: Code endpoints
+    r = timed_get("/code/modules")
+    check(r.status_code == 200, "GET /code/modules returns 200")
+    modules = r.json().get("result", [])
+    if modules and isinstance(modules[0], dict):
+        mid = modules[0]["id"]
+        r = timed_get(f"/code/{mid}")
+        check(r.status_code == 200, f"GET /code/{mid} returns 200")
+
+    r = timed_post("/code/find", {"query": "utility"})
+    check(r.status_code == 201, "POST /code/find returns 201")
+
+    r = timed_post("/code/similarity", {"source": "function test() { return 42; }", "language": "typescript"})
+    check(r.status_code == 201, "POST /code/similarity returns 201")
+
+    # D17-D19: Prevention endpoints
+    r = timed_post("/prevention/check", {"errorType": "Error", "message": "test"})
+    check(r.status_code == 201, "POST /prevention/check returns 201")
+
+    r = timed_post("/prevention/antipatterns", {"errorType": "Error", "message": "test"})
+    check(r.status_code == 201, "POST /prevention/antipatterns returns 201")
+
+    r = timed_post("/prevention/code", {"source": "let x = 1;", "filePath": "test.js"})
+    check(r.status_code == 201, "POST /prevention/code returns 201")
+
+    # D20-D23: Synapse endpoints
+    r = timed_get("/synapses/stats")
+    check(r.status_code == 200, "GET /synapses/stats returns 200")
+    syn_stats = r.json().get("result", {})
+    check(isinstance(syn_stats, dict), f"Synapse stats: {syn_stats.get('totalSynapses', '?')} synapses")
+
+    if errors:
+        eid = errors[0]["id"] if isinstance(errors[0], dict) else errors[0]
+        r = timed_get(f"/synapses/context/{eid}")
+        check(r.status_code == 200, f"GET /synapses/context/{eid} returns 200")
+
+    r = timed_post("/synapses/related", {"nodeType": "error", "nodeId": 1})
+    check(r.status_code == 201, "POST /synapses/related returns 201")
+
+    r = timed_post("/synapses/path", {"fromType": "error", "fromId": 1, "toType": "solution", "toId": 1})
+    check(r.status_code == 201, "POST /synapses/path returns 201")
+
+    # D24-D27: Research/Insights endpoints
+    r = timed_get("/research/insights")
+    check(r.status_code == 200, "GET /research/insights returns 200")
+
+    r = timed_get("/research/suggest", params={"context": "TypeError handling"})
+    check(r.status_code == 200, "GET /research/suggest returns 200")
+
+    r = timed_get("/research/trends")
+    check(r.status_code == 200, "GET /research/trends returns 200")
+
+    # D28-D29: Notifications
+    r = timed_get("/notifications")
+    check(r.status_code == 200, "GET /notifications returns 200")
+
+    # D30-D34: Analytics endpoints
+    r = timed_get("/analytics/summary")
+    check(r.status_code == 200, "GET /analytics/summary returns 200")
+    summary = r.json().get("result", {})
+    check(isinstance(summary, dict), "Analytics summary is a dict")
+
+    r = timed_get("/analytics/network")
+    check(r.status_code == 200, "GET /analytics/network returns 200")
+
+    r = timed_get("/analytics/health")
+    check(r.status_code == 200, "GET /analytics/health returns 200")
+
+    r = timed_get("/analytics/timeline")
+    check(r.status_code == 200, "GET /analytics/timeline returns 200")
+
+    if errors:
+        eid = errors[0]["id"] if isinstance(errors[0], dict) else errors[0]
+        r = timed_get(f"/analytics/explain/{eid}")
+        check(r.status_code == 200, f"GET /analytics/explain/{eid} returns 200")
+
+    # D35-D39: Git endpoints
+    r = timed_get("/git/context")
+    check(r.status_code == 200, "GET /git/context returns 200")
+
+    r = timed_get("/git/diff")
+    check(r.status_code == 200, "GET /git/diff returns 200")
+
+    # D40: Learning
+    r = timed_post("/learning/run")
+    check(r.status_code == 201, "POST /learning/run returns 201")
+
+    # ══════════════════════════════════════════════════════════
+    # Section E: CLI Commands
+    # ══════════════════════════════════════════════════════════
+    section("E: CLI Commands")
+
+    cli_commands = [
+        (["brain", "status"], "brain status"),
+        (["brain", "doctor"], "brain doctor"),
+        (["brain", "query", "TypeError"], "brain query TypeError"),
+        (["brain", "modules"], "brain modules"),
+        (["brain", "insights"], "brain insights"),
+        (["brain", "projects"], "brain projects"),
+        (["brain", "network"], "brain network"),
+        (["brain", "learn"], "brain learn"),
+    ]
+
+    for cmd, label in cli_commands:
+        try:
+            result = subprocess.run(
+                cmd, capture_output=True, text=True, timeout=30,
+                shell=(sys.platform == "win32"),
+            )
+            # Some commands may have non-zero exit codes if no data, but they shouldn't crash
+            check(result.returncode == 0 or result.returncode is not None,
+                  f"CLI '{label}' ran (exit={result.returncode})")
+        except FileNotFoundError:
+            check(False, f"CLI '{label}' (brain not found in PATH)")
+        except subprocess.TimeoutExpired:
+            check(False, f"CLI '{label}' (timeout)")
+
+    # ══════════════════════════════════════════════════════════
+    # Section F: Git Integration
+    # ══════════════════════════════════════════════════════════
+    section("F: Git Integration")
+
+    # F1: Git context
+    r = get("/git/context")
+    ctx = r.json().get("result", {})
+    check(r.status_code == 200, "Git context returns 200")
+    check(ctx.get("branch") is not None or ctx.get("branch") is None, f"Git branch: {ctx.get('branch')}")
+
+    # F2: Link error to commit
+    fake_hash = "abc1234567890def1234567890abcdef12345678"
+    if errors:
+        eid = errors[0]["id"] if isinstance(errors[0], dict) else errors[0]
+        # Get project ID from the error
+        err_detail = get(f"/errors/{eid}").json().get("result", {})
+        pid = err_detail.get("project_id", 1)
+        r = post("/git/link-error", {
+            "errorId": eid,
+            "projectId": pid,
+            "commitHash": fake_hash,
+            "relationship": "introduced_by",
+        })
+        check(r.status_code == 201, "Git link-error returns 201")
+
+        # F3: Query commits by error
+        r = get(f"/git/errors/{eid}/commits")
+        check(r.status_code == 200, f"Git errorCommits returns 200")
+
+        # F4: Query errors by commit
+        r = get(f"/git/commits/{fake_hash}/errors")
+        check(r.status_code == 200, "Git commitErrors returns 200")
+
+    # F5: Git diff
+    r = get("/git/diff")
+    check(r.status_code == 200, "Git diff returns 200")
+
+    # ══════════════════════════════════════════════════════════
+    # Section G: Terminal Lifecycle
+    # ══════════════════════════════════════════════════════════
+    section("G: Terminal Lifecycle")
+
+    term_uuid = str(uuid.uuid4())
+
+    # G1: Register terminal
+    r = post("/terminal/register", {
+        "uuid": term_uuid,
+        "pid": 12345,
+        "shell": "bash",
+        "cwd": "/tmp/test-project",
+    })
+    check(r.status_code == 201, "Terminal register returns 201")
+    term_id = r.json().get("result")
+    check(term_id is not None, f"Terminal registered (id={term_id})")
+
+    # G2: Heartbeat
+    r = post("/terminal/heartbeat", {"uuid": term_uuid})
+    check(r.status_code == 201, "Terminal heartbeat returns 201")
+
+    # G3: Disconnect
+    r = post("/terminal/disconnect", {"uuid": term_uuid})
+    check(r.status_code == 201, "Terminal disconnect returns 201")
+
+    # ══════════════════════════════════════════════════════════
+    # Section H: Notifications
+    # ══════════════════════════════════════════════════════════
+    section("H: Notifications")
+
+    # H1: List notifications
+    r = get("/notifications")
+    check(r.status_code == 200, "Notifications list returns 200")
+    notifs = r.json().get("result", [])
+    check(isinstance(notifs, list), f"Notifications: {len(notifs)} items")
+
+    # H2: Acknowledge a notification (if any exist)
+    if notifs and isinstance(notifs[0], dict):
+        nid = notifs[0].get("id")
+        if nid:
+            r = post(f"/notifications/{nid}/ack")
+            check(r.status_code == 201, f"Notification {nid} acknowledged")
+
+            # H3: Verify dismissal
+            r = get("/notifications")
+            new_notifs = r.json().get("result", [])
+            dismissed = all(n.get("id") != nid for n in new_notifs if isinstance(n, dict))
+            check(dismissed or len(new_notifs) <= len(notifs),
+                  f"Notification dismissed ({len(notifs)} → {len(new_notifs)})")
+    else:
+        check(True, "No notifications to acknowledge (OK)")
+        check(True, "No notifications to verify dismissal (OK)")
+
+    # ══════════════════════════════════════════════════════════
+    # Section I: Performance Benchmarks
+    # ══════════════════════════════════════════════════════════
+    section("I: Performance Benchmarks")
+
+    # Run a few extra timed calls for benchmark variety
+    for _ in range(3):
+        timed_get("/health")
+        timed_get("/analytics/summary")
+        timed_post("/rpc", {"method": "synapse.stats", "params": {}})
+
+    # Print performance table
+    if PERF:
+        print(f"\n  {'Endpoint':<45} {'Time (ms)':>10}")
+        print(f"  {'-' * 45} {'-' * 10}")
+
+        # Group by endpoint and compute averages
+        from collections import defaultdict
+        groups: dict[str, list[float]] = defaultdict(list)
+        for endpoint, ms in PERF:
+            groups[endpoint].append(ms)
+
+        for endpoint, times in sorted(groups.items()):
+            avg = sum(times) / len(times)
+            count = f" (×{len(times)})" if len(times) > 1 else ""
+            color = "\033[32m" if avg < 50 else "\033[33m" if avg < 200 else "\033[31m"
+            print(f"  {endpoint:<45} {color}{avg:>8.1f}ms\033[0m{count}")
+
+        all_times = [t for _, t in PERF]
+        avg_all = sum(all_times) / len(all_times)
+        max_time = max(all_times)
+        p95 = sorted(all_times)[int(len(all_times) * 0.95)]
+        print(f"\n  Average: {avg_all:.1f}ms | P95: {p95:.1f}ms | Max: {max_time:.1f}ms")
+
+        check(avg_all < 500, f"Average response time < 500ms ({avg_all:.1f}ms)")
+        check(max_time < 5000, f"Max response time < 5s ({max_time:.1f}ms)")
+
+    # ══════════════════════════════════════════════════════════
+    # Section J: Error Handling
+    # ══════════════════════════════════════════════════════════
+    section("J: Error Handling")
+
+    # J1: Invalid JSON body
+    r = httpx.post(f"{BASE}/errors", content=b"not json", headers={"Content-Type": "application/json"}, timeout=10)
+    check(r.status_code == 400, "Invalid JSON returns 400")
+
+    # J2: Missing required fields
+    r = post("/errors", {})
+    # Should still work but with empty/default values, or return error
+    check(r.status_code in (201, 400), f"Empty error body returns {r.status_code}")
+
+    # J3: Non-existent error ID
+    r = get("/errors/999999")
+    check(r.status_code in (200, 400, 404), f"Non-existent error ID returns {r.status_code}")
+
+    # J4: Non-existent code module
+    r = get("/code/999999")
+    check(r.status_code in (200, 400, 404), f"Non-existent module ID returns {r.status_code}")
+
+    # J5: Invalid RPC method type
+    r = post("/rpc", {"method": 12345, "params": {}})
+    check(r.status_code in (200, 400), f"Invalid method type returns {r.status_code}")
+
+    # ══════════════════════════════════════════════════════════
+    # Summary
+    # ══════════════════════════════════════════════════════════
+    print("\n" + "=" * 60)
+    total = PASS + FAIL
+    print(f"  Results: {PASS}/{total} passed, {FAIL} failed")
+    if ERRORS:
+        print(f"\n  Failed tests:")
+        for e in ERRORS:
+            print(f"    - {e}")
+    print("=" * 60 + "\n")
+
+    return 0 if FAIL == 0 else 1
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except httpx.ConnectError:
+        print("\n\033[31mERROR: Cannot connect to Brain daemon on port 7777.\033[0m")
+        print("Run 'brain start' or 'brain doctor' first.\n")
+        sys.exit(2)
+    except Exception as e:
+        print(f"\n\033[31mFATAL: {e}\033[0m\n")
+        import traceback
+        traceback.print_exc()
+        sys.exit(2)