aw-spec-cli 0.1.4 → 0.1.5

package/assets/.aw-spec/scripts/python/aw-parity.py

@@ -0,0 +1,380 @@
+#!/usr/bin/env python3
+"""aw-parity.py — the Brownfield Pipeline PARITY runner (the cutover gate).
+
+Parity is the migration-grade oracle: run every case in a parity manifest against
+BOTH the running LEGACY system and the new CANDIDATE, and diff observable output
+(response status + body, and optional resulting-state probes). The seam cuts over
+to the candidate only when parity HOLDS — i.e. no unaccepted diffs. Approved
+intended differences are whitelisted in the manifest's `expected_deviations` (each
+must become a MODIFIED spec item via the CR loop — parity never lets behaviour
+drift silently). See Brownfield-Pipeline.my-ref.md.
+
+Unlike Characterize (a frozen snapshot of behaviour, in CI), Parity compares
+against the LIVE legacy system, so it catches behaviour no static test captured.
+
+Stdlib only (urllib for http, subprocess for shell adapters). Subcommands print a
+single JSON object as the LAST stdout line (the orchestrator binds it); human
+detail goes to STDERR.
+
+  scaffold --feature-dir <dir>          -> {manifest_path, created}   (copy the template)
+  check    --manifest <path>            -> {ready, sides_configured, case_count, error}
+  run      --manifest <path> [--out <report.md>]
+           -> {ok, ready, total, matched, mismatched, accepted, errors, report_path}
+
+A side is invoked by `kind`:
+  http  : { "kind":"http", "base_url":"http://host:port" }  → base_url + case.request.path
+  shell : { "kind":"shell", "command":"./adapter.sh" }       → request JSON on stdin, output JSON on stdout
+"""
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import urllib.error
+import urllib.request
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+TEMPLATE = os.path.join(HERE, "..", "..", "templates", "parity-manifest-template.json")
+PLACEHOLDER = re.compile(r"REPLACE_WITH", re.IGNORECASE)
+
+
+# --------------------------------------------------------------------------- IO
+def load_manifest(path):
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+
+def strip_comments(obj):
+    """Drop `_comment*` keys (documentation) recursively so they never diff."""
+    if isinstance(obj, dict):
+        return {k: strip_comments(v) for k, v in obj.items() if not str(k).startswith("_comment")}
+    if isinstance(obj, list):
+        return [strip_comments(x) for x in obj]
+    return obj
+
+
+# ------------------------------------------------------------------- normalize
+def _drop_path(obj, parts):
+    """Remove one ignore-path (list of segments, '*' = wildcard) from obj in place."""
+    if not parts or obj is None:
+        return
+    seg, rest = parts[0], parts[1:]
+    if isinstance(obj, dict):
+        keys = list(obj.keys()) if seg == "*" else ([seg] if seg in obj else [])
+        for k in keys:
+            if not rest:
+                obj.pop(k, None)
+            else:
+                _drop_path(obj[k], rest)
+    elif isinstance(obj, list):
+        items = obj if seg in ("*", "[*]") else []
+        if not items and seg.isdigit() and int(seg) < len(obj):
+            items = [obj[int(seg)]]
+        for it in items:
+            if rest:
+                _drop_path(it, rest)
+
+
+def _parse_path(p):
+    # "$.a.b[*].c" / "$.*.timestamp" -> ["a","b","*","c"]
+    p = p.strip()
+    p = p[2:] if p.startswith("$.") else (p[1:] if p.startswith("$") else p)
+    p = p.replace("[*]", ".*")
+    return [seg for seg in p.split(".") if seg != ""]
+
+
+def normalize(body, ignore_paths):
+    """Return a deep copy of body with ignore_paths removed (JSON only)."""
+    if not isinstance(body, (dict, list)):
+        return body
+    clone = json.loads(json.dumps(body))
+    for ip in ignore_paths or []:
+        _drop_path(clone, _parse_path(ip))
+    return clone
+
+
+# ------------------------------------------------------------------ deep diff
+def deep_diff(a, b, path="$"):
+    """List of {path, legacy, candidate} where a (legacy) and b (candidate) differ."""
+    out = []
+    if isinstance(a, dict) and isinstance(b, dict):
+        for k in sorted(set(a) | set(b)):
+            if k not in a:
+                out.append({"path": f"{path}.{k}", "legacy": None, "candidate": b[k]})
+            elif k not in b:
+                out.append({"path": f"{path}.{k}", "legacy": a[k], "candidate": None})
+            else:
+                out += deep_diff(a[k], b[k], f"{path}.{k}")
+    elif isinstance(a, list) and isinstance(b, list):
+        if len(a) != len(b):
+            out.append({"path": f"{path}.length", "legacy": len(a), "candidate": len(b)})
+        for i in range(min(len(a), len(b))):
+            out += deep_diff(a[i], b[i], f"{path}[{i}]")
+    elif a != b:
+        out.append({"path": path, "legacy": a, "candidate": b})
+    return out
+
+
+# ------------------------------------------------------------------- invoke
+def invoke(side, request, timeout):
+    kind = (side or {}).get("kind", "http")
+    if kind == "http":
+        return _invoke_http(side, request, timeout)
+    if kind == "shell":
+        return _invoke_shell(side, request, timeout)
+    return {"error": f"unknown side kind '{kind}'"}
+
+
+def _invoke_http(side, request, timeout):
+    base = (side.get("base_url") or "").rstrip("/")
+    url = base + "/" + str(request.get("path", "")).lstrip("/")
+    method = str(request.get("method", "GET")).upper()
+    headers = request.get("headers", {}) or {}
+    data = None
+    if request.get("body") is not None and method in ("POST", "PUT", "PATCH", "DELETE"):
+        data = json.dumps(request["body"]).encode("utf-8")
+        headers.setdefault("Content-Type", "application/json")
+    req = urllib.request.Request(url, data=data, method=method, headers=headers)
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            raw, status = resp.read().decode("utf-8", "replace"), resp.status
+    except urllib.error.HTTPError as e:
+        raw, status = e.read().decode("utf-8", "replace"), e.code
+    except Exception as e:
+        return {"error": f"{type(e).__name__}: {e}", "url": url}
+    try:
+        body = json.loads(raw) if raw.strip() else ""
+    except json.JSONDecodeError:
+        body = raw
+    return {"status": status, "body": body}
+
+
+def _invoke_shell(side, request, timeout):
+    cmd = side.get("command")
+    if not cmd:
+        return {"error": "shell side has no 'command'"}
+    try:
+        proc = subprocess.run(cmd, shell=True, input=json.dumps(request),
+                              capture_output=True, text=True, timeout=timeout)
+    except subprocess.TimeoutExpired:
+        return {"error": f"timeout after {timeout}s"}
+    if proc.returncode != 0:
+        return {"error": (proc.stderr or "").strip()[:500], "status": proc.returncode}
+    out = (proc.stdout or "").strip()
+    try:
+        body = json.loads(out) if out else ""
+    except json.JSONDecodeError:
+        body = out
+    return {"status": 0, "body": body}
+
+
+# ------------------------------------------------------------------- compare
+def compare_observation(legacy, candidate, ignore_paths, ignore_status):
+    """Diff a single observation (one request's outputs) across the two sides."""
+    if "error" in legacy or "error" in candidate:
+        return [{"path": "$", "legacy": legacy.get("error", legacy.get("status")),
+                 "candidate": candidate.get("error", candidate.get("status")),
+                 "invoke_error": True}]
+    diffs = []
+    if not ignore_status and legacy.get("status") != candidate.get("status"):
+        diffs.append({"path": "$.status", "legacy": legacy.get("status"),
+                      "candidate": candidate.get("status")})
+    diffs += deep_diff(normalize(legacy.get("body"), ignore_paths),
+                       normalize(candidate.get("body"), ignore_paths), "$.body")
+    return diffs
+
+
+def _accepted(case_id, path, deviations):
+    for d in deviations or []:
+        if d.get("case") in (case_id, "*") and (d.get("path") == path or d.get("path") == "*"):
+            return d.get("reason", "approved deviation")
+    return None
+
+
+# ------------------------------------------------------------------- commands
+def cmd_scaffold(args):
+    parity_dir = os.path.join(args.feature_dir, "parity")
+    os.makedirs(parity_dir, exist_ok=True)
+    manifest = os.path.join(parity_dir, "parity-manifest.json")
+    created = False
+    if not os.path.exists(manifest):
+        with open(TEMPLATE, encoding="utf-8") as src:
+            content = src.read()
+        agg = os.path.basename(os.path.normpath(args.feature_dir))
+        content = content.replace('"AGGREGATE"', json.dumps(agg))
+        with open(manifest, "w", encoding="utf-8") as dst:
+            dst.write(content)
+        created = True
+    print(f"parity manifest {'created' if created else 'exists'}: {manifest}", file=sys.stderr)
+    print(json.dumps({"manifest_path": manifest, "created": "true" if created else "false"}))
+    return 0
+
+
+def _sides_configured(m):
+    for side in (m.get("sides") or {}).values():
+        side = strip_comments(side)
+        target = side.get("base_url") if side.get("kind", "http") == "http" else side.get("command")
+        if not target or PLACEHOLDER.search(str(target)):
+            return False
+    return bool(m.get("sides"))
+
+
+def cmd_check(args):
+    try:
+        m = strip_comments(load_manifest(args.manifest))
+    except FileNotFoundError:
+        print(json.dumps({"ready": "false", "sides_configured": "false", "case_count": 0,
+                          "error": f"manifest not found: {args.manifest}"}))
+        return 0
+    except json.JSONDecodeError as e:
+        print(json.dumps({"ready": "false", "sides_configured": "false", "case_count": 0,
+                          "error": f"manifest is not valid JSON: {e}"}))
+        return 0
+    cases = [c for c in (m.get("cases") or []) if not PLACEHOLDER.search(json.dumps(c))
+             and not str(c.get("id", "")).endswith("EXAMPLE")]
+    configured = _sides_configured(m)
+    ready = configured and len(cases) > 0
+    print(json.dumps({"ready": "true" if ready else "false",
+                      "sides_configured": "true" if configured else "false",
+                      "case_count": len(cases), "error": ""}))
+    return 0
+
+
+def cmd_run(args):
+    try:
+        m = strip_comments(load_manifest(args.manifest))
+    except (FileNotFoundError, json.JSONDecodeError) as e:
+        print(json.dumps({"ok": "false", "ready": "false", "total": 0, "matched": 0,
+                          "mismatched": 0, "accepted": 0, "errors": 1, "report_path": "",
+                          "error": str(e)}))
+        return 0
+    sides = m.get("sides") or {}
+    legacy_side, cand_side = strip_comments(sides.get("legacy") or {}), strip_comments(sides.get("candidate") or {})
+    comp = strip_comments(m.get("compare") or {})
+    ignore_paths = comp.get("ignore_paths") or []
+    ignore_status = bool(comp.get("ignore_status"))
+    timeout = float(comp.get("timeout_seconds") or 30)
+    deviations = [strip_comments(d) for d in (m.get("expected_deviations") or [])]
+    cases = [strip_comments(c) for c in (m.get("cases") or [])
+             if not str(c.get("id", "")).endswith("EXAMPLE")]
+
+    # Don't attempt calls against unconfigured (placeholder) sides — the workflow's
+    # readiness gate normally stops earlier, but guard here too.
+    if not _sides_configured(m):
+        print("parity: sides not configured (placeholder base_url/command) — nothing run",
+              file=sys.stderr)
+        print(json.dumps({"ok": "false", "ready": "false", "total": len(cases), "matched": 0,
+                          "mismatched": 0, "accepted": 0, "errors": 0, "report_path": "",
+                          "error": "sides not configured"}))
+        return 0
+
+    results, n_match, n_mismatch, n_accept, n_err = [], 0, 0, 0, 0
+    for case in cases:
+        cid = case.get("id", "?")
+        case_diffs, accepted, invoke_err = [], [], False
+        observations = [("request", case.get("request"))]
+        if case.get("state_probe"):
+            observations.append(("state", case["state_probe"]))
+        for label, request in observations:
+            if not request:
+                continue
+            lg = invoke(legacy_side, request, timeout)
+            cd = invoke(cand_side, request, timeout)
+            for d in compare_observation(lg, cd, ignore_paths, ignore_status):
+                d["observation"] = label
+                if d.get("invoke_error"):
+                    invoke_err = True
+                    case_diffs.append(d)
+                    continue
+                reason = _accepted(cid, d["path"], deviations)
+                if reason:
+                    d["accepted_reason"] = reason
+                    accepted.append(d)
+                else:
+                    case_diffs.append(d)
+        if invoke_err:
+            n_err += 1
+            status = "error"
+        elif case_diffs:
+            n_mismatch += 1
+            status = "mismatch"
+        else:
+            n_match += 1
+            status = "match"
+        n_accept += len(accepted)
+        results.append({"id": cid, "description": case.get("description", ""),
+                        "spec_ids": case.get("spec_ids", []), "status": status,
+                        "diffs": case_diffs, "accepted": accepted})
+
+    ok = (n_mismatch == 0 and n_err == 0 and len(cases) > 0)
+    report_path = args.out or os.path.join(os.path.dirname(args.manifest), "parity-report.md")
+    _write_report(report_path, m, results, n_match, n_mismatch, n_accept, n_err)
+
+    print(f"parity: {n_match} match · {n_mismatch} mismatch · {n_err} error · "
+          f"{n_accept} accepted-deviation(s) over {len(cases)} case(s)", file=sys.stderr)
+    print(json.dumps({"ok": "true" if ok else "false", "ready": "true" if cases else "false",
+                      "total": len(cases), "matched": n_match, "mismatched": n_mismatch,
+                      "accepted": n_accept, "errors": n_err, "report_path": report_path}))
+    return 0
+
+
+def _write_report(path, m, results, n_match, n_mismatch, n_accept, n_err):
+    from datetime import datetime, timezone
+    lines = [f"# Parity Report — {m.get('aggregate', '?')}",
+             f"_generated {datetime.now(timezone.utc).date().isoformat()} · "
+             f"baseline {m.get('baseline_commit', '-')}_", "",
+             f"**verdict:** {'PARITY HOLDS ✓' if (n_mismatch == 0 and n_err == 0) else 'PARITY FAILS ✗'} · "
+             f"{n_match} match · {n_mismatch} mismatch · {n_err} error · {n_accept} accepted deviation(s)", "",
+             "| case | spec ids | status | diffs |", "|---|---|---|---|"]
+    for r in results:
+        lines.append(f"| {r['id']} {r['description']} | {', '.join(r['spec_ids'])} | "
+                     f"{r['status']} | {len(r['diffs'])} |")
+    fails = [r for r in results if r["diffs"]]
+    if fails:
+        lines += ["", "## Unaccepted diffs (block cutover)"]
+        for r in fails:
+            lines.append(f"### {r['id']} — {r['description']}")
+            for d in r["diffs"]:
+                tag = " (INVOKE ERROR)" if d.get("invoke_error") else ""
+                lines.append(f"- `{d['path']}`{tag} [{d.get('observation','request')}]: "
+                             f"legacy=`{json.dumps(d['legacy'])}` vs candidate=`{json.dumps(d['candidate'])}`")
+    accepted = [(r, d) for r in results for d in r["accepted"]]
+    if accepted:
+        lines += ["", "## Accepted deviations (intended — must be MODIFIED spec items via CR)"]
+        for r, d in accepted:
+            lines.append(f"- {r['id']} `{d['path']}`: {d.get('accepted_reason')}")
+    lines += ["", "## How to proceed",
+              "- Each unaccepted diff is either a REGRESSION (fix the candidate to match legacy) or an "
+              "INTENDED change (add it to `expected_deviations` AND record it as a MODIFIED spec item via "
+              "/aw-spec-orch specify). Then re-run /aw-spec-orch parity.",
+              "- Parity holds → the seam is cutover-ready: route it to the candidate; roll back by re-routing."]
+    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        f.write("\n".join(lines) + "\n")
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Brownfield parity runner (cutover gate).")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    ps = sub.add_parser("scaffold", help="Create specs/<Agg>/parity/parity-manifest.json from the template.")
+    ps.add_argument("--feature-dir", required=True)
+    ps.set_defaults(func=cmd_scaffold)
+
+    pc = sub.add_parser("check", help="Report whether the manifest is ready to run (sides configured + cases).")
+    pc.add_argument("--manifest", required=True)
+    pc.set_defaults(func=cmd_check)
+
+    pr = sub.add_parser("run", help="Run every case against both sides, diff, write the report.")
+    pr.add_argument("--manifest", required=True)
+    pr.add_argument("--out", help="Report path (default: <manifest dir>/parity-report.md).")
+    pr.set_defaults(func=cmd_run)
+
+    args = ap.parse_args()
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())