@apexops/cli 0.10.0-next.5 → 0.11.0-next.1

package/assets/capability-packs/azure-governance-discovery/source/scripts/discover.py

@@ -1,1189 +0,0 @@
-#!/usr/bin/env python3
-# ruff: noqa: E501
-"""Deterministic Azure Policy discovery for 04g-Governance.
-
-Replaces the legacy governance-discovery-subagent. Lists effective policy
-assignments at subscription scope (including MG-inherited), pulls definitions,
-set-definitions, and exemptions, classifies effects, filters Defender
-auto-assignments, and writes a governance-constraints-v1 envelope.
-
-Stdout line 1 is always a one-line JSON status object. Optional Markdown
-preview follows on later lines. Stderr carries warnings and filter notes.
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import re
-import subprocess
-import sys
-import time
-import urllib.error
-import urllib.request
-from concurrent.futures import ThreadPoolExecutor
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Any, Callable
-
-# --------------------------------------------------------------------------- #
-# Constants                                                                   #
-# --------------------------------------------------------------------------- #
-
-API_ASSIGNMENTS = "2022-06-01"
-API_DEFINITIONS = "2021-06-01"
-API_EXEMPTIONS = "2022-07-01-preview"
-ARM = "https://management.azure.com"
-
-# L0 envelope: staleness threshold (days). Downstream consumers (Planner,
-# CodeGen, Deploy) treat envelopes older than this as STALE and refuse to
-# proceed without a fresh `--refresh` discovery.
-DEFAULT_TTL_DAYS = 7
-
-# Only these effects are plan-blocking. Everything else lives in the summary.
-BLOCKER_EFFECTS = {"Deny"}
-AUTO_REMEDIATE_EFFECTS = {"DeployIfNotExists", "Modify"}
-RELEVANT_EFFECTS = BLOCKER_EFFECTS | AUTO_REMEDIATE_EFFECTS
-
-# Defender-for-Cloud auto-assignment marker. See
-# references/effect-classification.md for background.
-DEFENDER_ASSIGNED_BY_VALUES = {
-    "Security Center",
-    "Microsoft Defender for Cloud",
-}
-
-# Resource-type to Bicep singular-lowercase mapping. Extend as needed; the
-# fallback derives from the full ARM type string.
-BICEP_TYPE_OVERRIDES = {
-    "Microsoft.Storage/storageAccounts": "storageAccounts",
-    "Microsoft.Sql/servers": "sqlServers",
-    "Microsoft.Sql/servers/databases": "sqlServers/databases",
-    "Microsoft.KeyVault/vaults": "keyVaults",
-    "Microsoft.Web/sites": "sites",
-    "Microsoft.Network/virtualNetworks": "virtualNetworks",
-    "Microsoft.Compute/virtualMachines": "virtualMachines",
-    "Microsoft.Resources/subscriptions/resourceGroups": "resourceGroups",
-}
-
-
-# --------------------------------------------------------------------------- #
-# Azure CLI shim (injectable for tests)                                       #
-# --------------------------------------------------------------------------- #
-
-# Parallelism cap for concurrent ARM calls. ARM throttles aggressively above
-# ~16 concurrent; 8 is a safe default that still gives ~4-6x speedup over
-# sequential `az rest` subprocess invocations.
-_PARALLEL_WORKERS = 8
-
-# Process-wide ARM token cache. `az account get-access-token` takes ~0.5-1s
-# per invocation; caching saves that overhead on every subsequent REST call.
-_TOKEN_CACHE: dict[str, Any] = {"token": None, "expires_at": 0.0}
-
-
-def _get_arm_token() -> str:
-    """Return a cached ARM bearer token, refreshing when near expiry."""
-    now = time.time()
-    if _TOKEN_CACHE["token"] and _TOKEN_CACHE["expires_at"] > now + 60:
-        return _TOKEN_CACHE["token"]
-    out = subprocess.check_output(  # noqa: S603 — trusted Azure CLI
-        [
-            "az",
-            "account",
-            "get-access-token",
-            "--resource",
-            f"{ARM}/",
-            "-o",
-            "json",
-        ],
-        text=True,
-        timeout=30,
-    )
-    data = json.loads(out)
-    _TOKEN_CACHE["token"] = data["accessToken"]
-    # Tokens are valid for ~60 min. Use 50-min safe window to avoid mid-run
-    # expiry. `expiresOn` format varies across az versions, so we compute
-    # relative expiry from wall-clock instead of parsing it.
-    _TOKEN_CACHE["expires_at"] = now + 50 * 60
-    return data["accessToken"]
-
-
-def _default_az_rest(url: str) -> dict[str, Any]:
-    """GET `url` with the cached ARM bearer token via stdlib urllib.
-
-    Replaces per-call `az rest` subprocess (saves ~0.5-1s × N calls). Returns
-    `{}` for 404 (missing policy definition) so callers can filter silently.
-    Raises on auth failures and other HTTP errors.
-    """
-    token = _get_arm_token()
-    req = urllib.request.Request(
-        url,
-        headers={
-            "Authorization": f"Bearer {token}",
-            "Content-Type": "application/json",
-        },
-        method="GET",
-    )
-    try:
-        with urllib.request.urlopen(req, timeout=60) as resp:  # noqa: S310 — ARM https
-            return json.loads(resp.read().decode("utf-8"))
-    except urllib.error.HTTPError as e:
-        if e.code == 404:
-            return {}
-        detail = ""
-        try:
-            detail = e.read().decode("utf-8", errors="replace")[:500]
-        except Exception:  # pragma: no cover
-            pass
-        raise RuntimeError(
-            f"ARM GET {url} failed: HTTP {e.code} {e.reason}; {detail}"
-        ) from e
-
-
-def _default_get_subscription() -> str:
-    out = subprocess.check_output(  # noqa: S603
-        ["az", "account", "show", "--query", "id", "-o", "tsv"],
-        text=True,
-        timeout=30,
-    )
-    return out.strip()
-
-
-def _default_check_auth() -> None:
-    """Raise if the ARM token cannot be obtained."""
-    # Warms the token cache as a side effect so subsequent REST calls are fast.
-    _get_arm_token()
-
-
-# --------------------------------------------------------------------------- #
-# Discovery                                                                   #
-# --------------------------------------------------------------------------- #
-
-
-def _list_all(az_rest: Callable[[str], dict[str, Any]], url: str) -> list[dict[str, Any]]:
-    """Follow `nextLink` pagination and return the union of `value` arrays."""
-    items: list[dict[str, Any]] = []
-    next_url: str | None = url
-    while next_url:
-        page = az_rest(next_url)
-        items.extend(page.get("value", []))
-        next_url = page.get("nextLink")
-    return items
-
-
-def _parallel_list(
-    az_rest: Callable[[str], dict[str, Any]],
-    urls: dict[str, str],
-) -> dict[str, list[dict[str, Any]]]:
-    """Run multiple paginated list calls concurrently. Preserves key mapping.
-
-    Uses a thread pool because `az_rest` is a blocking callable (either
-    urllib GET or subprocess `az rest`). Thread-per-URL is fine at this
-    concurrency level.
-    """
-    if not urls:
-        return {}
-    with ThreadPoolExecutor(max_workers=min(_PARALLEL_WORKERS, len(urls))) as pool:
-        futures = {key: pool.submit(_list_all, az_rest, url) for key, url in urls.items()}
-        return {key: fut.result() for key, fut in futures.items()}
-
-
-def _parallel_fetch_items(
-    az_rest: Callable[[str], dict[str, Any]],
-    urls: list[str],
-    expected_ids: list[str] | None = None,
-) -> list[dict[str, Any] | None]:
-    """Fetch individual items concurrently; tolerate both response shapes.
-
-    Azure returns a single object for GET `.../policyDefinitions/{name}`, but
-    injected test fakes commonly match on a URL substring and return the
-    wrapped list shape `{"value": [...]}`. This helper accepts both:
-
-    - dict with no `value` key → single-item payload (return as-is, unless empty)
-    - dict with `value` list → look for the item whose id matches the expected
-      id (preserved positionally via `expected_ids`); fall back to first
-    """
-    if not urls:
-        return []
-    if expected_ids is None:
-        expected_ids = [""] * len(urls)
-
-    def _fetch(url: str, expected_id: str) -> dict[str, Any] | None:
-        resp = az_rest(url)
-        if not isinstance(resp, dict) or not resp:
-            return None
-        if "value" in resp and isinstance(resp["value"], list):
-            if expected_id:
-                for item in resp["value"]:
-                    if (item.get("id") or "").lower() == expected_id.lower():
-                        return item
-            return resp["value"][0] if resp["value"] else None
-        if resp.get("id"):
-            return resp
-        return None
-
-    with ThreadPoolExecutor(max_workers=min(_PARALLEL_WORKERS, len(urls))) as pool:
-        return list(pool.map(_fetch, urls, expected_ids))
-
-
-_PARAMETER_EXPRESSION_RE = re.compile(r"^\[parameters\('([^']+)'\)\]$", re.IGNORECASE)
-
-
-def _resolve_parameter_value(value: Any, assignment_params: dict[str, Any]) -> Any:
-    if isinstance(value, str):
-        match = _PARAMETER_EXPRESSION_RE.match(value)
-        if match:
-            return assignment_params.get(match.group(1))
-    return value
-
-
-def _effect_of(defn: dict[str, Any], assignment_params: dict[str, Any] | None = None) -> str | None:
-    rule = (defn.get("properties") or {}).get("policyRule") or {}
-    then = rule.get("then") or {}
-    eff = then.get("effect")
-    assignment_params = assignment_params or {}
-    eff = _resolve_parameter_value(eff, assignment_params)
-    if eff is None:
-        expression = then.get("effect")
-        if isinstance(expression, str):
-            match = _PARAMETER_EXPRESSION_RE.match(expression)
-            if match:
-                eff = (((defn.get("properties") or {}).get("parameters") or {}).get(match.group(1)) or {}).get(
-                    "defaultValue"
-                )
-    if not isinstance(eff, str):
-        return None
-    # Canonicalise case (some definitions use lowercase).
-    mapping = {
-        "deny": "Deny",
-        "audit": "Audit",
-        "auditifnotexists": "AuditIfNotExists",
-        "deployifnotexists": "DeployIfNotExists",
-        "modify": "Modify",
-        "append": "Append",
-        "disabled": "Disabled",
-    }
-    return mapping.get(eff.lower(), eff)
-
-
-def _resource_types(defn: dict[str, Any]) -> list[str]:
-    rule = (defn.get("properties") or {}).get("policyRule") or {}
-    types: set[str] = set()
-    stack = [rule.get("if")]
-    while stack:
-        node = stack.pop()
-        if isinstance(node, dict):
-            if node.get("field") == "type" and isinstance(node.get("equals"), str):
-                types.add(node["equals"])
-            stack.extend(node.values())
-        elif isinstance(node, list):
-            stack.extend(node)
-    return sorted(types)
-
-
-def _required_value(defn: dict[str, Any]) -> Any:
-    """Extract the value a Deny/Modify policy requires."""
-    props = defn.get("properties") or {}
-    then = (props.get("policyRule") or {}).get("then") or {}
-    details = then.get("details") or {}
-    if not isinstance(details, dict):
-        details = {}
-
-    if "value" in details:
-        return details["value"]
-
-    ops = details.get("operations") or []
-    if isinstance(ops, list) and len(ops) == 1:
-        op = ops[0] or {}
-        if "value" in op:
-            return op["value"]
-
-    params = props.get("parameters") or {}
-    for pname, pval in params.items():
-        dv = (pval or {}).get("defaultValue")
-        key = pname.lower()
-        if dv is not None and key in {
-            "allowedlocations",
-            "listofallowedlocations",
-            "allowedskus",
-            "listofallowedskus",
-            "tagname",
-            "tagvalue",
-            "minimumtlsversion",
-            "requiredretentiondays",
-        }:
-            return dv
-    return None
-
-
-def _property_paths(defn: dict[str, Any], resource_types: list[str]) -> dict[str, str | None]:
-    """Best-effort extraction of azurePropertyPath / bicepPropertyPath.
-
-    Walks the policyRule looking for `field: "<Type>/<path>"` patterns, which
-    is how Azure Policy addresses resource properties. Falls back to the
-    `Modify` / `DeployIfNotExists` operation field when the top-level rule
-    does not name a field.
-    """
-    rule = (defn.get("properties") or {}).get("policyRule") or {}
-
-    # Tag policies address `tags['<name>']`, often without any `field: type`.
-    # Check before the resource-type short-circuit below.
-    category = ((((defn.get("properties") or {}).get("metadata") or {}).get("category")) or "").lower()
-    if _looks_like_tag_policy(rule) and category == "tags":
-        return {
-            "azurePropertyPath": "resourceGroup.tags",
-            "bicepPropertyPath": "resourceGroups::tags",
-            "pathSemantics": "tag-policy-non-property",
-        }
-
-    if not resource_types:
-        return {"azurePropertyPath": "", "bicepPropertyPath": ""}
-
-    primary_type = resource_types[0]
-
-    candidate_fields: list[str] = []
-    stack = [rule.get("if"), rule.get("then")]
-    while stack:
-        node = stack.pop()
-        if isinstance(node, dict):
-            f = node.get("field")
-            if isinstance(f, str) and "/" in f and f.startswith(primary_type):
-                candidate_fields.append(f)
-            stack.extend(node.values())
-        elif isinstance(node, list):
-            stack.extend(node)
-
-    # Also check modify operations for their `field` property.
-    then = rule.get("then") or {}
-    ops = (then.get("details") or {}).get("operations") or []
-    if isinstance(ops, list):
-        for op in ops:
-            f = (op or {}).get("field")
-            if isinstance(f, str) and f.startswith(primary_type):
-                candidate_fields.append(f)
-
-    if not candidate_fields:
-        return {"azurePropertyPath": "", "bicepPropertyPath": ""}
-
-    # Prefer the field that names the deepest property path (most slashes
-    # after the type prefix); otherwise take the first candidate.
-    prefix = primary_type + "/"
-    scored = [
-        (f[len(prefix):] if f.startswith(prefix) else f.rsplit("/", 1)[-1], f)
-        for f in candidate_fields
-    ]
-    # Drop empty tails and sort by descending depth.
-    scored = [(tail, orig) for tail, orig in scored if tail]
-    if not scored:
-        return {"azurePropertyPath": "", "bicepPropertyPath": ""}
-    scored.sort(key=lambda pair: pair[0].count("/"), reverse=True)
-    path = scored[0][0]
-    # Azure path: lowerCamel resource type + dot + property path.
-    provider_type = primary_type.split("/", 1)[-1]
-    camel_type = provider_type[:1].lower() + provider_type[1:]
-    azure_path = f"{camel_type}.{path}"
-    bicep_segment = BICEP_TYPE_OVERRIDES.get(primary_type, camel_type)
-    bicep_path = f"{bicep_segment}::{path}"
-    return {"azurePropertyPath": azure_path, "bicepPropertyPath": bicep_path}
-
-
-def _looks_like_tag_policy(rule: dict[str, Any]) -> bool:
-    operations = (((rule.get("then") or {}).get("details") or {}).get("operations") or [])
-    if isinstance(operations, list) and operations:
-        operation_fields = [
-            (operation or {}).get("field")
-            for operation in operations
-            if isinstance((operation or {}).get("field"), str)
-        ]
-        if operation_fields and not all(field.lower().startswith("tags[") for field in operation_fields):
-            return False
-    stack = [rule.get("if")]
-    while stack:
-        node = stack.pop()
-        if isinstance(node, dict):
-            f = node.get("field")
-            if isinstance(f, str) and f.lower().startswith("tags["):
-                return True
-            stack.extend(node.values())
-        elif isinstance(node, list):
-            stack.extend(node)
-    return False
-
-
-_TAG_FIELD_RE = re.compile(r"^tags\[(?:'|\")?([^'\"\]]+)(?:'|\")?\]$", re.IGNORECASE)
-
-
-def _extract_policy_rule_tag_keys(defn: dict[str, Any]) -> list[str]:
-    """Walk policyRule looking for hard-coded `tags['<key>']` field references.
-
-    Many custom deny-style tag policies (e.g. `JV-Enforce Resource Group Tags`)
-    hard-code the enforced tag list in `policyRule.if.allOf[*].anyOf[*].field`
-    instead of exposing it as `tagName*` assignment parameters. Without this
-    extraction the downstream `tag_contract.required_tag_keys` silently falls
-    back to a sibling Modify policy's keys, producing transcription drift
-    (e.g. `tech-contact` vs `technical-contact`). See
-    `tools/scripts/diagnose-governance-tag-drift.md` for the full root-cause.
-    """
-    rule = (defn.get("properties") or {}).get("policyRule") or {}
-    keys: list[str] = []
-    seen: set[str] = set()
-    stack: list[Any] = [rule.get("if")]
-    while stack:
-        node = stack.pop()
-        if isinstance(node, dict):
-            field = node.get("field")
-            if isinstance(field, str):
-                m = _TAG_FIELD_RE.match(field.strip())
-                if m:
-                    key = m.group(1).strip()
-                    if key and key not in seen:
-                        seen.add(key)
-                        keys.append(key)
-            stack.extend(node.values())
-        elif isinstance(node, list):
-            stack.extend(node)
-    return keys
-
-
-def _is_defender_auto(assignment: dict[str, Any]) -> bool:
-    metadata = (assignment.get("properties") or {}).get("metadata") or {}
-    assigned_by = metadata.get("assignedBy")
-    if isinstance(assigned_by, str) and assigned_by.strip() in DEFENDER_ASSIGNED_BY_VALUES:
-        return True
-    created_by = metadata.get("createdBy")
-    if isinstance(created_by, dict):
-        name = (created_by.get("displayName") or "").strip()
-        if name in DEFENDER_ASSIGNED_BY_VALUES:
-            return True
-    return False
-
-
-def _build_exemption_map(exemptions: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
-    """Map lowercase policyAssignmentId → exemption summary."""
-    out: dict[str, dict[str, Any]] = {}
-    for ex in exemptions:
-        props = ex.get("properties") or {}
-        aid = (props.get("policyAssignmentId") or "").lower()
-        if not aid:
-            continue
-        out[aid] = {
-            "exemptionCategory": props.get("exemptionCategory"),
-            "expiresOn": props.get("expiresOn"),
-            "description": props.get("description"),
-            "policyDefinitionReferenceIds": props.get("policyDefinitionReferenceIds") or [],
-        }
-    return out
-
-
-# --------------------------------------------------------------------------- #
-# Envelope enrichment helpers (Fix E)                                         #
-# --------------------------------------------------------------------------- #
-
-
-def _extract_tags_required(findings: list[dict[str, Any]]) -> list[dict[str, str]]:
-    """Extract tag-enforcement findings into a flat tags_required list.
-
-    Looks for actual tag key names in assignment parameters (tagName, tagNames)
-    and in required_value. Falls back to category-based matching for tag policies
-    that don't use pathSemantics.
-    """
-    seen: set[str] = set()
-    tags: list[dict[str, str]] = []
-
-    for f in findings:
-        is_tag = (
-            f.get("pathSemantics") == "tag-policy-non-property"
-            or (f.get("category") or "").lower() == "tags"
-        )
-        if not is_tag:
-            continue
-
-        # Try to extract actual tag keys from assignment parameters.
-        params = f.get("assignment_parameters") or {}
-        tag_keys: list[str] = []
-
-        # Deny-style tag policies often hard-code keys in policyRule rather
-        # than exposing them via tagName* parameters. Prefer those — they are
-        # the authoritative enforced list.
-        for key in (f.get("extracted_tag_keys") or []):
-            if isinstance(key, str) and key:
-                tag_keys.append(key)
-
-        # Common parameter names for tag policies (Modify/Append style).
-        for pname in ("tagName", "tagname", "tag_name"):
-            val = params.get(pname)
-            if isinstance(val, str) and val:
-                tag_keys.append(val)
-            elif isinstance(val, list):
-                tag_keys.extend(str(v) for v in val if v)
-
-        for pname, val in params.items():
-            if re.fullmatch(r"tagName\d+", pname, re.IGNORECASE) and isinstance(val, str) and val:
-                tag_keys.append(val)
-
-        # Some policies use tagNames (plural) or listOfTagNames
-        for pname in ("tagNames", "listOfTagNames", "tagnames"):
-            val = params.get(pname)
-            if isinstance(val, list):
-                tag_keys.extend(str(v) for v in val if v)
-
-        source = f.get("policy_id", "")
-        policy_name = (f.get("display_name") or "").strip()
-
-        if tag_keys:
-            for key in tag_keys:
-                key = key.strip()
-                if key and key not in seen:
-                    seen.add(key)
-                    tags.append({
-                        "name": key,
-                        "source_policy": source,
-                        "source_assignment": f.get("assignment_display_name", ""),
-                    })
-        elif policy_name and policy_name not in seen:
-            # No specific tag key found — record the policy as a tag requirement
-            # so the agent knows to investigate further.
-            seen.add(policy_name)
-            tags.append({
-                "name": f"[unresolved: {policy_name}]",
-                "source_policy": source,
-                "source_assignment": f.get("assignment_display_name", ""),
-                "unresolved": "true",
-            })
-    return tags
-
-
-def _is_location_constraint(defn: dict[str, Any]) -> bool:
-    props = defn.get("properties") or {}
-    if any(name.lower() in {"listofallowedlocations", "allowedlocations"} for name in (props.get("parameters") or {})):
-        return True
-    display_name = props.get("displayName") or ""
-    if not re.search(r"\b(?:locations?|regions?)\b", display_name, re.IGNORECASE):
-        return False
-    stack = [(props.get("policyRule") or {}).get("if")]
-    while stack:
-        node = stack.pop()
-        if isinstance(node, dict):
-            field = node.get("field")
-            if isinstance(field, str) and (field.lower() == "location" or field.lower().endswith("/location")):
-                return True
-            stack.extend(node.values())
-        elif isinstance(node, list):
-            stack.extend(node)
-    return False
-
-
-def _location_values(parameters: dict[str, Any], required_value: Any = None) -> list[str]:
-    locations: set[str] = set()
-    if isinstance(required_value, list):
-        locations.update(str(value) for value in required_value if value)
-    elif isinstance(required_value, str) and required_value:
-        locations.add(required_value)
-    for name, value in parameters.items():
-        if name.lower() in {"listofallowedlocations", "allowedlocations"} and isinstance(value, list):
-            locations.update(str(item) for item in value if item)
-    return sorted(locations)
-
-
-def _extract_allowed_locations(
-    findings: list[dict[str, Any]], location_constraints: list[dict[str, Any]] | None = None
-) -> list[str]:
-    """Extract allowed-location values from findings with location constraints.
-
-    Checks both required_value (from definition defaults) and assignment_parameters
-    (which carry the actual configured location list).
-    """
-    locations: set[str] = set()
-    for f in findings:
-        dn = (f.get("display_name") or "").lower()
-        cat = (f.get("category") or "").lower()
-        is_location = "location" in dn or "region" in dn or "location" in cat
-
-        # Check required_value from definition
-        rv = f.get("required_value")
-        if is_location:
-            if isinstance(rv, list):
-                locations.update(str(v) for v in rv)
-            elif isinstance(rv, str) and rv:
-                locations.add(rv)
-
-        # Check assignment parameters for location lists
-        params = f.get("assignment_parameters") or {}
-        for pname in (
-            "listOfAllowedLocations", "allowedLocations",
-            "listofallowedlocations", "allowedlocations",
-        ):
-            val = params.get(pname)
-            if isinstance(val, list):
-                locations.update(str(v) for v in val if v)
-    for constraint in location_constraints or []:
-        locations.update(str(value) for value in constraint.get("allowed_locations") or [] if value)
-    return sorted(locations)
-
-
-# --------------------------------------------------------------------------- #
-# L0 envelope helpers                                                         #
-#                                                                             #
-# The canonical implementations live in render_governance.py so the cached    #
-# baseline path (render_cached_governance.py) and the live discovery path     #
-# (this module) share one signature algorithm. Re-export under the legacy    #
-# private names so existing call sites and tests keep working unchanged.    #
-# --------------------------------------------------------------------------- #
-
-from render_governance import _build_discovery_metadata  # noqa: E402,F401 — shared L0 helper
-from render_governance import _completeness_signature  # noqa: E402,F401 — shared L0 helper
-from render_governance import _extract_management_groups  # noqa: E402,F401 — shared L0 helper
-
-
-def _self_check_assignments(
-    az_rest: Callable[[str], dict[str, Any]],
-    subscription_id: str,
-    expected_count: int,
-) -> tuple[bool, int]:
-    """Re-fetch the first page of policyAssignments and verify count.
-
-    Returns (ok, actual_count). On any network/parse failure, returns
-    (False, -1) so the caller can downgrade `discovery_status` to
-    `PARTIAL` without crashing.
-    """
-    try:
-        url = (
-            f"{ARM}/subscriptions/{subscription_id}/providers/Microsoft.Authorization/"
-            f"policyAssignments?$filter=atScope()&api-version={API_ASSIGNMENTS}"
-        )
-        # We deliberately re-use _default_az_rest here via the caller's
-        # az_rest, which is the same surface that just produced the original
-        # list — any drift now is real (filter change, RBAC drop, paging bug).
-        first = az_rest(url) or {}
-        items = first.get("value") or []
-        return len(items) == expected_count, len(items)
-    except Exception:  # noqa: BLE001 — self-check must never crash discovery
-        return False, -1
-
-
-# --------------------------------------------------------------------------- #
-# Core                                                                        #
-# --------------------------------------------------------------------------- #
-
-
-def _classify(effect: str) -> str:
-    if effect in BLOCKER_EFFECTS:
-        return "blocker"
-    if effect in AUTO_REMEDIATE_EFFECTS:
-        return "auto-remediate"
-    return "informational"
-
-
-def discover(
-    subscription_id: str,
-    *,
-    project: str,
-    include_defender_auto: bool = False,
-    az_rest: Callable[[str], dict[str, Any]] | None = None,
-    verbose: bool = False,
-) -> dict[str, Any]:
-    """Run discovery and return the full envelope (not written to disk).
-
-    ``verbose`` gates noisy informational stderr (per-assignment Defender
-    filter announcements). The count is always recorded in
-    ``discovery_summary.defender_auto_filtered`` regardless.
-    """
-    if az_rest is None:
-        # Resolve at call time so tests can monkeypatch `_default_az_rest`.
-        az_rest = _default_az_rest
-    base = f"{ARM}/subscriptions/{subscription_id}/providers/Microsoft.Authorization"
-
-    # Phase 1 — Parallel list calls for subscription-scope data + exemptions.
-    # The tenant-wide built-in definition list (~1500 items) is deliberately
-    # NOT fetched here; we fetch only the tenant definitions an assignment
-    # actually references (Phase 2). Saves a large, mostly-unused payload.
-    primary = _parallel_list(
-        az_rest,
-        {
-            "assignments": f"{base}/policyAssignments?$filter=atScope()&api-version={API_ASSIGNMENTS}",
-            "sub_defs": f"{base}/policyDefinitions?api-version={API_DEFINITIONS}",
-            "sub_sets": f"{base}/policySetDefinitions?api-version={API_DEFINITIONS}",
-            "exemptions": f"{base}/policyExemptions?$filter=atScope()&api-version={API_EXEMPTIONS}",
-        },
-    )
-    assignments = primary["assignments"]
-    defs: dict[str, dict[str, Any]] = {d["id"].lower(): d for d in primary["sub_defs"]}
-    sets: dict[str, dict[str, Any]] = {s["id"].lower(): s for s in primary["sub_sets"]}
-    exemptions = primary["exemptions"]
-
-    # Phase 2 — Fetch only the tenant-scoped initiatives and definitions that
-    # assignments actually reference. Initiatives come first because their
-    # members may themselves be tenant-scope definitions. We preserve the
-    # original-case ID for URL construction (ARM is case-insensitive but
-    # downstream consumers and test routers match by substring).
-    tenant_set_ids: dict[str, str] = {}  # lowercase → original-case
-    tenant_def_ids: dict[str, str] = {}
-    for a in assignments:
-        pid_original = (a.get("properties") or {}).get("policyDefinitionId") or ""
-        pid = pid_original.lower()
-        if not pid or not pid.startswith("/providers/microsoft.authorization/"):
-            continue
-        if "/policysetdefinitions/" in pid and pid not in sets:
-            tenant_set_ids.setdefault(pid, pid_original)
-        elif "/policydefinitions/" in pid and pid not in defs:
-            tenant_def_ids.setdefault(pid, pid_original)
-
-    if tenant_set_ids:
-        ordered = sorted(tenant_set_ids.items())
-        fetched_sets = _parallel_fetch_items(
-            az_rest,
-            [f"{ARM}{orig}?api-version={API_DEFINITIONS}" for _, orig in ordered],
-            expected_ids=[orig for _, orig in ordered],
-        )
-        for s in fetched_sets:
-            if s:
-                sets.setdefault((s.get("id") or "").lower(), s)
-                for m in (s.get("properties") or {}).get("policyDefinitions") or []:
-                    mid_original = (m.get("policyDefinitionId") or "")
-                    mid = mid_original.lower()
-                    if (
-                        mid
-                        and mid.startswith("/providers/microsoft.authorization/")
-                        and mid not in defs
-                    ):
-                        tenant_def_ids.setdefault(mid, mid_original)
-
-    if tenant_def_ids:
-        ordered = sorted(tenant_def_ids.items())
-        fetched_defs = _parallel_fetch_items(
-            az_rest,
-            [f"{ARM}{orig}?api-version={API_DEFINITIONS}" for _, orig in ordered],
-            expected_ids=[orig for _, orig in ordered],
-        )
-        for d in fetched_defs:
-            if d:
-                defs.setdefault((d.get("id") or "").lower(), d)
-
-    exemption_map = _build_exemption_map(exemptions)
-
-    # Assignment filtering (Defender auto-assignments).
-    filtered_defender: list[str] = []
-    kept_assignments: list[dict[str, Any]] = []
-    for a in assignments:
-        if _is_defender_auto(a) and not include_defender_auto:
-            filtered_defender.append(
-                (a.get("properties") or {}).get("displayName") or a.get("name") or a.get("id") or "<unknown>"
-            )
-            continue
-        kept_assignments.append(a)
-
-    for name in filtered_defender:
-        # Per-assignment filter announcements are noise by default — the
-        # aggregate count lives in discovery_summary.defender_auto_filtered.
-        # Surface the names only when --verbose is passed (Phase 6 fix).
-        if verbose:
-            print(f"filter: skipping Defender auto-assignment: {name}", file=sys.stderr)
-
-    assignment_inventory: list[dict[str, Any]] = []
-    findings: list[dict[str, Any]] = []
-    location_constraints: list[dict[str, Any]] = []
-    audit_count = 0
-    disabled_count = 0
-
-    for a in kept_assignments:
-        props = a.get("properties") or {}
-        display = props.get("displayName") or a.get("name") or a.get("id") or "<unknown>"
-        scope = props.get("scope") or ""
-        pid = (props.get("policyDefinitionId") or "").lower()
-        assignment_id = (a.get("id") or "").lower()
-        assignment_type = (
-            "management-group" if "/providers/microsoft.management/managementgroups/" in scope.lower() else "subscription"
-        )
-        assignment_params = {
-            key: (value or {}).get("value")
-            for key, value in (props.get("parameters") or {}).items()
-            if (value or {}).get("value") is not None
-        }
-        assignment_inventory.append(
-            {
-                "displayName": display,
-                "scope": scope,
-                "assignmentType": assignment_type,
-                "policyDefinitionId": pid,
-                "enforcementMode": props.get("enforcementMode") or "Default",
-                "parameters": assignment_params,
-            }
-        )
-
-        if not pid:
-            continue
-
-        # Resolve to member definitions (initiative → its members; policy → itself).
-        members: list[tuple[dict[str, Any], str | None, dict[str, Any]]] = []
-        if "/policysetdefinitions/" in pid and pid in sets:
-            initiative_defaults = {
-                key: (value or {}).get("defaultValue")
-                for key, value in ((sets[pid].get("properties") or {}).get("parameters") or {}).items()
-                if (value or {}).get("defaultValue") is not None
-            }
-            initiative_params = {**initiative_defaults, **assignment_params}
-            for m in (sets[pid].get("properties") or {}).get("policyDefinitions") or []:
-                mid = (m.get("policyDefinitionId") or "").lower()
-                if mid in defs:
-                    member_params = {
-                        key: _resolve_parameter_value((value or {}).get("value"), initiative_params)
-                        for key, value in (m.get("parameters") or {}).items()
-                        if _resolve_parameter_value((value or {}).get("value"), initiative_params) is not None
-                    }
-                    members.append((defs[mid], m.get("policyDefinitionReferenceId"), member_params))
-        elif pid in defs:
-            members.append((defs[pid], None, {}))
-        # Unknown definition id — nothing to emit; still counted in inventory.
-
-        for defn, member_ref_id, member_params in members:
-            effective_params = {**assignment_params, **member_params}
-            eff = _effect_of(defn, effective_params)
-            if eff is None:
-                continue
-            if eff == "Disabled":
-                disabled_count += 1
-                continue
-            if _is_location_constraint(defn):
-                location_constraints.append(
-                    {
-                        "policy_id": defn.get("id"),
-                        "display_name": (defn.get("properties") or {}).get("displayName")
-                        or defn.get("name")
-                        or defn.get("id"),
-                        "assignment_display_name": display,
-                        "assignment_id": a.get("id"),
-                        "scope": scope,
-                        "effect": eff[:1].lower() + eff[1:],
-                        "enforcement_mode": props.get("enforcementMode") or "Default",
-                        "allowed_locations": _location_values(effective_params, _required_value(defn)),
-                    }
-                )
-            if eff in {"Audit", "AuditIfNotExists"}:
-                audit_count += 1
-                continue
-            if eff not in RELEVANT_EFFECTS:
-                continue
-
-            rtypes = _resource_types(defn)
-            paths = _property_paths(defn, rtypes)
-            category = ((defn.get("properties") or {}).get("metadata") or {}).get(
-                "category"
-            ) or "Uncategorized"
-
-            exemption = None
-            if assignment_id in exemption_map:
-                candidate = exemption_map[assignment_id]
-                ref_ids = candidate.get("policyDefinitionReferenceIds") or []
-                if not ref_ids or member_ref_id in ref_ids:
-                    exemption = candidate
-
-            classification = _classify(eff)
-            if exemption is not None and classification == "blocker":
-                classification = "informational"
-
-            finding = {
-                "policy_id": defn.get("id"),
-                "display_name": (defn.get("properties") or {}).get("displayName")
-                or defn.get("name")
-                or defn.get("id"),
-                "effect": eff[:1].lower() + eff[1:],  # lower-camel for schema enum
-                "scope": scope,
-                "assignment_display_name": display,
-                "assignment_id": a.get("id"),
-                "classification": classification,
-                "category": category,
-                "resource_types": rtypes,
-                "required_value": _required_value(defn),
-                "azurePropertyPath": paths.get("azurePropertyPath"),
-                "bicepPropertyPath": paths.get("bicepPropertyPath"),
-                "exemption": exemption,
-                "override": None,
-            }
-            # Carry assignment-level parameter values (tag keys, location lists).
-            if effective_params:
-                finding["assignment_parameters"] = effective_params
-            if paths.get("pathSemantics"):
-                finding["pathSemantics"] = paths["pathSemantics"]
-            # For Tags-category policies, extract enforced tag keys from the
-            # policyRule body. Deny-style tag policies often hard-code keys
-            # in `policyRule.if.allOf[*].anyOf[*].field` rather than exposing
-            # them as `tagName*` assignment parameters; without this we get
-            # silent drift to sibling Modify-policy keys.
-            if (
-                finding.get("pathSemantics") == "tag-policy-non-property"
-                or (finding.get("category") or "").lower() == "tags"
-            ):
-                rule_tag_keys = _extract_policy_rule_tag_keys(defn)
-                if rule_tag_keys:
-                    finding["extracted_tag_keys"] = rule_tag_keys
-            findings.append(finding)
-
-    blockers = sum(1 for f in findings if f["classification"] == "blocker")
-    auto_remediate = sum(1 for f in findings if f["classification"] == "auto-remediate")
-    exempted = sum(1 for f in findings if f["exemption"] is not None)
-
-    # L0 envelope: signature, scope, and self-check.
-    management_groups = _extract_management_groups(kept_assignments)
-    discovery_status = "COMPLETE"
-    # Re-fetch page 1 of assignments and verify the count matches the
-    # initial pull. Drift here indicates RBAC change, filter mutation, or
-    # a paginated REST surface change between the two calls.
-    self_check_ok, self_check_count = _self_check_assignments(az_rest, subscription_id, len(assignments))
-    if not self_check_ok:
-        discovery_status = "PARTIAL"
-        print(
-            f"self-check: policyAssignments count drift "
-            f"(expected={len(assignments)}, observed={self_check_count}); "
-            f"marking discovery_status=PARTIAL",
-            file=sys.stderr,
-        )
-
-    discovered_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-    discovery_metadata = _build_discovery_metadata(
-        findings=findings,
-        subscription_id=subscription_id,
-        management_groups=management_groups,
-        page_counts={
-            # `_parallel_list` collapses pagination, so we record the
-            # final item counts per REST surface. The self-check above
-            # validates these against a fresh page-1 fetch.
-            "policyAssignments": len(assignments),
-            "policyDefinitions": len(defs),
-            "policyExemptions": len(exemptions),
-        },
-        discovered_at=discovered_at,
-        discovery_status=discovery_status,
-        source="live",
-    )
-
-    envelope = {
-        "schema_version": "governance-constraints-v1",
-        "project": project,
-        "subscription_id": subscription_id,
-        "discovered_at": discovered_at,
-        "source": "azure-policy-rest-api",
-        "discovery_status": discovery_status,
-        "discovery_metadata": discovery_metadata,
-        "discovery_summary": {
-            "assignment_total": len(assignments),
-            "assignment_kept": len(kept_assignments),
-            "defender_auto_filtered": len(filtered_defender),
-            "subscription_scope_count": sum(
-                1 for a in kept_assignments if "/providers/microsoft.management/" not in (a.get("properties", {}).get("scope") or "").lower()
-            ),
-            "management_group_inherited_count": sum(
-                1 for a in kept_assignments if "/providers/microsoft.management/" in (a.get("properties", {}).get("scope") or "").lower()
-            ),
-            "blocker_count": blockers,
-            "auto_remediate_count": auto_remediate,
-            "informational_count": sum(
-                1 for f in findings if f["classification"] == "informational"
-            ),
-            "audit_count": audit_count,
-            "disabled_count": disabled_count,
-            "exempted_count": exempted,
-        },
-        "assignment_inventory": assignment_inventory,
-        "findings": findings,
-        # Fix E: canonical aliases so the agent never needs to reshape JSON.
-        # `policies` is a reference alias of `findings` (not a copy).
-        "policies": findings,
-        # Authoritative reference set for the L3 policy-precheck subagent.
-        # Contains every policy definition ID we resolved during this run
-        # (subscription-scope + tenant-scope, both initiative members and
-        # standalone assignments), regardless of effect. The L3 subagent
-        # uses this to suppress false-positive "live policies missing from
-        # constraints" noise caused by initiative child policies that the
-        # findings[] filter (Deny + DeployIfNotExists + Modify only) drops.
-        # Values are lowercase resource IDs to match `az policy state list`
-        # output. See iac-common/references/policy-precheck-contract.md
-        # Phase 3 step 1 for the comparison contract.
-        "member_policy_index": sorted(defs.keys()),
-        "tags_required": _extract_tags_required(findings),
-        "location_constraints": location_constraints,
-        "allowed_locations": _extract_allowed_locations(findings, location_constraints),
-    }
-    return envelope
-
-
-# --------------------------------------------------------------------------- #
-# CLI                                                                         #
-# --------------------------------------------------------------------------- #
-
-
-def _emit_status(status: dict[str, Any]) -> None:
-    """Emit single-line JSON status as the first stdout line."""
-    sys.stdout.write(json.dumps(status, separators=(",", ":")) + "\n")
-    sys.stdout.flush()
-
-
-def _emit_preview(envelope: dict[str, Any], limit: int = 20) -> None:
-    """Human-readable Markdown preview. NEVER re-ingest into LLM context."""
-    summary = envelope["discovery_summary"]
-    print(
-        f"\n{envelope['subscription_id']}: {summary['assignment_kept']} kept "
-        f"assignments (of {summary['assignment_total']}; "
-        f"{summary['defender_auto_filtered']} Defender auto filtered); "
-        f"{summary['blocker_count']} blockers, "
-        f"{summary['auto_remediate_count']} auto-remediate, "
-        f"{summary['exempted_count']} exempted."
-    )
-    print()
-    print("| Effect | Classification | Category | Policy | Assignment |")
-    print("|---|---|---|---|---|")
-    for f in envelope["findings"][:limit]:
-        print(
-            f"| {f['effect']} | {f['classification']} | {f['category']} | "
-            f"{f['display_name']} | {f['assignment_display_name']} |"
-        )
-    if len(envelope["findings"]) > limit:
-        print(f"\n… {len(envelope['findings']) - limit} more in {envelope.get('_out_path')}")
-
-
-# Renderer functions extracted to render_governance.py — import and re-export
-# as module-level aliases so existing tests (test_discover.py) that call
-# discover._extract_arch_resources(...) continue to pass unchanged.
-from render_governance import extract_arch_resources as _extract_arch_resources  # noqa: F401
-from render_governance import emit_preview_md as _emit_preview_md  # noqa: F401
-
-
-def main(argv: list[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(
-        prog="discover",
-        description="Deterministic Azure Policy discovery for 04g-Governance.",
-    )
-    parser.add_argument("--project", required=True)
-    parser.add_argument("--out", required=True)
-    parser.add_argument("--subscription", default="default")
-    parser.add_argument("--refresh", action="store_true")
-    parser.add_argument(
-        "--arch",
-        default=None,
-        help="Path to 02-architecture-assessment.md. Enables policy→resource mapping in preview.md.",
-    )
-    parser.add_argument(
-        "--include-defender-auto",
-        action="store_true",
-        help="Retain Defender-for-Cloud auto-assignments (filtered by default).",
-    )
-    parser.add_argument(
-        "--verbose",
-        action="store_true",
-        help="Emit informational stderr (per-assignment Defender filter announcements). "
-        "Aggregate counts are always recorded in discovery_summary regardless.",
-    )
-    args = parser.parse_args(argv)
-
-    out_path = Path(args.out)
-    existing: dict[str, Any] | None = None
-    if out_path.exists():
-        try:
-            candidate = json.loads(out_path.read_text())
-            existing = candidate if isinstance(candidate, dict) else None
-        except json.JSONDecodeError:
-            existing = None
-    preserved_security_exceptions = (
-        existing.get("security_exceptions")
-        if isinstance(existing, dict) and isinstance(existing.get("security_exceptions"), list)
-        else []
-    )
-
-    # Cache short-circuit — reuse an existing COMPLETE snapshot unless --refresh.
-    if out_path.exists() and not args.refresh:
-        cached = existing
-        if (
-            isinstance(cached, dict)
-            and cached.get("discovery_status") == "COMPLETE"
-            and isinstance(cached.get("findings"), list)
-        ):
-            summary = cached.get("discovery_summary") or {}
-            status = {
-                "status": "COMPLETE",
-                "cache_hit": True,
-                "assignment_total": summary.get("assignment_total", 0),
-                "blockers": summary.get("blocker_count", 0),
-                "auto_remediate": summary.get("auto_remediate_count", 0),
-                "exempted": summary.get("exempted_count", 0),
-                "out_path": str(out_path),
-            }
-            _emit_status(status)
-            print(
-                f"cache hit: reusing {out_path} ({len(cached.get('findings', []))} findings; "
-                f"pass --refresh to re-discover)"
-            )
-            return 0
-
-    # Resolve subscription id.
-    try:
-        sub_id = _default_get_subscription() if args.subscription == "default" else args.subscription
-    except subprocess.CalledProcessError as e:
-        _emit_status({"status": "FAILED", "error": "subscription-resolution", "detail": str(e)})
-        print("ERROR: could not resolve subscription via `az account show`.", file=sys.stderr)
-        return 2
-
-    try:
-        _default_check_auth()
-    except subprocess.CalledProcessError as e:
-        _emit_status(
-            {
-                "status": "FAILED",
-                "error": "auth",
-                "detail": (e.output or "").strip() if hasattr(e, "output") else str(e),
-            }
-        )
-        print(
-            "ERROR: Azure ARM token unavailable. Run `az login --use-device-code`.",
-            file=sys.stderr,
-        )
-        return 2
-
-    try:
-        envelope = discover(
-            sub_id,
-            project=args.project,
-            include_defender_auto=args.include_defender_auto,
-            verbose=args.verbose,
-        )
-    except subprocess.CalledProcessError as e:
-        _emit_status(
-            {
-                "status": "FAILED",
-                "error": "rest",
-                "detail": str(e),
-            }
-        )
-        print(f"ERROR: Azure REST call failed: {e}", file=sys.stderr)
-        return 2
-    except Exception as e:  # pragma: no cover — defensive
-        _emit_status({"status": "FAILED", "error": "unexpected", "detail": str(e)})
-        print(f"ERROR: unexpected failure: {e}", file=sys.stderr)
-        return 2
-
-    out_path.parent.mkdir(parents=True, exist_ok=True)
-    if preserved_security_exceptions:
-        envelope["security_exceptions"] = preserved_security_exceptions
-    envelope["_out_path"] = str(out_path)
-    out_path.write_text(json.dumps(envelope, indent=2, sort_keys=False) + "\n")
-
-    # Fix C: emit sibling preview.md for agent annotation
-    arch_resources = _extract_arch_resources(args.arch) if args.arch else None
-    preview_path = _emit_preview_md(envelope, out_path, arch_resources=arch_resources)
-    if preview_path:
-        print(f"preview: wrote {preview_path}", file=sys.stderr)
-
-    summary = envelope["discovery_summary"]
-    status = {
-        "status": envelope["discovery_status"],
-        "cache_hit": False,
-        "assignment_total": summary["assignment_total"],
-        "blockers": summary["blocker_count"],
-        "auto_remediate": summary["auto_remediate_count"],
-        "exempted": summary["exempted_count"],
-        "out_path": str(out_path),
-    }
-    _emit_status(status)
-    _emit_preview(envelope)
-    return 0 if envelope["discovery_status"] == "COMPLETE" else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())