@mmerterden/multi-agent-pipeline 20.6.0 → 20.8.0

package/pipeline/skills/shared/external/localization-reuse-map/scripts/verify-map.py

@@ -1,26 +1,9 @@
 #!/usr/bin/env python3
-"""Fidelity check  -  cross-verify a mapping against the REAL sources before publishing.
-
-The map can drift from reality (stale keys parsed from an old doc, hallucinated keys,
-a wired key that never made it into a row, an empty value). This script reconciles the
-mapping JSON against the live repos and reports what doesn't hold up  -  so the page you
-publish reflects the code, not a guess. Read-only.
-
-Checks (each reported, never auto-fixed):
-  1. newKey ∈ screen code  -  every row's `newKey` should appear as a wired
-     `LocalizationStringKey.<...>` call site (or `"Key".localized`, or a web
-     `t('Key')` / `$t('Key')` i18n call) in --screen-path.
-     Component-owned keys (`(owned by ...)` / `(DS bileşen...)`) are exempt (they live in the
-     component, not the screen file) and listed separately.
-  2. code keys ∉ map  -  `LocalizationStringKey.<...>` wired in the screen but with no
-     row → missing coverage (add a row or confirm it's out of scope).
-  3. value gaps  -  `new.tr` empty (verdict≠new), or legacy entirely empty where
-     the verdict claims reuse/review.
-  4. untranslated  -  a row whose `new.tr` == `new.en` (flag, as the renderer does).
-  5. resources (optional)  -  with --resources-root, the new key's Suggested/<Key>.json exists.
-
-Exit code 2 when any hard fidelity issue is found (unverified non-component key, code key
-missing from the map, or a reuse/review row with no legacy value), so it can gate a run.
+"""Fidelity gate: reconcile a reuse-map mapping against the implemented screen.
+
+Checks that every mapped newKey is wired in the screen code, that every key
+the code wires is mapped, and that no row carries an empty or invented value.
+Exit 2 when a hard issue is found (blocks publishing), 0 otherwise.
 """
 import argparse
 import json
@@ -28,145 +11,178 @@ import os
 import re
 import sys
 
-EXTS = (".swift", ".kt", ".kts", ".xml", ".ts", ".tsx", ".js", ".jsx", ".vue")
-RE_LSK = re.compile(r"LocalizationStringKey\.([A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+)+)")
-RE_LITERAL = re.compile(r'"([A-Za-z0-9_.]+)"\s*\.\s*localized')
-# Web (react-i18next / next-intl `t('key')`; vue-i18n `$t('key')`)
-RE_WEB_T = re.compile(r"(?<!\$)\bt\(\s*['\"]([A-Za-z0-9_.\-]+)['\"]")
-RE_WEB_VUE_T = re.compile(r"\$t\(\s*['\"]([A-Za-z0-9_.\-]+)['\"]")
-# RE_LSK is greedy across dots, so `...StringKey.Ns.leaf.localized` captures the accessor
-# too. Strip it, or no call site ever matches its mapping row and every wired key is
-# reported unmapped.
-RE_ACCESSOR = re.compile(r"\.localized\w*$")
-MARKER = re.compile(r"\(.*(owned by|bileşen|component).*\)", re.I)
+EXTENSIONS = {".swift", ".kt", ".kts", ".xml", ".ts", ".tsx", ".js", ".jsx", ".vue"}
+SKIP_PARTS = ("/.git", "/Generated/")
 
+WORD = "A-Za-z0-9_"
+LSK = re.compile(rf"LocalizationStringKey((?:\.[A-Za-z_][{WORD}]*)+)")
+LITERAL_LOCALIZED = re.compile(rf"\"([{WORD}.]+)\"\.localized\b")
+WEB_T = re.compile(r"(?<![\w$])t\(\s*(['\"])([A-Za-z0-9_.\-]+)\1")
+WEB_DOLLAR_T = re.compile(r"\$t\(\s*(['\"])([A-Za-z0-9_.\-]+)\1")
+TRAILING_MARKER = re.compile(r"\s*\([^()]*\)\s*$")
+COMPONENT_MARKER = re.compile(r"\([^)]*(owned by|bile\u015fen|component)[^)]*\)", re.IGNORECASE)
+PLACEHOLDER = re.compile(r"<[^<>]+>")
 
-def clean_key(k):
-    """Strip a trailing '(owned by ...)' / '(DS bileşene ait)' marker → bare dotted key."""
-    return re.sub(r"\s*\(.*\)\s*$", "", (k or "")).strip()
 
+def strip_accessor(chain):
+    segments = chain.strip(".").split(".")
+    while segments and segments[-1].startswith("localized"):
+        segments.pop()
+    return ".".join(segments)
 
-def is_component(k):
-    return bool(MARKER.search(k or ""))
 
+def normalize_key(key):
+    return ".".join(seg[:1].upper() + seg[1:] for seg in key.split("."))
 
-def norm_key(k):
-    return clean_key(k).lower()
 
+def source_files(root):
+    if os.path.isfile(root):
+        yield root
+        return
+    for current, dirs, files in os.walk(root):
+        probe = current.replace(os.sep, "/") + "/"
+        if any(part in probe for part in SKIP_PARTS):
+            dirs[:] = []
+            continue
+        for name in sorted(files):
+            if os.path.splitext(name)[1] in EXTENSIONS:
+                yield os.path.join(current, name)
 
-def scan_code(screen_path):
-    """Return (set of wired dotted keys lowercased, full lowercased corpus text)."""
-    keys, corpus = set(), []
-    files = []
-    if os.path.isfile(screen_path):
-        files = [screen_path]
-    else:
-        for dp, _d, names in os.walk(screen_path):
-            if "/.git" in dp:
-                continue
-            files += [os.path.join(dp, n) for n in names if n.endswith(EXTS)]
-    for f in files:
+
+def scan_code(root):
+    keys = set()
+    corpus = []
+    scanned = 0
+    for path in source_files(root):
         try:
-            txt = open(f, encoding="utf-8", errors="replace").read()
-        except OSError:
+            with open(path, encoding="utf-8") as stream:
+                text = stream.read()
+        except (OSError, UnicodeDecodeError):
             continue
-        corpus.append(txt.lower())
-        for m in RE_LSK.finditer(txt):
-            keys.add(RE_ACCESSOR.sub("", m.group(1)).lower())
-        for m in RE_LITERAL.finditer(txt):
-            keys.add(m.group(1).lower())
-        for m in RE_WEB_T.finditer(txt):
-            keys.add(m.group(1).lower())
-        for m in RE_WEB_VUE_T.finditer(txt):
-            keys.add(m.group(1).lower())
-    return keys, "\n".join(corpus), len(files)
+        scanned += 1
+        corpus.append(text.lower())
+        chains = (strip_accessor(hit.group(1)) for hit in LSK.finditer(text))
+        keys.update(chain.lower() for chain in chains if chain)
+        for pattern, group in ((LITERAL_LOCALIZED, 1), (WEB_T, 2), (WEB_DOLLAR_T, 2)):
+            keys.update(hit.group(group).lower() for hit in pattern.finditer(text))
+    return keys, "\n".join(corpus), scanned
+
+
+def is_component(row):
+    text = " ".join(str(row.get(f) or "") for f in ("newKey", "element", "note"))
+    return bool(COMPONENT_MARKER.search(text))
 
 
-def leaf_in_corpus(key, corpus):
-    """Weak check: the key's last segment appears as a word in the code corpus."""
-    leaf = clean_key(key).split(".")[-1]
-    return bool(leaf) and re.search(r"\b" + re.escape(leaf.lower()) + r"\b", corpus) is not None
+def is_dynamic(row, key):
+    element = str(row.get("element") or "").lower()
+    note = str(row.get("note") or "").lower()
+    return (
+        key.startswith("(dynamic")
+        or bool(PLACEHOLDER.search(key))
+        or "dinamik" in element
+        or "dynamic" in element
+        or "enumerate at runtime" in note
+    )
+
+
+def suggested_exists(root, key):
+    name = normalize_key(key) + ".json"
+    candidates = [
+        os.path.join(root, "Localization", "Suggested", name),
+        os.path.join(root, "Resources", "Localization", "LocalizationStrings", "Sources", "Suggested", name),
+    ]
+    return any(os.path.isfile(p) for p in candidates)
 
 
 def main():
-    ap = argparse.ArgumentParser(description=__doc__,
-                                 formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("--mapping", required=True)
-    ap.add_argument("--screen-path", required=True, help="implemented screen dir/file")
-    ap.add_argument("--resources-root", help="resources / snapshot root (checks Suggested/<Key>.json)")
-    ap.add_argument("--out", help="write the full report JSON here (default: stdout summary only)")
-    args = ap.parse_args()
-
-    m = json.load(open(args.mapping, encoding="utf-8"))
-    rows = m.get("rows", [])
-    code_keys, corpus, nfiles = scan_code(args.screen_path)
-
-    verified, weak, unverified, component = [], [], [], []
-    value_gaps, untranslated = [], []
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--mapping", required=True, help="the reuse-map JSON")
+    parser.add_argument("--screen-path", required=True, help="folder or single file of the built screen")
+    parser.add_argument("--resources-root", help="resources checkout or snapshot; turns on the Suggested-file check")
+    parser.add_argument("--out", help="JSON report destination; stdout when omitted")
+    args = parser.parse_args()
+
+    with open(args.mapping, encoding="utf-8") as stream:
+        mapping = json.load(stream)
+    rows = mapping.get("rows") or []
+    code_keys, corpus, scanned = scan_code(args.screen_path)
+
+    verified = 0
+    exempt = 0
+    weak, unverified = [], []
+    gaps, untranslated = [], []
     map_keys = set()
 
-    for r in rows:
-        k = r.get("newKey", "")
-        ck = clean_key(k)
-        map_keys.add(norm_key(k))
-        # 1. newKey in code
-        if is_component(k) or k.startswith("(dynamic") or "dinamik" in (r.get("element", "").lower()):
-            component.append(k)
-        elif norm_key(k) in code_keys:
-            verified.append(k)
-        elif leaf_in_corpus(k, corpus):
-            weak.append(k)
+    for row in rows:
+        raw_key = str(row.get("newKey") or "").strip()
+        key = raw_key if raw_key.startswith("(dynamic") else TRAILING_MARKER.sub("", raw_key)
+        component = is_component(row)
+        dynamic = is_dynamic(row, key)
+        if key:
+            map_keys.add(key.lower())
+
+        if component or dynamic:
+            exempt += 1
+        elif key.lower() in code_keys:
+            verified += 1
         else:
-            unverified.append({"key": k, "element": r.get("element")})
-        # 3. value gaps
-        nw, lg, vd = r.get("new") or {}, r.get("legacy") or {}, r.get("verdict")
-        if vd != "new" and not (nw.get("tr") or "").strip() and not k.startswith("(dynamic"):
-            value_gaps.append({"key": k, "issue": "new.tr empty"})
-        if vd in ("reuse", "review") and not any((lg.get(l) or "").strip() for l in ("en", "tr")):
-            value_gaps.append({"key": k, "issue": f"verdict={vd} but no legacy value"})
-        # 4. untranslated
-        if (nw.get("tr") or "").strip() and nw.get("tr") == nw.get("en"):
-            untranslated.append(k)
-        # 5. resources
-        if args.resources_root and not is_component(k) and not k.startswith("(dynamic"):
-            seg = ".".join(s[:1].upper() + s[1:] if s else s for s in ck.split("."))
-            p1 = os.path.join(args.resources_root, "Localization", "Suggested", seg + ".json")
-            p2 = os.path.join(args.resources_root, "Resources", "Localization",
-                              "LocalizationStrings", "Sources", "Suggested", seg + ".json")
-            if not (os.path.isfile(p1) or os.path.isfile(p2)):
-                value_gaps.append({"key": k, "issue": "no Suggested/<Key>.json"})
-
-    # 2. code keys not in map
-    code_not_mapped = sorted(k for k in code_keys if k not in map_keys)
-
+            leaf = key.rsplit(".", 1)[-1].lower()
+            whole_word = r"\b%s\b" % re.escape(leaf)
+            if leaf and re.search(whole_word, corpus):
+                weak.append(key)
+            else:
+                unverified.append({"key": key, "element": row.get("element")})
+
+        verdict = row.get("verdict")
+        new = row.get("new") or {}
+        legacy = row.get("legacy") or {}
+        if verdict != "new" and not new.get("tr") and not dynamic:
+            gaps.append({"key": key, "issue": "new.tr empty"})
+        if verdict in ("reuse", "review") and not (legacy.get("en") or legacy.get("tr")):
+            gaps.append({"key": key, "issue": f"verdict={verdict} but no legacy value"})
+        if new.get("tr") and new.get("tr") == new.get("en"):
+            untranslated.append(key)
+        if args.resources_root and not component and not dynamic and key and not suggested_exists(args.resources_root, key):
+            gaps.append({"key": key, "issue": f"no Suggested/{normalize_key(key)}.json"})
+
+    not_in_map = sorted(code_keys - map_keys)
     report = {
-        "screen": m.get("screen"), "rows": len(rows), "filesScanned": nfiles,
-        "newKeyInCode": {"verified": len(verified), "weak": weak,
-                         "unverified": unverified, "component_or_dynamic": len(component)},
-        "codeKeysNotInMap": code_not_mapped,
-        "valueGaps": value_gaps, "untranslated": untranslated,
+        "screen": mapping.get("screen"),
+        "rows": len(rows),
+        "filesScanned": scanned,
+        "newKeyInCode": {
+            "verified": verified,
+            "weak": weak,
+            "unverified": unverified,
+            "component_or_dynamic": exempt,
+        },
+        "codeKeysNotInMap": not_in_map,
+        "valueGaps": gaps,
+        "untranslated": untranslated,
     }
-    hard = len(unverified) + len(code_not_mapped) + sum(1 for g in value_gaps if "no legacy" in g["issue"])
-
+    text = json.dumps(report, indent=2, ensure_ascii=False) + "\n"
     if args.out:
-        with open(args.out, "w", encoding="utf-8") as fh:
-            json.dump(report, fh, ensure_ascii=False, indent=2)
-            fh.write("\n")
-    # summary to stderr (always)
-    sys.stderr.write(
-        f"# fidelity: {len(rows)} rows · {nfiles} files\n"
-        f"#   newKey in code: {len(verified)} verified · {len(weak)} weak(leaf) · "
-        f"{len(unverified)} UNVERIFIED · {len(component)} component/dynamic (exempt)\n"
-        f"#   code keys not in map: {len(code_not_mapped)}\n"
-        f"#   value gaps: {len(value_gaps)} · untranslated(tr==en): {len(untranslated)}\n")
-    if unverified:
-        sys.stderr.write("#   UNVERIFIED keys: " + ", ".join(u["key"] for u in unverified[:12]) + "\n")
-    if code_not_mapped:
-        sys.stderr.write("#   wired-but-unmapped: " + ", ".join(code_not_mapped[:12]) + "\n")
-    if not args.out:
-        json.dump(report, sys.stdout, ensure_ascii=False, indent=2)
-        sys.stdout.write("\n")
-    sys.exit(2 if hard else 0)
+        with open(args.out, "w", encoding="utf-8") as sink:
+            sink.write(text)
+    else:
+        sys.stdout.write(text)
+
+    no_legacy = [g for g in gaps if "no legacy value" in g["issue"]]
+    hard = len(unverified) + len(not_in_map) + len(no_legacy)
+    print(
+        f"# rows {len(rows)}, files {scanned}: verified {verified}, weak {len(weak)}, "
+        f"UNVERIFIED {len(unverified)}, component/dynamic {exempt}, wired-but-unmapped {len(not_in_map)}, "
+        f"value gaps {len(gaps)} ({len(no_legacy)} hard), untranslated {len(untranslated)}",
+        file=sys.stderr,
+    )
+    for item in unverified[:12]:
+        print(f"#   UNVERIFIED {item['key']}  ({item['element']})", file=sys.stderr)
+    for key in not_in_map[:12]:
+        print(f"#   wired but unmapped: {key}", file=sys.stderr)
+    if hard:
+        print(f"# FAIL: {hard} hard issue(s); fix them or record a conscious waiver before publishing", file=sys.stderr)
+        sys.exit(2)
+    print("# PASS", file=sys.stderr)
 
 
 if __name__ == "__main__":