@mmerterden/multi-agent-pipeline 20.6.0 → 20.8.0

package/pipeline/skills/shared/external/localization-reuse-map/scripts/scan-screen-keys.py

@@ -1,31 +1,11 @@
 #!/usr/bin/env python3
-"""Scan an *implemented* screen for the localization keys the registry×code union misses.
-
-The mapper's primary new-key recipe is a STATIC cross-reference (design-export
-components × component-registry `localizationKeys` × the screen's wired call sites).
-That recipe is structurally blind to three things, which this scanner recovers by
-reading the screen's actual source tree:
-
-  1. **Error / validation / alert strings**  -  copy that lives in validators, error
-     mappers, alert/toast builders, or ViewModel error branches, not in the
-     VC→VM→cell chain the static recipe walks.
-  2. **Dynamic / interpolated keys**  -  keys built at runtime: `String(format:)`,
-     interpolated/variable `.localized` receivers, `LocalizationStringKey(<expr>)`,
-     Android `getString(<var>)`  -  none of which are string literals, so the literal
-     greps never see them.
-  3. **Keys wired in code but absent from the component registry** (and the inverse
-     is the agent's job to confirm)  -  surfaced here as plain `static` call sites so
-     the agent can diff them against the registry union.
-
-Heuristic + read-only: it greps `.swift`/`.kt`/`.xml`/`.ts`/`.tsx`/`.js`/`.jsx`/`.vue`
-under --screen-path and classifies each hit `static | error | dynamic` with
-file:line provenance. It does
-NOT resolve values or invent keys  -  it hands the agent a categorized worklist to
-fold into the mapping (errors and dynamics especially), where each still gets the
-normal new/legacy/CMS resolution.
-
-Output (to --out or stdout): { screenPath, counts:{static,error,dynamic},
-  keys:[ {key, category, platform, file, line, snippet} ] }.
+"""Find the localization keys a built screen references and sort them into categories.
+
+This is a pattern match over source text; it changes nothing. A key written
+out literally is "static", unless its line looks like an alert, validation or
+error path, in which case it is "error". A key assembled at runtime, which no
+static cross-check can resolve, is "dynamic". Swift, Kotlin, Android layout
+XML and web t() / $t() calls are understood.
 """
 import argparse
 import json
@@ -33,151 +13,160 @@ import os
 import re
 import sys
 
-EXTS = (".swift", ".kt", ".kts", ".xml", ".ts", ".tsx", ".js", ".jsx", ".vue")
-ERROR_CTX = re.compile(
-    r"alert|error|validat|warning|toast|snackbar|failure|exception|errorMessage|showError",
-    re.I)
-
-# iOS
-RE_LSK = re.compile(r"LocalizationStringKey\.([A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+)+)")
-RE_LSK_DYN = re.compile(r"LocalizationStringKey\s*\(")
-RE_LITERAL_LOCALIZED = re.compile(r'"([^"\\]+)"\s*\.\s*localized')
-RE_VAR_LOCALIZED = re.compile(r'(?<!")\b([A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)*)\s*\.\s*localized')
-RE_STRING_FORMAT = re.compile(r"String\s*\(\s*format\s*:")
-# Android
-RE_R_STRING = re.compile(r"R\.string\.([A-Za-z0-9_]+)")
-RE_AT_STRING = re.compile(r"@string/([A-Za-z0-9_]+)")
-RE_GETSTRING_DYN = re.compile(r"getString\s*\(\s*[A-Za-z_]")  # variable, not R.string.literal
-# Web (react-i18next / next-intl `t('key')`; vue-i18n `$t('key')`); `(?<!\$)\bt\(`
-# keeps a plain t(...) call from also matching inside a $t(...) call.
-RE_WEB_T = re.compile(r"(?<!\$)\bt\(\s*['\"]([A-Za-z0-9_.\-]+)['\"]")
-RE_WEB_VUE_T = re.compile(r"\$t\(\s*['\"]([A-Za-z0-9_.\-]+)['\"]")
-RE_WEB_T_DYN = re.compile(r"(?<!\$)\bt\(\s*[`$A-Za-z_]")   # template literal or variable, not a literal
-RE_WEB_VUE_T_DYN = re.compile(r"\$t\(\s*[`$A-Za-z_]")
+EXTENSIONS = {".swift", ".kt", ".kts", ".xml", ".ts", ".tsx", ".js", ".jsx", ".vue"}
+ANDROID = {".kt", ".kts", ".xml"}
+WEB = {".ts", ".tsx", ".js", ".jsx", ".vue"}
+SKIP_PARTS = ("/.git", "/Generated/")
+
+ERROR_WORDS = ("toast", "snackbar", "alert", "warning", "failure", "exception", "showerror", "errormessage", "error", "validat")
+ERROR_CONTEXT = re.compile("|".join(ERROR_WORDS), re.IGNORECASE)
+
+WORD = "A-Za-z0-9_"
+IDENT = rf"[A-Za-z_][{WORD}]*"
+LSK = re.compile(rf"LocalizationStringKey((?:\.{IDENT}){{2,}})")
+LITERAL_LOCALIZED = re.compile(r"\"([^\"\\\n]+)\"\.localized\b")
+R_STRING = re.compile(rf"\bR\.string\.([{WORD}]+)")
+XML_STRING = re.compile(rf"@string/([{WORD}.]+)")
+WEB_T = re.compile(r"(?<![\w$])t\(\s*(['\"])([A-Za-z0-9_.\-]+)\1")
+WEB_DOLLAR_T = re.compile(r"\$t\(\s*(['\"])([A-Za-z0-9_.\-]+)\1")
+WEB_T_DYNAMIC = re.compile(r"(?<![\w$])t\(\s*[`$A-Za-z_]")
+WEB_DOLLAR_T_DYNAMIC = re.compile(r"\$t\(\s*[`$A-Za-z_]")
+LSK_DYNAMIC = re.compile(r"LocalizationStringKey\(")
+FORMAT_DYNAMIC = re.compile(r"String\(format:")
+GETSTRING_DYNAMIC = re.compile(r"\bgetString\(\s*[A-Za-z_]")
+INTERPOLATED_LOCALIZED = re.compile(r"\"([^\"\n]*(?:\\\(|\$\{|\$[A-Za-z_])[^\"\n]*)\"\.localized\b")
+RECEIVER_LOCALIZED = re.compile(rf"(?<![\w.\"])({IDENT}(?:\.{IDENT})*)\.localized\b")
+
+
+def strip_accessor(chain):
+    segments = chain.strip(".").split(".")
+    while segments and segments[-1].startswith("localized"):
+        segments.pop()
+    return ".".join(segments)
 
 
 def platform_of(path):
-    if path.endswith((".kt", ".kts")):
-        return "android"
-    if path.endswith(".xml"):
+    ext = os.path.splitext(path)[1]
+    if ext in ANDROID:
         return "android"
-    if path.endswith((".ts", ".tsx", ".js", ".jsx", ".vue")):
+    if ext in WEB:
         return "web"
     return "ios"
 
 
-def classify(line):
-    return "error" if ERROR_CTX.search(line) else None
-
-
-def scan_line(path, lineno, line, out, seen):
-    plat = platform_of(path)
-    err = classify(line)
-
-    def add(key, category, dyn=False):
-        cat = "dynamic" if dyn else (category or "static")
-        # error context upgrades a static/dynamic hit's category to error only when
-        # it isn't already dynamic (dynamic is the more actionable label)
-        if err and cat == "static":
-            cat = "error"
-        sig = (key, cat, path, lineno)
-        if sig in seen:
-            return
-        seen.add(sig)
-        out.append({
-            "key": key, "category": cat, "platform": plat,
-            "file": path, "line": lineno, "snippet": line.strip()[:200],
-        })
-
-    if path.endswith(".xml"):
-        for m in RE_AT_STRING.finditer(line):
-            add(m.group(1), err)
-        return
-
-    if plat == "web":
-        for m in RE_WEB_T.finditer(line):
-            add(m.group(1), err)
-        for m in RE_WEB_VUE_T.finditer(line):
-            add(m.group(1), err)
-        if RE_WEB_T_DYN.search(line):
-            add("(dynamic: t(...))", err, dyn=True)
-        if RE_WEB_VUE_T_DYN.search(line):
-            add("(dynamic: $t(...))", err, dyn=True)
+def source_files(root):
+    if os.path.isfile(root):
+        yield root
         return
-
-    # iOS / Kotlin source
-    for m in RE_LSK.finditer(line):
-        add(m.group(1), err)
-    for m in RE_LITERAL_LOCALIZED.finditer(line):
-        add(m.group(1), err)
-    for m in RE_R_STRING.finditer(line):
-        add(m.group(1), err)
-
-    # dynamic constructions  -  record the expression, not a resolvable key
-    if RE_LSK_DYN.search(line):
-        add("(dynamic: LocalizationStringKey(...))", err, dyn=True)
-    if RE_STRING_FORMAT.search(line) and (".localized" in line or "LocalizationStringKey" in line):
-        add("(dynamic: String(format:...))", err, dyn=True)
-    if RE_GETSTRING_DYN.search(line) and "R.string." not in line:
-        add("(dynamic: getString(var))", err, dyn=True)
-    # variable .localized receiver (not a string literal, not the iOS key enum)
-    for m in RE_VAR_LOCALIZED.finditer(line):
-        recv = m.group(1)
-        if recv.startswith("LocalizationStringKey"):
+    for current, dirs, files in os.walk(root):
+        probe = current.replace(os.sep, "/") + "/"
+        if any(part in probe for part in SKIP_PARTS):
+            dirs[:] = []
             continue
-        add(f"(dynamic: {recv}.localized)", err, dyn=True)
-
-
-def main():
-    ap = argparse.ArgumentParser(description=__doc__,
-                                 formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("--screen-path", required=True,
-                    help="dir (or file) of the implemented screen to scan")
-    ap.add_argument("--out", help="write JSON here (default: stdout)")
-    ap.add_argument("--category", choices=["all", "error", "dynamic", "static"], default="all",
-                    help="filter the emitted keys (default all)")
-    args = ap.parse_args()
-
-    root = args.screen_path
-    if not os.path.exists(root):
-        ap.error(f"--screen-path not found: {root}")
-
-    files = []
-    if os.path.isfile(root):
-        files = [root]
-    else:
-        for dirpath, _dirs, names in os.walk(root):
-            if "/.git" in dirpath or "/Generated/" in dirpath:
-                continue
-            for n in names:
-                if n.endswith(EXTS):
-                    files.append(os.path.join(dirpath, n))
-
-    out, seen = [], set()
-    for path in sorted(files):
+        for name in sorted(files):
+            if os.path.splitext(name)[1] in EXTENSIONS:
+                yield os.path.join(current, name)
+
+
+def hits_in_line(line, ext):
+    static, dynamic = [], []
+    if ext == ".xml":
+        static.extend(m.group(1) for m in XML_STRING.finditer(line))
+        return static, dynamic
+    if ext in WEB:
+        static.extend(m.group(2) for m in WEB_T.finditer(line))
+        static.extend(m.group(2) for m in WEB_DOLLAR_T.finditer(line))
+        if WEB_T_DYNAMIC.search(line):
+            dynamic.append("(dynamic: t(...))")
+        if WEB_DOLLAR_T_DYNAMIC.search(line):
+            dynamic.append("(dynamic: $t(...))")
+        return static, dynamic
+    for match in LSK.finditer(line):
+        key = strip_accessor(match.group(1))
+        if key.count(".") >= 1:
+            static.append(key)
+    static.extend(m.group(1) for m in LITERAL_LOCALIZED.finditer(line))
+    static.extend(m.group(1) for m in R_STRING.finditer(line))
+    if LSK_DYNAMIC.search(line):
+        dynamic.append("(dynamic: LocalizationStringKey(...))")
+    mentions_key = any(marker in line for marker in (".localized", "LocalizationStringKey"))
+    if mentions_key and FORMAT_DYNAMIC.search(line):
+        dynamic.append("(dynamic: String(format:...))")
+    if "R.string." not in line and GETSTRING_DYNAMIC.search(line):
+        dynamic.append("(dynamic: getString(var))")
+    for match in INTERPOLATED_LOCALIZED.finditer(line):
+        dynamic.append(f'(dynamic: "{match.group(1)}".localized)')
+    for match in RECEIVER_LOCALIZED.finditer(line):
+        receiver = match.group(1)
+        if not receiver.startswith("LocalizationStringKey"):
+            dynamic.append(f"(dynamic: {receiver}.localized)")
+    return static, dynamic
+
+
+def scan(root):
+    keys = []
+    seen = set()
+    scanned = 0
+    for path in source_files(root):
         try:
-            with open(path, encoding="utf-8", errors="replace") as fh:
-                for i, line in enumerate(fh, 1):
-                    scan_line(path, i, line, out, seen)
-        except OSError:
+            with open(path, encoding="utf-8") as stream:
+                lines = stream.read().splitlines()
+        except (OSError, UnicodeDecodeError):
             continue
+        scanned += 1
+        ext = os.path.splitext(path)[1]
+        platform = platform_of(path)
+        for number, line in enumerate(lines, 1):
+            static, dynamic = hits_in_line(line, ext)
+            in_error = bool(ERROR_CONTEXT.search(line))
+            found = [(k, "error" if in_error else "static") for k in static] + [(k, "dynamic") for k in dynamic]
+            for key, category in found:
+                marker = (key, category, path, number)
+                if marker in seen:
+                    continue
+                seen.add(marker)
+                keys.append({
+                    "key": key,
+                    "category": category,
+                    "platform": platform,
+                    "file": path,
+                    "line": number,
+                    "snippet": line.strip()[:200],
+                })
+    return scanned, keys
+
+
+CATEGORIES = ("static", "error", "dynamic")
 
-    if args.category != "all":
-        out = [k for k in out if k["category"] == args.category]
-    out.sort(key=lambda k: (k["category"], k["file"], k["line"]))
-
-    counts = {c: sum(1 for k in out if k["category"] == c) for c in ("static", "error", "dynamic")}
-    result = {"screenPath": root, "filesScanned": len(files), "counts": counts, "keys": out}
 
-    text = json.dumps(result, ensure_ascii=False, indent=2)
-    if args.out:
-        with open(args.out, "w", encoding="utf-8") as fh:
-            fh.write(text + "\n")
+def main():
+    cli = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    cli.add_argument("--screen-path", required=True, help="folder or single file of the built screen")
+    cli.add_argument("--out", help="JSON destination; stdout when omitted")
+    cli.add_argument("--category", default="all", choices=["all", "error", "dynamic", "static"])
+    opts = cli.parse_args()
+
+    if not os.path.exists(opts.screen_path):
+        cli.error(f"--screen-path not found: {opts.screen_path}")
+
+    scanned, keys = scan(opts.screen_path)
+    if opts.category != "all":
+        keys = [entry for entry in keys if entry["category"] == opts.category]
+    keys.sort(key=lambda entry: (entry["category"], entry["file"], entry["line"]))
+    per_category = [entry["category"] for entry in keys]
+    counts = {name: per_category.count(name) for name in CATEGORIES}
+
+    text = json.dumps(
+        {"screenPath": opts.screen_path, "filesScanned": scanned, "counts": counts, "keys": keys},
+        indent=2, ensure_ascii=False,
+    ) + "\n"
+    if opts.out:
+        with open(opts.out, "w", encoding="utf-8") as sink:
+            sink.write(text)
     else:
-        sys.stdout.write(text + "\n")
-    sys.stderr.write(f"# scanned {len(files)} file(s): "
-                     f"{counts['static']} static · {counts['error']} error · "
-                     f"{counts['dynamic']} dynamic\n")
+        sys.stdout.write(text)
+    summary = ", ".join(f"{counts[name]} {name}" for name in CATEGORIES)
+    print(f"# {scanned} file(s) read: {summary}", file=sys.stderr)
 
 
 if __name__ == "__main__":

package/pipeline/skills/shared/external/localization-reuse-map/scripts/snapshot-resources.sh

@@ -1,26 +1,29 @@
 #!/usr/bin/env bash
-# Snapshot the redesign resources (component registry + Suggested translations) from
-# the resources repo into specs/resources, so the localization
-# mapper reads everything from one repo (like design-export). Generated, not authored  -  
-# refresh from resources; never hand-edit the snapshot. See reference/publish-and-snapshot.md.
-#
-# Usage: snapshot-resources.sh <resources-root> <specs-root>
+# Mirror the component registry and Suggested translations from a resources
+# checkout into <specs-root>/resources/ so every read comes from one repo.
 set -euo pipefail
 
-SRC="${1:?resources root}"; DST="${2:?specs root}"
-REG_SRC="$SRC/Resources/Figma/Components"
-SUG_SRC="$SRC/Resources/Localization/LocalizationStrings/Sources/Suggested"
-REG_DST="$DST/resources/Figma/Components"
-SUG_DST="$DST/resources/Localization/Suggested"
+src="${1:?resources root}"
+dst="${2:?specs root}"
 
-for d in "$REG_SRC" "$SUG_SRC"; do
-  [ -d "$d" ] || { echo "missing source dir: $d" >&2; exit 1; }
+components_src="$src/Resources/Figma/Components"
+suggested_src="$src/Resources/Localization/LocalizationStrings/Sources/Suggested"
+components_dst="$dst/resources/Figma/Components"
+suggested_dst="$dst/resources/Localization/Suggested"
+
+for dir in "$components_src" "$suggested_src"; do
+  if [[ ! -d "$dir" ]]; then
+    echo "missing source dir: $dir" >&2
+    exit 1
+  fi
 done
-mkdir -p "$REG_DST" "$SUG_DST"
 
-# --delete keeps the snapshot a faithful mirror (drops removed keys/components).
-rsync -a --delete "$REG_SRC/" "$REG_DST/"
-rsync -a --delete "$SUG_SRC/" "$SUG_DST/"
+mkdir -p "$components_dst" "$suggested_dst"
+rsync -a --delete "$components_src/" "$components_dst/"
+rsync -a --delete "$suggested_src/" "$suggested_dst/"
+
+component_count=$(ls "$components_dst" | wc -l | tr -d ' ')
+suggested_count=$(ls "$suggested_dst" | wc -l | tr -d ' ')
 
-echo "snapshot: $(ls "$REG_DST" | wc -l | tr -d ' ') components, $(ls "$SUG_DST" | wc -l | tr -d ' ') suggested keys"
-echo "→ $DST/resources  (commit like a design-export refresh)"
+echo "snapshot: $component_count components, $suggested_count suggested keys"
+echo "review $dst/resources and commit it the same way as a design-export refresh"