@mmerterden/multi-agent-pipeline 20.5.0 → 20.7.0

package/pipeline/skills/shared/external/localization-reuse-map/scripts/fetch-legacy-labels.py

@@ -1,25 +1,9 @@
 #!/usr/bin/env python3
-"""Refresh the LEGACY label snapshot: fetch a legacy label map for every language
-from your project's label endpoint and write it into the resources snapshot, so the
-mapper resolves legacy values offline instead of a network round-trip per run. This
-is the ONLY step that hits the network  -  run it manually / in CI to refresh, the same
-way you refresh a design export.
+"""Refresh the legacy label snapshot from the label endpoint.
 
-Writes <out>/<lang>.json = flat {<key>: value} (keys verbatim, any prefix kept).
-
-The endpoint is entirely project-specific, so nothing about it is baked in  -  you
-supply the URL template and the headers it needs:
-
-  fetch-legacy-labels.py --out resources/Localization/Legacy --langs all \\
-    --endpoint 'https://labels.example.internal/{env}/labels/{lang}' \\
-    --headers-file .secrets/label-headers.json
-
-Placeholders available in --endpoint: {lang} and {env}.
-
-Prefer --headers-file (JSON object) over repeated --header for anything secret, so
-tokens never land in shell history or CI logs. Response shape is discovered rather
-than assumed: the largest flat {str: str} map anywhere in the JSON wins, which covers
-both a bare map and one nested under an envelope.
+Writes one flat <out>/<lang>.json per language with keys exactly as the
+service returns them (backend prefix kept). This is the only step of the
+skill that needs the network.
 """
 import argparse
 import json
@@ -27,103 +11,103 @@ import os
 import sys
 import urllib.request
 
-LANGS_ALL = ["en", "tr", "ar", "de", "es", "fr", "it", "ru"]
+ALL_LANGS = ["en", "tr", "ar"] + sorted(["ru", "it", "fr", "es", "de"])
 
 
-def largest_str_map(obj):
-    best, stack = {}, [obj]
-    while stack:
-        cur = stack.pop()
-        if isinstance(cur, dict):
-            strs = {k: v for k, v in cur.items() if isinstance(v, str)}
-            if len(strs) > len(best):
-                best = strs
-            stack.extend(cur.values())
-        elif isinstance(cur, list):
-            stack.extend(cur)
-    return best
+def parse_langs(value):
+    if value.strip() == "all":
+        return list(ALL_LANGS)
+    return [part.strip() for part in value.split(",") if part.strip()]
 
 
-def parse_headers(a):
-    headers = {"Accept": "application/json"}
-    if a.headers_file:
-        with open(a.headers_file, encoding="utf-8") as fh:
-            loaded = json.load(fh)
-        if not isinstance(loaded, dict):
-            raise SystemExit(f"--headers-file must be a JSON object, got {type(loaded).__name__}")
-        headers.update({str(k): str(v) for k, v in loaded.items()})
-    for pair in a.header or []:
+def parse_pairs(pairs, flag):
+    result = {}
+    for pair in pairs or []:
         if "=" not in pair:
-            raise SystemExit(f"--header expects K=V, got {pair!r}")
-        k, v = pair.split("=", 1)
-        headers[k.strip()] = v.strip()
+            raise SystemExit(f"{flag} expects K=V, got {pair!r}")
+        key, value = pair.split("=", 1)
+        result[key.strip()] = value.strip()
+    return result
+
+
+def build_headers(args):
+    headers = {"Accept": "application/json"}
+    if args.headers_file:
+        with open(args.headers_file, encoding="utf-8") as stream:
+            extra = json.load(stream)
+        if not isinstance(extra, dict):
+            raise SystemExit(f"--headers-file must hold a JSON object: {args.headers_file}")
+        headers.update({str(k): str(v) for k, v in extra.items()})
+    headers.update(parse_pairs(args.header, "--header"))
     return headers
 
 
-def parse_fail_statuses(values):
-    out = {}
-    for pair in values or []:
-        if "=" not in pair:
-            raise SystemExit(f"--fail-status expects CODE=message, got {pair!r}")
-        code, message = pair.split("=", 1)
-        out[code.strip()] = message.strip()
-    return out
-
-
-def fetch(lang, headers, fail_statuses, a):
-    url = a.endpoint.format(env=a.env, lang=lang)
-    per_lang = dict(headers)
-    per_lang.setdefault("Accept-Language", lang)
-    req = urllib.request.Request(url, headers=per_lang)
-    with urllib.request.urlopen(req, timeout=a.timeout) as r:
-        data = json.loads(r.read().decode("utf-8"))
-    if isinstance(data, dict) and a.status_field in data:
-        code = str(data.get(a.status_field))
-        if code in fail_statuses:
-            raise RuntimeError(f"{url}: {fail_statuses[code]} (status {code})")
-    m = largest_str_map(data)
-    if not m:
-        raise RuntimeError(f"no label map found in the response for '{lang}' ({url})")
-    return m
+def largest_string_map(data):
+    best = {}
+    stack = [data]
+    while stack:
+        node = stack.pop()
+        if isinstance(node, dict):
+            if node and all(isinstance(k, str) and isinstance(v, str) for k, v in node.items()):
+                if len(node) > len(best):
+                    best = node
+            stack.extend(node.values())
+        elif isinstance(node, list):
+            stack.extend(node)
+    return best
+
+
+def fetch_labels(endpoint, lang, env, headers, status_field, fail_status, timeout):
+    url = endpoint.format(lang=lang, env=env)
+    request_headers = dict(headers)
+    request_headers.setdefault("Accept-Language", lang)
+    request = urllib.request.Request(url, headers=request_headers)
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        data = json.loads(response.read().decode("utf-8"))
+    if isinstance(data, dict) and status_field in data:
+        code = str(data[status_field])
+        if code in fail_status:
+            raise RuntimeError(f"{url}: {fail_status[code]} (status {code})")
+    labels = largest_string_map(data)
+    if not labels:
+        raise RuntimeError(f"the response for '{lang}' ({url}) contains no string-to-string label map")
+    return labels
 
 
 def main():
-    ap = argparse.ArgumentParser(description=__doc__,
-                                formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("--out", required=True, help="snapshot dir, e.g. resources/Localization/Legacy")
-    ap.add_argument("--endpoint", required=True,
-                    help="URL template for the legacy label map; may contain {lang} and {env}")
-    ap.add_argument("--header", action="append",
-                    help="extra request header as K=V (repeatable); avoid for secrets")
-    ap.add_argument("--headers-file", help="JSON object of request headers  -  use this for tokens")
-    ap.add_argument("--langs", default="all", help="'all' or a comma-separated list")
-    ap.add_argument("--env", default="", help="value substituted for {env} in --endpoint")
-    ap.add_argument("--status-field", default="statusCode",
-                    help="response field carrying a provider status code")
-    ap.add_argument("--fail-status", action="append",
-                    help="CODE=message for a provider error code that must abort (repeatable)")
-    ap.add_argument("--timeout", type=int, default=40)
-    a = ap.parse_args()
-
-    if "{lang}" not in a.endpoint:
-        print("warning: --endpoint has no {lang} placeholder; every language will hit the same URL",
-              file=sys.stderr)
-
-    headers = parse_headers(a)
-    fail_statuses = parse_fail_statuses(a.fail_status)
-    langs = LANGS_ALL if a.langs.strip() == "all" else [l.strip() for l in a.langs.split(",") if l.strip()]
-    os.makedirs(a.out, exist_ok=True)
+    cli = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    cli.add_argument("--out", dest="snapshot_dir", required=True, help="where the snapshot lives, for example resources/Localization/Legacy")
+    cli.add_argument("--endpoint", required=True, help="URL template containing {lang} and, if needed, {env}")
+    cli.add_argument("--header", action="append", help="additional request header as K=V; may be repeated")
+    cli.add_argument("--headers-file", help="request headers as a JSON object, so secrets stay out of argv")
+    cli.add_argument("--langs", default="all", help="languages separated by commas, or 'all'")
+    cli.add_argument("--env", default="", help="text put in place of {env}")
+    cli.add_argument("--status-field", default="statusCode", help="name of the response field that holds a status code")
+    cli.add_argument("--fail-status", action="append", help="CODE=message pair that marks a failed response; may be repeated")
+    cli.add_argument("--timeout", default=40, type=int)
+    opts = cli.parse_args()
+
+    if "{lang}" not in opts.endpoint:
+        print("# warning: --endpoint lacks a {lang} placeholder, so all languages request the same URL", file=sys.stderr)
+
+    headers = build_headers(opts)
+    fail_status = parse_pairs(opts.fail_status, "--fail-status")
+    langs = parse_langs(opts.langs)
+    os.makedirs(opts.snapshot_dir, exist_ok=True)
+
     total = 0
     for lang in langs:
-        m = fetch(lang, headers, fail_statuses, a)
-        path = os.path.join(a.out, f"{lang}.json")
-        with open(path, "w", encoding="utf-8") as fh:
-            json.dump(m, fh, ensure_ascii=False, indent=0, sort_keys=True)
-            fh.write("\n")
-        total += len(m)
-        print(f"  {lang}: {len(m)} labels → {path}")
-    print(f"legacy snapshot refreshed: {len(langs)} languages, ~{total // max(len(langs), 1)} keys each")
-    print("→ commit it like a design-export refresh")
+        labels = fetch_labels(opts.endpoint, lang, opts.env, headers, opts.status_field, fail_status, opts.timeout)
+        target = os.path.join(opts.snapshot_dir, lang + ".json")
+        with open(target, "w", encoding="utf-8") as sink:
+            json.dump(labels, sink, ensure_ascii=False, sort_keys=True, indent=0)
+            sink.write("\n")
+        total += len(labels)
+        print(f"  {lang}: {len(labels)} labels -> {target}")
+
+    average = total // len(langs) if langs else 0
+    print(f"snapshot refreshed: {len(langs)} languages, about {average} keys each")
+    print(f"commit {opts.snapshot_dir} so later lookups stay offline")
 
 
 if __name__ == "__main__":