@mmerterden/multi-agent-pipeline 14.2.1 → 14.2.2

package/pipeline/skills/shared/external/localization-reuse-map/scripts/fetch-legacy-labels.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Refresh the LEGACY label snapshot: fetch a legacy label map for every language
+from your project's label endpoint and write it into the resources snapshot, so the
+mapper resolves legacy values offline instead of a network round-trip per run. This
+is the ONLY step that hits the network  -  run it manually / in CI to refresh, the same
+way you refresh a design export.
+
+Writes <out>/<lang>.json = flat {<key>: value} (keys verbatim, any prefix kept).
+
+The endpoint is entirely project-specific, so nothing about it is baked in  -  you
+supply the URL template and the headers it needs:
+
+  fetch-legacy-labels.py --out resources/Localization/Legacy --langs all \\
+    --endpoint 'https://labels.example.internal/{env}/labels/{lang}' \\
+    --headers-file .secrets/label-headers.json
+
+Placeholders available in --endpoint: {lang} and {env}.
+
+Prefer --headers-file (JSON object) over repeated --header for anything secret, so
+tokens never land in shell history or CI logs. Response shape is discovered rather
+than assumed: the largest flat {str: str} map anywhere in the JSON wins, which covers
+both a bare map and one nested under an envelope.
+"""
+import argparse
+import json
+import os
+import sys
+import urllib.request
+
+LANGS_ALL = ["en", "tr", "ar", "de", "es", "fr", "it", "ru"]
+
+
+def largest_str_map(obj):
+    best, stack = {}, [obj]
+    while stack:
+        cur = stack.pop()
+        if isinstance(cur, dict):
+            strs = {k: v for k, v in cur.items() if isinstance(v, str)}
+            if len(strs) > len(best):
+                best = strs
+            stack.extend(cur.values())
+        elif isinstance(cur, list):
+            stack.extend(cur)
+    return best
+
+
+def parse_headers(a):
+    headers = {"Accept": "application/json"}
+    if a.headers_file:
+        with open(a.headers_file, encoding="utf-8") as fh:
+            loaded = json.load(fh)
+        if not isinstance(loaded, dict):
+            raise SystemExit(f"--headers-file must be a JSON object, got {type(loaded).__name__}")
+        headers.update({str(k): str(v) for k, v in loaded.items()})
+    for pair in a.header or []:
+        if "=" not in pair:
+            raise SystemExit(f"--header expects K=V, got {pair!r}")
+        k, v = pair.split("=", 1)
+        headers[k.strip()] = v.strip()
+    return headers
+
+
+def parse_fail_statuses(values):
+    out = {}
+    for pair in values or []:
+        if "=" not in pair:
+            raise SystemExit(f"--fail-status expects CODE=message, got {pair!r}")
+        code, message = pair.split("=", 1)
+        out[code.strip()] = message.strip()
+    return out
+
+
+def fetch(lang, headers, fail_statuses, a):
+    url = a.endpoint.format(env=a.env, lang=lang)
+    per_lang = dict(headers)
+    per_lang.setdefault("Accept-Language", lang)
+    req = urllib.request.Request(url, headers=per_lang)
+    with urllib.request.urlopen(req, timeout=a.timeout) as r:
+        data = json.loads(r.read().decode("utf-8"))
+    if isinstance(data, dict) and a.status_field in data:
+        code = str(data.get(a.status_field))
+        if code in fail_statuses:
+            raise RuntimeError(f"{url}: {fail_statuses[code]} (status {code})")
+    m = largest_str_map(data)
+    if not m:
+        raise RuntimeError(f"no label map found in the response for '{lang}' ({url})")
+    return m
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--out", required=True, help="snapshot dir, e.g. resources/Localization/Legacy")
+    ap.add_argument("--endpoint", required=True,
+                    help="URL template for the legacy label map; may contain {lang} and {env}")
+    ap.add_argument("--header", action="append",
+                    help="extra request header as K=V (repeatable); avoid for secrets")
+    ap.add_argument("--headers-file", help="JSON object of request headers  -  use this for tokens")
+    ap.add_argument("--langs", default="all", help="'all' or a comma-separated list")
+    ap.add_argument("--env", default="", help="value substituted for {env} in --endpoint")
+    ap.add_argument("--status-field", default="statusCode",
+                    help="response field carrying a provider status code")
+    ap.add_argument("--fail-status", action="append",
+                    help="CODE=message for a provider error code that must abort (repeatable)")
+    ap.add_argument("--timeout", type=int, default=40)
+    a = ap.parse_args()
+
+    if "{lang}" not in a.endpoint:
+        print("warning: --endpoint has no {lang} placeholder; every language will hit the same URL",
+              file=sys.stderr)
+
+    headers = parse_headers(a)
+    fail_statuses = parse_fail_statuses(a.fail_status)
+    langs = LANGS_ALL if a.langs.strip() == "all" else [l.strip() for l in a.langs.split(",") if l.strip()]
+    os.makedirs(a.out, exist_ok=True)
+    total = 0
+    for lang in langs:
+        m = fetch(lang, headers, fail_statuses, a)
+        path = os.path.join(a.out, f"{lang}.json")
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(m, fh, ensure_ascii=False, indent=0, sort_keys=True)
+            fh.write("\n")
+        total += len(m)
+        print(f"  {lang}: {len(m)} labels → {path}")
+    print(f"legacy snapshot refreshed: {len(langs)} languages, ~{total // max(len(langs), 1)} keys each")
+    print("→ commit it like a design-export refresh")
+
+
+if __name__ == "__main__":
+    main()

package/pipeline/skills/shared/external/localization-reuse-map/scripts/publish-confluence.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""Publish (or UPDATE) one screen's localization map to Confluence  -  live, idempotent.
+
+Targets a Confluence **Server/Data Center** instance with a **Bearer Personal Access
+Token** (Server/DC auth  -  NOT Atlassian Cloud `user:token`). Nothing about your wiki is
+baked in: pass `--base-url`, `--space` and `--parent`, or set `CONFLUENCE_BASE_URL`,
+`CONFLUENCE_SPACE` and `CONFLUENCE_PARENT` once in the environment.
+
+Idempotent by title under a parent page:
+  * search `space=<SPACE> and title="<title>"` (CQL);
+  * **exists → PUT** `/rest/api/content/<id>` with `version.number+1` → updates the
+    SAME page in place (re-running the mapper on a screen refreshes its page, never
+    duplicates it);
+  * **absent → POST** `/rest/api/content` under `--parent`.
+Then attaches `screenshot.png` + the overlay PNG + (via `--keyshots <manifest>`) every
+per-key screenpiece PNG from render-key-shots.py (the `<ac:image ri:filename=...>`
+references in the body resolve to these), creating or updating each by filename.
+`--attach <file>` (repeatable) attaches any additional document  -  e.g. the content
+team's `<slug>.localization.xlsx` (build-spreadsheet.py)  -  and, for office/spreadsheet
+files, appends a `view-file` macro section so it also renders inline on the page.
+
+HTTP goes through **curl** on purpose: curl uses the system trust store, so it works
+behind a corporate TLS proxy where Python's `urllib` fails cert verification. The token
+is read from the keychain (item `CONFLUENCE_API_TOKEN`) and passed to curl via a
+`chmod 600` `-K` config file  -  never on the argv / `ps`.
+
+The skill asks the user for the target (space / parent / formats) and passes them here.
+Read-only against the local artifacts; the only writes are to Confluence.
+
+Examples:
+  python3 publish-confluence.py --xml out/<slug>.confluence.xml --screen "<name>" \
+      --base-url https://wiki.example.com --space ENG --parent 12345 \
+      --screenshot out/screenshot.png --overlay out/<slug>.overlay.png
+  python3 publish-confluence.py --xml ... --title "Custom Title"   # base/space/parent from env
+  python3 publish-confluence.py --xml ... --screen "<name>" --dry-run
+"""
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import urllib.parse
+
+DEFAULT_BASE = os.environ.get("CONFLUENCE_BASE_URL", "")
+DEFAULT_SPACE = os.environ.get("CONFLUENCE_SPACE", "")
+DEFAULT_PARENT = os.environ.get("CONFLUENCE_PARENT", "")
+KEYCHAIN_ITEM = "CONFLUENCE_API_TOKEN"
+
+
+def get_token(args):
+    if args.token:
+        return args.token
+    v = os.environ.get("CONFLUENCE_API_TOKEN")
+    if v:
+        return v.strip()
+    try:
+        r = subprocess.run(
+            ["security", "find-generic-password", "-s", args.keychain_name, "-w"],
+            capture_output=True, text=True)
+        if r.returncode == 0 and r.stdout.strip():
+            return r.stdout.strip()
+    except Exception:
+        pass
+    sys.exit(f"No Confluence token  -  add keychain item '{args.keychain_name}', set "
+             "CONFLUENCE_API_TOKEN, or pass --token.")
+
+
+class Cf:
+    """Thin curl-backed Confluence Server REST client (Bearer auth via -K config)."""
+
+    def __init__(self, base, token):
+        self.base = base.rstrip("/")
+        self._cfg = tempfile.NamedTemporaryFile("w", suffix=".curlrc", delete=False)
+        os.chmod(self._cfg.name, 0o600)
+        self._cfg.write(f'header = "Authorization: Bearer {token}"\n')
+        self._cfg.close()
+
+    def close(self):
+        try:
+            os.unlink(self._cfg.name)
+        except OSError:
+            pass
+
+    def _run(self, args):
+        r = subprocess.run(["curl", "-sS", "-K", self._cfg.name, *args],
+                           capture_output=True, text=True)
+        if r.returncode != 0:
+            sys.exit(f"curl failed: {r.stderr.strip()}")
+        return r.stdout
+
+    def _json(self, args):
+        out = self._run(args)
+        try:
+            data = json.loads(out)
+        except json.JSONDecodeError:
+            sys.exit(f"Confluence returned non-JSON (auth/endpoint?):\n{out[:400]}")
+        if isinstance(data, dict) and data.get("statusCode") and data.get("message"):
+            sys.exit(f"Confluence error {data['statusCode']}: {data['message']}")
+        return data
+
+    def find_by_title(self, space, title):
+        # DB-backed content query (NOT /content/search  -  CQL hits the async search
+        # index, which lags seconds behind a just-created page and breaks idempotency).
+        qs = urllib.parse.urlencode({"spaceKey": space, "title": title, "type": "page",
+                                     "expand": "version", "limit": "5"})
+        data = self._json([f"{self.base}/rest/api/content?{qs}"])
+        results = data.get("results") or []
+        return results[0] if results else None
+
+    def create(self, space, parent, title, storage):
+        body = {"type": "page", "title": title, "space": {"key": space},
+                "ancestors": [{"id": str(parent)}],
+                "body": {"storage": {"value": storage, "representation": "storage"}}}
+        return self._post_json("POST", f"{self.base}/rest/api/content", body)
+
+    def update(self, page_id, title, storage, new_version, space):
+        body = {"id": str(page_id), "type": "page", "title": title,
+                "space": {"key": space},
+                "version": {"number": new_version},
+                "body": {"storage": {"value": storage, "representation": "storage"}}}
+        return self._post_json("PUT", f"{self.base}/rest/api/content/{page_id}", body)
+
+    def _post_json(self, method, url, body):
+        with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False, encoding="utf-8") as bf:
+            json.dump(body, bf, ensure_ascii=False)
+            bpath = bf.name
+        try:
+            return self._json(["-X", method, "-H", "Content-Type: application/json",
+                               "--data", f"@{bpath}", url])
+        finally:
+            os.unlink(bpath)
+
+    def existing_attachment(self, page_id, filename):
+        q = urllib.parse.quote(filename)
+        data = self._json([f"{self.base}/rest/api/content/{page_id}/child/attachment?filename={q}&limit=1"])
+        results = data.get("results") or []
+        return results[0] if results else None
+
+    def attach(self, page_id, path):
+        fn = os.path.basename(path)
+        existing = self.existing_attachment(page_id, fn)
+        if existing:
+            url = f"{self.base}/rest/api/content/{page_id}/child/attachment/{existing['id']}/data"
+        else:
+            url = f"{self.base}/rest/api/content/{page_id}/child/attachment"
+        return self._json(["-H", "X-Atlassian-Token: nocheck",
+                           "-F", f"file=@{path}", "-X", "POST", url])
+
+
+EMBED_EXTS = (".xlsx", ".xls", ".csv", ".docx", ".pdf")
+
+
+def attachment_section(paths):
+    """Storage-format section embedding a `view-file` macro for each office/spreadsheet
+    attachment, so the file renders inline on the page (not only in the drawer). Returns
+    "" when no path is an embeddable document."""
+    docs = [p for p in paths if os.path.splitext(p)[1].lower() in EMBED_EXTS]
+    if not docs:
+        return ""
+    macros = "".join(
+        '<p><ac:structured-macro ac:name="view-file">'
+        '<ac:parameter ac:name="name">'
+        f'<ri:attachment ri:filename="{os.path.basename(p)}"/>'
+        '</ac:parameter></ac:structured-macro></p>'
+        for p in docs
+    )
+    return "<h2>CMS Aktarım Dosyası</h2>" + macros
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--xml", required=True, help="<slug>.confluence.xml (storage body)")
+    ap.add_argument("--screen", help="screen name (used in the default title)")
+    ap.add_argument("--title", help="page title (default: '<title-prefix>  -  <screen>')")
+    ap.add_argument("--title-prefix", default=os.environ.get("LOCALIZATION_PAGE_PREFIX", "Localization"),
+                    help="prefix for the default title; set it to your wiki's wording "
+                         "(env LOCALIZATION_PAGE_PREFIX)")
+    ap.add_argument("--screenshot", help="screenshot PNG to attach")
+    ap.add_argument("--overlay", help="overlay PNG to attach")
+    ap.add_argument("--keyshots", metavar="MANIFEST",
+                    help="<slug>.keyshots.manifest.json (render-key-shots.py)  -  attaches every "
+                         "per-key screenpiece PNG it lists, so the table's Ekran Görüntüsü "
+                         "cells (<ri:attachment> refs) render on the page")
+    ap.add_argument("--attach", action="append", default=[], metavar="FILE",
+                    help="extra file to attach (repeatable); e.g. <slug>.localization.xlsx. "
+                         "Office/spreadsheet files also get a view-file macro on the page.")
+    ap.add_argument("--base", "--base-url", dest="base", default=DEFAULT_BASE,
+                    help="Confluence base URL, e.g. https://wiki.example.com (env CONFLUENCE_BASE_URL)")
+    ap.add_argument("--space", default=DEFAULT_SPACE, help="space key (env CONFLUENCE_SPACE)")
+    ap.add_argument("--parent", default=DEFAULT_PARENT,
+                    help="parent pageId the screen pages hang under (env CONFLUENCE_PARENT)")
+    ap.add_argument("--token", help="Confluence PAT (else keychain / env)")
+    ap.add_argument("--keychain-name", default=KEYCHAIN_ITEM)
+    ap.add_argument("--dry-run", action="store_true", help="print the plan, write nothing")
+    args = ap.parse_args()
+
+    title = args.title or (f"{args.title_prefix}  -  {args.screen}" if args.screen else None)
+    if not title:
+        ap.error("supply --title or --screen")
+    missing = [flag for flag, value in (("--base-url", args.base), ("--space", args.space),
+                                        ("--parent", args.parent)) if not value]
+    if missing and not args.dry_run:
+        ap.error("no Confluence target configured  -  pass " + ", ".join(missing) +
+                 " (or set CONFLUENCE_BASE_URL / CONFLUENCE_SPACE / CONFLUENCE_PARENT)")
+    if not os.path.isfile(args.xml):
+        ap.error(f"--xml not found: {args.xml}")
+    storage = open(args.xml, encoding="utf-8").read()
+
+    keyshot_files = []
+    if args.keyshots:
+        if not os.path.isfile(args.keyshots):
+            ap.error(f"--keyshots manifest not found: {args.keyshots}")
+        with open(args.keyshots, encoding="utf-8") as fh:
+            man = json.load(fh)
+        base = os.path.dirname(os.path.abspath(args.keyshots))
+        keyshot_files = [os.path.join(base, s["file"]) for s in man.get("shots", [])
+                         if s.get("file", "").endswith(".png")]
+
+    attachments = [p for p in (args.screenshot, args.overlay, *keyshot_files, *args.attach) if p]
+    for p in attachments:
+        if not os.path.isfile(p):
+            ap.error(f"attachment not found: {p}")
+
+    # Embed a view-file macro for the spreadsheet/office attachments so they render on the page.
+    storage += attachment_section(args.attach)
+
+    if args.dry_run:
+        print(json.dumps({
+            "base": args.base, "space": args.space, "parent": args.parent,
+            "title": title, "xmlBytes": len(storage),
+            "attachments": [os.path.basename(p) for p in attachments],
+            "action": "search-then-create-or-update (idempotent by title)",
+        }, ensure_ascii=False, indent=2))
+        return
+
+    cf = Cf(args.base, get_token(args))
+    try:
+        existing = cf.find_by_title(args.space, title)
+        if existing:
+            cur = int(existing.get("version", {}).get("number", 1))
+            page = cf.update(existing["id"], title, storage, cur + 1, args.space)
+            action = f"updated (v{cur}→v{cur + 1})"
+        else:
+            page = cf.create(args.space, args.parent, title, storage)
+            action = "created"
+        page_id = page["id"]
+
+        attached = []
+        for p in attachments:
+            cf.attach(page_id, p)
+            attached.append(os.path.basename(p))
+
+        webui = (page.get("_links") or {}).get("webui", "")
+        url = f"{args.base}{webui}" if webui else f"{args.base}/pages/viewpage.action?pageId={page_id}"
+        print(json.dumps({"action": action, "id": page_id, "title": title,
+                          "attachments": attached, "url": url}, ensure_ascii=False, indent=2))
+    finally:
+        cf.close()
+
+
+if __name__ == "__main__":
+    main()