@mmerterden/multi-agent-pipeline 20.6.0 → 20.7.0

package/pipeline/skills/shared/external/localization-reuse-map/scripts/render-key-shots.py

@@ -1,297 +1,232 @@
 #!/usr/bin/env python3
-"""Render a PER-KEY "screenpiece" for every mapping row: a cropped band of the screen
-with a RED BOX around exactly that element, so the table's "Ekran Görüntüsü" cell shows
-the content team where each key lives on screen (ported from the web skill's
-render-key-shots.mjs). No AI image generation  -  pure geometry + HTML + headless Chrome.
-
-Input modes (same split as render-overlay.py):
-  * REST (default)  -  Figma token (--token / env / keychain FIGMA_ACCESS_TOKEN):
-      geometry  GET /v1/files/<key>/nodes?ids=<frame>  → per-text absoluteBoundingBox,
-                with ANCESTOR VISIBILITY (`visible!=false`) + cumulative opacity tracked,
-                so hidden component variants with stale bboxes are never boxed;
-      image     GET /v1/images/<key>?ids=<frame>&format=png&scale=<n> → one render,
-                cropped per key.
-  * --spec <file> (repeatable; MCP / offline)  -  the SAME spec shape render-overlay.py
-      takes: { fileKey?, pages:[{nodeId, image, frame:{w,h}, nodes:[{id, characters,
-      x,y,w,h}]}] }. `image` may be a data URL **or a file path** (resolved relative to
-      the spec file)  -  a committed example spec can reference screenshot.png instead of
-      inlining base64. Several --spec files = several screen states; each row is boxed
-      on the FIRST spec/page where its node matches.
-
-Rows are matched to nodes exactly like the overlay: by `cmsNodeId` first, else by
-normalized value (new/cms/legacy tr+en, element label). Every matched row gets a shot;
-unmatched rows are listed in the manifest's `missing` (blank cell in the table  -  never
-a guessed crop).
-
-Output, into --out:
-  keyshots/keyshot__<NN>__<key>.png   one per matched row; NN = the row's 1-based
-                                      position in mapping.rows  -  the SAME number the
-                                      table and the overlay cards use.
-  <slug>.keyshots.manifest.json       {shots:[{row,key,element,file,...}], missing:[...]}
-                                      build-artifact.py reads this (mapping top-level
-                                      `keyshots`) to fill the table cells;
-                                      publish-confluence.py --keyshots attaches the PNGs.
-No headless Chrome → per-shot `.html` files are written instead (open/print manually);
-the manifest marks them `"fallback": "html"`. Read-only against every source.
+"""Cut one crop per mapped row, with the element framed in red, for the Screenshot column.
+
+A crop keeps the whole frame width and a strip of --pad pixels over and under
+the element, which gets a red outline. Output lands in
+keyshots/keyshot__NN__<key>.png, NN being the row's index in mapping.rows
+counted from 1. A row whose element cannot be found on screen is reported as
+missing and gets no picture; crops are never guessed. Node matching and the
+Figma calls are shared with render-overlay.py, loaded from this folder.
 """
 import argparse
-import base64
 import importlib.util
 import json
 import os
 import re
+import subprocess
 import sys
-import tempfile
-
-sys.dont_write_bytecode = True
-_HERE = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, _HERE)
 
-# Shared with the overlay renderer on purpose: SAME matching = the shot, the
-# overlay card and the table row can never disagree about which node a key is.
-# The filename is not a valid identifier, so it loads by path rather than by
-# `import`: spec_from_file_location yields a real module object, and no source
-# text is evaluated at runtime.
-_RO_PATH = os.path.join(_HERE, "render-overlay.py")
-_spec = importlib.util.spec_from_file_location("render_overlay", _RO_PATH)
-if _spec is None or _spec.loader is None:
-    sys.exit(f"render-key-shots: cannot load {_RO_PATH}")
-_ro = importlib.util.module_from_spec(_spec)
-sys.modules["render_overlay"] = _ro
-_spec.loader.exec_module(_ro)
+SCRIPT_DIR = os.path.split(os.path.abspath(__file__))[0]
+BOX_COLOUR = "#E81932"
+ID_FIELD = "cmsNodeId"
+INSET = 6
+UNSAFE = re.compile(r"[\\/:*?\"<>|\s]+")
+TRAILING_MARKER = re.compile(r"\s*\([^()]*\)\s*$")
 
-canon_id = _ro.canon_id
-slug = _ro.slug
-build_index = _ro.build_index
-match_row = _ro.match_row
-figma_token = _ro.figma_token
-http_get_json = _ro.http_get_json
-http_get_bytes = _ro.http_get_bytes
-find_chrome = _ro.find_chrome
-FIGMA_API = _ro.FIGMA_API
 
-RED = "#E81932"
+def load_overlay():
+    path = os.path.join(SCRIPT_DIR, "render-overlay.py")
+    try:
+        spec = importlib.util.spec_from_file_location("render_overlay", path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+    except Exception as exc:
+        sys.exit(f"cannot load {path}: {exc}")
+    return module
 
 
-def key_file(num, key):
-    # Drop a trailing "(owned by ...)" marker first: it is provenance, not identity, so
-    # letting it into the filename churns every PNG (and orphans the old ones) whenever
-    # a component is renamed or the tag is added. Matches build-spreadsheet's clean_key.
-    bare = re.sub(r"\s*\(.*\)\s*$", "", key or "").strip()
-    safe = re.sub(r'[/\\:*?"<>|\s]+', "-", bare or "key").strip("-")
-    return f"keyshot__{num:02d}__{safe}.png"
+ov = load_overlay()
 
 
-def build_shot_html(img_src, fw, fh, box, pad):
-    """Full-frame-width band: `pad` px of context above/below the element, red box on it."""
-    cy0 = max(0, box["y"] - pad)
-    cy1 = min(fh, box["y"] + box["h"] + pad)
-    crop_w, crop_h = fw, cy1 - cy0
-    bx = max(0, box["x"] - 6)
-    by = box["y"] - cy0 - 6
-    bw, bh = box["w"] + 12, box["h"] + 12
-    html = f"""<!doctype html><html><head><meta charset="utf-8"><style>
-*{{margin:0;padding:0;box-sizing:border-box}}
-body{{width:{crop_w}px;height:{crop_h}px;overflow:hidden;position:relative;background:#fff}}
-img{{position:absolute;left:0;top:{-cy0}px;width:{fw}px;height:{fh}px}}
-.box{{position:absolute;left:{bx}px;top:{by}px;width:{bw}px;height:{bh}px;
- border:3px solid {RED};border-radius:6px;box-shadow:0 0 0 2px rgba(232,25,50,.15)}}
-</style></head><body><img src="{img_src.replace('"', '&quot;')}"/><div class="box"></div></body></html>"""
-    return html, crop_w, crop_h
+def shot_name(row_number, key):
+    clean = UNSAFE.sub("-", TRAILING_MARKER.sub("", key).strip()).strip("-")
+    return f"keyshot__{row_number:02d}__{clean}"
 
 
-def render_shot_png(chrome, html, w, h, out_path, scale):
-    """One headless-Chrome screenshot. Returns None on success, else an error string."""
-    import subprocess
-    with tempfile.NamedTemporaryFile("w", suffix=".html", delete=False, encoding="utf-8") as tf:
-        tf.write(html)
-        tmp = tf.name
-    try:
-        r = subprocess.run(
-            [chrome, "--headless", "--disable-gpu", "--hide-scrollbars",
-             f"--force-device-scale-factor={scale}",
-             f"--window-size={w},{int(h)}",
-             f"--screenshot={out_path}", tmp],
-            capture_output=True, text=True, timeout=120)
-        if not os.path.isfile(out_path):
-            return (r.stderr or "no output")[:200]
-        return None
-    except Exception as exc:  # timeout / spawn failure  -  fall back per shot, not per run
-        return str(exc)[:200]
-    finally:
-        try:
-            os.unlink(tmp)
-        except OSError:
-            pass
-
-
-def spec_image_src(page, spec_path):
-    """`image` as a browser src: pass a data URL through; inline a file path (relative
-    to the spec file) as base64 so the crop HTML stays self-contained for Chrome."""
-    img = page.get("image") or ""
-    if img.startswith("data:"):
-        return img
-    cand = img if os.path.isabs(img) else os.path.join(os.path.dirname(os.path.abspath(spec_path)), img)
-    if not os.path.isfile(cand):
-        sys.exit(f"spec image not found: {cand}")
-    with open(cand, "rb") as fh:
-        return "data:image/png;base64," + base64.b64encode(fh.read()).decode("ascii")
+def spec_pages(paths):
+    pages = []
+    for path in paths:
+        spec, loaded = ov.load_spec_pages(path)
+        if not loaded:
+            sys.exit(f"--spec lists no pages: {path}")
+        pages.extend(loaded)
+    return pages
 
 
-def collect_rest_pages(file_key, node_ids, token, ca_file, scale):
-    """Fetch each frame's VISIBLE text nodes (+ image render)  -  visibility is ancestor
-    `visible` AND cumulative opacity, exactly the web port's rule, because a node tree
-    carries text that is not rendered in this state and its stale bbox would land the
-    red box on the wrong content."""
-    headers = {"X-Figma-Token": token}
-    import urllib.parse
+def visible_text_nodes(document):
+    frame = document.get("absoluteBoundingBox") or {}
+    fx, fy = frame.get("x", 0), frame.get("y", 0)
+    found = []
+    pending = [(document, True, 1.0)]
+    while pending:
+        node, shown, alpha = pending.pop()
+        shown = shown and node.get("visible", True) is not False
+        alpha = alpha * float(node.get("opacity", 1.0))
+        if not shown or alpha <= 0:
+            continue
+        box = node.get("absoluteBoundingBox")
+        text = node.get("characters")
+        is_text = node.get("type") == "TEXT"
+        if is_text and box and text and text.strip():
+            found.append({
+                "id": node.get("id"),
+                "characters": text,
+                "x": round(box["x"] - fx),
+                "y": round(box["y"] - fy),
+                "w": round(box["width"]),
+                "h": round(box["height"]),
+            })
+        pending.extend((child, shown, alpha) for child in reversed(node.get("children") or []))
+    return found
+
+
+def rest_pages(file_key, node_ids, scale, token, ca_file):
     pages = []
-    for nid in node_ids:
-        data = http_get_json(f"{FIGMA_API}/files/{file_key}/nodes?ids={urllib.parse.quote(nid, safe='')}",
-                             headers, ca_file)
-        doc = ((data.get("nodes") or {}).get(nid) or {}).get("document")
-        if not doc:
-            sys.stderr.write(f"# warning: node {nid} not found  -  skipped\n")
+    for raw in node_ids:
+        node_id = ov.canon_id(raw)
+        document = ov.fetch_node(file_key, node_id, token, ca_file)
+        if not document:
+            print(f"# warning: {file_key} has no frame {node_id}; skipping it", file=sys.stderr)
             continue
-        fb = doc.get("absoluteBoundingBox") or {"x": 0, "y": 0, "width": 0, "height": 0}
-        nodes = []
-
-        def walk(n, anc_visible, opacity):
-            vis = anc_visible and n.get("visible") is not False
-            op = opacity * (n.get("opacity") if isinstance(n.get("opacity"), (int, float)) else 1)
-            b = n.get("absoluteBoundingBox")
-            is_text = n.get("type") == "TEXT" and isinstance(n.get("characters"), str) and n["characters"].strip()
-            if is_text and b and vis and op > 0:
-                nodes.append({"id": n.get("id"), "characters": n["characters"],
-                              "x": round(b["x"] - fb["x"]), "y": round(b["y"] - fb["y"]),
-                              "w": round(b["width"]), "h": round(b["height"])})
-            for c in n.get("children") or []:
-                walk(c, vis, op)
-
-        walk(doc, True, 1)
-        img = http_get_json(
-            f"{FIGMA_API}/images/{file_key}?ids={urllib.parse.quote(nid, safe='')}&format=png&scale={scale}",
-            headers, ca_file)
-        url = (img.get("images") or {}).get(nid)
-        if not url:
-            sys.stderr.write(f"# warning: no image render for {nid}  -  skipped\n")
+        image = ov.render_image(file_key, node_id, scale, token, ca_file)
+        if not image:
+            print(f"# warning: Figma returned no render of {node_id}; skipping it", file=sys.stderr)
             continue
-        raw = http_get_bytes(url, ca_file)
-        pages.append({"nodeId": nid,
-                      "image": "data:image/png;base64," + base64.b64encode(raw).decode("ascii"),
-                      "frame": {"w": round(fb.get("width", 0)), "h": round(fb.get("height", 0))},
-                      "nodes": nodes, "_src": "rest"})
+        box = document.get("absoluteBoundingBox") or {}
+        pages.append({
+            "nodeId": node_id,
+            "name": document.get("name"),
+            "image": image,
+            "frame": {"w": round(box.get("width", 0)), "h": round(box.get("height", 0))},
+            "nodes": visible_text_nodes(document),
+        })
+    if not pages:
+        sys.exit("none of the frames could be rendered")
     return pages
 
 
-def main():
-    ap = argparse.ArgumentParser(description=__doc__,
-                                 formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("--mapping", required=True, help="mapping JSON (rows + figmaFileKey/figmaNodes + screen)")
-    ap.add_argument("--spec", action="append", default=[],
-                    help="spec JSON (repeatable  -  one per screen state; overlay-compatible shape; "
-                         "image may be a data URL or a file path relative to the spec)")
-    ap.add_argument("--file", help="Figma fileKey (REST; else mapping figmaFileKey)")
-    ap.add_argument("--nodes", help="comma-separated frame node-ids (REST; else mapping figmaNodes)")
-    ap.add_argument("--pad", type=int, default=150, help="context band above/below the element (px)")
-    ap.add_argument("--scale", type=int, default=2)
-    ap.add_argument("--token")
-    ap.add_argument("--ca-file")
-    ap.add_argument("--out", default=".", help="output dir (shots go under <out>/keyshots/)")
-    ap.add_argument("--slug", help="override slug (default from screen name)")
-    args = ap.parse_args()
+def assign(rows, pages):
+    index = ov.build_index(rows)
+    hits = {}
+    for page in pages:
+        ordered = sorted(page.get("nodes") or [], key=lambda item: float(item.get("y", 0)))
+        for node in ordered:
+            position = ov.match_row(node, index)
+            if position is not None:
+                hits.setdefault(position, (page, node))
+    return hits
+
+
+def crop_html(page, node, pad):
+    fw, fh = (float(page["frame"][side]) for side in ("w", "h"))
+    x, y, w, h = (float(node.get(axis, 0)) for axis in ("x", "y", "w", "h"))
+    top = max(0.0, y - pad)
+    height = max(1.0, min(fh, y + h + pad) - top)
+    pin_left = "left:0;"
+    box_class = "box"
+    style = (
+        f"html,body{{margin:0;padding:0;}}body{{width:{fw:.0f}px;height:{height:.0f}px;position:relative;overflow:hidden;}}"
+        f"img{{position:absolute;{pin_left}top:{-top:.0f}px;width:{fw:.0f}px;height:{fh:.0f}px;}}"
+        f".{box_class}{{position:absolute;left:{x - INSET:.0f}px;top:{y - top - INSET:.0f}px;width:{w + 2 * INSET:.0f}px;"
+        f"height:{h + 2 * INSET:.0f}px;border:3px solid {BOX_COLOUR};border-radius:6px;box-sizing:border-box;}}"
+    )
+    document = (
+        f"{ov.PAGE_OPEN}{ov.META_UTF8}<style>{style}</style></head>"
+        f'<body><img src="{page["image"]}"><div class="{box_class}"></div></body></html>'
+    )
+    return document, fw, height
 
-    with open(args.mapping, encoding="utf-8") as fh:
-        m = json.load(fh)
-    rows = m.get("rows") or []
-    screen = m.get("screen") or "screen"
-    sl = args.slug or slug(screen)
-    shots_dir = os.path.join(args.out, "keyshots")
-    os.makedirs(shots_dir, exist_ok=True)
 
-    # ---- geometry pages, in precedence order (first match wins per row) ----
-    if args.spec:
-        pages = []
-        for sp in args.spec:
-            with open(sp, encoding="utf-8") as fh:
-                spec = json.load(fh)
-            for page in spec.get("pages") or []:
-                page["_img_src"] = spec_image_src(page, sp)
-                pages.append(page)
-        if not pages:
-            sys.exit("--spec files carry no pages")
+def main():
+    cli = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    cli.add_argument("--mapping", required=True, help="the reuse-map JSON")
+    cli.add_argument("--spec", action="append", help="geometry spec for one screen state; may be repeated")
+    cli.add_argument("--file", help="Figma file key; falls back to figmaFileKey in the mapping")
+    cli.add_argument("--nodes", help="frame node-ids separated by commas; falls back to figmaNodes")
+    cli.add_argument("--pad", default=150, type=int, help="pixels of surrounding context kept over and under the element")
+    cli.add_argument("--scale", default=2, type=int)
+    cli.add_argument("--token", help="Figma token; FIGMA_ACCESS_TOKEN or the keychain is safer since other processes can read argv")
+    cli.add_argument("--ca-file", help="PEM roots to trust when a proxy intercepts TLS")
+    cli.add_argument("--out", dest="target_dir", default=".", help="folder for the manifest; the crops go into <out>/keyshots/")
+    cli.add_argument("--slug", help="stem of the manifest name; derived from the screen name when omitted")
+    opts = cli.parse_args()
+
+    with open(opts.mapping, encoding="utf-8") as stream:
+        mapping = json.load(stream)
+    rows = mapping.get("rows") or []
+    stem = opts.slug or ov.slug(mapping.get("screen"))
+
+    if opts.spec:
+        pages = spec_pages(opts.spec)
     else:
-        file_key = args.file or m.get("figmaFileKey")
-        node_ids = [canon_id(n) for n in (args.nodes or "").split(",") if n.strip()] or \
-            [canon_id(str(n)) for n in (m.get("figmaNodes") or [])]
-        if not (file_key and node_ids):
-            ap.error("REST needs a fileKey + node-ids (via --file/--nodes or the mapping). "
-                     "Or pass --spec for the offline path.")
-        token = figma_token(args)
+        file_key = opts.file or mapping.get("figmaFileKey")
+        nodes = ov.split_nodes(opts.nodes, mapping)
+        if not (file_key and nodes):
+            cli.error("need a Figma file key and frame node-ids (flags or mapping), or --spec")
+        token = ov.figma_token(opts.token)
         if not token:
-            sys.exit("No Figma token  -  pass --token, set FIGMA_ACCESS_TOKEN, or add the "
-                     "keychain item FIGMA_ACCESS_TOKEN (or use --spec).")
-        pages = collect_rest_pages(file_key, node_ids, token, args.ca_file, args.scale)
-        for page in pages:
-            page["_img_src"] = page["image"]
-        if not pages:
-            sys.exit("no renderable frames")
-
-    # ---- choose one node per row: first page (in order) whose nodes match it ----
-    by = [build_index([r]) for r in rows]  # per-row index → exact same matcher as overlay
-    chosen = {}   # row idx (0-based) → {page, node}
-    for page in pages:
-        for n in sorted(page["nodes"], key=lambda x: x.get("y", 0)):
-            for i, r in enumerate(rows):
-                if i in chosen or not r.get("newKey"):
-                    continue
-                if match_row(n, *by[i]) is r:
-                    chosen[i] = {"page": page, "node": n}
-    missing = [{"row": i + 1, "key": r.get("newKey"),
-                "reason": "no visible node matched (cmsNodeId/value)"}
-               for i, r in enumerate(rows) if r.get("newKey") and i not in chosen]
+            sys.exit("No Figma token found (FIGMA_ACCESS_TOKEN, FIGMA_TOKEN, keychain FIGMA_ACCESS_TOKEN or --token). "
+                     "Without one, build --spec files from the screenshots instead.")
+        pages = rest_pages(file_key, nodes, opts.scale, token, opts.ca_file)
+
+    crops_dir = os.path.join(opts.target_dir, "keyshots")
+    os.makedirs(crops_dir, exist_ok=True)
+    chrome = ov.find_chrome()
+    hits = assign(rows, pages)
+    shots, missing = [], []
+    failed = 0
+
+    for number, row in enumerate(rows, 1):
+        key = str(row.get("newKey") or "").strip()
+        if not key:
+            continue
+        if number - 1 not in hits:
+            missing.append({"row": number, "key": key, "reason": f"no visible node matched ({ID_FIELD}/value)"})
+            continue
+        page, node = hits[number - 1]
+        document, width, height = crop_html(page, node, opts.pad)
+        base = shot_name(number, key)
+        entry = {
+            "row": number,
+            "key": TRAILING_MARKER.sub("", key),
+            "element": row.get("element"),
+            "pageNode": page.get("nodeId"),
+            "cropW": int(round(width)),
+            "cropH": int(round(height)),
+        }
+        if not chrome:
+            with open(os.path.join(crops_dir, base + ".html"), "w", encoding="utf-8") as sink:
+                sink.write(document)
+            entry.update(file=f"keyshots/{base}.html", fallback="html")
+            shots.append(entry)
+            continue
+        try:
+            ov.chrome_screenshot(chrome, document, os.path.join(crops_dir, base + ".png"), width, height, opts.scale)
+        except (RuntimeError, subprocess.SubprocessError) as exc:
+            failed += 1
+            print(f"# row {number} {key}: {exc}", file=sys.stderr)
+            continue
+        entry["file"] = f"keyshots/{base}.png"
+        shots.append(entry)
+
+    manifest_name = f"{stem}.keyshots.manifest.json"
+    manifest = {
+        "screen": mapping.get("screen"),
+        "slug": stem,
+        "pad": opts.pad,
+        "scale": opts.scale,
+        "dir": "keyshots",
+        "shots": shots,
+        "missing": missing,
+    }
+    with open(os.path.join(opts.target_dir, manifest_name), "w", encoding="utf-8") as sink:
+        json.dump(manifest, sink, indent=2, ensure_ascii=False)
+        sink.write("\n")
 
-    chrome = find_chrome()
     if not chrome:
-        sys.stderr.write("# no headless Chrome  -  writing per-shot .html fallbacks\n")
-
-    manifest = {"screen": screen, "slug": sl, "pad": args.pad, "scale": args.scale,
-                "dir": "keyshots", "shots": [], "missing": missing}
-    ok = fail = 0
-    for i in sorted(chosen):
-        r, c = rows[i], chosen[i]
-        page, n = c["page"], c["node"]
-        num = i + 1
-        html, w, h = build_shot_html(page["_img_src"], page["frame"]["w"], page["frame"]["h"],
-                                     {"x": n["x"], "y": n["y"], "w": n["w"], "h": n["h"]}, args.pad)
-        fname = key_file(num, r["newKey"])
-        out_png = os.path.join(shots_dir, fname)
-        entry = {"row": num, "key": r["newKey"], "element": r.get("element"),
-                 "file": f"keyshots/{fname}", "pageNode": page.get("nodeId"),
-                 "cropW": w, "cropH": h}
-        if chrome:
-            err = render_shot_png(chrome, html, w, h, out_png, args.scale)
-            if err:
-                fail += 1
-                sys.stderr.write(f"# ✗ row {num} {r['newKey']}: {err}\n")
-                continue
-        else:
-            alt = out_png[:-4] + ".html"
-            with open(alt, "w", encoding="utf-8") as fh:
-                fh.write(html)
-            entry["file"] = f"keyshots/{os.path.basename(alt)}"
-            entry["fallback"] = "html"
-        ok += 1
-        manifest["shots"].append(entry)
-
-    mpath = os.path.join(args.out, sl + ".keyshots.manifest.json")
-    with open(mpath, "w", encoding="utf-8") as fh:
-        json.dump(manifest, fh, ensure_ascii=False, indent=2)
-        fh.write("\n")
-    sys.stderr.write(f"# keyshots: {ok} written, {len(missing)} unmatched, {fail} failed → "
-                     f"{shots_dir}/ · manifest: {mpath}\n")
-    sys.stderr.write(f'# set the mapping\'s top-level "keyshots": "{os.path.basename(mpath)}" '
-                     "so build-artifact.py fills the table cells\n")
+        print("# Chrome, Chromium and Edge are all missing; the crops were saved as .html", file=sys.stderr)
+    print(f"# keyshots: {len(shots)} written, {len(missing)} unmatched, {failed} failed", file=sys.stderr)
+    print(f'# point the mapping\'s top-level "keyshots" field at "{manifest_name}"', file=sys.stderr)
 
 
 if __name__ == "__main__":