@mmerterden/multi-agent-pipeline 20.8.0 → 20.8.2

package/pipeline/skills/shared/external/localization-reuse-map/scripts/_shared.py

@@ -0,0 +1,201 @@
+"""Key patterns, mapping-row marker rules and text helpers shared by the scripts.
+
+Every script that scans keys, names output files or orders languages imports
+this module, so the scanner, the gate, the CMS sheet and the overlay agree on
+what a wired key is, which rows are component-owned or dynamic, which files are
+source, how a screen name becomes a file slug and which order languages take.
+The key accessor type (`--key-type`, default `L10n`) and the language list
+(`--langs`, default the languages present in the data) are inputs, never a
+built-in list of one app's.
+"""
+import argparse
+import os
+import re
+
+EXTENSIONS = {".swift", ".kt", ".kts", ".xml", ".ts", ".tsx", ".js", ".jsx", ".vue"}
+SKIP_DIRS = {".git", "Generated"}
+FALLBACK_LANG = "en"
+RTL_LANGS = {"ar", "fa", "he", "ur"}
+
+WORD = "A-Za-z0-9_"
+IDENT = rf"[A-Za-z_][{WORD}]*"
+DEFAULT_KEY_TYPES = ("L10n",)
+KEY_TYPE_NAME = re.compile(rf"^{IDENT}$")
+LITERAL_KEY = re.compile(rf"\"([{WORD}.]+)\"\.localized\b")
+WEB_T = re.compile(r"(?<![\w$])t\(\s*(['\"])([A-Za-z0-9_.\-]+)\1")
+WEB_DOLLAR_T = re.compile(r"\$t\(\s*(['\"])([A-Za-z0-9_.\-]+)\1")
+INTERPOLATION = re.compile(r"\\\(|\$\{|\$[A-Za-z_]")
+
+TRAILING_MARKER = re.compile(r"\s*\([^()]*\)\s*$")
+COMPONENT_MARKER = re.compile(r"\([^)]*(owned by|bile\u015fen|component)[^)]*\)", re.IGNORECASE)
+PLACEHOLDER = re.compile(r"<[^<>]+>")
+DYNAMIC_WORDS = ("dynamic", "dinamik")
+RUNTIME_NOTE = "enumerate at runtime"
+
+XML_INVALID = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f]")
+
+ACCESSOR_EXTENSIONS = {".swift", ".kt", ".kts"}
+COMMON_ACCESSORS = (
+    "L10n", "LocalizationKey", "LocalizationKeys",
+    "LocalizedStringKey", "LocalizedStrings", "Localizable", "Localization", "Strings", "Loc",
+)
+ACCESSOR_CHAIN = re.compile(rf"(?<![{WORD}.])([A-Z][{WORD}]*)((?:\.{IDENT}){{2,}})")
+
+
+def strip_accessor(chain):
+    segments = chain.strip(".").split(".")
+    while segments and segments[-1].startswith("localized"):
+        segments.pop()
+    return ".".join(segments)
+
+
+def key_types(values, fallback=DEFAULT_KEY_TYPES):
+    """The accessor type names from repeated or comma-separated --key-type values.
+
+    Raises ValueError on a name that is not an identifier.
+    """
+    names = [part.strip() for value in values or [] for part in str(value).split(",") if part.strip()]
+    for name in names:
+        if not KEY_TYPE_NAME.match(name):
+            raise ValueError(f"--key-type expects an identifier, got {name!r}")
+    return tuple(dict.fromkeys(names)) or tuple(fallback)
+
+
+def key_type_alternation(types):
+    return "|".join(re.escape(name) for name in sorted(types, key=len, reverse=True))
+
+
+def key_chain_pattern(types):
+    return re.compile(rf"(?:{key_type_alternation(types)})((?:\.{IDENT})+)")
+
+
+def lsk_keys(text, types=DEFAULT_KEY_TYPES):
+    for hit in key_chain_pattern(types).finditer(text):
+        key = strip_accessor(hit.group(1))
+        if key.count(".") >= 1:
+            yield key
+
+
+def accessor_roots(texts, types=DEFAULT_KEY_TYPES):
+    """Roots of key-like member chains that are not one of `types`, with their counts.
+
+    A chain counts when its root is a common accessor name or when it ends in
+    `.localized`, and it keeps two or more segments once that suffix is dropped.
+    """
+    counts = {}
+    for text in texts:
+        for hit in ACCESSOR_CHAIN.finditer(text):
+            root, members = hit.group(1), hit.group(2)
+            if root in types:
+                continue
+            segments = members.strip(".").split(".")
+            localized = segments[-1].startswith("localized")
+            if strip_accessor(members).count(".") < 1:
+                continue
+            if root in COMMON_ACCESSORS or localized:
+                counts[root] = counts.get(root, 0) + 1
+    return dict(sorted(counts.items(), key=lambda item: (-item[1], item[0])))
+
+
+def key_type_warning(texts, types=DEFAULT_KEY_TYPES, mapping_hint=False):
+    """A warning when no chain of `types` is found but chains of another accessor are.
+
+    `texts` are the Swift and Kotlin sources read. Returns None when the
+    configured type matched at least once or nothing resembles an accessor.
+    """
+    texts = list(texts)
+    if any(True for text in texts for _ in lsk_keys(text, types)):
+        return None
+    roots = accessor_roots(texts, types)
+    if not roots:
+        return None
+    seen = ", ".join(f"{root} ({count})" for root, count in list(roots.items())[:5])
+    where = "--key-type <Name> (or \"keyType\" in the mapping)" if mapping_hint else "--key-type <Name>"
+    return (
+        f"WARNING: no key chain of {', '.join(types)} was found, but the code has accessor chains rooted at "
+        f"{seen}. Set {where} to the project's key accessor type; until then its keys are "
+        f"missed or counted as weak."
+    )
+
+
+def order_langs(langs):
+    """en first when present, then the rest sorted; en alone when nothing is given."""
+    unique = list(dict.fromkeys(lang for lang in langs if lang))
+    rest = sorted(lang for lang in unique if lang != FALLBACK_LANG)
+    ordered = ([FALLBACK_LANG] if FALLBACK_LANG in unique else []) + rest
+    return ordered or [FALLBACK_LANG]
+
+
+def parse_langs(value, present=()):
+    """A comma list in the order given, or for 'all' every language in `present`."""
+    text = str(value or "all").strip()
+    if text == "all":
+        return order_langs(present)
+    return [part.strip() for part in text.split(",") if part.strip()]
+
+
+def reject_token_flag(cli, opts, env_hint):
+    """Exit 2 when a credential was passed on argv, which other processes can read."""
+    if getattr(opts, "token", None) is not None:
+        cli.error(f"--token is not accepted, because other processes can read argv; {env_hint}")
+
+
+def add_token_flag(cli):
+    cli.add_argument("--token", help=argparse.SUPPRESS)
+
+
+def is_interpolated(literal):
+    return bool(INTERPOLATION.search(literal))
+
+
+def normalize_key(key):
+    return ".".join(seg[:1].upper() + seg[1:] for seg in key.split("."))
+
+
+def source_files(root):
+    if os.path.isfile(root):
+        yield root
+        return
+    for current, dirs, files in os.walk(root):
+        relative = os.path.relpath(current, root)
+        parts = [] if relative == os.curdir else relative.split(os.sep)
+        if SKIP_DIRS.intersection(parts):
+            dirs[:] = []
+            continue
+        for name in sorted(files):
+            if os.path.splitext(name)[1] in EXTENSIONS:
+                yield os.path.join(current, name)
+
+
+def raw_key(row):
+    return str(row.get("newKey") or "").strip()
+
+
+def clean_key(new_key):
+    text = str(new_key or "").strip()
+    if text.startswith("(dynamic"):
+        return text
+    return TRAILING_MARKER.sub("", text)
+
+
+def is_component(row):
+    return bool(COMPONENT_MARKER.search(raw_key(row)))
+
+
+def key_is_dynamic(key):
+    return key.startswith("(dynamic") or bool(PLACEHOLDER.search(key))
+
+
+def is_dynamic(row, key):
+    element = str(row.get("element") or "").lower()
+    note = str(row.get("note") or "").lower()
+    return key_is_dynamic(key) or any(word in element for word in DYNAMIC_WORDS) or RUNTIME_NOTE in note
+
+
+def xml_text(value):
+    return XML_INVALID.sub("", str(value))
+
+
+def slug(text):
+    value = re.sub(r"[\W_]+", "-", str(text or "").strip().lower()).strip("-")
+    return value or "screen"

package/pipeline/skills/shared/external/localization-reuse-map/scripts/build-artifact.py

@@ -11,6 +11,7 @@ import html
 import json
 import os
 import re
+import shlex
 import shutil
 import struct
 import subprocess
@@ -22,7 +23,8 @@ import zipfile
 from collections import namedtuple
 from xml.sax.saxutils import escape, quoteattr
 
-LANGS = ["en", "tr", "ar"] + sorted(["ru", "it", "fr", "es", "de"])
+from _shared import RTL_LANGS, order_langs, parse_langs, slug, xml_text
+
 WARN = "\u26a0\ufe0f"
 DASH = "-"
 VERDICTS = ("reuse", "review", "new")
@@ -120,11 +122,6 @@ def look(row):
     return LOOKS.get(row.get("verdict"), UNKNOWN_LOOK)
 
 
-def slug(text):
-    value = re.sub(r"[\W_]+", "-", str(text or "").strip().lower()).strip("-")
-    return value or "screen"
-
-
 def norm(value):
     text = unicodedata.normalize("NFC", str(value or "")).strip()
     text = re.sub(r"\s+", " ", text)
@@ -189,8 +186,12 @@ def load_keyshots(mapping, mapping_dir, out_dir):
     return shots
 
 
+def xml_escape(value):
+    return escape(xml_text(value))
+
+
 def node(name, attrs=(), *children):
-    opening = name + "".join(f" {k}={quoteattr(str(v))}" for k, v in attrs)
+    opening = name + "".join(f" {k}={quoteattr(xml_text(v))}" for k, v in attrs)
     if not children:
         return f"<{opening}/>"
     return f"<{opening}>{''.join(children)}</{name}>"
@@ -209,14 +210,23 @@ def storage_image(filename, dimension, pixels):
     return node("ac:image", [(f"ac:{dimension}", pixels)], target)
 
 
+def mapping_langs(rows):
+    found = []
+    for row in rows:
+        for side in ("new", "legacy", "cms"):
+            found.extend((row.get(side) or {}).keys())
+    return order_langs(found)
+
+
 class Doc:
-    def __init__(self, mapping, t, prefix, shots, out_dir):
+    def __init__(self, mapping, t, prefix, shots, out_dir, langs=None):
         self.m = mapping
         self.t = t
         self.prefix = prefix
         self.shots = shots
         self.out = out_dir
         self.rows = mapping.get("rows") or []
+        self.langs = list(langs) if langs else mapping_langs(self.rows)
         self.counts = {v: sum(1 for r in self.rows if r.get("verdict") == v) for v in VERDICTS}
         self.cms_rows = sum(1 for r in self.rows if (r.get("cms") or {}).get("tr") or (r.get("cms") or {}).get("en"))
         self.drift_rows = sum(1 for r in self.rows if row_drifts(r))
@@ -245,7 +255,7 @@ class Doc:
         ]
 
     def detail_langs(self, row):
-        return LANGS + [lang for lang in (row.get("cms") or {}) if lang not in LANGS]
+        return self.langs + [lang for lang in (row.get("cms") or {}) if lang not in self.langs]
 
     def md(self):
         t, m = self.t, self.m
@@ -291,12 +301,12 @@ class Doc:
         return storage_macro(
             "status",
             node("ac:parameter", [("ac:name", "colour")], look(row).status),
-            node("ac:parameter", [("ac:name", "title")], escape(self.verdict(row))),
+            node("ac:parameter", [("ac:name", "title")], xml_escape(self.verdict(row))),
         )
 
     def xml(self):
         t, m = self.t, self.m
-        e = escape
+        e = xml_escape
 
         def heading(label):
             return f"<h2>{e(t[label])}</h2>"
@@ -504,7 +514,7 @@ def para(text, bold=False, size=None, rtl=False):
     if rtl:
         styling.append(w("rtl"))
     run = [w("rPr", *styling)] if styling else []
-    run.append(node("w:t", [("xml:space", "preserve")], escape(str(text))))
+    run.append(node("w:t", [("xml:space", "preserve")], xml_escape(text)))
     lead = [w("pPr", w("bidi"))] if rtl else []
     return w("p", *lead, w("r", *run))
 
@@ -571,7 +581,7 @@ def detail_blocks(doc, t):
         blocks.append(para(f"{t['key']}: {row.get('newKey') or ''}"))
         lines = [table_row(tcell(c, fill="F5F6F8") for c in t["detail_columns"])]
         for lang in doc.detail_langs(row):
-            lines.append(table_row(tcell(v, rtl=lang == "ar") for v in (lang, *lang_values(row, lang))))
+            lines.append(table_row(tcell(v, rtl=lang in RTL_LANGS) for v in (lang, *lang_values(row, lang))))
         blocks.append(table(lines))
     return blocks
 
@@ -663,17 +673,21 @@ def chrome_candidates():
 
 def pdf_libreoffice(docx_path, out_dir):
     binaries = [shutil.which("soffice"), shutil.which("libreoffice"), mac_app("LibreOffice", "soffice")]
-    expected = os.path.splitext(docx_path)[0] + ".pdf"
+    name = os.path.splitext(os.path.basename(docx_path))[0] + ".pdf"
     for binary in binaries:
         if not (binary and os.path.isfile(binary)):
             continue
-        try:
-            subprocess.run([binary, "--headless", "--convert-to", "pdf", "--outdir", out_dir, docx_path],
-                           capture_output=True, timeout=180)
-        except (OSError, subprocess.SubprocessError):
-            continue
-        if os.path.isfile(expected) and os.path.getsize(expected) > 0:
-            return expected
+        with tempfile.TemporaryDirectory() as scratch:
+            try:
+                subprocess.run([binary, "--headless", "--convert-to", "pdf", "--outdir", scratch, docx_path],
+                               capture_output=True, timeout=180)
+            except (OSError, subprocess.SubprocessError):
+                continue
+            rendered = os.path.join(scratch, name)
+            if os.path.isfile(rendered) and os.path.getsize(rendered) > 0:
+                target = os.path.join(out_dir, name)
+                shutil.copyfile(rendered, target)
+                return target
     return None
 
 
@@ -698,7 +712,7 @@ def pdf_chrome(preview_path, pdf_path):
             with tempfile.TemporaryDirectory() as profile:
                 if os.path.exists(pdf_path):
                     os.remove(pdf_path)
-                argv = [chrome, headless, "--disable-gpu", f"--user-data-dir={profile}", "--no-pdf-header-footer",
+                argv = [chrome, headless, "--no-sandbox", "--disable-gpu", f"--user-data-dir={profile}", "--no-pdf-header-footer",
                         f"--print-to-pdf={pdf_path}", page]
                 try:
                     proc = subprocess.Popen(argv, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
@@ -721,17 +735,31 @@ def pdf_wkhtml(preview_path, pdf_path):
     return pdf_path if os.path.isfile(pdf_path) and os.path.getsize(pdf_path) > 0 else None
 
 
-def print_upload(name, mapping, screen):
+def upload_path(name, mapping_dir, out_dir):
+    if os.path.isabs(name):
+        return name
+    for folder in (mapping_dir, out_dir):
+        if os.path.exists(os.path.join(folder, name)):
+            return os.path.join(folder, name)
+    return os.path.join(out_dir, name)
+
+
+def print_upload(name, mapping, screen, mapping_dir, out_dir):
     prefix = os.environ.get("LOCALIZATION_PAGE_PREFIX", "Localization")
-    flags = [f"--xml {name}.confluence.xml", f'--screen "{screen}"']
-    flags += [f"--{field} {mapping[field]}" for field in ("screenshot", "overlay", "keyshots") if mapping.get(field)]
-    flags.append(f"--attach {name}.localization.xlsx")
+    quote = shlex.quote
+    flags = [f"--xml {quote(os.path.join(out_dir, name + '.confluence.xml'))}", f"--screen {quote(str(screen))}"]
+    flags += [
+        f"--{field} {quote(upload_path(str(mapping[field]), mapping_dir, out_dir))}"
+        for field in ("screenshot", "overlay", "keyshots") if mapping.get(field)
+    ]
+    flags.append(f"--attach {quote(os.path.join(out_dir, name + '.localization.xlsx'))}")
     print(" \\\n  ".join(["python3 scripts/publish-confluence.py"] + flags))
     print()
+    title = json.dumps(f"{prefix} - {screen}", ensure_ascii=False).replace("'", "'\\''")
     print("# The same call by hand, for Server/DC with a Bearer PAT read from the keychain (argv never sees the token):")
-    print("#   printf 'header = \"Authorization: Bearer %s\"\\n' \"$(security find-generic-password -s CONFLUENCE_API_TOKEN -w)\" > auth.curlrc; chmod 600 auth.curlrc")
+    print("#   (umask 077 && rm -f auth.curlrc && printf 'header = \"Authorization: Bearer %s\"\\n' \"$(security find-generic-password -s CONFLUENCE_API_TOKEN -w)\" > auth.curlrc)")
     print("#   curl -sS -K auth.curlrc -X POST -H 'Content-Type: application/json' \"$CONFLUENCE_BASE_URL/rest/api/content\" \\")
-    print(f"#     --data '{{\"type\":\"page\",\"title\":\"{prefix} - {screen}\",\"space\":{{\"key\":\"'$CONFLUENCE_SPACE'\"}},"
+    print(f"#     --data '{{\"type\":\"page\",\"title\":{title},\"space\":{{\"key\":\"'$CONFLUENCE_SPACE'\"}},"
           "\"ancestors\":[{\"id\":\"'$CONFLUENCE_PARENT'\"}],\"body\":{\"storage\":{\"value\":\"...\",\"representation\":\"storage\"}}}'")
     print("#   rm auth.curlrc")
 
@@ -766,6 +794,11 @@ def main():
     cli.add_argument("--pdf", action="store_true", help="add a PDF to the outputs")
     cli.add_argument("--all", action="store_true", help="shorthand for --docx plus --pdf")
     cli.add_argument("--print-upload", action="store_true", help="show the publish command and a hand-rolled curl version of it")
+    cli.add_argument(
+        "--langs", default="all",
+        help="languages of the details tables, comma list in display order; 'all' (default) takes every language "
+             "in the mapping, en first then the rest sorted",
+    )
     opts = cli.parse_args()
 
     mapping, mapping_dir = read_mapping(opts.mapping)
@@ -776,7 +809,8 @@ def main():
     out_dir = os.path.abspath(opts.target_dir)
     os.makedirs(out_dir, exist_ok=True)
     shots = load_keyshots(mapping, mapping_dir, out_dir)
-    doc = Doc(mapping, T[opts.ui_lang], mapping.get("legacyKeyPrefix") or "", shots, out_dir)
+    langs = None if opts.langs.strip() == "all" else parse_langs(opts.langs)
+    doc = Doc(mapping, T[opts.ui_lang], mapping.get("legacyKeyPrefix") or "", shots, out_dir, langs)
     doc.base = mapping_dir
 
     renderers = [("md", doc.md), ("confluence.xml", doc.xml), ("preview.html", doc.preview)]
@@ -797,7 +831,7 @@ def main():
         make_pdf(docx_path, out_dir, stem, keep_docx)
 
     if opts.print_upload:
-        print_upload(stem, mapping, mapping.get("screen"))
+        print_upload(stem, mapping, mapping.get("screen"), mapping_dir, out_dir)
 
 
 if __name__ == "__main__":

package/pipeline/skills/shared/external/localization-reuse-map/scripts/build-spreadsheet.py

@@ -15,6 +15,8 @@ import sys
 import zipfile
 from xml.sax.saxutils import escape
 
+from _shared import clean_key, key_is_dynamic, slug, xml_text
+
 COLUMNS = [
     "Channel", "Property Group", "Property Module", "Key",
     "EN Value", "TR Value", "AR Value", "Anotation EN", "Anotation TR",
@@ -37,11 +39,6 @@ DEFAULT_TAXONOMY = {
 }
 
 
-def slug(text):
-    value = re.sub(r"[\W_]+", "-", str(text or "").strip().lower()).strip("-")
-    return value or "screen"
-
-
 def read_json(source):
     with open(source, encoding="utf-8") as stream:
         return json.load(stream)
@@ -62,12 +59,6 @@ def load_taxonomy(path):
     return taxonomy
 
 
-def clean_key(new_key):
-    text = str(new_key or "")
-    cut = text.find("(owned by")
-    return (text[:cut] if cut >= 0 else text).strip()
-
-
 def bucket(key, taxonomy):
     lowered = key.lower()
     if "error" in lowered:
@@ -88,9 +79,12 @@ def bucket(key, taxonomy):
 
 
 def table_rows(mapping, channel, taxonomy):
-    out = []
-    for row in mapping.get("rows") or []:
+    out, skipped = [], []
+    for number, row in enumerate(mapping.get("rows") or [], 1):
         key = clean_key(row.get("newKey"))
+        if not key or key_is_dynamic(key):
+            skipped.append((number, key))
+            continue
         group, module = bucket(key, taxonomy)
         if row.get("propertyGroup"):
             group = row["propertyGroup"]
@@ -103,7 +97,7 @@ def table_rows(mapping, channel, taxonomy):
             new.get("en") or "", new.get("tr") or "", new.get("ar") or "",
             cms.get("en") or "", cms.get("tr") or "",
         ])
-    return out
+    return out, skipped
 
 
 def column_letter(index):
@@ -137,7 +131,7 @@ def part(root, attrs, *children):
 
 
 def cell(ref, value, style):
-    text = escape(str(value)).replace("\n", "
")
+    text = escape(xml_text(value)).replace("\n", "
")
     attrs = [("r", ref), ("t", "inlineStr")]
     if style:
         attrs.append(("s", style))
@@ -276,7 +270,7 @@ def main():
     if not isinstance(mapping, dict) or "rows" not in mapping:
         cli.error("mapping must be a JSON object with rows")
 
-    rows = table_rows(mapping, opts.channel, load_taxonomy(opts.taxonomy))
+    rows, skipped = table_rows(mapping, opts.channel, load_taxonomy(opts.taxonomy))
     stem = opts.slug or slug(mapping.get("screen"))
     os.makedirs(opts.target_dir, exist_ok=True)
 
@@ -294,6 +288,10 @@ def main():
     summary = {"slug": stem, "rows": len(rows), "columns": COLUMNS, "channel": opts.channel, "written": produced}
     json.dump(summary, sys.stdout, indent=2, ensure_ascii=False)
     sys.stdout.write("\n")
+    if skipped:
+        print(f"# {len(skipped)} row(s) left out of the sheet: no literal key (empty, dynamic or <placeholder>)", file=sys.stderr)
+        for number, key in skipped:
+            print(f"#   row {number}: {key or '(empty)'}", file=sys.stderr)
 
 
 if __name__ == "__main__":

package/pipeline/skills/shared/external/localization-reuse-map/scripts/fetch-annotations.py

@@ -17,6 +17,8 @@ import urllib.parse
 import urllib.request
 from datetime import datetime, timezone
 
+from _shared import add_token_flag, reject_token_flag
+
 API = "https://api.figma.com/v1"
 BATCH = 4
 SEP = "[:\\-\u2013]"
@@ -24,8 +26,12 @@ TR_RE = re.compile("^(?:tr|tr[kc]e|t[\u00fcu]rk[\u00e7c]e)\\s*" + SEP + "\\s*(.+
 EN_RE = re.compile("^(?:en|eng|ing|[i\u0130]ngilizce)\\s*" + SEP + "\\s*(.+)$", re.IGNORECASE)
 
 
+URL_NODE_ID = re.compile(r"^\d+-\d+$")
+
+
 def canon_id(node_id):
-    return str(node_id).strip().replace("-", ":")
+    text = "" if node_id is None else str(node_id).strip()
+    return text.replace("-", ":") if URL_NODE_ID.match(text) else text
 
 
 def split_lang(text):
@@ -68,7 +74,7 @@ def pick_label(labels):
 def record(node_id, name, node_type, design_text, labels):
     raw, parsed = pick_label(labels)
     return {
-        "nodeId": canon_id(node_id),
+        "nodeId": canon_id(node_id) or None,
         "name": name,
         "type": node_type,
         "designText": design_text,
@@ -113,9 +119,7 @@ def keychain_secret(service):
     return secret if lookup.returncode == 0 and secret else None
 
 
-def figma_token(explicit):
-    if explicit:
-        return explicit
+def figma_token():
     from_env = next((os.environ[v] for v in ("FIGMA_ACCESS_TOKEN", "FIGMA_TOKEN") if os.environ.get(v)), None)
     return from_env or keychain_secret("FIGMA_ACCESS_TOKEN")
 
@@ -206,11 +210,11 @@ def pick_source(opts, cli, file_key, nodes):
     if opts.source == "rest":
         if not have_target:
             cli.error("--source rest needs a file key and node-ids (--file/--nodes or --mapping)")
-        token = figma_token(opts.token)
+        token = figma_token()
         if not token:
-            sys.exit("No Figma token found. Export FIGMA_ACCESS_TOKEN or FIGMA_TOKEN, keep it in the keychain under FIGMA_ACCESS_TOKEN, or use --token.")
+            sys.exit("No Figma token found. Export FIGMA_ACCESS_TOKEN or FIGMA_TOKEN, or keep it in the keychain under FIGMA_ACCESS_TOKEN.")
         return ("rest",) + from_rest(file_key, nodes, token, opts.ca_file)
-    token = figma_token(opts.token) if have_target else None
+    token = figma_token() if have_target else None
     if have_target and token:
         try:
             return ("rest",) + from_rest(file_key, nodes, token, opts.ca_file)
@@ -232,10 +236,11 @@ def main():
     cli.add_argument("--from-mcp", help="JSON dump of annotations taken through MCP")
     cli.add_argument("--local", help="screen folder of a design export that contains tree.json")
     cli.add_argument("--source", default="auto", choices=["auto", "rest", "mcp", "local"])
-    cli.add_argument("--token", help="Figma token; FIGMA_ACCESS_TOKEN or the keychain is safer since other processes can read argv")
+    add_token_flag(cli)
     cli.add_argument("--ca-file", help="PEM roots to trust when a proxy intercepts TLS")
     cli.add_argument("--out", help="file to write; stdout when omitted")
     opts = cli.parse_args()
+    reject_token_flag(cli, opts, "export FIGMA_ACCESS_TOKEN (or FIGMA_TOKEN), or keep it in the macOS keychain under FIGMA_ACCESS_TOKEN")
 
     file_key = opts.file
     nodes = [piece for piece in (opts.nodes or "").split(",") if piece.strip()]
@@ -246,10 +251,13 @@ def main():
 
     source, found, files, audited = pick_source(opts, cli, file_key, nodes)
 
-    first_seen = {}
+    unique, seen = [], set()
     for item in found:
-        first_seen.setdefault(item["nodeId"], item)
-    unique = list(first_seen.values())
+        if item["nodeId"]:
+            if item["nodeId"] in seen:
+                continue
+            seen.add(item["nodeId"])
+        unique.append(item)
 
     def having(lang):
         return sum(1 for entry in unique if entry[lang])

package/pipeline/skills/shared/external/localization-reuse-map/scripts/fetch-legacy-labels.py

@@ -11,13 +11,13 @@ import os
 import sys
 import urllib.request
 
-ALL_LANGS = ["en", "tr", "ar"] + sorted(["ru", "it", "fr", "es", "de"])
+from _shared import parse_langs
 
 
-def parse_langs(value):
-    if value.strip() == "all":
-        return list(ALL_LANGS)
-    return [part.strip() for part in value.split(",") if part.strip()]
+def snapshot_langs(snapshot_dir):
+    if not os.path.isdir(snapshot_dir):
+        return []
+    return [name[:-5] for name in os.listdir(snapshot_dir) if name.endswith(".json")]
 
 
 def parse_pairs(pairs, flag):
@@ -80,7 +80,7 @@ def main():
     cli.add_argument("--endpoint", required=True, help="URL template containing {lang} and, if needed, {env}")
     cli.add_argument("--header", action="append", help="additional request header as K=V; may be repeated")
     cli.add_argument("--headers-file", help="request headers as a JSON object, so secrets stay out of argv")
-    cli.add_argument("--langs", default="all", help="languages separated by commas, or 'all'")
+    cli.add_argument("--langs", default="all", help="languages separated by commas, or 'all' for every language already in the snapshot (en when it is empty)")
     cli.add_argument("--env", default="", help="text put in place of {env}")
     cli.add_argument("--status-field", default="statusCode", help="name of the response field that holds a status code")
     cli.add_argument("--fail-status", action="append", help="CODE=message pair that marks a failed response; may be repeated")
@@ -92,7 +92,7 @@ def main():
 
     headers = build_headers(opts)
     fail_status = parse_pairs(opts.fail_status, "--fail-status")
-    langs = parse_langs(opts.langs)
+    langs = parse_langs(opts.langs, snapshot_langs(opts.snapshot_dir))
     os.makedirs(opts.snapshot_dir, exist_ok=True)
 
     total = 0