@mmerterden/multi-agent-pipeline 8.6.0

package/pipeline/skills/figma-ios/figma-to-component/scripts/backfill-status.py

@@ -0,0 +1,609 @@
+#!/usr/bin/env python3
+"""backfill-status.py — populate status blocks in existing registry files.
+
+Walks every JSON in the registry directory and, for each component,
+infers the initial `status.ios.*` block from filesystem signals:
+
+- status.ios.implemented:
+    True if a Swift source file with the component's iOS name exists
+    anywhere under the uicomponents Sources tree. When set, version
+    starts at 1.
+
+- status.ios.codeConnect:
+    Mirrors the legacy top-level `codeConnect.ios` field. Only set if
+    the legacy value was True. Recorded against the current
+    impl.version — so if impl was just set to 1 in this same run,
+    codeConnect lands green (against=1), not yellow.
+
+- status.ios.tested and status.ios.wiki:
+    NOT backfilled in this pass. These signals live in separate repos
+    and trees, and the heuristics are fragile. They'll naturally
+    populate on the next pipeline run of each component.
+
+- status.android.*:
+    NOT backfilled. Android source tree and component index are
+    out-of-scope for tonight. Future enhancement.
+
+The legacy top-level `codeConnect.{ios,android}` field is left intact.
+set_flag's built-in mirror keeps the two representations in sync going
+forward, and the backfill never explicitly downgrades anything.
+
+Usage:
+    # See what would change — no writes
+    python3 backfill-status.py \\
+        --registry-dir /path/to/Shared/Figma/Components \\
+        --sources-dir  /path/to/uicomponents/Sources \\
+        --dry-run
+
+    # Smoke test on 5 entries
+    python3 backfill-status.py \\
+        --registry-dir ... --sources-dir ... \\
+        --dry-run --limit 5
+
+    # Real backfill (only run after you've reviewed a --dry-run)
+    python3 backfill-status.py \\
+        --registry-dir ... --sources-dir ...
+
+Exit codes:
+    0 — Success (full walk completed)
+    1 — Error (bad paths, read/write failure)
+"""
+
+import argparse
+import json
+import subprocess
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+
+sys.path.insert(0, str(Path(__file__).parent))
+from lib.registry_writer import set_flag, set_implemented  # noqa: E402
+
+
+def find_repo_root() -> Path:
+    p = Path.cwd()
+    while p != p.parent:
+        if (p / ".git").exists():
+            return p
+        p = p.parent
+    return Path.cwd()
+
+
+def default_registry_dir() -> Path:
+    return (
+        find_repo_root()
+        / "{repos.packagesContainer}"
+        / "Packages"
+        / "{project.slug}-common"
+        / "Shared"
+        / "Figma"
+        / "Components"
+    )
+
+
+def default_sources_dir() -> Path:
+    return (
+        find_repo_root()
+        / "{repos.packagesContainer}"
+        / "Packages"
+        / "{project.slug}-components"
+        / "Sources"
+    )
+
+
+def default_tests_dir() -> Path:
+    return (
+        find_repo_root()
+        / "{repos.packagesContainer}"
+        / "Packages"
+        / "{project.slug}-components"
+        / "Tests"
+    )
+
+
+def default_wiki_dir() -> Path:
+    return (
+        find_repo_root()
+        / "{repos.packagesContainer}"
+        / "Packages"
+        / "{project.slug}-wiki"
+    )
+
+
+def default_uicomponents_repo_dir() -> Path:
+    return (
+        find_repo_root()
+        / "{repos.packagesContainer}"
+        / "Packages"
+        / "{project.slug}-components"
+    )
+
+
+# --------------------------------------------------------------------
+# Source-tree indexing (separate from backfill for testability)
+# --------------------------------------------------------------------
+
+def build_swift_file_index(sources_dir: Path) -> Dict[str, Path]:
+    """Walk the Sources tree and return { base_name: first_matching_path }.
+
+    Base name is the filename without the .swift extension.
+    Only the first match is kept if the same name appears twice
+    (unlikely in a well-organized source tree).
+    """
+    index: Dict[str, Path] = {}
+    if not sources_dir.exists():
+        return index
+
+    for swift in sources_dir.rglob("*.swift"):
+        base = swift.stem
+        if base not in index:
+            index[base] = swift
+    return index
+
+
+# Test-file name suffixes we recognize as "this component has tests".
+TEST_SUFFIXES = (
+    "SnapshotTests",
+    "Tests",
+    "XCTests",
+    "ViewInspectorTests",
+    "UnitTests",
+)
+
+
+def build_test_file_index(tests_dir: Path) -> Dict[str, List[Path]]:
+    """Walk the Tests tree and return { ComponentName: [test_path, ...] }.
+
+    For each `{ComponentName}{suffix}.swift` we find, strip the suffix
+    and index under the stripped component name. A component is
+    considered tested if at least one test file maps to it.
+    """
+    index: Dict[str, List[Path]] = {}
+    if not tests_dir.exists():
+        return index
+
+    for swift in tests_dir.rglob("*.swift"):
+        stem = swift.stem
+        for suffix in TEST_SUFFIXES:
+            if stem.endswith(suffix) and len(stem) > len(suffix):
+                comp = stem[: -len(suffix)]
+                index.setdefault(comp, []).append(swift)
+                break
+    return index
+
+
+def build_wiki_index(wiki_dir: Path) -> Dict[str, Path]:
+    """Walk the wiki tree and return { ComponentName: path_to_md_file }.
+
+    Matches `**/{ComponentName}/{ComponentName}.md` (the canonical
+    per-component layout used in the uicomponents-wiki submodule).
+    Also accepts `**/{ComponentName}.md` as a fallback for flat
+    structures.
+    """
+    index: Dict[str, Path] = {}
+    if not wiki_dir.exists():
+        return index
+
+    # Prefer per-component directories first
+    for md in wiki_dir.rglob("*.md"):
+        stem = md.stem
+        parent_name = md.parent.name
+        if parent_name == stem:
+            # {Component}/{Component}.md — canonical
+            index.setdefault(stem, md)
+        elif stem not in index:
+            # {Component}.md at any depth — fallback
+            index[stem] = md
+    return index
+
+
+def detect_primary_author(
+    source_file: Path,
+    repo_root: Path,
+) -> Optional[str]:
+    """Return the git author (most commits on `source_file`) inside `repo_root`.
+
+    Runs `git -C repo_root log --format=%an -- <rel_path>`, counts
+    unique authors, picks the one with the most commits.
+
+    Returns None if:
+      - source file doesn't exist
+      - git command fails
+      - no commits touch the file
+      - author name is empty
+    """
+    if not source_file.exists():
+        return None
+    try:
+        rel = source_file.relative_to(repo_root)
+    except ValueError:
+        rel = source_file
+    try:
+        r = subprocess.run(
+            ["git", "-C", str(repo_root), "log", "--format=%an", "--", str(rel)],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=10,
+        )
+    except (subprocess.SubprocessError, FileNotFoundError):
+        return None
+    if r.returncode != 0:
+        return None
+    lines = [line.strip() for line in r.stdout.splitlines() if line.strip()]
+    if not lines:
+        return None
+    counter = Counter(lines)
+    author, _ = counter.most_common(1)[0]
+    return author or None
+
+
+# --------------------------------------------------------------------
+# Per-entry decision logic (pure, testable)
+# --------------------------------------------------------------------
+
+def _safe_dict(value) -> dict:
+    """Return value if it's a dict, otherwise an empty dict.
+
+    Real registry data in the wild contains a small number of legacy
+    entries where fields like `status` are strings (e.g. "implemented")
+    or `componentName` is a bare string. This helper lets the decision
+    logic treat those the same as missing/empty without raising.
+    """
+    return value if isinstance(value, dict) else {}
+
+
+def _component_name_ios(raw) -> Optional[str]:
+    """Handle both shapes: {"ios": "X", "android": "X"} and bare "X"."""
+    if isinstance(raw, dict):
+        return raw.get("ios")
+    if isinstance(raw, str) and raw:
+        return raw
+    return None
+
+
+def decide_backfill(
+    entry: dict,
+    swift_index: Dict[str, Path],
+    test_index: Optional[Dict[str, List[Path]]] = None,
+    wiki_index: Optional[Dict[str, Path]] = None,
+) -> dict:
+    """Decide what the backfill should do for one registry entry.
+
+    Pure function — no I/O. Returns a dict describing the intended
+    action per flag, used by both dry-run and real-write paths.
+
+    test_index and wiki_index are optional for backward compat — when
+    None, tested/wiki detection is skipped and only implemented/cc are
+    considered (original Step 5 behavior).
+
+    Returned shape (added fields are marked NEW):
+        {
+            "node_id": str,
+            "component_name_ios": Optional[str],
+            "has_status_block": bool,
+            "impl_action": ...,
+            "codeconnect_action": ...,
+            "tested_action": ...,        # NEW — "set" | "skip_*"
+            "wiki_action": ...,          # NEW — "set" | "skip_*"
+            "swift_path": Optional[str],
+            "test_paths": List[str],     # NEW — 0+ paths that match the component
+            "wiki_path": Optional[str],  # NEW — the detected wiki .md file
+        }
+    """
+    node_id = entry.get("nodeId")
+    component_name = _component_name_ios(entry.get("componentName"))
+
+    # Legacy shape: `status` was a bare string like "implemented". Leave
+    # these for manual migration — do not overwrite with the new block.
+    raw_status = entry.get("status")
+    if isinstance(raw_status, str):
+        return {
+            "node_id": node_id,
+            "component_name_ios": component_name,
+            "has_status_block": True,
+            "impl_action": "skip_legacy_status_string",
+            "codeconnect_action": "skip_legacy_status_string",
+            "tested_action": "skip_legacy_status_string",
+            "wiki_action": "skip_legacy_status_string",
+            "swift_path": None,
+            "test_paths": [],
+            "wiki_path": None,
+        }
+
+    status_block = _safe_dict(raw_status)
+    ios_status = _safe_dict(status_block.get("ios"))
+    implemented_block = _safe_dict(ios_status.get("implemented"))
+    has_status = bool(implemented_block.get("value"))
+
+    legacy_cc_ios = bool(_safe_dict(entry.get("codeConnect")).get("ios"))
+    skip = bool(entry.get("skip"))
+
+    result: dict = {
+        "node_id": node_id,
+        "component_name_ios": component_name,
+        "has_status_block": has_status,
+        "impl_action": None,
+        "codeconnect_action": None,
+        "tested_action": None,
+        "wiki_action": None,
+        "swift_path": None,
+        "test_paths": [],
+        "wiki_path": None,
+    }
+
+    # Skip entries marked skip=true
+    if skip:
+        for k in ("impl_action", "codeconnect_action", "tested_action", "wiki_action"):
+            result[k] = "skip_marked_skip"
+        return result
+
+    if not component_name:
+        for k in ("impl_action", "codeconnect_action", "tested_action", "wiki_action"):
+            result[k] = "skip_no_name" if k == "impl_action" else "skip_no_impl"
+        return result
+
+    # tested / wiki can be set independently of impl (they're orthogonal
+    # signals) but we compute them regardless so the decision dict is
+    # fully populated for the report.
+    if test_index is not None:
+        test_paths = test_index.get(component_name, [])
+        if test_paths:
+            result["test_paths"] = [str(p) for p in test_paths]
+    if wiki_index is not None:
+        wiki_path = wiki_index.get(component_name)
+        if wiki_path:
+            result["wiki_path"] = str(wiki_path)
+
+    # If the new status block already has implemented.value=true, don't
+    # overwrite the impl/cc flags. tested/wiki can still be added if
+    # they haven't been populated yet (backfill is additive for those).
+    if has_status:
+        result["impl_action"] = "skip_existing_status"
+        result["codeconnect_action"] = "skip_existing_status"
+        # Populate swift_path even when skipping impl — author detection
+        # needs it, and we want to be able to fill in implementedBy for
+        # components that were backfilled before author detection existed.
+        swift_path = swift_index.get(component_name) if component_name else None
+        if swift_path:
+            result["swift_path"] = str(swift_path)
+        # tested/wiki: still set them if not already set
+        tested_set = bool(_safe_dict(ios_status.get("tested")).get("value"))
+        wiki_set = bool(_safe_dict(ios_status.get("wiki")).get("value"))
+        if result["test_paths"] and not tested_set:
+            result["tested_action"] = "set"
+        else:
+            result["tested_action"] = "skip_existing_status" if tested_set else "skip_no_tests"
+        if result["wiki_path"] and not wiki_set:
+            result["wiki_action"] = "set"
+        else:
+            result["wiki_action"] = "skip_existing_status" if wiki_set else "skip_no_wiki"
+        return result
+
+    # Fresh entry — check swift file for implementation
+    swift_path = swift_index.get(component_name)
+    if not swift_path:
+        result["impl_action"] = "skip_no_swift"
+        result["codeconnect_action"] = "skip_no_impl"
+        result["tested_action"] = "skip_no_impl"
+        result["wiki_action"] = "skip_no_impl"
+        return result
+
+    result["impl_action"] = "set"
+    result["swift_path"] = str(swift_path)
+    result["codeconnect_action"] = "set" if legacy_cc_ios else "skip_legacy_false"
+    result["tested_action"] = "set" if result["test_paths"] else "skip_no_tests"
+    result["wiki_action"] = "set" if result["wiki_path"] else "skip_no_wiki"
+
+    return result
+
+
+# --------------------------------------------------------------------
+# Backfill execution
+# --------------------------------------------------------------------
+
+def apply_decision(
+    registry_dir: Path,
+    decision: dict,
+    uicomponents_repo: Optional[Path] = None,
+) -> None:
+    """Execute the decision (skip if action is 'skip_*').
+
+    Writes are atomic per flag — a crash between writes leaves a
+    partially-backfilled entry, recoverable on re-run. Order:
+        1. set_implemented (establishes version=1)
+        2. set_implemented_by (author)
+        3. set_flag codeConnect (if legacy was true)
+        4. set_flag tested (if tests detected)
+        5. set_flag wiki (if wiki detected)
+    """
+    from lib.registry_writer import set_implemented_by  # local to avoid circular
+
+    node_id = decision["node_id"]
+
+    if decision.get("impl_action") == "set":
+        set_implemented(
+            registry_dir=registry_dir,
+            node_id=node_id,
+            platform="ios",
+            updater="backfill-status",
+            source_file=None,
+        )
+
+    # Author detection — only meaningful when we actually set impl OR
+    # impl was already set (so the registry entry exists and we just
+    # want to populate the missing field).
+    if uicomponents_repo and decision.get("swift_path"):
+        swift_file = Path(decision["swift_path"])
+        author = detect_primary_author(swift_file, uicomponents_repo)
+        if author:
+            set_implemented_by(
+                registry_dir=registry_dir,
+                node_id=node_id,
+                platform="ios",
+                author=author,
+                updater="backfill-status",
+            )
+            decision["author"] = author
+
+    if decision.get("codeconnect_action") == "set":
+        set_flag(
+            registry_dir=registry_dir,
+            node_id=node_id,
+            platform="ios",
+            flag="codeConnect",
+            updater="backfill-status",
+        )
+
+    if decision.get("tested_action") == "set":
+        set_flag(
+            registry_dir=registry_dir,
+            node_id=node_id,
+            platform="ios",
+            flag="tested",
+            updater="backfill-status",
+        )
+
+    if decision.get("wiki_action") == "set":
+        set_flag(
+            registry_dir=registry_dir,
+            node_id=node_id,
+            platform="ios",
+            flag="wiki",
+            updater="backfill-status",
+        )
+
+
+def run_backfill(
+    registry_dir: Path,
+    sources_dir: Path,
+    dry_run: bool,
+    limit: Optional[int] = None,
+    tests_dir: Optional[Path] = None,
+    wiki_dir: Optional[Path] = None,
+    uicomponents_repo: Optional[Path] = None,
+) -> dict:
+    """Top-level backfill. Returns a summary dict."""
+    swift_index = build_swift_file_index(sources_dir)
+    test_index = build_test_file_index(tests_dir) if tests_dir else {}
+    wiki_index = build_wiki_index(wiki_dir) if wiki_dir else {}
+
+    all_files = sorted(registry_dir.glob("*.json"))
+    if limit is not None:
+        all_files = all_files[:limit]
+
+    counts: Dict[str, int] = {}
+    flag_counts = {
+        "impl_set": 0,
+        "tested_set": 0,
+        "codeconnect_set": 0,
+        "wiki_set": 0,
+        "author_set": 0,
+    }
+    decisions: List[dict] = []
+    errors: List[dict] = []
+
+    for path in all_files:
+        try:
+            entry = json.loads(path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError) as e:
+            errors.append({"file": path.name, "error": f"read: {type(e).__name__}: {e}"})
+            continue
+
+        decision = decide_backfill(entry, swift_index, test_index, wiki_index)
+        decisions.append(decision)
+
+        key = f"{decision['impl_action']}/{decision['codeconnect_action']}/{decision['tested_action']}/{decision['wiki_action']}"
+        counts[key] = counts.get(key, 0) + 1
+
+        # per-flag set counts (dry-run accurate)
+        for flag_name, counter_key in (
+            ("impl_action", "impl_set"),
+            ("tested_action", "tested_set"),
+            ("codeconnect_action", "codeconnect_set"),
+            ("wiki_action", "wiki_set"),
+        ):
+            if decision.get(flag_name) == "set":
+                flag_counts[counter_key] += 1
+
+        if not dry_run:
+            try:
+                apply_decision(registry_dir, decision, uicomponents_repo=uicomponents_repo)
+                if decision.get("author"):
+                    flag_counts["author_set"] += 1
+            except (OSError, ValueError) as e:
+                errors.append({
+                    "file": path.name,
+                    "node_id": decision["node_id"],
+                    "error": f"write: {type(e).__name__}: {e}",
+                })
+
+    return {
+        "status": "ok" if not errors else "partial",
+        "dry_run": dry_run,
+        "registry_dir": str(registry_dir),
+        "sources_dir": str(sources_dir),
+        "tests_dir": str(tests_dir) if tests_dir else None,
+        "wiki_dir": str(wiki_dir) if wiki_dir else None,
+        "total_files": len(all_files),
+        "swift_files_indexed": len(swift_index),
+        "test_files_indexed": len(test_index),
+        "wiki_files_indexed": len(wiki_index),
+        "flag_set_counts": flag_counts,
+        "action_counts": counts,
+        "errors": errors,
+        "sample_decisions": decisions[:5],
+    }
+
+
+# --------------------------------------------------------------------
+# CLI
+# --------------------------------------------------------------------
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Backfill status blocks in existing registry entries",
+    )
+    parser.add_argument("--registry-dir", help="Override registry path")
+    parser.add_argument("--sources-dir", help="Override uicomponents Sources path")
+    parser.add_argument("--tests-dir", help="Override uicomponents Tests path")
+    parser.add_argument("--wiki-dir", help="Override uicomponents-wiki path")
+    parser.add_argument("--uicomponents-repo", help="Override uicomponents repo root (for git-log author detection)")
+    parser.add_argument("--skip-authors", action="store_true", help="Skip git log author detection (fast mode)")
+    parser.add_argument("--dry-run", action="store_true", help="No writes, print summary only")
+    parser.add_argument("--limit", type=int, help="Process at most N registry files (smoke-test safety)")
+    args = parser.parse_args()
+
+    registry_dir = Path(args.registry_dir) if args.registry_dir else default_registry_dir()
+    sources_dir = Path(args.sources_dir) if args.sources_dir else default_sources_dir()
+    tests_dir = Path(args.tests_dir) if args.tests_dir else default_tests_dir()
+    wiki_dir = Path(args.wiki_dir) if args.wiki_dir else default_wiki_dir()
+    uicomponents_repo = None
+    if not args.skip_authors:
+        uicomponents_repo = (
+            Path(args.uicomponents_repo) if args.uicomponents_repo else default_uicomponents_repo_dir()
+        )
+
+    if not registry_dir.exists():
+        print(f"ERROR: registry directory does not exist: {registry_dir}", file=sys.stderr)
+        return 1
+    if not sources_dir.exists():
+        print(f"ERROR: sources directory does not exist: {sources_dir}", file=sys.stderr)
+        return 1
+
+    summary = run_backfill(
+        registry_dir=registry_dir,
+        sources_dir=sources_dir,
+        dry_run=args.dry_run,
+        limit=args.limit,
+        tests_dir=tests_dir,
+        wiki_dir=wiki_dir,
+        uicomponents_repo=uicomponents_repo,
+    )
+    print(json.dumps(summary, indent=2, ensure_ascii=False))
+    return 0 if summary["status"] == "ok" else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())