@harmonyos-arkts/d2h 0.1.20 → 0.1.21

package/skills/d2h-app-solution/scripts/scan_repositories.py

@@ -0,0 +1,350 @@
+#!/usr/bin/env python3
+"""Create a bounded, auditable evidence index for application repositories."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import subprocess
+from collections import Counter, defaultdict
+from pathlib import Path
+
+
+SKIP_DIRS = {
+    ".git", ".gradle", ".idea", ".harmonyos", ".dart_tool", ".swiftpm",
+    "node_modules", "build", "dist", "out", "target", "vendor", "Pods",
+    "DerivedData", "coverage", "generated", "tmp", "temp",
+}
+TEXT_SUFFIXES = {
+    ".kt", ".kts", ".java", ".xml", ".gradle", ".ets", ".ts", ".tsx",
+    ".js", ".jsx", ".swift", ".m", ".mm", ".dart", ".json", ".json5",
+    ".yaml", ".yml", ".toml", ".plist", ".storyboard", ".xib", ".html",
+    ".vue", ".svelte", ".md", ".txt", ".arb", ".properties", ".c", ".cc", ".cpp", ".cxx", ".h", ".hpp",
+}
+SPECIAL_TEXT_NAMES = {"AndroidManifest.xml", "Podfile", "Podfile.lock", "Package.swift", "Package.resolved", "CMakeLists.txt", "Makefile"}
+TEST_PARTS = {"test", "tests", "androidtest", "ohostest", "uitest", "__tests__"}
+DOC_PARTS = {"doc", "docs", "documentation", "user-manual", "manual"}
+
+
+PATTERNS = [
+    ("entry", "android", r"android\.intent\.action\.MAIN|android\.intent\.category\.LAUNCHER"),
+    ("entry", "harmonyos", r"loadContent\s*\(|@Entry\b|srcEntry\b"),
+    ("entry", "ios", r"@main\b|WindowGroup\s*\{|UIApplicationMain"),
+    ("entry", "flutter", r"runApp\s*\(|MaterialApp\s*\(|CupertinoApp\s*\("),
+    ("entry", "react-native", r"AppRegistry\.registerComponent|NavigationContainer"),
+    ("route_declaration", "android", r"\bNavHost\s*\(|\bcomposable\s*\(|<fragment\b|<activity\b|<deepLink\b"),
+    ("route_declaration", "harmonyos", r"NavDestination\s*\(|enum\s+Route|main_pages|\"pages\"\s*:"),
+    ("route_declaration", "ios", r"NavigationStack\s*\{|NavigationLink\s*\(|navigationDestination\s*\("),
+    ("route_declaration", "flutter", r"GoRoute\s*\(|routes\s*:|onGenerateRoute\s*:|AutoRoute\s*\("),
+    ("route_declaration", "react-native", r"\.Screen\s+name=|create(?:Native)?StackNavigator|createBottomTabNavigator|createDrawerNavigator"),
+    ("route_declaration", "web", r"<Route\b|createBrowserRouter\s*\(|defineNuxtRouteMiddleware|pages/"),
+    ("navigation_call", "android", r"\.navigate\s*\(|startActivity\s*\(|findNavController\s*\(|FragmentTransaction|fragmentManager\b"),
+    ("navigation_call", "harmonyos", r"pushPath(?:ByName)?\s*\(|router\.(?:pushUrl|replaceUrl|back)\s*\(|startAbility\s*\("),
+    ("navigation_call", "ios", r"present\s*\(|pushViewController\s*\(|\.sheet\s*\(|fullScreenCover\s*\("),
+    ("navigation_call", "flutter", r"Navigator\.(?:push|pushNamed|pop)|context\.(?:push|go)\s*\("),
+    ("navigation_call", "react-native", r"navigation\.(?:navigate|push|replace|goBack)\s*\(|router\.(?:push|replace|back)\s*\("),
+    ("navigation_call", "web", r"router\.(?:push|replace)\s*\(|navigate\s*\(|<Link\b"),
+    ("ui_interaction", "generic", r"\bonClick\b|\.onClick\s*\(|setOnClickListener|\bclickable\s*\{|\bonTap\b|\bonPressed\b|\bonPress\b|addEventListener\s*\(\s*['\"]click"),
+    ("ui_interaction", "generic", r"\bonSubmit\b|\.onSubmit\s*\(|\bonSelected\b|\.onChange\s*\(|onOptionsItemSelected"),
+    ("tab", "generic", r"\bTabs?\s*\(|TabContent\s*\(|BottomNavigation|NavigationBar\s*\(|createBottomTabNavigator"),
+    ("dialog", "generic", r"AlertDialog|DialogFragment|\.sheet\s*\(|showDialog\s*\(|showModal|Popover"),
+    ("visibility_condition", "generic", r"\bvisible\s*=|android:visible=|\.visibility\b|\benabled\s*=|featureFlag|feature_flag"),
+    ("effect", "generic", r"showToast|Snackbar|save\s*\(|delete\s*\(|remove\s*\(|send\s*\(|share\s*\(|write\s*\(|persist"),
+    ("dependency_declaration", "generic", r"\b(?:implementation|api|compileOnly|runtimeOnly|ksp|classpath)\s*\(|\bpod\s+['\"]|\.package\s*\(|\bdependencies\s*:|\bdev_dependencies\s*:|\"dependencies\"\s*:|\"peerDependencies\"\s*:"),
+    ("native_library", "android", r"System\.loadLibrary\s*\(|ReLinker\.loadLibrary\s*\(|external\s+fun\b|jniLibs|externalNativeBuild|cmake\s*\{|ndkBuild"),
+    ("native_library", "generic", r"JNIEXPORT\b|RegisterNatives\s*\(|\bdlopen\s*\(|\.so\b|XCFramework|\.framework\b"),
+    ("permission_declaration", "android", r"<uses-permission\b|<uses-feature\b|android\.permission\.[A-Z0-9_]+"),
+    ("permission_request", "android", r"requestPermissions\s*\(|ActivityResultContracts\.RequestPermission|ActivityResultContracts\.RequestMultiplePermissions|checkSelfPermission\s*\(|shouldShowRequestPermissionRationale\s*\("),
+    ("tech_stack", "android", r"AndroidView\s*\(|WebView\s*\(|loadUrl\s*\(|addJavascriptInterface\s*\("),
+    ("tech_stack", "flutter", r"MethodChannel\s*\(|EventChannel\s*\(|PlatformViewLink\s*\("),
+    ("tech_stack", "react-native", r"ReactContextBaseJavaModule|RCT_EXPORT_MODULE|NativeModules\.|requireNativeComponent\s*\("),
+]
+COMPILED_PATTERNS = [(kind, fw, re.compile(expr, re.IGNORECASE)) for kind, fw, expr in PATTERNS]
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repo", action="append", required=True, metavar="ID=PATH")
+    parser.add_argument("--output", required=True, type=Path)
+    parser.add_argument("--max-file-bytes", type=int, default=1_500_000)
+    parser.add_argument("--max-records-per-file", type=int, default=500)
+    parser.add_argument("--exclude-tests", action="store_true")
+    parser.add_argument("--all-locales", action="store_true", help="index every locale instead of base/Chinese resources")
+    return parser.parse_args()
+
+
+def parse_repo(value: str) -> tuple[str, Path]:
+    if "=" not in value:
+        raise ValueError(f"--repo must use ID=PATH: {value}")
+    repo_id, raw_path = value.split("=", 1)
+    if not re.fullmatch(r"[a-z][a-z0-9_]*", repo_id):
+        raise ValueError(f"invalid repository ID: {repo_id}")
+    path = Path(raw_path).expanduser().resolve()
+    if not path.is_dir():
+        raise ValueError(f"repository path is not a directory: {path}")
+    return repo_id, path
+
+
+def detect_frameworks(repo: Path, files: list[Path]) -> list[str]:
+    rels = {p.relative_to(repo).as_posix() for p in files}
+    names = {p.name for p in files}
+    suffixes = Counter(p.suffix.lower() for p in files)
+    frameworks: list[str] = []
+    if "AndroidManifest.xml" in names or suffixes[".kt"] + suffixes[".java"] > 0 and any("gradle" in n for n in names):
+        frameworks.append("android")
+    if "module.json5" in names and suffixes[".ets"] > 0:
+        frameworks.append("harmonyos")
+    if "pubspec.yaml" in names and suffixes[".dart"] > 0:
+        frameworks.append("flutter")
+    if suffixes[".swift"] > 0 or any(r.endswith(".xcodeproj/project.pbxproj") for r in rels):
+        frameworks.append("ios")
+    package_files = [p for p in files if p.name == "package.json"]
+    package_text = ""
+    for package in package_files[:5]:
+        package_text += read_text(package, 300_000)
+    if "react-native" in package_text or suffixes[".tsx"] + suffixes[".jsx"] > 0:
+        frameworks.append("react-native")
+    if package_files and any(token in package_text for token in ("next", "nuxt", "vite", "react", "vue", "svelte")):
+        frameworks.append("web")
+    return frameworks or ["generic"]
+
+
+def iter_files(repo: Path, max_bytes: int) -> list[Path]:
+    result: list[Path] = []
+    for root, dirs, files in os.walk(repo):
+        dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS and not d.startswith(".cache"))
+        for name in sorted(files):
+            path = Path(root) / name
+            if path.suffix.lower() not in TEXT_SUFFIXES and name not in SPECIAL_TEXT_NAMES:
+                continue
+            try:
+                if path.stat().st_size <= max_bytes:
+                    result.append(path)
+            except OSError:
+                continue
+    return result
+
+
+def is_secondary_locale(rel: str) -> bool:
+    """Keep base and Chinese resources by default; suppress duplicate translations."""
+    normalized = "/" + rel.replace("\\", "/")
+    android_match = re.search(r"/res/(values-[^/]+)/", normalized)
+    if android_match and "zh" not in android_match.group(1).lower():
+        return True
+    harmony_match = re.search(r"/resources/([^/]+)/element/string\.json$", normalized)
+    if harmony_match:
+        locale = harmony_match.group(1).lower()
+        if locale != "base" and "zh" not in locale:
+            return True
+    name = Path(rel).name.lower()
+    if name.endswith(".arb") and not any(token in name for token in ("_en", "-en", "_zh", "-zh")):
+        return True
+    return False
+
+
+def read_text(path: Path, max_bytes: int) -> str:
+    try:
+        return path.read_bytes()[:max_bytes].decode("utf-8", errors="replace")
+    except OSError:
+        return ""
+
+
+def source_role(rel: str) -> str:
+    parts = {part.lower() for part in Path(rel).parts}
+    if parts & TEST_PARTS or re.search(r"(?:Test|Spec)\.[^.]+$", rel):
+        return "test"
+    if parts & DOC_PARTS or Path(rel).suffix.lower() == ".md":
+        return "doc"
+    if any(part.lower() in {"sample", "samples", "demo", "preview", "catalog", "benchmark"} for part in Path(rel).parts):
+        return "non-production"
+    if "baseline" in Path(rel).name.lower() or "config/detekt/" in rel.lower() or "config/lint/" in rel.lower():
+        return "non-production"
+    return "production"
+
+
+def line_number(text: str, offset: int) -> int:
+    return text.count("\n", 0, offset) + 1
+
+
+def clean_excerpt(value: str, limit: int = 360) -> str:
+    value = re.sub(r"\s+", " ", value).strip()
+    return value[:limit]
+
+
+def record(repo_id: str, framework: str, kind: str, rel: str, line: int, role: str, excerpt: str, symbol: str = "") -> dict:
+    return {
+        "repo_id": repo_id,
+        "framework": framework,
+        "type": kind,
+        "path": rel,
+        "line": line,
+        "source_role": role,
+        "symbol": symbol,
+        "excerpt": clean_excerpt(excerpt),
+    }
+
+
+def source_technology(path: Path, frameworks: list[str]) -> str | None:
+    suffix = path.suffix.lower()
+    mapping = {
+        ".kt": "kotlin", ".kts": "kotlin", ".java": "java",
+        ".c": "cpp", ".cc": "cpp", ".cpp": "cpp", ".cxx": "cpp", ".h": "cpp", ".hpp": "cpp",
+        ".dart": "flutter", ".swift": "swift", ".m": "objective-c", ".mm": "objective-c",
+        ".ets": "arkts", ".ts": "typescript", ".tsx": "typescript", ".js": "javascript", ".jsx": "javascript",
+        ".html": "h5", ".vue": "h5", ".svelte": "h5",
+    }
+    technology = mapping.get(suffix)
+    if technology in {"typescript", "javascript"} and "react-native" in frameworks:
+        return "react-native"
+    return technology
+
+
+def extract_structured_labels(repo_id: str, framework: str, rel: str, role: str, text: str) -> list[dict]:
+    found: list[dict] = []
+    if rel.endswith(".xml"):
+        for match in re.finditer(r"<string\b[^>]*\bname=['\"]([^'\"]+)['\"][^>]*>(.*?)</string>", text, re.I | re.S):
+            value = re.sub(r"<[^>]+>", "", match.group(2))
+            found.append(record(repo_id, framework, "ui_label", rel, line_number(text, match.start()), role, value, match.group(1)))
+        for match in re.finditer(r"<(?:item|Preference\w*)\b.*?(?:android:)?title=['\"]([^'\"]+)['\"].*?>", text, re.I | re.S):
+            tag = match.group(0)
+            id_match = re.search(r"(?:android:)?id=['\"]([^'\"]+)", tag)
+            found.append(record(repo_id, framework, "menu_item", rel, line_number(text, match.start()), role, tag, id_match.group(1) if id_match else match.group(1)))
+    if rel.endswith("string.json"):
+        try:
+            parsed = json.loads(text)
+            for item in parsed.get("string", []):
+                if isinstance(item, dict) and isinstance(item.get("value"), str):
+                    needle = json.dumps(item.get("name", ""), ensure_ascii=False)
+                    offset = text.find(needle)
+                    found.append(record(repo_id, framework, "ui_label", rel, line_number(text, max(offset, 0)), role, item["value"], str(item.get("name", ""))))
+        except json.JSONDecodeError:
+            pass
+    if rel.endswith((".strings", ".properties")) or "/locales/" in rel or "/i18n/" in rel:
+        for match in re.finditer(r"^\s*['\"]?([\w.:-]+)['\"]?\s*[=:]\s*['\"](.+?)['\"]\s*[,;]?\s*$", text, re.M):
+            found.append(record(repo_id, framework, "ui_label", rel, line_number(text, match.start()), role, match.group(2), match.group(1)))
+    return found
+
+
+def git_revision(repo: Path) -> tuple[str, bool | None]:
+    try:
+        revision = subprocess.run(["git", "-C", str(repo), "rev-parse", "HEAD"], check=True, capture_output=True, text=True, timeout=10).stdout.strip()
+        dirty = bool(subprocess.run(["git", "-C", str(repo), "status", "--porcelain"], check=True, capture_output=True, text=True, timeout=15).stdout.strip())
+        return revision, dirty
+    except (OSError, subprocess.SubprocessError):
+        return "working-tree", None
+
+
+def main() -> int:
+    args = parse_args()
+    repos = [parse_repo(value) for value in args.repo]
+    if len({repo_id for repo_id, _ in repos}) != len(repos):
+        raise ValueError("repository IDs must be unique")
+    args.output.mkdir(parents=True, exist_ok=True)
+
+    all_records: list[dict] = []
+    summaries: list[dict] = []
+    candidate_scores: dict[tuple[str, str], Counter] = defaultdict(Counter)
+
+    for repo_id, repo in repos:
+        files = iter_files(repo, args.max_file_bytes)
+        frameworks = detect_frameworks(repo, files)
+        counts = Counter()
+        for path in files:
+            rel = path.relative_to(repo).as_posix()
+            if not args.all_locales and is_secondary_locale(rel):
+                continue
+            role = source_role(rel)
+            if args.exclude_tests and role == "test":
+                continue
+            text = read_text(path, args.max_file_bytes)
+            if not text:
+                continue
+            file_records = extract_structured_labels(repo_id, frameworks[0], rel, role, text)
+            for kind, framework, regex in COMPILED_PATTERNS:
+                if framework != "generic" and framework not in frameworks:
+                    continue
+                for match in regex.finditer(text):
+                    start = text.rfind("\n", 0, match.start()) + 1
+                    end = text.find("\n", match.end())
+                    if end == -1:
+                        end = len(text)
+                    excerpt = text[start:end].strip()
+                    if excerpt.startswith(("//", "<!--", "*", "#")):
+                        continue
+                    file_records.append(record(repo_id, framework, kind, rel, line_number(text, match.start()), role, excerpt))
+                    if len(file_records) >= args.max_records_per_file:
+                        break
+                if len(file_records) >= args.max_records_per_file:
+                    break
+            strong_types = {"entry", "route_declaration", "navigation_call", "ui_interaction", "menu_item", "tab", "dialog", "effect"}
+            technology = source_technology(path, frameworks)
+            if technology and any(item["type"] in strong_types for item in file_records):
+                file_records.append(record(repo_id, frameworks[0], "tech_stack", rel, 1, role, f"source language/framework: {technology}", technology))
+            unique = {}
+            for item in file_records[: args.max_records_per_file]:
+                key = (item["type"], item["line"], item["symbol"], item["excerpt"])
+                unique[key] = item
+            for item in unique.values():
+                all_records.append(item)
+                counts[item["type"]] += 1
+                weight = {
+                    "entry": 8, "route_declaration": 7, "navigation_call": 7,
+                    "menu_item": 6, "ui_interaction": 5, "tab": 5,
+                    "dialog": 4, "effect": 1, "visibility_condition": 1,
+                    "ui_label": 0, "tech_stack": 1, "dependency_declaration": 2,
+                    "native_library": 3, "permission_declaration": 2, "permission_request": 3,
+                }.get(item["type"], 1)
+                if role == "production":
+                    candidate_scores[(repo_id, rel)]["score"] += weight
+                candidate_scores[(repo_id, rel)][item["type"]] += 1
+        revision, dirty = git_revision(repo)
+        fingerprint = hashlib.sha256("\n".join(sorted(p.relative_to(repo).as_posix() for p in files)).encode()).hexdigest()[:16]
+        summaries.append({
+            "id": repo_id,
+            "path": str(repo),
+            "frameworks": frameworks,
+            "revision": revision,
+            "dirty": dirty,
+            "scanned_files": len(files),
+            "file_list_fingerprint": fingerprint,
+            "evidence_counts": dict(sorted(counts.items())),
+        })
+
+    all_records.sort(key=lambda item: (item["repo_id"], item["path"], item["line"], item["type"]))
+    with (args.output / "evidence.jsonl").open("w", encoding="utf-8") as handle:
+        for item in all_records:
+            handle.write(json.dumps(item, ensure_ascii=False) + "\n")
+    (args.output / "scan-summary.json").write_text(json.dumps({"schema_version": "1.1", "repositories": summaries, "evidence_records": len(all_records)}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+    header = ["repo_id", "score", "path", "entry", "route", "navigation", "interaction", "menu", "label", "effect"]
+    rows = ["\t".join(header)]
+    for (repo_id, rel), counter in sorted(candidate_scores.items(), key=lambda item: (-item[1]["score"], item[0])):
+        strong = sum(counter[kind] for kind in ("entry", "route_declaration", "navigation_call", "ui_interaction", "menu_item", "tab", "dialog"))
+        if counter["score"] <= 0 or strong <= 0:
+            continue
+        rows.append("\t".join(map(str, [
+            repo_id, counter["score"], rel, counter["entry"], counter["route_declaration"],
+            counter["navigation_call"], counter["ui_interaction"], counter["menu_item"],
+            counter["ui_label"], counter["effect"],
+        ])))
+    (args.output / "candidate-files.tsv").write_text("\n".join(rows) + "\n", encoding="utf-8")
+    technical_header = ["repo_id", "score", "path", "tech_stack", "dependency", "native_library", "permission_declaration", "permission_request"]
+    technical_rows = ["\t".join(technical_header)]
+    for (repo_id, rel), counter in sorted(candidate_scores.items(), key=lambda item: (-item[1]["score"], item[0])):
+        technical_count = sum(counter[kind] for kind in ("tech_stack", "dependency_declaration", "native_library", "permission_declaration", "permission_request"))
+        if technical_count <= 0:
+            continue
+        technical_rows.append("\t".join(map(str, [
+            repo_id, counter["score"], rel, counter["tech_stack"], counter["dependency_declaration"],
+            counter["native_library"], counter["permission_declaration"], counter["permission_request"],
+        ])))
+    (args.output / "technical-files.tsv").write_text("\n".join(technical_rows) + "\n", encoding="utf-8")
+    print(json.dumps({"output": str(args.output.resolve()), "repositories": len(repos), "evidence_records": len(all_records)}, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())