@agentunion/kite 1.4.0 → 1.6.0

package/launcher/count_lines.py

@@ -2,6 +2,7 @@
 """
 代码行数统计工具
 统计 Kite 项目的代码行数并记录到 JSONL 文件
+支持文件级缓存，只统计修改过的文件
 """
 import json
 import sys
@@ -18,6 +19,65 @@ if sys.platform == "win32":
         pass
 
 
+# 排除的目录（防御性措施）
+EXCLUDE_DIRS = {
+    # Python 相关
+    "__pycache__",
+    ".venv",
+    "venv",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".tox",
+    ".eggs",
+    "*.egg-info",
+    # JavaScript 相关
+    "node_modules",
+    # 构建产物
+    "dist",
+    "build",
+    # 版本控制
+    ".git",
+    ".svn",
+    ".hg",
+    # IDE 配置
+    ".idea",
+    ".vscode",
+    # 项目特定
+    ".dev",          # 开发相关（变更日志、发布备份）
+    ".claude",       # Claude 工作目录
+}
+
+
+def load_cache(cache_file: Path) -> dict:
+    """加载文件缓存
+
+    缓存格式: {
+        "file_path": {
+            "mtime": 1234567890.123,
+            "lines": 100
+        }
+    }
+    """
+    if not cache_file.exists():
+        return {}
+
+    try:
+        with open(cache_file, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError):
+        return {}
+
+
+def save_cache(cache: dict, cache_file: Path):
+    """保存文件缓存"""
+    try:
+        cache_file.parent.mkdir(parents=True, exist_ok=True)
+        with open(cache_file, "w", encoding="utf-8") as f:
+            json.dump(cache, f, ensure_ascii=False, indent=2)
+    except OSError:
+        pass
+
+
 def count_lines_in_file(file_path: Path) -> int:
     """统计单个文件的行数"""
     try:
@@ -27,61 +87,138 @@ def count_lines_in_file(file_path: Path) -> int:
         return 0
 
 
-def count_lines(root_dir: Path, pattern: str, exclude_dirs: set[str]) -> int:
-    """统计指定模式的文件行数"""
+def count_lines_cached(root_dir: Path, pattern: str, exclude_dirs: set[str], cache: dict) -> tuple[int, dict]:
+    """统计指定模式的文件行数（使用缓存）
+
+    Returns:
+        (total_lines, updated_cache)
+    """
     total = 0
+    updated_cache = {}
+    cache_hits = 0
+    cache_misses = 0
+
     for file_path in root_dir.rglob(pattern):
         # 检查是否在排除目录中
         if any(excluded in file_path.parts for excluded in exclude_dirs):
             continue
-        total += count_lines_in_file(file_path)
-    return total
 
+        # 获取文件修改时间
+        try:
+            mtime = file_path.stat().st_mtime
+        except OSError:
+            continue
 
-def count_all(root_dir: Path) -> dict:
-    """统计所有类型的代码行数"""
-    stats = {}
+        # 转换为相对路径作为缓存键
+        rel_path = str(file_path.relative_to(root_dir))
 
-    # 排除的目录
-    exclude_dirs = {
-        "__pycache__",
-        "node_modules",
-        ".git",
-        ".venv",
-        "venv",
-        ".idea",
-        ".vscode",
-        ".dev",          # 开发相关（变更日志、发布备份）
-        ".claude",       # Claude 工作目录
-    }
+        # 检查缓存
+        cached = cache.get(rel_path)
+        if cached and cached.get("mtime") == mtime:
+            # 缓存命中
+            lines = cached["lines"]
+            cache_hits += 1
+        else:
+            # 缓存未命中，重新统计
+            lines = count_lines_in_file(file_path)
+            cache_misses += 1
 
-    # Python 代码
-    stats["python"] = count_lines(root_dir, "*.py", exclude_dirs)
+        total += lines
+        updated_cache[rel_path] = {"mtime": mtime, "lines": lines}
 
-    # JavaScript 代码
-    stats["javascript"] = count_lines(root_dir, "*.js", exclude_dirs)
+    return total, updated_cache
 
-    # Markdown 文档
-    stats["markdown"] = count_lines(root_dir, "*.md", exclude_dirs)
 
-    # YAML 配置
-    stats["yaml"] = count_lines(root_dir, "*.yaml", exclude_dirs) + \
-                    count_lines(root_dir, "*.yml", exclude_dirs)
+def count_all(root_dir: Path, cache_file: Path) -> tuple[dict, dict]:
+    """统计所有类型的代码行数（使用缓存）
 
-    # JSON 配置（排除 package-lock.json）
-    json_total = 0
-    for file_path in root_dir.rglob("*.json"):
-        if any(excluded in file_path.parts for excluded in exclude_dirs):
-            continue
-        if file_path.name == "package-lock.json":
-            continue
-        json_total += count_lines_in_file(file_path)
-    stats["json"] = json_total
+    Returns:
+        (stats, cache_info)
+    """
+    import time
 
-    # 总计
+    # 加载缓存
+    t0 = time.time()
+    cache = load_cache(cache_file)
+    print(f"DEBUG: 加载缓存耗时 {time.time()-t0:.3f}秒")
+
+    # 一次性遍历所有文件，按扩展名分类（用 os.scandir 跳过排除目录）
+    import os
+    t0 = time.time()
+    target_exts = {'.py', '.js', '.md', '.yaml', '.yml', '.json'}
+    files_by_ext = {ext: [] for ext in target_exts}
+
+    def _scan(dir_path):
+        try:
+            entries = os.scandir(dir_path)
+        except PermissionError:
+            return
+        with entries:
+            for entry in entries:
+                if entry.is_dir(follow_symlinks=False):
+                    if entry.name not in EXCLUDE_DIRS:
+                        _scan(entry.path)
+                elif entry.is_file(follow_symlinks=False):
+                    ext = os.path.splitext(entry.name)[1].lower()
+                    if ext in target_exts:
+                        files_by_ext[ext].append(Path(entry.path))
+
+    _scan(str(root_dir))
+    print(f"DEBUG: 目录遍历耗时 {time.time()-t0:.3f}秒")
+
+    stats = {}
+    new_cache = {}
+
+    # 统计各类型文件
+    def count_files(files, label):
+        t = time.time()
+        total = 0
+        for file_path in files:
+            if file_path.name == "package-lock.json":
+                continue
+
+            try:
+                mtime = file_path.stat().st_mtime
+            except OSError:
+                continue
+
+            rel_path = str(file_path.relative_to(root_dir))
+            cached = cache.get(rel_path)
+
+            if cached and cached.get("mtime") == mtime:
+                lines = cached["lines"]
+            else:
+                lines = count_lines_in_file(file_path)
+
+            total += lines
+            new_cache[rel_path] = {"mtime": mtime, "lines": lines}
+
+        print(f"DEBUG: {label}统计耗时 {time.time()-t:.3f}秒 ({len(files)}个文件)")
+        return total
+
+    stats["python"] = count_files(files_by_ext['.py'], "Python")
+    stats["javascript"] = count_files(files_by_ext['.js'], "JavaScript")
+    stats["markdown"] = count_files(files_by_ext['.md'], "Markdown")
+    stats["yaml"] = count_files(files_by_ext['.yaml'], "YAML") + count_files(files_by_ext['.yml'], "YML")
+    stats["json"] = count_files(files_by_ext['.json'], "JSON")
     stats["total"] = sum(stats.values())
 
-    return stats
+    # 计算缓存命中率
+    total_files = len(new_cache)
+    cached_files = sum(1 for k in new_cache if k in cache and cache[k].get("mtime") == new_cache[k]["mtime"])
+    cache_info = {
+        "total_files": total_files,
+        "cache_hits": cached_files,
+        "cache_misses": total_files - cached_files,
+        "hit_rate": f"{cached_files / total_files * 100:.1f}%" if total_files > 0 else "0%"
+    }
+
+    # 保存新缓存
+    t0 = time.time()
+    save_cache(new_cache, cache_file)
+    print(f"DEBUG: 保存缓存耗时 {time.time()-t0:.3f}秒")
+
+    return stats, cache_info
 
 
 def save_record(stats: dict, record_file: Path):
@@ -113,10 +250,11 @@ def save_record(stats: dict, record_file: Path):
         f.write(json.dumps(record, ensure_ascii=False) + "\n")
 
 
-def print_stats(stats: dict):
+def print_stats(stats: dict, cache_info: dict = None):
     """打印统计结果"""
     # ANSI 颜色代码
     BOLD = "\033[1m"
+    DIM = "\033[2m"
     RESET = "\033[0m"
 
     print("\n" + "=" * 50)
@@ -129,6 +267,8 @@ def print_stats(stats: dict):
     print(f"JSON 配置:        {BOLD}{stats['json']:>8,} 行{RESET}")
     print("-" * 50)
     print(f"总计:             {BOLD}{stats['total']:>8,} 行{RESET}")
+    if cache_info:
+        print(f"{DIM}缓存命中率:       {cache_info['hit_rate']} ({cache_info['cache_hits']}/{cache_info['total_files']}){RESET}")
     print("=" * 50 + "\n")
 
 
@@ -222,18 +362,25 @@ def show_history(record_file: Path, limit: int = 10):
 
 def run_stats():
     """Run code stats from main.py entry point (simplified output)."""
+    import time
+    start_time = time.time()
+
     script_dir = Path(__file__).parent
     root_dir = script_dir.parent
     record_file = root_dir / "data" / "stats" / "lines.jsonl"
+    cache_file = root_dir / "data" / "stats" / "cache.json"
 
     print("[launcher] 正在统计代码行数...")
     try:
-        stats = count_all(root_dir)
+        stats, cache_info = count_all(root_dir, cache_file)
+        elapsed = time.time() - start_time
+        print(f"[launcher] 统计耗时: {elapsed:.2f}秒")
         save_record(stats, record_file)
 
         # Print stats
         BRIGHT_GREEN = "\033[92m"
         BOLD = "\033[1m"
+        DIM = "\033[2m"
         RESET = "\033[0m"
 
         print("")
@@ -247,6 +394,7 @@ def run_stats():
         print(f"JSON 配置:        {BRIGHT_GREEN}{stats['json']:>8,}{RESET} 行")
         print("-" * 50)
         print(f"总计:             {BOLD}{BRIGHT_GREEN}{stats['total']:>8,}{RESET} 行")
+        print(f"{DIM}缓存命中率:       {cache_info['hit_rate']} ({cache_info['cache_hits']}/{cache_info['total_files']}){RESET}")
         print("=" * 50)
         print("")
 
@@ -263,6 +411,7 @@ def main():
 
     # 记录文件路径
     record_file = root_dir / "data" / "stats" / "lines.jsonl"
+    cache_file = root_dir / "data" / "stats" / "cache.json"
 
     # 解析命令行参数
     show_hist = "--history" in sys.argv or "-h" in sys.argv
@@ -276,14 +425,14 @@ def main():
     # 统计代码行数
     if not quiet:
         print("正在统计代码行数...")
-    stats = count_all(root_dir)
+    stats, cache_info = count_all(root_dir, cache_file)
 
     # 保存记录
     save_record(stats, record_file)
 
     # 打印结果
     if not quiet:
-        print_stats(stats)
+        print_stats(stats, cache_info)
         # 显示最近 8 次记录
         show_history(record_file, limit=8)