@agentunion/kite 1.4.0 → 1.6.0

package/scripts/python-env.js

@@ -0,0 +1,79 @@
+/**
+ * Python 环境工具模块
+ *
+ * 提供获取虚拟环境 Python 路径的工具函数
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+/**
+ * 获取 KITE_DATA 路径
+ */
+function getKiteDataPath() {
+  return path.join(os.homedir(), '.kite');
+}
+
+/**
+ * 获取虚拟环境路径
+ */
+function getVenvPath() {
+  return path.join(getKiteDataPath(), 'venv');
+}
+
+/**
+ * 获取虚拟环境的 Python 可执行文件路径
+ */
+function getVenvPython() {
+  const venvPath = getVenvPath();
+
+  if (process.platform === 'win32') {
+    return path.join(venvPath, 'Scripts', 'python.exe');
+  } else {
+    return path.join(venvPath, 'bin', 'python');
+  }
+}
+
+/**
+ * 获取虚拟环境的 pip 可执行文件路径
+ */
+function getVenvPip() {
+  const venvPath = getVenvPath();
+
+  if (process.platform === 'win32') {
+    return path.join(venvPath, 'Scripts', 'pip.exe');
+  } else {
+    return path.join(venvPath, 'bin', 'pip');
+  }
+}
+
+/**
+ * 检查虚拟环境是否存在
+ */
+function venvExists() {
+  const pythonPath = getVenvPython();
+  return fs.existsSync(pythonPath);
+}
+
+/**
+ * 检查 Python 环境是否就绪
+ */
+function isPythonReady() {
+  if (!venvExists()) {
+    return false;
+  }
+
+  // 检查 pip 是否存在
+  const pipPath = getVenvPip();
+  return fs.existsSync(pipPath);
+}
+
+module.exports = {
+  getKiteDataPath,
+  getVenvPath,
+  getVenvPython,
+  getVenvPip,
+  venvExists,
+  isPythonReady
+};

package/scripts/scan_dependencies.py

@@ -0,0 +1,461 @@
+"""
+依赖扫描工具
+
+在开发环境执行，扫描所有模块的依赖库，生成 dependencies_lock.json
+
+用法：
+    python scripts/scan_dependencies.py
+"""
+
+import ast
+import json
+import os
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, List, Set
+
+
+def get_project_root() -> Path:
+    """获取项目根目录"""
+    return Path(__file__).parent.parent
+
+
+def get_python_version() -> str:
+    """获取当前 Python 版本"""
+    v = sys.version_info
+    return f"{v.major}.{v.minor}.{v.micro}"
+
+
+def scan_imports_in_file(file_path: Path) -> Set[str]:
+    """扫描单个 Python 文件的 import 语句"""
+    imports = set()
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            tree = ast.parse(f.read(), filename=str(file_path))
+
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Import):
+                for alias in node.names:
+                    # import xxx
+                    imports.add(alias.name.split('.')[0])
+            elif isinstance(node, ast.ImportFrom):
+                # from xxx import yyy
+                if node.module:
+                    imports.add(node.module.split('.')[0])
+
+    except Exception as e:
+        print(f"[警告] 解析文件失败: {file_path} - {e}")
+
+    return imports
+
+
+def scan_module_imports(module_path: Path) -> Set[str]:
+    """扫描模块目录下所有 Python 文件的 import"""
+    all_imports = set()
+
+    # 排除的目录（不扫描这些目录中的依赖）
+    exclude_dirs = {'__pycache__', 'samples', 'vendor', 'tests', 'test', '.dev'}
+
+    for py_file in module_path.rglob("*.py"):
+        # 跳过排除的目录
+        if any(excluded in py_file.parts for excluded in exclude_dirs):
+            continue
+
+        imports = scan_imports_in_file(py_file)
+        all_imports.update(imports)
+
+    return all_imports
+
+
+def is_stdlib_module(module_name: str) -> bool:
+    """判断是否是标准库模块"""
+    stdlib_modules = {
+        '__future__', 'abc', 'argparse', 'asyncio', 'atexit', 'base64', 'builtins',
+        'calendar', 'collections', 'concurrent', 'contextlib', 'copy', 'ctypes',
+        'dataclasses', 'datetime', 'enum', 'fcntl', 'fnmatch', 'functools', 'glob', 'gzip',
+        'hashlib', 'hmac', 'http', 'importlib', 'inspect', 'io', 'json', 'locale',
+        'logging', 'msvcrt', 'ntpath', 'numbers', 'os', 'pathlib', 'platform',
+        'queue', 'random', 're', 'secrets', 'shutil', 'signal', 'socket',
+        'sqlite3', 'ssl', 'statistics', 'string', 'struct', 'subprocess',
+        'sys', 'tempfile', 'threading', 'time', 'traceback', 'types', 'typing',
+        'typing_extensions', 'urllib', 'uuid', 'wave', 'wsgiref', 'zipfile',
+        'zlib'
+    }
+
+    return module_name in stdlib_modules
+
+
+# import 名称到 package 名称的映射表（仅包含不一致的情况）
+IMPORT_TO_PACKAGE = {
+    # 配置和数据格式
+    'yaml': 'pyyaml',
+    'dotenv': 'python-dotenv',
+    'json5': 'json5',
+
+    # 图像处理
+    'cv2': 'opencv-python',
+    'cv2_headless': 'opencv-python-headless',
+    'PIL': 'pillow',
+    'skimage': 'scikit-image',
+
+    # 机器学习和数据科学
+    'sklearn': 'scikit-learn',
+
+    # Web 开发
+    'bs4': 'beautifulsoup4',
+    'flask_cors': 'flask-cors',
+    'flask_restful': 'flask-restful',
+    'flask_sqlalchemy': 'flask-sqlalchemy',
+    'flask_login': 'flask-login',
+    'flask_migrate': 'flask-migrate',
+    'flask_wtf': 'flask-wtf',
+    'django_rest_framework': 'djangorestframework',
+    'rest_framework': 'djangorestframework',
+
+    # 网络和 WebSocket
+    'websocket': 'websocket-client',  # 单数版本
+    # 'websockets' 包名和导入名一致，不需要映射
+
+    # 日期时间
+    'dateutil': 'python-dateutil',
+
+    # 加密和安全
+    'OpenSSL': 'pyopenssl',
+    'nacl': 'pynacl',
+    'jwt': 'pyjwt',
+    'Crypto': 'pycryptodome',
+    'Cryptodome': 'pycryptodomex',
+
+    # 文件和文档处理
+    'magic': 'python-magic',
+    'docx': 'python-docx',
+    'pptx': 'python-pptx',
+    'slugify': 'python-slugify',
+
+    # 数据库
+    'MySQLdb': 'mysqlclient',
+    'psycopg2': 'psycopg2-binary',
+    'bson': 'pymongo',
+
+    # 硬件和系统
+    'serial': 'pyserial',
+    'usb': 'pyusb',
+    'gi': 'pygobject',
+    'win32com': 'pywin32',
+    'win32api': 'pywin32',
+    'win32con': 'pywin32',
+    'win32file': 'pywin32',
+    'win32event': 'pywin32',
+    'win32process': 'pywin32',
+    'win32security': 'pywin32',
+    'win32service': 'pywin32',
+    'pywintypes': 'pywin32',
+    'pythoncom': 'pywin32',
+
+    # GUI
+    'wx': 'wxpython',
+    'tkinter': 'tk',
+    'PyQt5': 'pyqt5',
+    'PyQt6': 'pyqt6',
+    'PySide2': 'pyside2',
+    'PySide6': 'pyside6',
+
+    # 网络和协议
+    'dns': 'dnspython',
+    'google': 'google-api-python-client',
+    'googleapiclient': 'google-api-python-client',
+
+    # 日志和监控
+    'sentry_sdk': 'sentry-sdk',
+
+    # 测试和开发工具
+    'pytest_cov': 'pytest-cov',
+    'pytest_django': 'pytest-django',
+    'pytest_mock': 'pytest-mock',
+
+    # 异步和任务队列
+    'aiomysql': 'aiomysql',
+    'aiopg': 'aiopg',
+
+    # 其他常用
+    'attr': 'attrs',
+    'cattr': 'cattrs',
+    'msgpack': 'msgpack-python',
+    'ruamel': 'ruamel.yaml',
+}
+
+
+def get_installed_version(import_name: str) -> str:
+    """获取已安装包的版本"""
+    # 先尝试映射
+    package_name = IMPORT_TO_PACKAGE.get(import_name, import_name)
+
+    try:
+        import importlib.metadata
+        return importlib.metadata.version(package_name)
+    except:
+        return "unknown"
+
+
+def is_project_module(module_name: str) -> bool:
+    """判断是否是项目内部模块"""
+    project_modules = {
+        # 核心模块
+        'core', 'kernel', 'launcher', 'extensions', 'kite_cli',
+        # 子模块
+        'config', 'storage', 'tools', 'conversation', 'task', 'bluetooth',
+        'asr', 'llm', 'tts', 'vad', 'webhook', 'routes', 'server',
+        'registry', 'event_hub', 'dedup', 'router', 'rpc_router', 'registry_store',
+        'relay', 'pairing', 'entry', 'logging_setup', 'module_scanner', 'process_manager',
+        'types', 'utils', 'models', 'exceptions', 'context', 'log', 'store',
+        'presenter', 'userPresenter', 'configPresenter', 'config_loader',
+        'console_auth', 'proxy_config', 'monitoring_service', 'metrics_store',
+        'llm_server', 'sliding_window', 'standalone_reader', 'ws_logger',
+        'mermaid', 'version', 'vendor', 'scripts', 'evol'
+    }
+
+    return module_name in project_modules
+
+
+def scan_all_modules() -> Dict[str, Set[str]]:
+    """扫描所有模块的依赖"""
+    project_root = get_project_root()
+
+    modules_to_scan = {
+        "core": project_root / "core",
+        "kernel": project_root / "kernel",
+        "launcher": project_root / "launcher",
+        "extensions": project_root / "extensions",
+        "kite_cli": project_root / "kite_cli",
+    }
+
+    module_imports = {}
+
+    for module_name, module_path in modules_to_scan.items():
+        if not module_path.exists():
+            continue
+
+        print(f"[扫描] {module_name}...")
+        imports = scan_module_imports(module_path)
+
+        # 过滤掉标准库和项目内部模块
+        third_party = {
+            imp for imp in imports
+            if not is_stdlib_module(imp)
+            and not is_project_module(imp)
+        }
+
+        if third_party:
+            module_imports[module_name] = third_party
+            print(f"  发现 {len(third_party)} 个第三方依赖")
+
+    return module_imports
+
+
+def analyze_skipped_package(import_name: str, module_imports: Dict[str, Set[str]]) -> Dict:
+    """分析被跳过的包的使用情况"""
+    project_root = get_project_root()
+
+    # 找出哪些模块使用了这个包
+    used_by_modules = [
+        module_name
+        for module_name, imports in module_imports.items()
+        if import_name in imports
+    ]
+
+    # 搜索具体的使用位置
+    usage_locations = []
+    for module_name in used_by_modules:
+        module_path = project_root / module_name
+        if not module_path.exists():
+            continue
+
+        # 搜索 import 语句
+        for py_file in module_path.rglob("*.py"):
+            # 跳过排除的目录
+            exclude_dirs = {'__pycache__', 'samples', 'vendor', 'tests', 'test', '.dev'}
+            if any(excluded in py_file.parts for excluded in exclude_dirs):
+                continue
+
+            try:
+                with open(py_file, "r", encoding="utf-8") as f:
+                    content = f.read()
+
+                # 检查是否包含该包的 import
+                if f"import {import_name}" in content or f"from {import_name}" in content:
+                    # 判断是否是条件导入
+                    is_conditional = False
+                    lines = content.split('\n')
+                    for i, line in enumerate(lines):
+                        if f"import {import_name}" in line or f"from {import_name}" in line:
+                            # 检查前几行是否有 try
+                            for j in range(max(0, i-5), i):
+                                if 'try:' in lines[j]:
+                                    is_conditional = True
+                                    break
+                            break
+
+                    # 获取相对路径
+                    rel_path = py_file.relative_to(project_root)
+                    usage_locations.append({
+                        "file": str(rel_path),
+                        "conditional": is_conditional
+                    })
+            except:
+                pass
+
+    # 检查是否有包名映射
+    package_name = IMPORT_TO_PACKAGE.get(import_name, import_name)
+    has_mapping = package_name != import_name
+
+    return {
+        "import_name": import_name,
+        "package_name": package_name,
+        "has_mapping": has_mapping,
+        "used_by_modules": used_by_modules,
+        "usage_locations": usage_locations,
+        "is_conditional": any(loc["conditional"] for loc in usage_locations) if usage_locations else False
+    }
+
+
+def build_dependency_lock(module_imports: Dict[str, Set[str]]) -> tuple[Dict, List[Dict]]:
+    """构建依赖锁定文件，返回 (lock_data, skipped_packages_analysis)"""
+
+    # 收集所有第三方包
+    all_packages = set()
+    for imports in module_imports.values():
+        all_packages.update(imports)
+
+    # 获取每个包的版本和使用者
+    dependencies = {}
+    skipped_packages = []
+
+    for import_name in sorted(all_packages):
+        version = get_installed_version(import_name)
+
+        # 收集未安装的包信息
+        if version == "unknown":
+            print(f"[警告] 跳过未安装的包: {import_name}")
+
+            # 分析这个包的使用情况
+            analysis = analyze_skipped_package(import_name, module_imports)
+            skipped_packages.append(analysis)
+            continue
+
+        # 获取真实的 package 名称（用于 pip install）
+        package_name = IMPORT_TO_PACKAGE.get(import_name, import_name)
+
+        # 找出哪些模块使用了这个包
+        used_by = [
+            module_name
+            for module_name, imports in module_imports.items()
+            if import_name in imports
+        ]
+
+        dependencies[package_name] = {
+            "version": version,
+            "used_by": used_by
+        }
+
+        print(f"[依赖] {package_name}=={version} (used by: {', '.join(used_by)})")
+
+    lock_data = {
+        "scan_time": datetime.now(timezone.utc).isoformat(),
+        "python_version": get_python_version(),
+        "dependencies": dependencies
+    }
+
+    return lock_data, skipped_packages
+
+
+def main():
+    print("=" * 60)
+    print("  Kite 依赖扫描工具")
+    print("=" * 60)
+    print()
+
+    # 1. 扫描所有模块
+    print("[步骤 1] 扫描模块依赖...")
+    module_imports = scan_all_modules()
+    print()
+
+    # 2. 构建依赖锁定
+    print("[步骤 2] 构建依赖锁定...")
+    lock_data, skipped_packages = build_dependency_lock(module_imports)
+    print()
+
+    # 3. 保存到文件
+    output_file = get_project_root() / "dependencies_lock.json"
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(lock_data, f, indent=2, ensure_ascii=False)
+
+    print(f"[完成] 依赖锁定文件已生成: {output_file}")
+    print(f"[统计] 共 {len(lock_data['dependencies'])} 个第三方依赖")
+    print()
+
+    # 4. 同步更新 python_version.json
+    version_file = get_project_root() / "python_version.json"
+    version_data = {
+        "version": get_python_version(),
+        "note": "此文件在开发环境打包时生成，记录开发环境的 Python 版本。生产环境必须使用相同版本。"
+    }
+
+    with open(version_file, "w", encoding="utf-8") as f:
+        json.dump(version_data, f, indent=2, ensure_ascii=False)
+
+    print(f"[完成] Python 版本文件已更新: {version_file}")
+    print(f"[版本] Python {get_python_version()}")
+    print()
+
+    # 5. 输出跳过包的详细分析
+    if skipped_packages:
+        print("=" * 60)
+        print("  未安装包分析报告")
+        print("=" * 60)
+        print()
+        print(f"共发现 {len(skipped_packages)} 个未安装的包，详细信息如下：")
+        print()
+
+        for pkg in skipped_packages:
+            print(f"包名: {pkg['import_name']}")
+            if pkg['has_mapping']:
+                print(f"  → 映射到: {pkg['package_name']}")
+            print(f"  使用模块: {', '.join(pkg['used_by_modules'])}")
+            print(f"  导入类型: {'条件导入 (try-except)' if pkg['is_conditional'] else '直接导入'}")
+
+            if pkg['usage_locations']:
+                print(f"  使用位置:")
+                for loc in pkg['usage_locations'][:3]:  # 最多显示 3 个位置
+                    conditional_mark = " [条件]" if loc['conditional'] else ""
+                    print(f"    - {loc['file']}{conditional_mark}")
+                if len(pkg['usage_locations']) > 3:
+                    print(f"    ... 还有 {len(pkg['usage_locations']) - 3} 个位置")
+            else:
+                print(f"  使用位置: 未找到（可能已被排除）")
+
+            # 给出建议
+            if pkg['is_conditional']:
+                print(f"  建议: ✅ 可选依赖，跳过即可")
+            elif not pkg['usage_locations']:
+                print(f"  建议: ✅ 已被排除（samples/vendor），跳过即可")
+            elif not pkg['has_mapping']:
+                print(f"  建议: ⚠️ 检查是否需要添加包名映射")
+            else:
+                print(f"  建议: ⚠️ 检查是否需要安装该包")
+
+            print()
+
+        print("=" * 60)
+        print()
+        print("如果有需要处理的包，请：")
+        print("  1. 检查是否需要添加到 IMPORT_TO_PACKAGE 映射表")
+        print("  2. 检查是否需要安装该包")
+        print("  3. 检查是否需要标记为可选依赖")
+        print()
+
+
+if __name__ == "__main__":
+    main()