@agentunion/kite 1.5.0 → 1.6.1

package/extensions/services/evol/async_http.py

@@ -0,0 +1,551 @@
+from typing import Any, Awaitable, Callable, Dict, Optional
+import asyncio
+import ssl
+import uuid
+import os
+import threading
+from urllib.parse import urlparse
+
+import aiohttp
+import certifi
+
+from ..presenter.configPresenter import configPresenter
+from ..version import __version__, __cmp_version__
+
+
+def _ensure_no_proxy_for_local_env():
+    local_addrs = 'localhost,127.0.0.1,::1'
+    for key in ('NO_PROXY', 'no_proxy'):
+        existing = (os.environ.get(key) or '').strip()
+        if not existing:
+            os.environ[key] = local_addrs
+            continue
+
+        existing_parts = [p.strip() for p in existing.replace(';', ',').split(',') if p.strip()]
+        existing_lower = {p.lower() for p in existing_parts}
+        for addr in local_addrs.split(','):
+            if addr.lower() not in existing_lower:
+                existing_parts.append(addr)
+        os.environ[key] = ','.join(existing_parts)
+
+
+def _is_local_url(url: str) -> bool:
+    try:
+        parsed = urlparse(url)
+        host = parsed.hostname
+        if not host:
+            return False
+        host = host.strip('[]').lower()
+        return host in ('localhost', '127.0.0.1', '::1', '0.0.0.0')
+    except Exception:
+        return False
+
+
+_ensure_no_proxy_for_local_env()
+
+
+class AsyncHttpClient:
+    # SSL验证配置：False=禁用验证(开发环境), True=启用验证(生产环境)
+    VERIFY_SSL = False
+
+    # 连接池配置
+    _connector: Optional[aiohttp.TCPConnector] = None
+    _connector_no_proxy: Optional[aiohttp.TCPConnector] = None
+    _connector_loop: Optional[asyncio.AbstractEventLoop] = None
+    _connector_no_proxy_loop: Optional[asyncio.AbstractEventLoop] = None
+    _lock = threading.Lock()
+
+    # 超时配置（秒）
+    CONNECT_TIMEOUT = 10
+    TOTAL_TIMEOUT = 60
+
+    @classmethod
+    def _is_connector_valid(cls, connector: Optional[aiohttp.TCPConnector], connector_loop: Optional[asyncio.AbstractEventLoop]) -> bool:
+        """
+        检查 connector 是否有效
+        - connector 不为 None
+        - connector 未关闭
+        - connector 绑定的 loop 与当前 loop 一致
+        - 当前 loop 未关闭
+        """
+        if connector is None or connector.closed:
+            return False
+
+        try:
+            current_loop = asyncio.get_running_loop()
+        except RuntimeError:
+            # 没有运行中的 event loop
+            return False
+
+        if current_loop is None or current_loop.is_closed():
+            return False
+
+        if connector_loop is not current_loop:
+            return False
+
+        return True
+
+    @classmethod
+    def _safe_close_connector(cls, connector: Optional[aiohttp.TCPConnector]) -> None:
+        """安全关闭 connector（同步方式，用于 event loop 变更时的清理）"""
+        if connector is None:
+            return
+        try:
+            if not connector.closed:
+                # 使用内部方法同步关闭，避免在错误的 loop 中 await
+                connector._close()
+        except Exception:
+            pass
+
+    @classmethod
+    def _get_connector(cls, use_proxy: bool = True) -> aiohttp.TCPConnector:
+        """
+        获取共享的 TCPConnector（连接池）
+        - limit: 总连接数限制
+        - limit_per_host: 每个主机的连接数限制
+        线程安全：使用锁保护 connector 的创建
+
+        修复 Event loop is closed 问题：
+        - 检查 connector 绑定的 event loop 是否与当前 loop 一致
+        - 如果 loop 变更，自动重建 connector
+        """
+        with cls._lock:
+            ssl_context = cls._get_ssl_context()
+
+            # 获取当前 event loop
+            try:
+                current_loop = asyncio.get_running_loop()
+            except RuntimeError:
+                current_loop = None
+
+            if use_proxy:
+                # 检查 connector 是否有效（包括 event loop 检查）
+                if not cls._is_connector_valid(cls._connector, cls._connector_loop):
+                    # 清理旧的 connector
+                    cls._safe_close_connector(cls._connector)
+                    # 创建新的 connector
+                    cls._connector = aiohttp.TCPConnector(
+                        ssl=ssl_context,
+                        limit=100,  # 总连接数
+                        limit_per_host=30,  # 每个主机最大连接数
+                        enable_cleanup_closed=True,
+                    )
+                    cls._connector_loop = current_loop
+                return cls._connector
+            else:
+                # 检查 no_proxy connector 是否有效
+                if not cls._is_connector_valid(cls._connector_no_proxy, cls._connector_no_proxy_loop):
+                    # 清理旧的 connector
+                    cls._safe_close_connector(cls._connector_no_proxy)
+                    # 创建新的 connector
+                    cls._connector_no_proxy = aiohttp.TCPConnector(
+                        ssl=ssl_context,
+                        limit=100,
+                        limit_per_host=30,
+                        enable_cleanup_closed=True,
+                    )
+                    cls._connector_no_proxy_loop = current_loop
+                return cls._connector_no_proxy
+
+    @classmethod
+    async def close(cls):
+        """
+        关闭所有连接池，应在应用退出时调用
+        """
+        with cls._lock:
+            if cls._connector is not None and not cls._connector.closed:
+                await cls._connector.close()
+            cls._connector = None
+            cls._connector_loop = None
+            if cls._connector_no_proxy is not None and not cls._connector_no_proxy.closed:
+                await cls._connector_no_proxy.close()
+            cls._connector_no_proxy = None
+            cls._connector_no_proxy_loop = None
+
+    @classmethod
+    def reset_connectors(cls):
+        """
+        重置所有 connector（同步方法，用于 event loop 变更后的恢复）
+        """
+        with cls._lock:
+            cls._safe_close_connector(cls._connector)
+            cls._connector = None
+            cls._connector_loop = None
+            cls._safe_close_connector(cls._connector_no_proxy)
+            cls._connector_no_proxy = None
+            cls._connector_no_proxy_loop = None
+
+    @classmethod
+    def _is_connector_closed_error(cls, error: Exception) -> bool:
+        message = str(error).lower()
+        return "connector is closed" in message or "event loop is closed" in message
+
+    @classmethod
+    async def _request_with_retry(cls, request_fn: Callable[[], Awaitable[Any]]) -> Any:
+        try:
+            return await request_fn()
+        except Exception as e:
+            if cls._is_connector_closed_error(e):
+                cls.reset_connectors()
+                return await request_fn()
+            raise
+
+    @classmethod
+    def _get_timeout(cls, total_timeout: Optional[int] = None) -> aiohttp.ClientTimeout:
+        """获取超时配置"""
+        effective_total_timeout = total_timeout if total_timeout is not None else cls.TOTAL_TIMEOUT
+        return aiohttp.ClientTimeout(
+            total=effective_total_timeout,
+            connect=cls.CONNECT_TIMEOUT,
+        )
+
+    @classmethod
+    def _get_ssl_context(cls):
+        """
+        创建 SSL 上下文
+        根据 VERIFY_SSL 配置决定是否验证SSL证书
+
+        - VERIFY_SSL=False: 禁用SSL验证，适用于开发环境或自签名证书
+        - VERIFY_SSL=True: 使用certifi证书验证，适用于生产环境
+        """
+        if not cls.VERIFY_SSL:
+            # 开发环境：禁用SSL验证
+            return False
+        else:
+            # 生产环境：使用certifi证书包
+            try:
+                ssl_context = ssl.create_default_context(cafile=certifi.where())
+                return ssl_context
+            except Exception:
+                # certifi加载失败，降级为禁用验证
+                return False
+
+    @classmethod
+    def _get_trust_env(cls, url: str) -> bool:
+        """获取是否信任环境变量（包括代理环境变量）；localhost 永远直连。"""
+        return configPresenter.get_use_system_proxy() and (not _is_local_url(url))
+
+    @classmethod
+    def _get_device_id(cls) -> str:
+        """
+        获取设备ID：优先从配置读取，否则根据系统生成
+        参考 userPresenter._get_device_id() 实现
+        """
+        import sys
+
+        is_mac = sys.platform == "darwin"
+
+        # 1. 尝试从配置获取
+        device_id = configPresenter.getSetting("device_id")
+        if device_id:
+            # Mac 系统如果是 mac_ 开头，转换为 uuid_ 格式
+            if is_mac and device_id.startswith("mac_"):
+                device_id = f"uuid_{uuid.uuid4().hex}"
+                configPresenter.setSetting("device_id", device_id)
+            return device_id
+
+        # 根据系统选择生成策略
+        if is_mac:
+            # Mac: 优先 UUID，备选 MAC 地址
+            try:
+                device_id = f"uuid_{uuid.uuid4().hex}"
+            except:
+                try:
+                    mac = uuid.getnode()
+                    device_id = f"mac_{mac:012x}"
+                except:
+                    device_id = f"uuid_{uuid.uuid4().hex}"
+        else:
+            # Windows/Linux: 优先 MAC 地址，备选 UUID
+            try:
+                mac = uuid.getnode()
+                device_id = f"mac_{mac:012x}"
+            except:
+                device_id = f"uuid_{uuid.uuid4().hex}"
+
+        # 保存到配置
+        configPresenter.setSetting("device_id", device_id)
+        return device_id
+
+    @classmethod
+    def _get_default_headers(cls, include_token: bool = True) -> Dict[str, str]:
+        headers = {
+            "User-Agent": f"Evol/{__version__}",
+            "Accept": "application/json",
+            "version": __version__,
+            "cmp_version": __cmp_version__,
+            "source": "evol_client",
+            "deviceId": cls._get_device_id(),
+        }
+
+        if include_token:
+            # 每次都从配置文件读取最新的token，避免使用缓存的旧token
+            token = configPresenter.get_token()
+            if token:
+                headers["Authorization"] = f"Bearer {token}"
+        return headers
+
+    @classmethod
+    async def get(cls, url: str, headers: Optional[Dict[str, str]] = None, include_token: bool = True, **kwargs) -> Any:
+        merged_headers = cls._merge_headers(headers, include_token)
+        total_timeout = kwargs.pop("total_timeout", None)
+
+        async def _do_request() -> Any:
+            trust_env = cls._get_trust_env(url)
+            connector = cls._get_connector(use_proxy=trust_env)
+            timeout = cls._get_timeout(total_timeout=total_timeout)
+            async with aiohttp.ClientSession(connector=connector, trust_env=trust_env, connector_owner=False) as session:
+                async with session.get(url, headers=merged_headers, timeout=timeout, **kwargs) as response:
+                    try:
+                        json_data = await response.json()
+                    except Exception:
+                        response.raise_for_status()
+                        raise
+
+                    if response.status >= 400:
+                        if isinstance(json_data, dict) and 'code' in json_data:
+                            return json_data
+                        response.raise_for_status()
+
+                    return json_data
+
+        return await cls._request_with_retry(_do_request)
+
+    @classmethod
+    async def post(
+        cls, url: str, data: Optional[Any] = None, headers: Optional[Dict[str, str]] = None, include_token: bool = True, **kwargs
+    ) -> Any:
+        merged_headers = cls._merge_headers(headers, include_token)
+        total_timeout = kwargs.pop("total_timeout", None)
+
+        async def _do_request() -> Any:
+            trust_env = cls._get_trust_env(url)
+            connector = cls._get_connector(use_proxy=trust_env)
+            timeout = cls._get_timeout(total_timeout=total_timeout)
+            async with aiohttp.ClientSession(connector=connector, trust_env=trust_env, connector_owner=False) as session:
+                async with session.post(url, json=data, headers=merged_headers, timeout=timeout, **kwargs) as response:
+                    try:
+                        json_data = await response.json()
+                    except Exception:
+                        response.raise_for_status()
+                        raise
+
+                    if response.status >= 400:
+                        if isinstance(json_data, dict) and 'code' in json_data:
+                            return json_data
+                        response.raise_for_status()
+
+                    return json_data
+
+        return await cls._request_with_retry(_do_request)
+
+    @classmethod
+    async def post_form(
+        cls, url: str, data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, include_token: bool = False, **kwargs
+    ) -> Any:
+        """
+        发送 application/x-www-form-urlencoded 格式的 POST 请求
+        """
+        merged_headers = cls._merge_headers(headers, include_token)
+        # 移除 Content-Type，让 aiohttp 自动设置为 application/x-www-form-urlencoded
+        merged_headers.pop("Content-Type", None)
+
+        async def _do_request() -> Any:
+            trust_env = cls._get_trust_env(url)
+            connector = cls._get_connector(use_proxy=trust_env)
+            timeout = cls._get_timeout()
+            async with aiohttp.ClientSession(connector=connector, trust_env=trust_env, connector_owner=False) as session:
+                async with session.post(url, data=data, headers=merged_headers, timeout=timeout, **kwargs) as response:
+                    try:
+                        json_data = await response.json()
+                    except Exception:
+                        response.raise_for_status()
+                        raise
+
+                    if response.status >= 400:
+                        if isinstance(json_data, dict) and 'code' in json_data:
+                            return json_data
+                        response.raise_for_status()
+
+                    return json_data
+
+        return await cls._request_with_retry(_do_request)
+
+    @classmethod
+    async def put(
+        cls, url: str, data: Optional[Any] = None, headers: Optional[Dict[str, str]] = None, include_token: bool = True, **kwargs
+    ) -> Any:
+        merged_headers = cls._merge_headers(headers, include_token)
+        total_timeout = kwargs.pop("total_timeout", None)
+
+        async def _do_request() -> Any:
+            trust_env = cls._get_trust_env(url)
+            connector = cls._get_connector(use_proxy=trust_env)
+            timeout = cls._get_timeout(total_timeout=total_timeout)
+            async with aiohttp.ClientSession(connector=connector, trust_env=trust_env, connector_owner=False) as session:
+                async with session.put(url, json=data, headers=merged_headers, timeout=timeout, **kwargs) as response:
+                    try:
+                        json_data = await response.json()
+                    except Exception:
+                        response.raise_for_status()
+                        raise
+
+                    if response.status >= 400:
+                        if isinstance(json_data, dict) and 'code' in json_data:
+                            return json_data
+                        response.raise_for_status()
+
+                    return json_data
+
+        return await cls._request_with_retry(_do_request)
+
+    @classmethod
+    async def delete(
+        cls, url: str, headers: Optional[Dict[str, str]] = None, include_token: bool = True, **kwargs
+    ) -> Any:
+        merged_headers = cls._merge_headers(headers, include_token)
+        total_timeout = kwargs.pop("total_timeout", None)
+
+        async def _do_request() -> Any:
+            trust_env = cls._get_trust_env(url)
+            connector = cls._get_connector(use_proxy=trust_env)
+            timeout = cls._get_timeout(total_timeout=total_timeout)
+            async with aiohttp.ClientSession(connector=connector, trust_env=trust_env, connector_owner=False) as session:
+                async with session.delete(url, headers=merged_headers, timeout=timeout, **kwargs) as response:
+                    try:
+                        json_data = await response.json()
+                    except Exception:
+                        response.raise_for_status()
+                        raise
+
+                    if response.status >= 400:
+                        if isinstance(json_data, dict) and 'code' in json_data:
+                            return json_data
+                        response.raise_for_status()
+
+                    return json_data
+
+        return await cls._request_with_retry(_do_request)
+
+    @classmethod
+    async def upload_file(
+        cls, url: str, form_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, include_token: bool = True, **kwargs
+    ) -> Any:
+        default_headers = cls._get_default_headers(include_token)
+        default_headers.pop("Accept", None)
+
+        if headers:
+            default_headers.update(headers)
+
+        data = aiohttp.FormData()
+        for key, value in form_data.items():
+            if isinstance(value, tuple):
+                filename, content, content_type = value
+                data.add_field(key, content, filename=filename, content_type=content_type)
+            else:
+                data.add_field(key, str(value))
+
+        async def _do_request() -> Any:
+            trust_env = cls._get_trust_env(url)
+            connector = cls._get_connector(use_proxy=trust_env)
+            timeout = cls._get_timeout()
+            async with aiohttp.ClientSession(connector=connector, trust_env=trust_env, connector_owner=False) as session:
+                async with session.post(url, data=data, headers=default_headers, timeout=timeout, **kwargs) as response:
+                    try:
+                        json_data = await response.json()
+                    except Exception:
+                        response.raise_for_status()
+                        raise
+
+                    if response.status >= 400:
+                        if isinstance(json_data, dict) and 'code' in json_data:
+                            return json_data
+                        response.raise_for_status()
+
+                    return json_data
+
+        return await cls._request_with_retry(_do_request)
+
+    @classmethod
+    async def download_file(
+        cls, url: str, save_path: str, headers: Optional[Dict[str, str]] = None, include_token: bool = True, **kwargs
+    ) -> Dict[str, Any]:
+        """
+        下载文件到本地
+
+        Args:
+            url: 下载接口地址
+            save_path: 保存文件的路径
+            headers: 额外的请求头
+            include_token: 是否包含认证token
+            **kwargs: 其他请求参数
+
+        Returns:
+            Dict: {"success": True, "file_path": save_path, "filename": filename} 或 {"success": False, "error": error_msg}
+        """
+        merged_headers = cls._merge_headers(headers, include_token)
+        # 下载文件不需要 Accept: application/json
+        merged_headers.pop("Accept", None)
+
+        trust_env = cls._get_trust_env(url)
+        connector = cls._get_connector(use_proxy=trust_env)
+        timeout = cls._get_timeout()
+
+        try:
+            async with aiohttp.ClientSession(connector=connector, trust_env=trust_env, connector_owner=False) as session:
+                async with session.get(url, headers=merged_headers, timeout=timeout, **kwargs) as response:
+                    if response.status >= 400:
+                        # 尝试获取错误信息
+                        try:
+                            json_data = await response.json()
+                            if isinstance(json_data, dict) and 'msg' in json_data:
+                                return {"success": False, "error": json_data.get('msg', '下载失败')}
+                        except:
+                            pass
+                        return {"success": False, "error": f"下载失败，状态码: {response.status}"}
+
+                    # 从 Content-Disposition 获取文件名
+                    content_disposition = response.headers.get('Content-Disposition', '')
+                    filename = None
+                    if content_disposition:
+                        import re
+                        # 尝试匹配 filename*=UTF-8''xxx 格式
+                        match = re.search(r"filename\*=UTF-8''(.+)", content_disposition)
+                        if match:
+                            from urllib.parse import unquote
+                            filename = unquote(match.group(1))
+                        else:
+                            # 尝试匹配 filename="xxx" 或 filename=xxx 格式
+                            match = re.search(r'filename[^;=\n]*=(["\']?)([^"\';\n]+)\1', content_disposition)
+                            if match:
+                                filename = match.group(2)
+
+                    # 如果没有获取到文件名，使用默认名称
+                    if not filename:
+                        filename = os.path.basename(save_path) or 'downloaded_file'
+
+                    # 确保保存目录存在
+                    save_dir = os.path.dirname(save_path)
+                    if save_dir and not os.path.exists(save_dir):
+                        os.makedirs(save_dir, exist_ok=True)
+
+                    # 如果 save_path 是目录，则使用获取到的文件名
+                    if os.path.isdir(save_path):
+                        save_path = os.path.join(save_path, filename)
+
+                    # 写入文件
+                    with open(save_path, 'wb') as f:
+                        async for chunk in response.content.iter_chunked(8192):
+                            f.write(chunk)
+
+                    return {"success": True, "file_path": save_path, "filename": filename}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @classmethod
+    def _merge_headers(cls, headers: Optional[Dict[str, str]], include_token: bool = True) -> Dict[str, str]:
+        if headers:
+            merged = cls._get_default_headers(include_token)
+            merged.update(headers)
+            return merged
+        return cls._get_default_headers(include_token)