myagent-ai 1.0.0

package/core/logger.py

@@ -0,0 +1,241 @@
+"""
+core/logger.py - 日志管理模块
+==============================
+提供统一的日志记录功能，支持:
+  - 文件 + 控制台双输出
+  - 按时间/大小自动轮转
+  - ANSI 彩色控制台输出
+  - JSON 结构化日志输出（可选）
+  - 动态日志级别调整
+"""
+from __future__ import annotations
+
+import os
+import sys
+import json
+import logging
+from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
+from pathlib import Path
+from typing import Optional, Dict, Any
+from datetime import datetime
+
+
+# ANSI 颜色码
+COLORS = {
+    "DEBUG": "\033[36m",     # 青色
+    "INFO": "\033[32m",      # 绿色
+    "WARNING": "\033[33m",   # 黄色
+    "ERROR": "\033[31m",     # 红色
+    "CRITICAL": "\033[35m",  # 紫色
+    "RESET": "\033[0m",
+}
+
+
+class ColorFormatter(logging.Formatter):
+    """带颜色的控制台日志格式化器"""
+
+    def format(self, record):
+        color = COLORS.get(record.levelname, COLORS["RESET"])
+        record.levelname = f"{color}{record.levelname:8s}{COLORS['RESET']}"
+        return super().format(record)
+
+
+class FileFormatter(logging.Formatter):
+    """文件日志格式化器(无颜色)"""
+
+    def format(self, record):
+        # 去除 ANSI 颜色码
+        record.msg = str(record.msg)
+        for code in COLORS.values():
+            record.msg = record.msg.replace(code, "")
+        return super().format(record)
+
+
+class JsonFormatter(logging.Formatter):
+    """
+    JSON 结构化日志格式化器。
+
+    输出格式:
+    {"timestamp": "...", "level": "INFO", "logger": "...", "message": "...", "extra": {}}
+    """
+
+    def format(self, record):
+        log_entry: Dict[str, Any] = {
+            "timestamp": datetime.utcnow().isoformat() + "Z",
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "module": record.module,
+            "function": record.funcName,
+            "line": record.lineno,
+        }
+
+        # 附加异常信息
+        if record.exc_info and record.exc_info[0] is not None:
+            log_entry["exception"] = {
+                "type": record.exc_info[0].__name__,
+                "message": str(record.exc_info[1]),
+            }
+
+        # 附加自定义字段
+        if hasattr(record, "extra_fields"):
+            log_entry.update(record.extra_fields)
+
+        return json.dumps(log_entry, ensure_ascii=False, default=str)
+
+
+class LevelFilter(logging.Filter):
+    """只允许指定级别以上的日志通过"""
+
+    def __init__(self, min_level: str = "DEBUG"):
+        super().__init__()
+        self.min_level = getattr(logging, min_level.upper(), logging.DEBUG)
+
+    def filter(self, record):
+        return record.levelno >= self.min_level
+
+
+# ==============================================================================
+# 全局日志器注册表（支持动态级别调整）
+# ==============================================================================
+
+_loggers: Dict[str, logging.Logger] = {}
+_log_configs: Dict[str, Dict[str, Any]] = {}
+
+
+def setup_logger(
+    name: str = "myagent",
+    log_dir: Optional[str] = None,
+    level: str = "INFO",
+    console: bool = True,
+    json_format: bool = False,
+    rotation: str = "size",
+    max_bytes: int = 10 * 1024 * 1024,  # 10MB
+    backup_count: int = 5,
+    rotation_when: str = "midnight",
+) -> logging.Logger:
+    """
+    初始化日志系统。
+
+    Args:
+        name: 日志器名称
+        log_dir: 日志文件目录，默认为 ~/.myagent/logs/
+        level: 日志级别 (DEBUG/INFO/WARNING/ERROR/CRITICAL)
+        console: 是否输出到控制台
+        json_format: 是否使用 JSON 结构化格式输出到文件
+        rotation: 轮转策略 "size"(按大小) | "time"(按时间) | "none"(不轮转)
+        max_bytes: 按大小轮转时，单个文件最大字节数
+        backup_count: 保留的备份文件数量
+        rotation_when: 按时间轮转的时间点 ("midnight"/"H"/"D"/"W0" 等)
+
+    Returns:
+        配置好的 Logger 实例
+    """
+    logger = logging.getLogger(name)
+
+    # 避免重复添加 handler
+    if logger.handlers:
+        return logger
+
+    logger.setLevel(getattr(logging, level.upper(), logging.INFO))
+    # 防止日志向上层传播（避免重复输出）
+    logger.propagate = False
+
+    # 日志格式
+    fmt = "[%(asctime)s] %(levelname)s %(name)s - %(message)s"
+    datefmt = "%Y-%m-%d %H:%M:%S"
+
+    # 控制台输出
+    if console:
+        ch = logging.StreamHandler(sys.stdout)
+        ch.setFormatter(ColorFormatter(fmt, datefmt=datefmt))
+        ch.setLevel(logging.DEBUG)  # 控制台始终显示所有级别
+        logger.addHandler(ch)
+
+    # 文件输出
+    if log_dir:
+        log_path = Path(log_dir)
+        log_path.mkdir(parents=True, exist_ok=True)
+
+        if rotation == "size":
+            # 按大小轮转
+            log_file = log_path / f"{name}.log"
+            fh = RotatingFileHandler(
+                log_file,
+                maxBytes=max_bytes,
+                backupCount=backup_count,
+                encoding="utf-8",
+            )
+            fh.setFormatter(
+                JsonFormatter() if json_format
+                else FileFormatter(fmt, datefmt=datefmt)
+            )
+            logger.addHandler(fh)
+
+        elif rotation == "time":
+            # 按时间轮转
+            log_file = log_path / f"{name}.log"
+            fh = TimedRotatingFileHandler(
+                log_file,
+                when=rotation_when,
+                backupCount=backup_count,
+                encoding="utf-8",
+            )
+            fh.setFormatter(
+                JsonFormatter() if json_format
+                else FileFormatter(fmt, datefmt=datefmt)
+            )
+            # 日志文件名后缀
+            fh.suffix = "%Y%m%d"
+            logger.addHandler(fh)
+
+        else:
+            # 不轮转，按日期命名
+            log_file = log_path / f"{name}_{datetime.now().strftime('%Y%m%d')}.log"
+            fh = logging.FileHandler(log_file, encoding="utf-8")
+            fh.setFormatter(
+                JsonFormatter() if json_format
+                else FileFormatter(fmt, datefmt=datefmt)
+            )
+            logger.addHandler(fh)
+
+    # 保存配置
+    _loggers[name] = logger
+    _log_configs[name] = {
+        "level": level,
+        "log_dir": log_dir,
+        "console": console,
+        "json_format": json_format,
+        "rotation": rotation,
+    }
+
+    return logger
+
+
+def get_logger(name: str = "myagent") -> logging.Logger:
+    """获取已存在的 Logger，如果不存在则创建默认的"""
+    logger = logging.getLogger(name)
+    if not logger.handlers:
+        return setup_logger(name)
+    return logger
+
+
+def set_level(name: str = "myagent", level: str = "INFO"):
+    """
+    动态调整日志级别。
+
+    Args:
+        name: 日志器名称，"all" 表示调整所有已注册的日志器
+        level: 新的日志级别
+    """
+    if name == "all":
+        for logger_name, logger in _loggers.items():
+            logger.setLevel(getattr(logging, level.upper(), logging.INFO))
+    else:
+        logger = logging.getLogger(name)
+        logger.setLevel(getattr(logging, level.upper(), logging.INFO))
+
+
+def get_log_config(name: str = "myagent") -> Dict[str, Any]:
+    """获取日志配置"""
+    return _log_configs.get(name, {})

package/core/task_queue.py

@@ -0,0 +1,362 @@
+"""
+core/task_queue.py - 任务队列管理
+===================================
+提供线程安全的异步任务队列，支持优先级、超时、取消操作。
+"""
+from __future__ import annotations
+
+import asyncio
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Callable, Coroutine, Optional, Dict, List, Set
+from datetime import datetime
+
+from core.logger import get_logger
+from core.utils import generate_id, timestamp
+
+logger = get_logger("myagent.task_queue")
+
+
+class TaskStatus(str, Enum):
+    """任务状态"""
+    PENDING = "pending"
+    RUNNING = "running"
+    SUCCESS = "success"
+    FAILED = "failed"
+    TIMEOUT = "timeout"
+    CANCELLED = "cancelled"
+
+
+@dataclass
+class TaskItem:
+    """任务项"""
+    id: str = field(default_factory=lambda: generate_id("task"))
+    name: str = ""
+    description: str = ""
+    func: Optional[Callable] = None
+    args: tuple = ()
+    kwargs: dict = field(default_factory=dict)
+    priority: int = 5                    # 1(最高) ~ 10(最低)
+    status: str = TaskStatus.PENDING
+    result: Any = None
+    error: str = ""
+    created_at: str = field(default_factory=timestamp)
+    started_at: str = ""
+    finished_at: str = ""
+    timeout: int = 300                   # 超时秒数
+    retry_count: int = 0
+    max_retries: int = 2
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    _done_event: Optional[asyncio.Event] = field(default=None, repr=False)
+
+    def get_done_event(self, loop: asyncio.AbstractEventLoop) -> asyncio.Event:
+        """获取或创建任务完成事件（用于 wait_for_task）"""
+        if self._done_event is None:
+            self._done_event = asyncio.Event()
+        return self._done_event
+
+    def mark_done(self):
+        """标记任务完成，通知等待者"""
+        if self._done_event:
+            self._done_event.set()
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "name": self.name,
+            "description": self.description,
+            "status": self.status,
+            "priority": self.priority,
+            "created_at": self.created_at,
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "result": self.result,
+            "error": self.error,
+            "retry_count": self.retry_count,
+        }
+
+
+class TaskQueue:
+    """
+    异步任务队列。
+
+    特性:
+      - 优先级调度
+      - 超时控制
+      - 自动重试
+      - 任务取消
+      - 回调通知
+      - 并发限制
+
+    使用示例:
+        queue = TaskQueue(max_workers=3)
+        await queue.start()
+
+        task = await queue.submit(
+            name="测试任务",
+            func=my_function,
+            args=(arg1, arg2),
+            priority=3,
+        )
+
+        result = await task.result
+        print(f"状态: {task.status}, 结果: {task.result}")
+    """
+
+    def __init__(
+        self,
+        max_workers: int = 3,
+        default_timeout: int = 300,
+        default_retries: int = 2,
+    ):
+        self.max_workers = max_workers
+        self.default_timeout = default_timeout
+        self.default_retries = default_retries
+
+        self._queue: asyncio.PriorityQueue = asyncio.PriorityQueue()
+        self._tasks: Dict[str, TaskItem] = {}
+        self._running_tasks: Set[str] = set()
+        self._workers: List[asyncio.Task] = []
+        self._lock: Optional[asyncio.Lock] = None  # 延迟初始化，需在事件循环中
+        self._running = False
+        self._event_callbacks: List[Callable] = []
+
+        # 统计
+        self._total_submitted = 0
+        self._total_completed = 0
+        self._total_failed = 0
+
+    async def start(self):
+        """启动任务队列工作线程"""
+        if self._running:
+            return
+        self._lock = asyncio.Lock()
+        self._running = True
+        for i in range(self.max_workers):
+            worker = asyncio.create_task(self._worker(f"worker-{i}"))
+            self._workers.append(worker)
+        logger.info(f"任务队列已启动 (workers={self.max_workers})")
+
+    async def stop(self, wait: bool = True):
+        """停止任务队列"""
+        self._running = False
+        # 取消所有 pending 任务
+        for task in self._tasks.values():
+            if task.status == TaskStatus.PENDING:
+                task.status = TaskStatus.CANCELLED
+
+        if wait:
+            for worker in self._workers:
+                if not worker.done():
+                    worker.cancel()
+                    try:
+                        await worker
+                    except asyncio.CancelledError:
+                        pass
+        self._workers.clear()
+        logger.info("任务队列已停止")
+
+    async def submit(
+        self,
+        func: Callable,
+        name: str = "",
+        description: str = "",
+        args: tuple = (),
+        kwargs: Optional[dict] = None,
+        priority: int = 5,
+        timeout: Optional[int] = None,
+        max_retries: Optional[int] = None,
+        callback: Optional[Callable] = None,
+        metadata: Optional[dict] = None,
+    ) -> TaskItem:
+        """
+        提交任务到队列。
+
+        Args:
+            func: 要执行的异步/同步函数
+            name: 任务名称
+            description: 任务描述
+            args: 位置参数
+            kwargs: 关键字参数
+            priority: 优先级 (1=最高, 10=最低)
+            timeout: 超时秒数
+            max_retries: 最大重试次数
+            callback: 完成回调
+            metadata: 附加元数据
+
+        Returns:
+            TaskItem 对象
+        """
+        task = TaskItem(
+            name=name or func.__name__,
+            description=description,
+            func=func,
+            args=args,
+            kwargs=kwargs or {},
+            priority=priority,
+            timeout=timeout or self.default_timeout,
+            max_retries=max_retries or self.default_retries,
+            metadata=metadata or {},
+        )
+        self._tasks[task.id] = task
+        self._total_submitted += 1
+        await self._queue.put((task.priority, task.created_at, task.id))
+        logger.debug(f"任务已提交: {task.name} (id={task.id}, priority={task.priority})")
+        return task
+
+    async def get_task(self, task_id: str) -> Optional[TaskItem]:
+        """获取任务状态"""
+        return self._tasks.get(task_id)
+
+    async def cancel_task(self, task_id: str) -> bool:
+        """取消任务"""
+        task = self._tasks.get(task_id)
+        if task and task.status in (TaskStatus.PENDING, TaskStatus.RUNNING):
+            task.status = TaskStatus.CANCELLED
+            logger.info(f"任务已取消: {task.name} (id={task_id})")
+            return True
+        return False
+
+    async def wait_for_task(self, task_id: str, timeout: Optional[int] = None) -> Optional[TaskItem]:
+        """等待任务完成（使用 asyncio.Event，无轮询）"""
+        task = self._tasks.get(task_id)
+        if not task:
+            return None
+
+        # 如果已经完成，直接返回
+        if task.status in (
+            TaskStatus.SUCCESS, TaskStatus.FAILED,
+            TaskStatus.TIMEOUT, TaskStatus.CANCELLED,
+        ):
+            return task
+
+        # 使用 asyncio.Event 高效等待
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = None
+
+        done_event = task.get_done_event(loop)
+        try:
+            await asyncio.wait_for(done_event.wait(), timeout=timeout)
+        except asyncio.TimeoutError:
+            pass
+
+        return self._tasks.get(task_id)
+
+    def get_all_tasks(self) -> List[Dict]:
+        """获取所有任务状态"""
+        return [t.to_dict() for t in self._tasks.values()]
+
+    def get_stats(self) -> Dict[str, int]:
+        """获取队列统计"""
+        return {
+            "total_submitted": self._total_submitted,
+            "total_completed": self._total_completed,
+            "total_failed": self._total_failed,
+            "pending": sum(1 for t in self._tasks.values() if t.status == TaskStatus.PENDING),
+            "running": sum(1 for t in self._tasks.values() if t.status == TaskStatus.RUNNING),
+        }
+
+    def on_event(self, callback: Callable):
+        """注册事件回调"""
+        self._event_callbacks.append(callback)
+
+    def _notify(self, task: TaskItem, event: str):
+        """通知事件回调"""
+        for cb in self._event_callbacks:
+            try:
+                cb(task, event)
+            except Exception as e:
+                logger.error(f"事件回调执行失败: {e}")
+
+    async def _worker(self, name: str):
+        """工作线程主循环"""
+        logger.debug(f"{name} 已启动")
+        while self._running:
+            try:
+                # 从队列取任务(带超时，防止无法退出)
+                try:
+                    _, _, task_id = await asyncio.wait_for(
+                        self._queue.get(), timeout=1.0
+                    )
+                except asyncio.TimeoutError:
+                    continue
+
+                task = self._tasks.get(task_id)
+                if not task or task.status == TaskStatus.CANCELLED:
+                    continue
+
+                await self._execute_task(task)
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"{name} 异常: {e}")
+
+        logger.debug(f"{name} 已退出")
+
+    async def _execute_task(self, task: TaskItem):
+        """执行单个任务(含重试和超时)"""
+        task.status = TaskStatus.RUNNING
+        task.started_at = timestamp()
+        self._running_tasks.add(task.id)
+        self._notify(task, "start")
+        logger.info(f"开始执行任务: {task.name} (id={task.id})")
+
+        try:
+            while task.retry_count <= task.max_retries:
+                try:
+                    if asyncio.iscoroutinefunction(task.func):
+                        result = await asyncio.wait_for(
+                            task.func(*task.args, **task.kwargs),
+                            timeout=task.timeout,
+                        )
+                    else:
+                        loop = asyncio.get_running_loop()
+                        result = await asyncio.wait_for(
+                            loop.run_in_executor(
+                                None,
+                                lambda: task.func(*task.args, **task.kwargs),
+                            ),
+                            timeout=task.timeout,
+                        )
+
+                    task.result = result
+                    task.status = TaskStatus.SUCCESS
+                    task.finished_at = timestamp()
+                    self._total_completed += 1
+                    self._notify(task, "success")
+                    logger.info(f"任务完成: {task.name} (id={task.id})")
+                    return
+
+                except asyncio.TimeoutError:
+                    task.retry_count += 1
+                    if task.retry_count > task.max_retries:
+                        task.status = TaskStatus.TIMEOUT
+                        task.finished_at = timestamp()
+                        task.error = f"执行超时 ({task.timeout}s)，已重试 {task.max_retries} 次"
+                        self._total_failed += 1
+                        self._notify(task, "timeout")
+                        logger.error(f"任务超时: {task.name} (id={task.id})")
+                        return
+                    logger.warning(f"任务超时，重试 {task.retry_count}/{task.max_retries}: {task.name}")
+
+                except Exception as e:
+                    task.retry_count += 1
+                    task.error = str(e)
+                    if task.retry_count > task.max_retries:
+                        task.status = TaskStatus.FAILED
+                        task.finished_at = timestamp()
+                        task.error = f"{str(e)} (已重试 {task.max_retries} 次)"
+                        self._total_failed += 1
+                        self._notify(task, "failed")
+                        logger.error(f"任务失败: {task.name} (id={task.id}) - {e}")
+                        return
+                    logger.warning(f"任务异常，重试 {task.retry_count}/{task.max_retries}: {task.name} - {e}")
+
+        finally:
+            self._running_tasks.discard(task.id)
+            task.finished_at = task.finished_at or timestamp()
+            task.mark_done()