foxclaw 0.1.0 → 0.1.3

package/python/foxclaw/core/paths.py

@@ -0,0 +1,103 @@
+"""文件路径准入 —— 把「调用方给的路径」收敛到允许的根目录内。
+
+为什么需要单独一层:领域导入接口(`/api/{finance,learning,itinerary}/import`)接受
+请求体里的 ``csv_path``。如果把它原样交给 ``open()``,HTTP 面就等于开放了「读本机
+任意文件」的能力 —— 而且读到的内容会被解析进知识库/账本,再从 ``/api`` 读回来,
+构成完整的信息泄露链(``csv_path=/etc/passwd`` 这类)。
+
+分层原则(与项目其余部分一致):
+- **HTTP 边界是信任边界**,用户可控的路径必须校验;
+- **本地库调用(CLI / 测试 / 示例)不校验**:调用方代码本身已有文件权限,
+  给它加限制只会破坏用法,不增加安全性。
+
+允许的根 = ``FOXCLAW_DATA_DIR``(默认 ``./foxclaw_data``)
+          + ``FOXCLAW_IMPORT_ROOTS``(逗号或分号分隔,给「从下载目录导入」留出口)。
+
+校验用 ``resolve(strict=True)``:先解掉符号链接再比前缀,避免造一个指向 ``/etc``
+的软链绕过检查。比较用 ``normcase``:Windows 路径大小写不敏感,直接比字符串会漏。
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from typing import Iterable, Sequence
+
+__all__ = [
+    "DEFAULT_IMPORT_SUFFIXES",
+    "allowed_import_roots",
+    "is_within",
+    "resolve_import_path",
+]
+
+_DEFAULT_DATA_DIR = "./foxclaw_data"
+DEFAULT_IMPORT_SUFFIXES: tuple[str, ...] = (".csv", ".txt")
+
+
+def _split_paths(raw: str) -> list[str]:
+    """按 , ; 切分路径列表(Windows 的盘符冒号不能当分隔符,故不用 :)。"""
+    return [x.strip().strip("\"'") for x in re.split(r"[;,]", raw or "") if x.strip()]
+
+
+def allowed_import_roots() -> list[Path]:
+    """允许作为导入来源的根目录(绝对路径、已 resolve)。"""
+    raw = [os.environ.get("FOXCLAW_DATA_DIR") or _DEFAULT_DATA_DIR]
+    raw += _split_paths(os.environ.get("FOXCLAW_IMPORT_ROOTS", ""))
+    roots: list[Path] = []
+    for item in raw:
+        try:
+            roots.append(Path(item).expanduser().resolve())
+        except (OSError, RuntimeError):
+            continue
+    return roots
+
+
+def is_within(path: Path | str, roots: Sequence[Path]) -> bool:
+    """path 是否落在 roots 的任意一个之下(已按平台规范化大小写)。"""
+    try:
+        target = Path(path).resolve()
+    except (OSError, RuntimeError):
+        return False
+    for root in roots:
+        try:
+            if target.is_relative_to(root):
+                return True
+        except (OSError, RuntimeError):
+            continue
+    return False
+
+
+def resolve_import_path(raw: str,
+                        suffixes: Iterable[str] = DEFAULT_IMPORT_SUFFIXES,
+                        roots: Sequence[Path] | None = None) -> Path:
+    """校验并返回可安全导入的绝对路径;不合法一律抛 ``ValueError``。
+
+    依次检查:非空 → 存在且是普通文件 → 后缀在白名单 → 落在允许根内。
+    """
+    text = (raw or "").strip().strip("\"'")
+    if not text:
+        raise ValueError("路径为空")
+
+    try:
+        path = Path(text).expanduser().resolve(strict=True)
+    except (OSError, RuntimeError) as exc:
+        raise ValueError(f"文件不存在或不可访问: {text}") from exc
+
+    if not path.is_file():
+        raise ValueError(f"不是普通文件: {path}")
+
+    allowed_suffixes = {s.lower() for s in suffixes}
+    if allowed_suffixes and path.suffix.lower() not in allowed_suffixes:
+        raise ValueError(
+            f"不支持的扩展名 {path.suffix or '(无)'};允许:"
+            f"{' '.join(sorted(allowed_suffixes))}")
+
+    bases = list(roots) if roots is not None else allowed_import_roots()
+    if not is_within(path, bases):
+        shown = ", ".join(str(b) for b in bases) or "(无)"
+        raise ValueError(
+            f"路径不在允许的导入目录内: {path}\n"
+            f"允许的目录: {shown}\n"
+            f"如需从其它目录导入,设置 FOXCLAW_IMPORT_ROOTS=<目录>(多个用逗号分隔)")
+    return path

package/python/foxclaw/core/recovery.py

@@ -0,0 +1,349 @@
+"""
+Foxclaw Core — 崩溃/重启恢复 (Restart Recovery)
+
+借鉴 OpenClaw 的原理,适配 Foxclaw。核心思想:
+  系统跑长任务到一半崩溃/重启,那个任务不该凭空消失。
+  启动时主动识别"上次还在进行、但现在没有活跃进程持有"的任务,
+  诚实标记为"被中断",而不是装无事或留下半完成的脏状态。
+
+这与 Foxclaw"可审计、可信"的定位高度契合——可信系统不该悄悄吞掉崩溃的任务。
+
+机制:
+  · 任务开始 → 落一条 in_progress 记录(带 pid + 启动时间)
+  · 任务正常结束 → 标记 done(或从活跃集移除)
+  · 启动时 → 扫描所有 in_progress 记录:
+      - 若记录的 pid 不是当前进程、且那个 pid 已不存在 → 判定被中断
+      - 标记为 interrupted,交给上层决定(通知用户 / 尝试恢复)
+
+存储:JSONL 追加写(崩溃安全:写一条是原子 append,不会损坏整个文件)。
+"""
+from __future__ import annotations
+import json
+import os
+import sys
+import time
+from dataclasses import dataclass, field, asdict
+
+
+def _pid_alive(pid: int) -> bool:
+    """判断进程是否存活(只查询、绝不影响目标进程)。跨平台:
+
+    · POSIX:os.kill(pid, 0) —— 发 0 号信号,内核只做权限/存在性检查,不真正投递。
+    · Windows:**决不能**用 os.kill(pid, 0)。CPython 在 Windows 上把非 CTRL 信号
+      翻译成 TerminateProcess(handle, sig),因此 os.kill(pid, 0) 会真的以退出码 0
+      终止该 pid(已实测:会误杀 pid 被 OS 复用后的无辜进程,且 os.kill 正常返回、
+      不抛异常,连"探测失败"都察觉不到)。改用 OpenProcess 只申请查询权限、绝不
+      终止,再用 GetExitCodeProcess 看是否仍 STILL_ACTIVE(259)。
+
+    非破坏性是硬约束:本函数只在启动恢复扫描里做诊断记录,任何情况下都不得
+    对被探测进程产生副作用。
+    """
+    if pid <= 0:
+        return False
+
+    if sys.platform == "win32":
+        import ctypes
+        from ctypes import wintypes
+        PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
+        STILL_ACTIVE = 259
+        kernel32 = ctypes.windll.kernel32
+        handle = kernel32.OpenProcess(
+            PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
+        if not handle:
+            return False
+        try:
+            code = wintypes.DWORD()
+            if not kernel32.GetExitCodeProcess(handle, ctypes.byref(code)):
+                return False
+            return code.value == STILL_ACTIVE
+        finally:
+            kernel32.CloseHandle(handle)
+
+    try:
+        os.kill(pid, 0)
+        return True
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    except Exception:
+        return False
+
+
+@dataclass
+class TaskRecord:
+    task_id: str
+    description: str
+    pid: int
+    started: float
+    status: str = "in_progress"
+    session_id: str = ""
+    channel: str = ""
+    finished: float = 0.0
+    boot_id: str = ""
+    retryable: bool = False
+    start_count: int = 1
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class RecoveryPlan:
+    """【P2-4】恢复计划:把中断任务分类,调用方(API 层/前端)据此决定动作。"""
+    retryable: list = field(default_factory=list)
+    notify_user: list = field(default_factory=list)
+    needs_manual: list = field(default_factory=list)
+    details: list = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+class RestartRecovery:
+    """任务生命周期日志 + 启动恢复扫描。"""
+
+    def __init__(self, log_path: str, current_pid: int | None = None, boot_id: str | None = None,
+                 spine=None, fsync: bool = True):
+        self.log_path = log_path
+        self.current_pid = current_pid or os.getpid()
+        import uuid
+        self.boot_id = boot_id or uuid.uuid4().hex[:16]
+        self._active: dict[str, TaskRecord] = {}
+        self._spine = spine
+        self._fsync = fsync
+        os.makedirs(os.path.dirname(log_path) or ".", exist_ok=True)
+
+    def task_started(self, task_id: str, description: str,
+                     session_id: str = "", channel: str = "",
+                     retryable: bool = False) -> TaskRecord:
+        rec = TaskRecord(
+            task_id=task_id, description=description[:200],
+            pid=self.current_pid, started=time.time(),
+            session_id=session_id, channel=channel, boot_id=self.boot_id,
+            retryable=retryable)
+        self._active[task_id] = rec
+        self._append({"event": "start", **rec.to_dict()})
+        return rec
+
+    def task_finished(self, task_id: str, status: str = "done"):
+        rec = self._active.pop(task_id, None)
+        if rec:
+            rec.status = status
+            rec.finished = time.time()
+        self._append({"event": "finish", "task_id": task_id,
+                      "status": status, "pid": self.current_pid,
+                      "finished": time.time()})
+
+    def record_step(self, task_id: str, step: dict) -> None:
+        """记一条「已完成的步骤」。管线每完成一个 stage、工具轮每完成一个工具调用各落一条,
+        崩溃/断连后 `steps_of()` 能回答「跑到哪一步」,也让续跑知道哪些步骤不必重做。
+
+        fail-safe:写日志失败绝不影响任务执行(与任务生命周期记账同一定位)。
+        """
+        if not task_id or not isinstance(step, dict):
+            return
+        self._append({"event": "step", "task_id": task_id,
+                      "ts": time.time(), **step})
+
+    def steps_of(self, task_id: str, limit: int = 200) -> list:
+        """回放日志,取该 task 已记录的步骤(写入顺序;只留最近 limit 条)。
+
+        只读、容错:文件不存在/行损坏/中途写一半都跳过,不抛异常。
+        """
+        out: list = []
+        if not task_id or not os.path.exists(self.log_path):
+            return out
+        try:
+            with open(self.log_path, encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        ev = json.loads(line)
+                    except Exception:
+                        continue
+                    if ev.get("event") == "step" and ev.get("task_id") == task_id:
+                        out.append({k: v for k, v in ev.items()
+                                    if k not in ("event", "task_id")})
+        except Exception:
+            return out
+        return out[-limit:] if limit else out
+
+    def scan_interrupted(self) -> list[TaskRecord]:
+        """启动时调用:重建日志状态,找出被中断的任务。
+
+        判定(双重,避免 pid 复用误判):
+          1. 任务仍是 in_progress
+          2. 它的 boot_id 不是本次启动(即属于上一次运行)
+        boot_id 随机不复用,所以"非本次启动的进行中任务"必然是被中断的。
+        """
+        records = self._replay()
+        interrupted = []
+        for task_id, rec in records.items():
+            if rec.status != "in_progress":
+                continue
+            if rec.boot_id == self.boot_id:
+                continue
+            rec.status = "interrupted"
+            interrupted.append(rec)
+            self._append({"event": "interrupted", "task_id": task_id,
+                          "pid": rec.pid, "prev_boot_id": rec.boot_id,
+                          "pid_still_alive": _pid_alive(rec.pid),
+                          "detected_by": self.boot_id,
+                          "detected_at": time.time()})
+        return interrupted
+
+    def _replay(self) -> dict[str, TaskRecord]:
+        """回放日志,重建每个 task 的最终状态。"""
+        records: dict[str, TaskRecord] = {}
+        if not os.path.exists(self.log_path):
+            return records
+        try:
+            with open(self.log_path, encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        ev = json.loads(line)
+                    except Exception:
+                        continue
+                    self._apply_event(records, ev)
+        except Exception:
+            pass
+        return records
+
+    @staticmethod
+    def _apply_event(records: dict, ev: dict):
+        tid = ev.get("task_id")
+        if not tid:
+            return
+        kind = ev.get("event")
+        if kind == "start":
+            existing = records.get(tid)
+            start_count = (existing.start_count + 1) if existing else 1
+            records[tid] = TaskRecord(
+                task_id=tid, description=ev.get("description", ""),
+                pid=ev.get("pid", 0), started=ev.get("started", 0),
+                status="in_progress", session_id=ev.get("session_id", ""),
+                channel=ev.get("channel", ""), boot_id=ev.get("boot_id", ""),
+                retryable=ev.get("retryable", False),
+                start_count=start_count)
+        elif kind in ("finish", "interrupted") and tid in records:
+            records[tid].status = ("interrupted" if kind == "interrupted"
+                                   else ev.get("status", "done"))
+            if kind == "finish":
+                records[tid].finished = ev.get("finished", 0)
+
+    def plan_recovery(self, interrupted: list | None = None) -> RecoveryPlan:
+        """【P2-4】给中断任务分类,返回恢复计划(不改 scan_interrupted 的返回,兼容)。
+
+        分类规则:
+          · retryable=True → retryable(调用方标记过可安全重试)
+          · channel 非空   → notify_user(任务来自某个渠道,应通知该渠道用户)
+          · 其他           → needs_manual(保守:人工处理)
+        每个分类决定都发 recovery.plan 审计事件(有 spine 时)。
+        """
+        if interrupted is None:
+            interrupted = self.scan_interrupted()
+        plan = RecoveryPlan()
+        for rec in interrupted:
+            if rec.retryable:
+                plan.retryable.append(rec.task_id)
+                action = "retry"
+            elif rec.channel:
+                plan.notify_user.append(rec.task_id)
+                action = "notify_user"
+            else:
+                plan.needs_manual.append(rec.task_id)
+                action = "manual"
+            plan.details.append({
+                "task_id": rec.task_id,
+                "description": rec.description,
+                "channel": rec.channel,
+                "session_id": rec.session_id,
+                "start_count": rec.start_count,
+                "reason": "crash_or_restart",
+                "action": action,
+            })
+            self._emit_recovery_event(rec, action)
+        return plan
+
+    def _emit_recovery_event(self, rec: TaskRecord, action: str):
+        if self._spine is None:
+            return
+        try:
+            self._spine.publish("recovery.plan", {
+                "task_id": rec.task_id,
+                "action": action,
+                "channel": rec.channel,
+                "boot_id": self.boot_id,
+            }, source="recovery")
+        except Exception:
+            pass
+
+    def _append(self, entry: dict):
+        """追加一条日志(崩溃安全:单行 append)。fsync 可配:默认开(崩溃安全),
+        高频场景可关(牺牲最后一条的持久性换吞吐)。"""
+        try:
+            with open(self.log_path, "a", encoding="utf-8") as f:
+                f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+                f.flush()
+                if self._fsync:
+                    os.fsync(f.fileno())
+        except Exception:
+            pass
+
+    def compact_log(self, keep_recent_days: float = 7):
+        """压实日志:丢弃已完成且超过 N 天的记录,防文件无限膨胀。
+        【P2-4】保留判据用 max(started, finished):旧逻辑只看 started,
+        会丢"开始得早但刚完成"的记录(其 finish 审计线索一并丢失)。
+        步骤级 checkpoint(step 事件)属于被保留的任务 → 一并搬运,否则压实即丢断点。"""
+        records = self._replay()
+        cutoff = time.time() - keep_recent_days * 86400
+        keep = []
+        for rec in records.values():
+            last_activity = max(rec.started, rec.finished or 0)
+            if rec.status == "in_progress" or last_activity >= cutoff:
+                keep.append({"event": "start", **rec.to_dict()})
+                if rec.status != "in_progress":
+                    keep.append({"event": "finish", "task_id": rec.task_id,
+                                 "status": rec.status, "finished": rec.finished})
+        keep_ids = {e["task_id"] for e in keep if e.get("event") == "start"}
+        steps = self._raw_steps(keep_ids)
+        try:
+            tmp = self.log_path + ".tmp"
+            with open(tmp, "w", encoding="utf-8") as f:
+                for e in keep:
+                    f.write(json.dumps(e, ensure_ascii=False) + "\n")
+                for e in steps:
+                    f.write(json.dumps(e, ensure_ascii=False) + "\n")
+            os.replace(tmp, self.log_path)
+        except Exception:
+            pass
+
+    def _raw_steps(self, task_ids: set) -> list:
+        """扫原始日志取指定任务的 step 事件(compact 用;task_ids 为空直接返回)。"""
+        out: list = []
+        if not task_ids or not os.path.exists(self.log_path):
+            return out
+        try:
+            with open(self.log_path, encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        ev = json.loads(line)
+                    except Exception:
+                        continue
+                    if ev.get("event") == "step" and ev.get("task_id") in task_ids:
+                        out.append(ev)
+        except Exception:
+            return out
+        return out
+
+    def active_count(self) -> int:
+        return len(self._active)

package/python/foxclaw/core/scheduler.py

@@ -0,0 +1,141 @@
+"""
+认知调度器 (Cognitive Scheduler)
+
+把任务调度建模成真正的 OS 调度:特征化 → 打分 → 负载调整 → 路由 + 抢占判定。
+借鉴 Codex 版 Foxclaw 的 CognitiveScheduler 思想,适配本项目的五 Agent 架构。
+
+主管在分诊前先过这一层,得到:优先级、执行模式(抢占/标准)、预算、调度理由。
+让主管从"关键词分诊"升级为"特征感知 + 负载感知的 OS 式调度器"。
+"""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class TaskIntent:
+    """任务意图:粗粒度类别 + 三个亲和度提示。"""
+    category: str
+    confidence: float
+    memory_affinity: float
+    reasoning_depth: float
+
+
+@dataclass
+class TaskFeatures:
+    complexity: float
+    memory_dependency: float
+    tool_cost: float
+    reasoning_depth: float
+    urgency: float
+    expected_runtime: float
+
+
+@dataclass
+class SystemContext:
+    """系统负载上下文。主管据此调整优先级、决定是否抢占。"""
+    active_agents: int = 0
+    queue_depth: int = 0
+    system_load: float = 0.1
+
+
+@dataclass
+class ScheduleDecision:
+    priority: float
+    route_hint: str
+    execution_mode: str
+    preemptible: bool
+    expected_budget: float
+    intent: Optional[TaskIntent] = None
+    features: Optional[TaskFeatures] = None
+    preemption_reason: Optional[str] = None
+    reason: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "priority": self.priority, "route_hint": self.route_hint,
+            "execution_mode": self.execution_mode, "preemptible": self.preemptible,
+            "expected_budget": self.expected_budget,
+            "preemption_reason": self.preemption_reason, "reason": self.reason,
+            "intent": self.intent.category if self.intent else None,
+        }
+
+
+class CognitiveScheduler:
+    """意图感知的认知调度器。"""
+
+    _REASONING = ["规划", "计划", "分析", "设计", "推理", "为什么", "怎么办", "策略", "方案"]
+    _MEMORY = ["记得", "记忆", "之前", "上次", "回忆", "查一下我", "我说过"]
+    _WORKFLOW = ["然后", "接着", "之后", "依次", "批量", "流程", "再然后", "下一步"]
+
+    def parse_intent(self, task: str) -> TaskIntent:
+        t = (task or "").lower()
+        if any(k in t for k in self._REASONING):
+            return TaskIntent("reasoning", 0.8, 0.6, 0.95)
+        if any(k in t for k in self._MEMORY):
+            return TaskIntent("memory", 0.85, 0.9, 0.4)
+        if any(k in t for k in self._WORKFLOW):
+            return TaskIntent("workflow", 0.75, 0.5, 0.6)
+        return TaskIntent("tool", 0.7, 0.2, 0.3)
+
+    def extract_features(self, task: str, intent: TaskIntent) -> TaskFeatures:
+        complexity, tool_cost, urgency, runtime = 0.4, 0.3, 0.6, 0.5
+        if intent.category == "reasoning":
+            complexity, tool_cost, urgency, runtime = 0.9, 0.4, 0.8, 0.9
+        elif intent.category == "workflow":
+            complexity, tool_cost, urgency, runtime = 0.7, 0.6, 0.65, 0.8
+        elif intent.category == "memory":
+            complexity, tool_cost, urgency, runtime = 0.5, 0.2, 0.55, 0.6
+        if any(w in task for w in ["紧急", "立刻", "马上", "尽快", "急"]):
+            urgency = min(1.0, urgency + 0.3)
+        return TaskFeatures(complexity, intent.memory_affinity, tool_cost,
+                            intent.reasoning_depth, urgency, runtime)
+
+    def score(self, f: TaskFeatures) -> float:
+        """基础分:加权特征。"""
+        weighted = (f.complexity * 0.3 + f.memory_dependency * 0.2
+                    + f.tool_cost * 0.15 + f.reasoning_depth * 0.25
+                    + f.urgency * 0.1)
+        return round(weighted, 3)
+
+    def adjust(self, base: float, ctx: SystemContext, intent: TaskIntent,
+               f: TaskFeatures) -> float:
+        """负载调整:负载/队列越深,优先级被压低;紧急/记忆亲和度抬升。"""
+        memory_affinity = max(0.25, intent.memory_affinity)
+        load_penalty = 1 + ctx.system_load + (ctx.queue_depth * 0.05)
+        urgency_boost = max(0.5, f.urgency)
+        final = (base * urgency_boost * memory_affinity) / load_penalty
+        return min(1.0, round(final, 3))
+
+    def route_hint(self, priority: float, intent: TaskIntent) -> str:
+        if intent.category == "workflow":
+            return "workflow"
+        if intent.category == "reasoning":
+            return "agent"
+        if intent.category == "memory" and priority < 0.3:
+            return "tool"
+        return "agent" if priority > 0.35 else "tool"
+
+    def preemption_reason(self, priority: float, ctx: SystemContext) -> str | None:
+        if priority > 0.35 and ctx.active_agents >= 3:
+            return "高优先级 + Agent 压力大"
+        if priority > 0.4 and ctx.queue_depth >= 2:
+            return "高优先级 + 队列积压"
+        return None
+
+    def schedule(self, task: str, ctx: SystemContext | None = None) -> ScheduleDecision:
+        """主入口:给一个任务做完整调度决策。"""
+        ctx = ctx or SystemContext()
+        intent = self.parse_intent(task)
+        features = self.extract_features(task, intent)
+        base = self.score(features)
+        priority = self.adjust(base, ctx, intent, features)
+        route = self.route_hint(priority, intent)
+        preempt = self.preemption_reason(priority, ctx)
+        mode = "preemptive" if priority > 0.4 else "standard"
+        return ScheduleDecision(
+            priority=priority, route_hint=route, execution_mode=mode,
+            preemptible=preempt is not None,
+            expected_budget=round(features.expected_runtime * (1 + priority), 2),
+            intent=intent, features=features, preemption_reason=preempt,
+            reason=f"意图={intent.category} 优先级={priority} 模式={mode}")