@agentunion/kite 1.5.0 → 1.6.0

package/extensions/services/dataclaw/services/oauth_service.py

@@ -0,0 +1,275 @@
+"""
+OAuth 第三方认证服务
+- GitHub OAuth 授权与用户信息获取
+- Google OAuth 授权与用户信息获取
+- 通过 OAuth 查找或创建 agent
+"""
+import urllib.parse
+from typing import Optional, Tuple
+
+import httpx
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from core.config import get_config
+from utils.id_generator import gen_agent_id, gen_api_key, gen_key_id, gen_uuid
+from utils.crypto import hash_api_key
+
+
+# ──────────────────────────────────────────────
+#  GitHub OAuth
+# ──────────────────────────────────────────────
+
+def get_github_authorize_url(redirect_uri: str) -> str:
+    """构造 GitHub OAuth 授权 URL"""
+    cfg = get_config()
+    github_cfg = cfg.get("oauth", {}).get("github", {})
+    client_id = github_cfg.get("client_id", "")
+
+    if not client_id:
+        print("[dataclaw][oauth] WARNING: github client_id 未配置")
+        return ""
+
+    params = {
+        "client_id": client_id,
+        "redirect_uri": redirect_uri,
+        "scope": "read:user user:email",
+    }
+    return f"https://github.com/login/oauth/authorize?{urllib.parse.urlencode(params)}"
+
+
+async def exchange_github_code(code: str, redirect_uri: str) -> Optional[dict]:
+    """
+    用 GitHub 授权码换取 access_token，再获取用户信息。
+    返回格式：{"id": str, "login": str, "name": str, "email": str, "avatar_url": str}
+    失败返回 None。
+    """
+    cfg = get_config()
+    github_cfg = cfg.get("oauth", {}).get("github", {})
+    client_id = github_cfg.get("client_id", "")
+    client_secret = github_cfg.get("client_secret", "")
+
+    if not client_id or not client_secret:
+        print("[dataclaw][oauth] WARNING: github OAuth 未配置完整，跳过")
+        return None
+
+    try:
+        async with httpx.AsyncClient(timeout=15) as client:
+            # 1. 用 code 换取 access_token
+            token_resp = await client.post(
+                "https://github.com/login/oauth/access_token",
+                json={
+                    "client_id": client_id,
+                    "client_secret": client_secret,
+                    "code": code,
+                    "redirect_uri": redirect_uri,
+                },
+                headers={"Accept": "application/json"},
+            )
+            token_data = token_resp.json()
+            access_token = token_data.get("access_token")
+            if not access_token:
+                print(f"[dataclaw][oauth] github token exchange failed: {token_data}")
+                return None
+
+            # 2. 用 access_token 获取用户信息
+            user_resp = await client.get(
+                "https://api.github.com/user",
+                headers={
+                    "Authorization": f"Bearer {access_token}",
+                    "Accept": "application/json",
+                },
+            )
+            if user_resp.status_code != 200:
+                print(f"[dataclaw][oauth] github user info failed: {user_resp.status_code}")
+                return None
+
+            user_data = user_resp.json()
+            return {
+                "id": str(user_data.get("id", "")),
+                "login": user_data.get("login", ""),
+                "name": user_data.get("name") or user_data.get("login", ""),
+                "email": user_data.get("email", ""),
+                "avatar_url": user_data.get("avatar_url", ""),
+            }
+    except Exception as e:
+        print(f"[dataclaw][oauth] github exchange error: {e}")
+        return None
+
+
+# ──────────────────────────────────────────────
+#  Google OAuth
+# ──────────────────────────────────────────────
+
+def get_google_authorize_url(redirect_uri: str) -> str:
+    """构造 Google OAuth 授权 URL"""
+    cfg = get_config()
+    google_cfg = cfg.get("oauth", {}).get("google", {})
+    client_id = google_cfg.get("client_id", "")
+
+    if not client_id:
+        print("[dataclaw][oauth] WARNING: google client_id 未配置")
+        return ""
+
+    params = {
+        "client_id": client_id,
+        "redirect_uri": redirect_uri,
+        "response_type": "code",
+        "scope": "openid email profile",
+        "access_type": "offline",
+        "prompt": "consent",
+    }
+    return f"https://accounts.google.com/o/oauth2/v2/auth?{urllib.parse.urlencode(params)}"
+
+
+async def exchange_google_code(code: str, redirect_uri: str) -> Optional[dict]:
+    """
+    用 Google 授权码换取 access_token，再获取用户信息。
+    返回格式：{"id": str, "name": str, "email": str, "picture": str}
+    失败返回 None。
+    """
+    cfg = get_config()
+    google_cfg = cfg.get("oauth", {}).get("google", {})
+    client_id = google_cfg.get("client_id", "")
+    client_secret = google_cfg.get("client_secret", "")
+
+    if not client_id or not client_secret:
+        print("[dataclaw][oauth] WARNING: google OAuth 未配置完整，跳过")
+        return None
+
+    try:
+        async with httpx.AsyncClient(timeout=15) as client:
+            # 1. 用 code 换取 access_token
+            token_resp = await client.post(
+                "https://oauth2.googleapis.com/token",
+                data={
+                    "client_id": client_id,
+                    "client_secret": client_secret,
+                    "code": code,
+                    "redirect_uri": redirect_uri,
+                    "grant_type": "authorization_code",
+                },
+            )
+            token_data = token_resp.json()
+            access_token = token_data.get("access_token")
+            if not access_token:
+                print(f"[dataclaw][oauth] google token exchange failed: {token_data}")
+                return None
+
+            # 2. 用 access_token 获取用户信息
+            user_resp = await client.get(
+                "https://www.googleapis.com/oauth2/v2/userinfo",
+                headers={"Authorization": f"Bearer {access_token}"},
+            )
+            if user_resp.status_code != 200:
+                print(f"[dataclaw][oauth] google user info failed: {user_resp.status_code}")
+                return None
+
+            user_data = user_resp.json()
+            return {
+                "id": str(user_data.get("id", "")),
+                "name": user_data.get("name", ""),
+                "email": user_data.get("email", ""),
+                "picture": user_data.get("picture", ""),
+            }
+    except Exception as e:
+        print(f"[dataclaw][oauth] google exchange error: {e}")
+        return None
+
+
+# ──────────────────────────────────────────────
+#  通用：查找或创建 agent
+# ──────────────────────────────────────────────
+
+async def find_or_create_agent_by_oauth(
+    provider: str,
+    oauth_user_info: dict,
+    session: AsyncSession,
+) -> Tuple[str, bool]:
+    """
+    通过 OAuth 提供商和用户信息查找或创建 agent。
+
+    参数：
+        provider      - OAuth 提供商标识，如 "github" / "google"
+        oauth_user_info - 上游返回的用户信息字典，必须包含 "id" 字段
+        session       - SQLAlchemy 异步会话
+
+    返回：
+        (agent_id, is_new) — agent_id 和是否为新创建
+    """
+    oauth_id = str(oauth_user_info.get("id", ""))
+    if not oauth_id:
+        raise ValueError("oauth_user_info 必须包含 id 字段")
+
+    # 1. 在 agent_contacts 表中查找已有绑定
+    row = (await session.execute(
+        text("SELECT agent_id FROM agent_contacts "
+             "WHERE contact_type=:provider AND contact_value=:oauth_id LIMIT 1"),
+        {"provider": provider, "oauth_id": oauth_id},
+    )).fetchone()
+
+    if row:
+        # 已有绑定，直接返回
+        return (row.agent_id, False)
+
+    # 2. 未找到绑定，创建新 agent
+    agent_id = gen_agent_id()
+    display_name = oauth_user_info.get("name") or oauth_user_info.get("login") or f"{provider}_{oauth_id}"
+    email = oauth_user_info.get("email", "")
+
+    cfg = get_config()
+    initial_credits = cfg.get("credits", {}).get("initial_credits", 100.0)
+
+    # 创建 agent 记录
+    await session.execute(
+        text("INSERT INTO agents (id, name, type, description, verified, credits, status) "
+             "VALUES (:id, :name, 'custom', :desc, 1, :credits, 'active')"),
+        {
+            "id": agent_id,
+            "name": display_name,
+            "desc": f"通过 {provider} OAuth 注册",
+            "credits": initial_credits,
+        },
+    )
+
+    # 创建 OAuth 联系方式记录
+    await session.execute(
+        text("INSERT INTO agent_contacts (agent_id, contact_type, contact_value, verified) "
+             "VALUES (:aid, :provider, :oauth_id, 1)"),
+        {"aid": agent_id, "provider": provider, "oauth_id": oauth_id},
+    )
+
+    # 如果有邮箱，也写入联系方式（忽略重复）
+    if email:
+        try:
+            await session.execute(
+                text("INSERT INTO agent_contacts (agent_id, contact_type, contact_value, verified) "
+                     "VALUES (:aid, 'email', :email, 1)"),
+                {"aid": agent_id, "email": email},
+            )
+        except Exception:
+            # 邮箱可能已被其他账户绑定，忽略
+            pass
+
+    # 创建默认 API Key
+    api_key = gen_api_key()
+    key_id = gen_key_id()
+    await session.execute(
+        text("INSERT INTO api_keys (id, agent_id, key_hash, key_prefix, name, permissions, status) "
+             "VALUES (:kid, :aid, :hash, :prefix, 'oauth_default', '[\"read\",\"write\"]', 'active')"),
+        {"kid": key_id, "aid": agent_id, "hash": hash_api_key(api_key), "prefix": api_key[:10]},
+    )
+
+    # 记录注册奖励积分流水
+    tid = gen_uuid()[:32]
+    await session.execute(
+        text("INSERT INTO credit_transactions (id, agent_id, type, amount, balance_after, description) "
+             "VALUES (:tid, :aid, 'bonus', :amt, :bal, :desc)"),
+        {"tid": tid, "aid": agent_id, "amt": initial_credits, "bal": initial_credits,
+         "desc": f"{provider} OAuth 注册奖励"},
+    )
+
+    await session.commit()
+    print(f"[dataclaw][oauth] 新 agent 创建成功: {agent_id} (via {provider})")
+
+    return (agent_id, True)

package/extensions/services/dataclaw/services/pricing.py

@@ -0,0 +1,102 @@
+"""
+三阶段定价算法服务
+
+根据订阅人数实现动态定价：
+- 早期阶段 (n<30):   collect_cost * 0.42（约半价，激励早期采用者）
+- 成长阶段 (30≤n<300): collect_cost * 0.15（规模效应开始显现）
+- 规模阶段 (n≥300):  collect_cost * 0.05（大规模摊薄成本）
+
+收益分配：贡献者 85%，平台 15%
+"""
+from typing import Dict
+
+# 最低单次价格（积分）
+MIN_PRICE = 0.01
+
+# 平台抽成比例
+PLATFORM_COMMISSION = 0.15
+
+# 贡献者分成比例
+CONTRIBUTOR_SHARE = 1 - PLATFORM_COMMISSION  # 0.85
+
+# 三阶段定价系数
+PRICING_TIERS = [
+    {"max_subscribers": 30,  "multiplier": 0.42, "label": "早期阶段"},
+    {"max_subscribers": 300, "multiplier": 0.15, "label": "成长阶段"},
+    {"max_subscribers": None, "multiplier": 0.05, "label": "规模阶段"},
+]
+
+# 成本折算参数：1 token ≈ 0.0000002 积分（基于 Claude Haiku 定价）
+TOKEN_UNIT_COST = 0.0000002
+# 基础调用数：假设每次查询消耗约 5000 tokens
+BASE_QUERY_TOKENS = 5000
+
+
+def _calculate_collect_cost(token_cost: int) -> float:
+    """计算采集成本（积分）"""
+    return token_cost * TOKEN_UNIT_COST * BASE_QUERY_TOKENS
+
+
+def _get_tier(subscriber_count: int) -> Dict:
+    """根据订阅人数获取定价阶段"""
+    for tier in PRICING_TIERS:
+        if tier["max_subscribers"] is None or subscriber_count < tier["max_subscribers"]:
+            return tier
+    return PRICING_TIERS[-1]
+
+
+def calculate_price(token_cost: int, subscriber_count: int) -> Dict:
+    """
+    计算单次查询价格
+
+    Args:
+        token_cost: feed 的 token 消耗量
+        subscriber_count: 当前订阅人数
+
+    Returns:
+        包含价格信息的字典：
+        - price: 单次价格（积分）
+        - collect_cost: 采集成本
+        - tier: 当前定价阶段
+        - subscriber_count: 订阅人数
+    """
+    n = max(subscriber_count, 1)
+    collect_cost = _calculate_collect_cost(token_cost or 1000)
+    tier = _get_tier(n)
+
+    price = round(collect_cost * tier["multiplier"], 4)
+    price = max(price, MIN_PRICE)
+
+    return {
+        "price": price,
+        "collect_cost": round(collect_cost, 4),
+        "tier": tier["label"],
+        "multiplier": tier["multiplier"],
+        "subscriber_count": n,
+    }
+
+
+def calculate_revenue_split(price: float) -> Dict:
+    """
+    计算收益分配
+
+    Args:
+        price: 单次查询价格（积分）
+
+    Returns:
+        包含分配信息的字典：
+        - total: 总价
+        - contributor_amount: 贡献者所得
+        - platform_amount: 平台所得
+        - contributor_ratio / platform_ratio: 分配比例
+    """
+    contributor_amount = round(price * CONTRIBUTOR_SHARE, 6)
+    platform_amount = round(price * PLATFORM_COMMISSION, 6)
+
+    return {
+        "total": price,
+        "contributor_amount": contributor_amount,
+        "platform_amount": platform_amount,
+        "contributor_ratio": CONTRIBUTOR_SHARE,
+        "platform_ratio": PLATFORM_COMMISSION,
+    }

package/extensions/services/dataclaw/services/quality.py

@@ -0,0 +1,79 @@
+"""
+数据质量治理服务
+- SLA 监控（更新频率、数据新鲜度）
+- 数据质量评分
+- 自动降级/警告
+"""
+from datetime import datetime, timedelta
+from typing import Optional
+
+
+class QualityMonitor:
+    """数据质量监控器"""
+
+    @staticmethod
+    def check_sla(feed_data: dict) -> dict:
+        """检查 SLA 合规性"""
+        update_cycle = feed_data.get("update_cycle", "1h")
+        last_data_at = feed_data.get("last_data_at")
+
+        if not last_data_at:
+            return {"compliant": False, "reason": "no_data", "score": 0}
+
+        # 解析更新周期
+        cycle_minutes = {"5m": 5, "1h": 60, "24h": 1440, "realtime": 1}.get(update_cycle, 60)
+        threshold = timedelta(minutes=cycle_minutes * 2)  # 允许 2 倍延迟
+
+        if isinstance(last_data_at, str):
+            last_data_at = datetime.fromisoformat(last_data_at.replace("Z", "+00:00"))
+
+        age = datetime.now(last_data_at.tzinfo) - last_data_at
+        compliant = age < threshold
+
+        return {
+            "compliant": compliant,
+            "age_minutes": age.total_seconds() / 60,
+            "threshold_minutes": threshold.total_seconds() / 60,
+            "score": 100 if compliant else max(0, 100 - int(age.total_seconds() / 60))
+        }
+
+    @staticmethod
+    def calculate_quality_score(feed_stats: dict) -> float:
+        """计算综合质量分"""
+        reputation = float(feed_stats.get("reputation_score", 50))
+        subscriber_count = int(feed_stats.get("subscriber_count", 0))
+        total_queries = int(feed_stats.get("total_queries", 0))
+
+        # 加权计算
+        score = reputation * 0.6
+        score += min(subscriber_count / 10, 20)  # 订阅数贡献最多 20 分
+        score += min(total_queries / 100, 20)    # 调用数贡献最多 20 分
+
+        return min(100, score)
+
+    @staticmethod
+    def should_warn(feed_data: dict) -> Optional[str]:
+        """判断是否需要警告"""
+        sla = QualityMonitor.check_sla(feed_data)
+        if not sla["compliant"]:
+            return f"数据更新延迟 {sla['age_minutes']:.0f} 分钟"
+
+        reputation = float(feed_data.get("reputation_score", 50))
+        if reputation < 30:
+            return f"信誉分过低 ({reputation:.0f})"
+
+        return None
+
+    @staticmethod
+    def auto_downgrade(feed_data: dict) -> str:
+        """自动降级判断"""
+        sla = QualityMonitor.check_sla(feed_data)
+        reputation = float(feed_data.get("reputation_score", 50))
+
+        if not sla["compliant"] and sla["age_minutes"] > 1440:  # 超过 1 天
+            return "inactive"
+
+        if reputation < 20:
+            return "warning"
+
+        return "active"

package/extensions/services/dataclaw/services/reputation.py

@@ -0,0 +1,142 @@
+"""
+信誉评分系统
+
+综合多维度指标计算 Feed 信誉分（0-100）：
+- 平均评分   40%  — 用户评价直接反映数据质量
+- 订阅数     20%  — 市场认可度
+- 数据活跃度 20%  — 近期更新频率
+- 投诉记录   20%  — 负面信号扣分
+"""
+from typing import Dict, Optional
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+# 各因子权重
+WEIGHT_RATING = 0.40
+WEIGHT_SUBSCRIBERS = 0.20
+WEIGHT_ACTIVITY = 0.20
+WEIGHT_COMPLAINTS = 0.20
+
+# 订阅数归一化上限（达到此值得满分）
+SUBSCRIBER_CAP = 500
+
+# 活跃度：近 7 天有更新版本视为活跃
+ACTIVITY_DAYS = 7
+
+# 投诉数扣分阈值
+COMPLAINT_PENALTY_EACH = 10  # 每条未解决投诉扣 10 分
+
+
+def calculate_reputation(feed_stats: dict) -> float:
+    """
+    基于统计数据计算信誉分
+
+    Args:
+        feed_stats: 包含以下字段的字典
+            - avg_rating: float, 平均评分（1-5），无评价时为 None
+            - subscriber_count: int, 订阅人数
+            - recent_versions: int, 近 N 天内的版本更新数
+            - pending_complaints: int, 未解决投诉数
+
+    Returns:
+        0-100 的信誉分
+    """
+    # 1. 平均评分维度（0-100）
+    avg_rating = feed_stats.get("avg_rating")
+    if avg_rating is not None:
+        rating_score = (float(avg_rating) / 5.0) * 100
+    else:
+        rating_score = 50.0  # 无评价时给中间分
+
+    # 2. 订阅数维度（0-100）
+    subscriber_count = int(feed_stats.get("subscriber_count", 0))
+    subscriber_score = min(subscriber_count / SUBSCRIBER_CAP, 1.0) * 100
+
+    # 3. 数据活跃度维度（0-100）
+    recent_versions = int(feed_stats.get("recent_versions", 0))
+    if recent_versions >= 5:
+        activity_score = 100.0
+    elif recent_versions >= 1:
+        activity_score = 60.0 + (recent_versions - 1) * 10  # 1 次 60，5 次 100
+    else:
+        activity_score = 20.0  # 无更新但不至于零分
+
+    # 4. 投诉记录维度（满分往下扣）
+    pending_complaints = int(feed_stats.get("pending_complaints", 0))
+    complaint_score = max(0, 100 - pending_complaints * COMPLAINT_PENALTY_EACH)
+
+    # 加权汇总
+    total = (
+        rating_score * WEIGHT_RATING
+        + subscriber_score * WEIGHT_SUBSCRIBERS
+        + activity_score * WEIGHT_ACTIVITY
+        + complaint_score * WEIGHT_COMPLAINTS
+    )
+
+    return round(min(100, max(0, total)), 2)
+
+
+async def update_reputation_after_review(
+    feed_id: str,
+    new_rating: int,
+    session: AsyncSession,
+) -> float:
+    """
+    评价提交后重新计算并更新 Feed 信誉分
+
+    Args:
+        feed_id: Feed ID
+        new_rating: 本次评分（1-5），仅作为触发信号，实际从 DB 聚合
+        session: 数据库 session
+
+    Returns:
+        更新后的信誉分
+    """
+    # 聚合平均评分
+    avg_rating = (await session.execute(
+        text("SELECT AVG(rating) FROM reviews WHERE feed_id=:fid"),
+        {"fid": feed_id}
+    )).scalar()
+
+    # 获取订阅数
+    row = (await session.execute(
+        text("SELECT subscriber_count FROM feeds WHERE id=:fid"),
+        {"fid": feed_id}
+    )).fetchone()
+    subscriber_count = row.subscriber_count if row else 0
+
+    # 近 N 天版本更新数
+    recent_versions = (await session.execute(
+        text(
+            "SELECT COUNT(*) FROM feed_versions WHERE feed_id=:fid "
+            "AND created_at >= DATE_SUB(NOW(), INTERVAL :days DAY)"
+        ),
+        {"fid": feed_id, "days": ACTIVITY_DAYS}
+    )).scalar() or 0
+
+    # 未解决投诉数
+    pending_complaints = (await session.execute(
+        text(
+            "SELECT COUNT(*) FROM reports WHERE feed_id=:fid "
+            "AND status IN ('pending', 'reviewing')"
+        ),
+        {"fid": feed_id}
+    )).scalar() or 0
+
+    # 计算信誉分
+    stats = {
+        "avg_rating": float(avg_rating) if avg_rating else None,
+        "subscriber_count": subscriber_count,
+        "recent_versions": recent_versions,
+        "pending_complaints": pending_complaints,
+    }
+    score = calculate_reputation(stats)
+
+    # 写回 feeds 表
+    await session.execute(
+        text("UPDATE feeds SET reputation_score=:score WHERE id=:fid"),
+        {"score": score, "fid": feed_id}
+    )
+
+    return score