@comate/zulu 1.2.1-beta.1 → 1.3.0

package/comate-engine/assets/skills/code-security-comate/scripts/scan_vulnerability.py

@@ -0,0 +1,444 @@
+#!/usr/bin/env python3
+"""
+漏洞扫描工具 - 将项目代码上传至 Comate 安全服务端进行 SAST 扫描。
+
+用法:
+    python3 scan_vulnerability.py --root-path <项目目录> --username <用户名>
+
+流程:
+    1. 获取扫描配置（支持的文件类型）
+    2. 遍历项目目录，收集文件哈希
+    3. 创建 bundle 并上传缺失文件
+    4. 发起扫描并轮询结果
+    5. 输出 SARIF 格式漏洞报告
+"""
+
+import argparse
+import json
+import logging
+import os
+import re
+import sys
+import time
+
+import http_client  # noqa: F401 (triggers shared logging config)
+import utils
+
+logger = logging.getLogger("scan")
+
+# 扫描轮询最大次数（每次间隔 3 秒，约 15 分钟）
+MAX_SCAN_POLLS = 300
+SCAN_POLL_INTERVAL = 3
+# AI 误报分析等待超时（秒，默认 20 分钟）
+AI_ANALYSIS_TIMEOUT = 1200
+
+
+def parse_gitignore(gitignore_path):
+    # type: (str) -> tuple
+    """解析 .gitignore 文件，返回 (ignore_patterns, negation_patterns) 正则表达式元组。
+
+    支持:
+    - `**` 匹配任意多级目录
+    - `!` 取消忽略
+    - `*` / `?` 通配符
+    - `/` 开头锚定根目录
+    - `/` 结尾匹配目录
+    """
+    ignore_patterns = []
+    negation_patterns = []
+    try:
+        with open(gitignore_path, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+
+                is_negation = line.startswith("!")
+                if is_negation:
+                    line = line[1:]
+
+                regex = line
+                regex = regex.replace(".", r"\.")
+                # 先处理 ** 再处理 *，避免 ** 被错误替换
+                regex = regex.replace("/**/", r"(?:/|/.+/)")
+                regex = regex.replace("**/", r"(?:|.+/)")
+                regex = regex.replace("/**", r"(?:|.*)")
+                regex = regex.replace("**", r".*")
+                regex = regex.replace("*", "[^/]*")
+                regex = regex.replace("?", "[^/]")
+
+                if line.startswith("/"):
+                    regex = "^" + regex[1:]
+                if line.endswith("/"):
+                    regex = regex[:-1] + r"($|/.*)"
+
+                compiled = re.compile(regex)
+                if is_negation:
+                    negation_patterns.append(compiled)
+                else:
+                    ignore_patterns.append(compiled)
+    except FileNotFoundError:
+        pass
+    return ignore_patterns, negation_patterns
+
+
+def is_ignored(path, gitignore_patterns, root_dir):
+    # type: (str, tuple, str) -> bool
+    rel_path = os.path.relpath(path, root_dir)
+    ignore_patterns, negation_patterns = gitignore_patterns
+    # 先检查是否被忽略
+    matched = any(p.search(rel_path) for p in ignore_patterns)
+    if not matched:
+        return False
+    # 再检查是否被取消忽略
+    if any(p.search(rel_path) for p in negation_patterns):
+        return False
+    return True
+
+
+def walk_dir(directory, extensions=None, gitignore_patterns=None):
+    # type: (str, list | None, list | None) -> list
+    """递归遍历目录，按扩展名过滤并排除 .gitignore 中的文件。"""
+    files = []
+    resolved_dir = os.path.realpath(directory)
+    for root, dirs, filenames in os.walk(resolved_dir):
+        # 跳过常见大目录
+        dirs[:] = [d for d in dirs if d not in utils.SKIP_DIRS]
+
+        if gitignore_patterns:
+            dirs[:] = [
+                d for d in dirs
+                if not is_ignored(os.path.join(root, d), gitignore_patterns, resolved_dir)
+            ]
+        for name in filenames:
+            full_path = os.path.join(root, name)
+            # 跳过过大的文件
+            try:
+                if os.path.getsize(full_path) > utils.MAX_FILE_SIZE:
+                    continue
+            except OSError:
+                continue
+            if gitignore_patterns and is_ignored(full_path, gitignore_patterns, resolved_dir):
+                continue
+            if extensions:
+                if not any(name.lower().endswith(ext.lower()) for ext in extensions):
+                    continue
+            files.append(full_path)
+    return files
+
+
+def get_settings(username, user_id):
+    # type: (str, str) -> dict
+    """获取扫描配置。"""
+    return http_client.get(
+        "{}/api/v2/analysis/settings".format(utils.HOST),
+        headers=utils.build_headers(username, user_id),
+    )
+
+
+def create_bundle(file_hashes, username, user_id):
+    # type: (dict, str, str) -> tuple
+    """创建扫描 bundle，返回 (bundle_hash, missing_files)。"""
+    data = http_client.post(
+        "{}/api/v1/bundle".format(utils.HOST),
+        headers=utils.build_headers(username, user_id),
+        json_body=file_hashes,
+    )
+
+    if isinstance(data.get("data"), list):
+        return data.get("bundleHash", ""), data["data"]
+    return (
+        data.get("data", {}).get("bundleHash", ""),
+        data.get("data", {}).get("missingFiles", []),
+    )
+
+
+def upload_files(bundle_hash, root_path, missing_files, file_hashes, username, user_id, upload_type="scan"):
+    # type: (str, str, list, dict, str, str, str) -> tuple
+    """上传缺失文件，返回 (bundle_hash, remaining_missing_files)。"""
+    payload = {"files": {}}
+    for name in missing_files:
+        file_path = os.path.join(root_path, name)
+        try:
+            content = utils.read_file_content(file_path)
+            payload["files"][name] = {"hash": file_hashes.get(name, ""), "content": content}
+        except Exception as e:
+            print("警告: 读取文件失败 {}: {}".format(file_path, e), file=sys.stderr)
+
+    if upload_type == "scan":
+        url = "{}/api/v1/bundle/{}".format(utils.HOST, bundle_hash)
+    else:
+        url = "{}/api/v1/upload".format(utils.HOST)
+
+    data = http_client.put(url, headers=utils.build_headers(username, user_id), json_body=payload)
+
+    if isinstance(data.get("data"), list):
+        return data.get("bundleHash", bundle_hash), data["data"]
+    return (
+        data.get("data", {}).get("bundleHash", bundle_hash),
+        data.get("data", {}).get("missingFiles", []),
+    )
+
+
+def scan_init(root_path, username, user_id):
+    # type: (str, str, str) -> tuple
+    """初始化扫描：获取配置、收集文件、创建 bundle、上传文件。返回 (bundle_hash, file_hashes)。"""
+    # 获取配置
+    cfg = get_settings(username, user_id)
+    scan_config = cfg.get("data", {}).get("scanConfiguration", {})
+    extensions = list(
+        set(scan_config.get("sca", {}).get("supportedLanguages", []))
+        | set(scan_config.get("sast", {}).get("supportedLanguages", []))
+    )
+
+    # 解析 .gitignore
+    gitignore_path = os.path.join(root_path, ".gitignore")
+    gitignore_patterns = parse_gitignore(gitignore_path)
+
+    # 遍历目录，收集文件哈希
+    all_files = walk_dir(root_path, extensions, gitignore_patterns)
+    file_hashes = {}  # type: dict
+    for f in all_files:
+        rel = os.path.relpath(f, root_path)
+        try:
+            file_hashes[rel] = utils.calc_sha256(f)
+        except Exception as e:
+            print("警告: 计算哈希失败 {}: {}".format(f, e), file=sys.stderr)
+
+    print("收集文件: {} 个".format(len(file_hashes)), file=sys.stderr)
+
+    # 创建 bundle
+    bundle_hash, missing_files = create_bundle(file_hashes, username, user_id)
+    print("Bundle: {}, 待上传: {} 个文件".format(bundle_hash, len(missing_files)), file=sys.stderr)
+
+    # 循环上传缺失文件
+    upload_retry = 0
+    MAX_UPLOAD_RETRIES = 10
+    while missing_files:
+        bundle_hash, missing_files = upload_files(
+            bundle_hash, root_path, missing_files, file_hashes, username, user_id, "scan"
+        )
+        upload_retry += 1
+        if missing_files:
+            print("剩余待上传: {} 个文件 (重试 {}/{})".format(len(missing_files), upload_retry,
+                                                     MAX_UPLOAD_RETRIES), file=sys.stderr)
+            if upload_retry >= MAX_UPLOAD_RETRIES:
+                print("错误: 上传重试 {} 次后仍有 {} 个文件未上传".format(MAX_UPLOAD_RETRIES, len(missing_files)), file=sys.stderr)
+                sys.exit(1)
+
+    return bundle_hash, file_hashes
+
+
+def get_analyzing_count(result):
+    # type: (dict) -> int
+    """统计处于 AI 分析中状态的漏洞数量。aiAnalysisStatus: 0-无需分析, 1-分析中, 2-真实漏洞, 3-误报"""
+    data = result.get("data", {})
+    runs = data.get("sarif", {}).get("runs") or data.get("runs") or []
+    count = 0
+    for run in runs:
+        results = run.get("results") or run.get("result") or []
+        for r in results:
+            status = r.get("properties", {}).get("aiAnalysisStatus", 0)
+            if status == 1:
+                count += 1
+    return count
+
+
+def _poll_ai_analysis(scan_info, chat_id, username, user_id, ai_analysis_timeout):
+    # type: (dict, str, str, str, int) -> dict
+    """轮询等待 AI 分析完成，返回最新结果。"""
+    start_time = time.time()
+    poll_interval = 10
+
+    while True:
+        elapsed = time.time() - start_time
+        if elapsed >= ai_analysis_timeout:
+            print("\nAI 分析等待超时（已等待 {} 分钟），将使用当前结果继续".format(
+                int(elapsed // 60)), file=sys.stderr)
+            logger.warning("AI analysis timeout after %d seconds", int(elapsed))
+            break
+
+        time.sleep(poll_interval)
+
+        result = http_client.post(
+            "{}/api/v2/analysis".format(utils.HOST),
+            headers=utils.build_headers(username, user_id, chat_id),
+            json_body=scan_info,
+        )
+
+        analyzing_count = get_analyzing_count(result)
+        if analyzing_count == 0:
+            print("\nAI 分析完成！", file=sys.stderr)
+            logger.info("AI analysis completed after %d seconds", int(time.time() - start_time))
+            return result
+
+        elapsed_min = int(elapsed // 60)
+        elapsed_sec = int(elapsed % 60)
+        print("AI 分析中... 剩余 {} 个漏洞待分析（已等待 {}分{}秒）".format(
+            analyzing_count, elapsed_min, elapsed_sec), file=sys.stderr)
+
+    return result
+
+
+def scan_vulnerability(root_path, chat_id="", username="", user_id="", wait_ai=True):
+    # type: (str, str, str, str, bool) -> tuple
+    """执行漏洞扫描，返回 (SARIF 格式结果, bundleHash)。
+
+    Args:
+        root_path: 项目根目录
+        chat_id: 对话 ID
+        username: Comate 用户名
+        user_id: 用户 ID
+        wait_ai: 是否等待 AI 分析完成，False 则扫描完成后立即返回
+    """
+    bundle_hash, file_hashes = scan_init(root_path, username, user_id)
+
+    git_info = utils.get_git_info(root_path)
+    scan_info = {
+        "key": {"hash": bundle_hash, "type": "file"},
+        "scan": 3,
+        "analysisContext": {
+            "initiator": "",
+            "trigger": "manual",
+            "workspaceName": "",
+            "workspacePath": "",
+            "gitInfo": [{
+                "path": "",
+                "gitURL": git_info["gitURL"],
+                "gitBranch": git_info["gitBranch"],
+                "gitCommitID": git_info["gitCommitID"],
+            }],
+        },
+    }
+
+    headers = utils.build_headers(username, user_id, chat_id)
+
+    print("开始扫描...", file=sys.stderr)
+    logger.info("scan start: chat_id=%s, bundle_hash=%s", chat_id, bundle_hash)
+
+    # 第一阶段：等待扫描完成
+    poll_count = 0
+    while poll_count < MAX_SCAN_POLLS:
+        result = http_client.post(
+            "{}/api/v2/analysis".format(utils.HOST), headers=headers, json_body=scan_info
+        )
+        if result.get("status") != 1:
+            break
+        poll_count += 1
+        print("扫描中，等待结果... ({}/{})".format(poll_count, MAX_SCAN_POLLS), file=sys.stderr)
+        time.sleep(SCAN_POLL_INTERVAL)
+
+    if result.get("status") == 1:
+        print("错误: 扫描超时，已轮询 {} 次仍未完成".format(MAX_SCAN_POLLS), file=sys.stderr)
+        return {"status": -1, "message": "扫描超时"}, bundle_hash
+
+    # 第二阶段：等待 AI 分析完成（可选）
+    analyzing_count = get_analyzing_count(result)
+    if analyzing_count > 0:
+        if wait_ai:
+            print("\n检测到 {} 个漏洞正在进行 AI 误报分析，等待分析完成...".format(analyzing_count), file=sys.stderr)
+            result = _poll_ai_analysis(scan_info, chat_id, username, user_id, AI_ANALYSIS_TIMEOUT)
+        else:
+            print("\n扫描完成，{} 个漏洞正在 AI 分析中（已跳过等待）".format(analyzing_count), file=sys.stderr)
+
+    return result, bundle_hash
+
+
+def wait_ai_analysis(root_path, scan_result_path, chat_id="", username="", user_id=""):
+    # type: (str, str, str, str, str) -> tuple
+    """仅等待 AI 分析完成，不重新扫描。从已有 scan_result.json 恢复上下文。"""
+    with open(scan_result_path, "r", encoding="utf-8") as f:
+        result = json.load(f)
+
+    bundle_hash = result.get("bundleHash", "")
+    if not bundle_hash:
+        print("错误: scan_result.json 中未找到 bundleHash", file=sys.stderr)
+        sys.exit(1)
+
+    analyzing_count = get_analyzing_count(result)
+    if analyzing_count == 0:
+        print("所有漏洞 AI 分析已完成，无需等待", file=sys.stderr)
+        return result, bundle_hash
+
+    git_info = utils.get_git_info(root_path)
+    scan_info = {
+        "key": {"hash": bundle_hash, "type": "file"},
+        "scan": 3,
+        "analysisContext": {
+            "initiator": "",
+            "trigger": "manual",
+            "workspaceName": "",
+            "workspacePath": "",
+            "gitInfo": [{
+                "path": "",
+                "gitURL": git_info["gitURL"],
+                "gitBranch": git_info["gitBranch"],
+                "gitCommitID": git_info["gitCommitID"],
+            }],
+        },
+    }
+
+    print("等待 {} 个漏洞的 AI 分析完成...".format(analyzing_count), file=sys.stderr)
+    result = _poll_ai_analysis(scan_info, chat_id, username, user_id, AI_ANALYSIS_TIMEOUT)
+    result["bundleHash"] = bundle_hash
+    return result, bundle_hash
+
+
+def main():
+    parser = argparse.ArgumentParser(description="代码安全漏洞扫描工具")
+    parser.add_argument("--root-path", required=True, help="待扫描项目根目录")
+    parser.add_argument("--username", required=True, help="Comate 用户名")
+    parser.add_argument("--chat-id", default="", help="对话唯一标识 (COMATE_SESSION_ID)")
+    parser.add_argument("--output-dir", default=None, help="结果输出目录，默认为 skill 临时目录")
+    parser.add_argument("--no-wait-ai", action="store_true",
+                        help="扫描完成后不等待 AI 分析，立即返回结果")
+    parser.add_argument("--wait-ai-only", action="store_true",
+                        help="仅等待 AI 分析完成（需配合 --scan-result 使用）")
+    parser.add_argument("--scan-result", default=None,
+                        help="已有扫描结果文件路径（与 --wait-ai-only 配合使用）")
+    args = parser.parse_args()
+
+    user_id = utils.make_user_id(args.username)
+
+    logger.info("scan_vulnerability start: chat_id=%s, username=%s, root_path=%s",
+                args.chat_id, args.username, args.root_path)
+
+    root_path = os.path.realpath(args.root_path)
+    if not os.path.isdir(root_path):
+        print("错误: 目录不存在 {}".format(root_path), file=sys.stderr)
+        sys.exit(1)
+
+    if args.wait_ai_only:
+        # --wait-ai-only 模式：仅等待 AI 分析，不重新扫描
+        if not args.scan_result:
+            print("错误: --wait-ai-only 需要配合 --scan-result 使用", file=sys.stderr)
+            sys.exit(1)
+        result, bundle_hash = wait_ai_analysis(
+            root_path, args.scan_result, chat_id=args.chat_id,
+            username=args.username, user_id=user_id,
+        )
+        # 原地更新 scan_result.json
+        output_file = os.path.realpath(args.scan_result)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(result, f, ensure_ascii=False, indent=2)
+        print(output_file)
+    else:
+        # 正常扫描模式
+        output_dir = utils.get_output_dir(root_path, args.output_dir)
+
+        result, bundle_hash = scan_vulnerability(
+            root_path, chat_id=args.chat_id, username=args.username, user_id=user_id,
+            wait_ai=not args.no_wait_ai,
+        )
+
+        # 将 bundleHash 写入结果，供后续解析和修复使用
+        result["bundleHash"] = bundle_hash
+
+        output_file = os.path.join(output_dir, "scan_result.json")
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(result, f, ensure_ascii=False, indent=2)
+        print(output_file)
+
+
+if __name__ == "__main__":
+    main()

package/comate-engine/assets/skills/code-security-comate/scripts/utils.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""
+公共工具函数 - 各脚本共享的常量和辅助函数。
+"""
+
+import base64
+import hashlib
+import os
+import shutil
+import subprocess
+import time
+import uuid
+from typing import Optional
+
+HOST = "https://comate-sec.baidu-int.com"
+WS_HOST = "wss://comate-sec.baidu-int.com"
+
+# 常见应跳过的大目录（即使 .gitignore 未列出）
+SKIP_DIRS = {".git", ".svn", ".hg", "node_modules", "__pycache__", "vendor", ".idea", ".vscode"}
+
+# 单文件最大扫描大小（10MB）
+MAX_FILE_SIZE = 10 * 1024 * 1024
+
+# 临时数据过期时间（秒），超过此时间的 .tmp 子目录将被自动清理
+TMP_EXPIRE_SECONDS = 24 * 60 * 60  # 24 小时
+
+
+def calc_sha256(file_path):
+    # type: (str) -> str
+    """计算文件 SHA256 哈希。"""
+    sha256_hash = hashlib.sha256()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            sha256_hash.update(chunk)
+    return sha256_hash.hexdigest()
+
+
+def calc_sha256_safe(file_path):
+    # type: (str) -> str
+    """计算文件 SHA256 哈希，失败返回空字符串。"""
+    try:
+        return calc_sha256(file_path)
+    except Exception:
+        return ""
+
+
+def read_file_content(file_path):
+    # type: (str) -> str
+    """读取文件内容，二进制文件用 base64 编码。"""
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            return f.read()
+    except UnicodeDecodeError:
+        with open(file_path, "rb") as f:
+            return base64.b64encode(f.read()).decode("ascii")
+
+
+def make_user_id(username):
+    # type: (str) -> str
+    """从用户名生成 USER_ID（MD5 前 12 位）。"""
+    return hashlib.md5(username.encode("utf-8")).hexdigest()[:12]
+
+
+def build_headers(username, user_id, chat_id=""):
+    # type: (str, str, str) -> dict
+    """构建统一的 HTTP 请求头。"""
+    headers = {
+        "Comate-Username": username,
+        "Comate-User-Id": user_id,
+        "SAST-Request-ID": str(uuid.uuid4()),
+    }
+    if chat_id:
+        headers["SAST-Chat-ID"] = chat_id
+    return headers
+
+
+def get_git_info(root_path):
+    # type: (str) -> dict
+    """从项目目录获取 git 信息（URL、分支、commitID）。"""
+    info = {"gitURL": "", "gitBranch": "", "gitCommitID": ""}
+    try:
+        info["gitURL"] = subprocess.check_output(
+            ["git", "remote", "get-url", "origin"], cwd=root_path, stderr=subprocess.DEVNULL
+        ).decode("utf-8").strip()
+    except Exception:
+        pass
+    try:
+        info["gitBranch"] = subprocess.check_output(
+            ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=root_path, stderr=subprocess.DEVNULL
+        ).decode("utf-8").strip()
+    except Exception:
+        pass
+    try:
+        info["gitCommitID"] = subprocess.check_output(
+            ["git", "rev-parse", "HEAD"], cwd=root_path, stderr=subprocess.DEVNULL
+        ).decode("utf-8").strip()
+    except Exception:
+        pass
+    return info
+
+
+def get_output_dir(root_path, output_dir=None):
+    # type: (str, Optional[str]) -> str
+    """获取输出目录，默认为 skill 临时目录下的项目隔离子目录。
+
+    每次调用时自动清理超过 24 小时的过期临时数据。
+    """
+    skill_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    project_name = os.path.basename(os.path.realpath(root_path))
+    path_hash = hashlib.md5(os.path.realpath(root_path).encode("utf-8")).hexdigest()[:8]
+    default_output = os.path.join(skill_dir, ".tmp", "{}_{}".format(project_name, path_hash))
+    result = os.path.realpath(output_dir) if output_dir else default_output
+
+    # 清理过期临时数据
+    _cleanup_expired_tmp(skill_dir)
+
+    os.makedirs(result, exist_ok=True)
+    return result
+
+
+def _cleanup_expired_tmp(skill_dir):
+    # type: (str) -> None
+    """清理 .tmp 目录下超过 24 小时的子目录。"""
+    tmp_dir = os.path.join(skill_dir, ".tmp")
+    if not os.path.isdir(tmp_dir):
+        return
+
+    now = time.time()
+    try:
+        for entry in os.listdir(tmp_dir):
+            entry_path = os.path.join(tmp_dir, entry)
+            if not os.path.isdir(entry_path):
+                continue
+            try:
+                mtime = os.path.getmtime(entry_path)
+                if now - mtime > TMP_EXPIRE_SECONDS:
+                    shutil.rmtree(entry_path)
+            except OSError:
+                pass
+    except OSError:
+        pass
+
+
+def safe_rmtree(dir_path):
+    # type: (str) -> None
+    """安全删除临时目录，校验路径位于 .tmp 下防止误删。"""
+    dir_path = os.path.realpath(dir_path)
+    # 安全校验：确保路径在 .tmp 目录下
+    if ".tmp" not in dir_path.split(os.sep):
+        return
+    if not os.path.isdir(dir_path):
+        return
+    shutil.rmtree(dir_path)