@comate/zulu 1.2.1-beta.2 → 1.3.0

package/comate-engine/assets/skills/code-security/scripts/report_chat.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""
+对话结果上报工具 - 扫描完成后向服务端上报本次对话信息。
+
+用法:
+    python3 report_chat.py --username <用户名> --chat-id <对话ID> --scan-result <扫描结果JSON文件> [--status <状态码>] [--err-message <错误信息>] [--git-url <仓库URL>] [--git-branch <分支>] [--ide <IDE名称>] [--query <用户输入>]
+
+接口: POST /api/v1/chats
+"""
+
+import argparse
+import hashlib
+import json
+import logging
+import os
+import subprocess
+import sys
+import uuid
+
+import http_client  # noqa: F401 (triggers shared logging config)
+
+HOST = "https://comate-sec.baidu-int.com"
+
+USERNAME = ""
+USER_ID = ""
+
+logger = logging.getLogger("report")
+
+
+def build_headers(chat_id):
+    # type: (str) -> dict
+    return {
+        "Comate-Username": USERNAME,
+        "Comate-User-Id": USER_ID,
+        "SAST-Request-ID": str(uuid.uuid4()),
+        "SAST-Chat-ID": chat_id,
+    }
+
+
+def calc_file_sha256(file_path):
+    # type: (str) -> str
+    """计算文件 SHA256。"""
+    sha256_hash = hashlib.sha256()
+    try:
+        with open(file_path, "rb") as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                sha256_hash.update(chunk)
+        return sha256_hash.hexdigest()
+    except Exception:
+        return ""
+
+
+def build_vuls_from_scan_result(scan_result, root_path=""):
+    # type: (dict, str) -> list
+    """从扫描结果中提取漏洞信息，构造 vuls 数组。"""
+    vuls = []
+    data = scan_result.get("data", {})
+    # 兼容两种结构: data.sarif.runs 或 data.runs
+    runs = data.get("sarif", {}).get("runs", []) or data.get("runs", [])
+    if not runs:
+        return vuls
+
+    # 按文件聚合漏洞
+    file_vuls = {}  # type: dict
+    for run in runs:
+        # 兼容 "results"(标准SARIF) 和 "result"(服务端实际返回)
+        results = run.get("results", []) or run.get("result", [])
+        for result in results:
+            locations = result.get("locations", [])
+            vul_hash = result.get("properties", {}).get("hash", "")
+            if not vul_hash:
+                continue
+            for loc in locations:
+                file_path = loc.get("physicalLocation", {}).get("artifactLocation", {}).get("uri", "")
+                if not file_path:
+                    continue
+                if file_path not in file_vuls:
+                    file_vuls[file_path] = {
+                        "fileName": file_path,
+                        "fileHash": "",
+                        "vuls": [],
+                    }
+                file_vuls[file_path]["vuls"].append({"vulHash": vul_hash})
+
+    # 填充 fileHash
+    bundle_files = scan_result.get("data", {}).get("bundleFiles", {})
+    for file_path, file_info in file_vuls.items():
+        fh = bundle_files.get(file_path, "")
+        if not fh and root_path:
+            fh = calc_file_sha256(os.path.join(root_path, file_path))
+        file_info["fileHash"] = fh
+
+    vuls = list(file_vuls.values())
+    return vuls
+
+
+def get_git_info(root_path):
+    # type: (str) -> dict
+    """从项目目录获取 git URL 和分支。"""
+    info = {"url": "", "branch": ""}
+    if not root_path:
+        return info
+    try:
+        info["url"] = subprocess.check_output(
+            ["git", "remote", "get-url", "origin"], cwd=root_path, stderr=subprocess.DEVNULL
+        ).decode("utf-8").strip()
+    except Exception:
+        pass
+    try:
+        info["branch"] = subprocess.check_output(
+            ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=root_path, stderr=subprocess.DEVNULL
+        ).decode("utf-8").strip()
+    except Exception:
+        pass
+    return info
+
+
+def report_chat(chat_id, scan_result, status, err_message, git_url, git_branch, ide, query, root_path=""):
+    # type: (str, dict, int, str, str, str, str, str, str) -> dict
+    """上报对话结果到服务端。"""
+    # git_url/git_branch 为空时，自动从 root_path 获取
+    if root_path and (not git_url or not git_branch):
+        git_info = get_git_info(root_path)
+        if not git_url:
+            git_url = git_info["url"]
+        if not git_branch:
+            git_branch = git_info["branch"]
+
+    vuls = build_vuls_from_scan_result(scan_result, root_path)
+
+    body = {
+        "type": 1,
+        "gitInfo": {
+            "url": git_url,
+            "branch": git_branch,
+        },
+        "ide": ide,
+        "query": query,
+        "status": status,
+        "errMessage": err_message,
+        "vuls": vuls,
+    }
+
+    headers = build_headers(chat_id)
+    logger.info("report_chat: chat_id=%s, vuls_count=%d, git_url=%s, git_branch=%s",
+                chat_id, len(vuls), git_url, git_branch)
+    return http_client.post("{}/api/v1/chats".format(HOST), headers=headers, json_body=body)
+
+
+def main():
+    global USERNAME, USER_ID
+    parser = argparse.ArgumentParser(description="对话结果上报工具")
+    parser.add_argument("--username", required=True, help="Comate 用户名")
+    parser.add_argument("--chat-id", required=True, help="对话唯一标识 (COMATE_SESSION_ID)")
+    parser.add_argument("--scan-result", required=True, help="扫描结果 JSON 文件路径")
+    parser.add_argument("--status", type=int, default=0, help="执行状态码，0=成功，1=失败 (默认 0)")
+    parser.add_argument("--err-message", default="", help="错误信息 (默认空)")
+    parser.add_argument("--root-path", default="", help="项目根目录，用于计算文件哈希")
+    parser.add_argument("--git-url", default="", help="Git 仓库 URL")
+    parser.add_argument("--git-branch", default="", help="Git 分支")
+    parser.add_argument("--ide", default="", help="IDE 名称")
+    parser.add_argument("--query", default="", help="用户输入的查询文本")
+    args = parser.parse_args()
+
+    USERNAME = args.username
+    USER_ID = hashlib.md5(USERNAME.encode("utf-8")).hexdigest()[:12]
+
+    logger.info("report_chat start: chat_id=%s, username=%s, scan_result=%s",
+                args.chat_id, USERNAME, args.scan_result)
+
+    # 读取扫描结果
+    scan_result = {}
+    try:
+        with open(args.scan_result, "r", encoding="utf-8") as f:
+            scan_result = json.load(f)
+    except Exception as e:
+        print("警告: 读取扫描结果失败 {}: {}".format(args.scan_result, e), file=sys.stderr)
+
+    try:
+        result = report_chat(
+            chat_id=args.chat_id,
+            scan_result=scan_result,
+            status=args.status,
+            err_message=args.err_message,
+            git_url=args.git_url,
+            git_branch=args.git_branch,
+            ide=args.ide,
+            query=args.query,
+            root_path=args.root_path,
+        )
+        print(json.dumps(result, ensure_ascii=False))
+    except Exception as e:
+        print("上报失败: {}".format(e), file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/comate-engine/assets/skills/code-security/scripts/scan_vulnerability.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+"""
+漏洞扫描工具 - 将项目代码上传至 Comate 安全服务端进行 SAST 扫描。
+
+用法:
+    python3 scan_vulnerability.py --root-path <项目目录> --username <用户名>
+
+流程:
+    1. 获取扫描配置（支持的文件类型）
+    2. 遍历项目目录，收集文件哈希
+    3. 创建 bundle 并上传缺失文件
+    4. 发起扫描并轮询结果
+    5. 输出 SARIF 格式漏洞报告
+"""
+
+import argparse
+import base64
+import hashlib
+import json
+import logging
+import os
+import re
+import sys
+import time
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+
+import http_client  # noqa: F401 (triggers shared logging config)
+
+HOST = "https://comate-sec.baidu-int.com"
+
+USERNAME = ""
+USER_ID = ""
+
+logger = logging.getLogger("scan")
+
+
+def build_headers(chat_id=""):
+    # type: (str) -> Dict[str, str]
+    headers = {
+        "Comate-Username": USERNAME,
+        "Comate-User-Id": USER_ID,
+        "SAST-Request-ID": str(uuid.uuid4()),
+    }
+    if chat_id:
+        headers["SAST-Chat-ID"] = chat_id
+    return headers
+
+
+def calc_sha256(file_path):
+    # type: (str) -> str
+    sha256_hash = hashlib.sha256()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            sha256_hash.update(chunk)
+    return sha256_hash.hexdigest()
+
+
+def parse_gitignore(gitignore_path):
+    # type: (str) -> List[Any]
+    """解析 .gitignore 文件，返回正则表达式列表。"""
+    patterns = []
+    try:
+        with open(gitignore_path, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                regex = line
+                regex = regex.replace(".", r"\.")
+                regex = regex.replace("*", ".*")
+                regex = regex.replace("?", ".")
+                if line.startswith("/"):
+                    regex = "^" + regex[1:]
+                if line.endswith("/"):
+                    regex = regex[:-1] + r"($|/.*)"
+                patterns.append(re.compile(regex))
+    except FileNotFoundError:
+        pass
+    return patterns
+
+
+def is_ignored(path, gitignore_patterns, root_dir):
+    # type: (str, List[Any], str) -> bool
+    rel_path = os.path.relpath(path, root_dir)
+    for pattern in gitignore_patterns:
+        if pattern.search(rel_path) or pattern.search(path):
+            return True
+    return False
+
+
+def walk_dir(directory, extensions=None, gitignore_patterns=None):
+    # type: (str, Optional[List[str]], Optional[List[Any]]) -> List[str]
+    """递归遍历目录，按扩展名过滤并排除 .gitignore 中的文件。"""
+    files = []
+    resolved_dir = os.path.realpath(directory)
+    for root, dirs, filenames in os.walk(resolved_dir):
+        if gitignore_patterns:
+            dirs[:] = [
+                d for d in dirs
+                if not is_ignored(os.path.join(root, d), gitignore_patterns, resolved_dir)
+            ]
+        for name in filenames:
+            if name.lower().endswith(".json"):
+                continue
+            full_path = os.path.join(root, name)
+            if gitignore_patterns and is_ignored(full_path, gitignore_patterns, resolved_dir):
+                continue
+            if extensions:
+                if not any(name.lower().endswith(ext.lower()) for ext in extensions):
+                    continue
+            files.append(full_path)
+    return files
+
+
+def read_file_content(file_path):
+    # type: (str) -> str
+    """读取文件内容，二进制文件用 base64 编码。"""
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            return f.read()
+    except UnicodeDecodeError:
+        with open(file_path, "rb") as f:
+            return base64.b64encode(f.read()).decode("ascii")
+
+
+def get_settings():
+    # type: () -> Dict
+    """获取扫描配置。"""
+    return http_client.get("{}/api/v2/analysis/settings".format(HOST), headers=build_headers())
+
+
+def create_bundle(file_hashes):
+    # type: (Dict[str, str]) -> Tuple[str, List[str]]
+    """创建扫描 bundle，返回 (bundle_hash, missing_files)。"""
+    data = http_client.post("{}/api/v1/bundle".format(HOST), headers=build_headers(), json_body=file_hashes)
+
+    if isinstance(data.get("data"), list):
+        return data.get("bundleHash", ""), data["data"]
+    return (
+        data.get("data", {}).get("bundleHash", ""),
+        data.get("data", {}).get("missingFiles", []),
+    )
+
+
+def upload_files(bundle_hash, root_path, missing_files, file_hashes, upload_type="scan"):
+    # type: (str, str, List[str], Dict[str, str], str) -> Tuple[str, List[str]]
+    """上传缺失文件，返回 (bundle_hash, remaining_missing_files)。"""
+    payload = {"files": {}}
+    for name in missing_files:
+        file_path = os.path.join(root_path, name)
+        try:
+            content = read_file_content(file_path)
+            payload["files"][name] = {"hash": file_hashes.get(name, ""), "content": content}
+        except Exception as e:
+            print("警告: 读取文件失败 {}: {}".format(file_path, e), file=sys.stderr)
+
+    if upload_type == "scan":
+        url = "{}/api/v1/bundle/{}".format(HOST, bundle_hash)
+    else:
+        url = "{}/api/v1/upload".format(HOST)
+
+    data = http_client.put(url, headers=build_headers(), json_body=payload)
+
+    if isinstance(data.get("data"), list):
+        return data.get("bundleHash", bundle_hash), data["data"]
+    return (
+        data.get("data", {}).get("bundleHash", bundle_hash),
+        data.get("data", {}).get("missingFiles", []),
+    )
+
+
+def scan_init(root_path):
+    # type: (str) -> Tuple[str, Dict[str, str]]
+    """初始化扫描：获取配置、收集文件、创建 bundle、上传文件。返回 (bundle_hash, file_hashes)。"""
+    # 获取配置
+    cfg = get_settings()
+    scan_config = cfg.get("data", {}).get("scanConfiguration", {})
+    extensions = list(
+        set(scan_config.get("sca", {}).get("supportedLanguages", []))
+        | set(scan_config.get("sast", {}).get("supportedLanguages", []))
+    )
+
+    # 解析 .gitignore
+    gitignore_path = os.path.join(root_path, ".gitignore")
+    gitignore_patterns = parse_gitignore(gitignore_path)
+
+    # 遍历目录，收集文件哈希
+    all_files = walk_dir(root_path, extensions, gitignore_patterns)
+    file_hashes = {}  # type: Dict[str, str]
+    for f in all_files:
+        rel = os.path.relpath(f, root_path)
+        try:
+            file_hashes[rel] = calc_sha256(f)
+        except Exception as e:
+            print("警告: 计算哈希失败 {}: {}".format(f, e), file=sys.stderr)
+
+    print("收集文件: {} 个".format(len(file_hashes)), file=sys.stderr)
+
+    # 创建 bundle
+    bundle_hash, missing_files = create_bundle(file_hashes)
+    print("Bundle: {}, 待上传: {} 个文件".format(bundle_hash, len(missing_files)), file=sys.stderr)
+
+    # 循环上传缺失文件
+    while missing_files:
+        bundle_hash, missing_files = upload_files(
+            bundle_hash, root_path, missing_files, file_hashes, "scan"
+        )
+        if missing_files:
+            print("剩余待上传: {} 个文件".format(len(missing_files)), file=sys.stderr)
+
+    return bundle_hash, file_hashes
+
+
+def get_git_info(root_path):
+    # type: (str) -> Dict[str, str]
+    """从项目目录获取 git 信息（URL、分支、commitID）。"""
+    import subprocess
+    info = {"gitURL": "", "gitBranch": "", "gitCommitID": ""}
+    try:
+        info["gitURL"] = subprocess.check_output(
+            ["git", "remote", "get-url", "origin"], cwd=root_path, stderr=subprocess.DEVNULL
+        ).decode("utf-8").strip()
+    except Exception:
+        pass
+    try:
+        info["gitBranch"] = subprocess.check_output(
+            ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=root_path, stderr=subprocess.DEVNULL
+        ).decode("utf-8").strip()
+    except Exception:
+        pass
+    try:
+        info["gitCommitID"] = subprocess.check_output(
+            ["git", "rev-parse", "HEAD"], cwd=root_path, stderr=subprocess.DEVNULL
+        ).decode("utf-8").strip()
+    except Exception:
+        pass
+    return info
+
+
+def scan_vulnerability(root_path, chat_id=""):
+    # type: (str, str) -> Tuple[Dict, str]
+    """执行漏洞扫描，返回 (SARIF 格式结果, bundleHash)。"""
+    bundle_hash, file_hashes = scan_init(root_path)
+
+    git_info = get_git_info(root_path)
+    scan_info = {
+        "key": {"hash": bundle_hash, "type": "file"},
+        "scan": 3,
+        "analysisContext": {
+            "initiator": "",
+            "trigger": "manual",
+            "workspaceName": "",
+            "workspacePath": "",
+            "gitInfo": [{
+                "path": "",
+                "gitURL": git_info["gitURL"],
+                "gitBranch": git_info["gitBranch"],
+                "gitCommitID": git_info["gitCommitID"],
+            }],
+        },
+    }
+
+    print("开始扫描...", file=sys.stderr)
+    logger.info("scan start: chat_id=%s, bundle_hash=%s", chat_id, bundle_hash)
+    while True:
+        result = http_client.post(
+            "{}/api/v2/analysis".format(HOST), headers=build_headers(chat_id), json_body=scan_info
+        )
+        if result.get("status") != 1:
+            return result, bundle_hash
+        print("扫描中，等待结果...", file=sys.stderr)
+        time.sleep(3)
+
+
+def main():
+    global USERNAME, USER_ID
+    parser = argparse.ArgumentParser(description="代码安全漏洞扫描工具")
+    parser.add_argument("--root-path", required=True, help="待扫描项目根目录")
+    parser.add_argument("--username", required=True, help="Comate 用户名")
+    parser.add_argument("--chat-id", default="", help="对话唯一标识 (COMATE_SESSION_ID)")
+    parser.add_argument("--output-dir", default=None, help="结果输出目录，默认为 skill 临时目录")
+    args = parser.parse_args()
+
+    USERNAME = args.username
+    USER_ID = hashlib.md5(USERNAME.encode("utf-8")).hexdigest()[:12]
+
+    logger.info("scan_vulnerability start: chat_id=%s, username=%s, root_path=%s",
+                args.chat_id, USERNAME, args.root_path)
+
+    root_path = os.path.realpath(args.root_path)
+    if not os.path.isdir(root_path):
+        print("错误: 目录不存在 {}".format(root_path), file=sys.stderr)
+        sys.exit(1)
+
+    # 默认输出到 skill 临时目录，按项目路径隔离子目录避免并发冲突
+    skill_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    project_name = os.path.basename(root_path)
+    path_hash = hashlib.md5(root_path.encode("utf-8")).hexdigest()[:8]
+    default_output = os.path.join(skill_dir, ".tmp", "{}_{}".format(project_name, path_hash))
+    output_dir = os.path.realpath(args.output_dir) if args.output_dir else default_output
+    os.makedirs(output_dir, exist_ok=True)
+
+    result, bundle_hash = scan_vulnerability(root_path, chat_id=args.chat_id)
+
+    # 将 bundleHash 写入结果，供后续解析和修复使用
+    result["bundleHash"] = bundle_hash
+
+    output_file = os.path.join(output_dir, "scan_result.json")
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+    print(output_file)
+
+
+if __name__ == "__main__":
+    main()