@comate/zulu 1.2.1-beta.2 → 1.3.1

package/comate-engine/assets/skills/code-security-comate/scripts/http_client.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""
+基于 urllib 的轻量 HTTP 客户端，零第三方依赖。
+提供 GET / POST / PUT 方法，支持 JSON 请求和响应。
+当 ssl 模块不可用时（如 libssl 版本不匹配），自动回退到 curl 子进程。
+所有脚本共用日志配置，写入 skill/logs/<日期>.log，自动清理 7 天前的日志。
+"""
+
+import datetime
+import glob
+import json
+import logging
+import os
+import subprocess
+
+try:
+    import ssl
+    _SSL_AVAILABLE = True
+except ImportError:
+    _SSL_AVAILABLE = False
+
+try:
+    from urllib.request import Request, urlopen
+    from urllib.error import HTTPError, URLError
+except ImportError:
+    from urllib2 import Request, urlopen, HTTPError, URLError
+
+# ---- 统一日志配置 ----
+_scripts_dir = os.path.dirname(os.path.abspath(__file__))
+_skill_dir = os.path.dirname(_scripts_dir)
+_log_dir = os.path.join(_skill_dir, "logs")
+os.makedirs(_log_dir, exist_ok=True)
+
+# 清理超过 7 天的日志文件
+_now = datetime.datetime.now()
+for _f in glob.glob(os.path.join(_log_dir, "*.log")):
+    try:
+        _mtime = datetime.datetime.fromtimestamp(os.path.getmtime(_f))
+        if (_now - _mtime).days > 7:
+            os.remove(_f)
+    except OSError:
+        pass
+
+_log_file = os.path.join(_log_dir, _now.strftime("%Y-%m-%d") + ".log")
+logging.basicConfig(
+    filename=_log_file,
+    level=logging.INFO,
+    format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
+)
+logger = logging.getLogger("http")
+
+
+def _build_request(url, method="GET", headers=None, json_body=None):
+    """构造 urllib Request 对象。"""
+    data = None
+    if json_body is not None:
+        data = json.dumps(json_body).encode("utf-8")
+
+    req = Request(url, data=data)
+    req.get_method = lambda: method
+
+    if headers:
+        for k, v in headers.items():
+            req.add_header(k, v)
+
+    if data is not None and not req.has_header("Content-type"):
+        req.add_header("Content-Type", "application/json; charset=utf-8")
+
+    return req
+
+
+def _create_ssl_context():
+    """创建 SSL 上下文。"""
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+
+def _log_safe_headers(headers):
+    """提取可安全记录的请求头。"""
+    safe = {}
+    if headers:
+        for k in ("SAST-Chat-ID", "SAST-Request-ID", "Comate-User-Id", "Comate-Username"):
+            if k in headers:
+                safe[k] = headers[k]
+    return safe
+
+
+def _request_curl(url, method="GET", headers=None, json_body=None, timeout=120):
+    """ssl 不可用时，通过 curl 子进程发送请求。"""
+    cmd = ["curl", "-s", "-S", "--insecure", "-X", method, "--max-time", str(timeout)]
+
+    if headers:
+        for k, v in headers.items():
+            cmd += ["-H", "{}: {}".format(k, v)]
+
+    if json_body is not None:
+        cmd += ["-H", "Content-Type: application/json; charset=utf-8"]
+        cmd += ["-d", json.dumps(json_body)]
+
+    cmd.append(url)
+
+    logger.info("[curl] %s %s headers=%s", method, url, json.dumps(_log_safe_headers(headers)))
+    try:
+        proc = subprocess.run(cmd, capture_output=True, timeout=timeout + 10)
+    except FileNotFoundError:
+        raise RuntimeError(
+            "ssl 模块不可用且未找到 curl 命令。"
+            "请安装 curl 或修复 Python ssl 依赖（安装 libssl1.1 或重新编译 Python）。"
+        )
+
+    if proc.returncode != 0:
+        stderr_text = proc.stderr.decode("utf-8", errors="replace").strip()
+        logger.error("[curl] %s %s -> curl exit %d: %s", method, url, proc.returncode, stderr_text)
+        raise RuntimeError("curl 请求失败 (exit {}): {}".format(proc.returncode, stderr_text))
+
+    body = proc.stdout.decode("utf-8")
+    logger.info("[curl] %s %s -> OK (%d bytes)", method, url, len(body))
+    return json.loads(body)
+
+
+def request(url, method="GET", headers=None, json_body=None, timeout=120):
+    """
+    发送 HTTP 请求，返回解析后的 JSON dict。
+    当 ssl 模块不可用时自动回退到 curl。
+
+    :param url: 请求 URL
+    :param method: HTTP 方法 (GET/POST/PUT)
+    :param headers: 请求头 dict
+    :param json_body: JSON 请求体 (自动序列化)
+    :param timeout: 超时秒数
+    :return: 响应 JSON dict
+    :raises: HTTPError, URLError, RuntimeError
+    """
+    if not _SSL_AVAILABLE:
+        return _request_curl(url, method, headers, json_body, timeout)
+
+    # 记录请求（仅 URL 和关键 header，不记录 body 避免日志过大）
+    logger.info("%s %s headers=%s", method, url, json.dumps(_log_safe_headers(headers)))
+
+    req = _build_request(url, method, headers, json_body)
+    ctx = _create_ssl_context()
+
+    try:
+        resp = urlopen(req, timeout=timeout, context=ctx)
+        status = resp.getcode()
+        body = resp.read().decode("utf-8")
+        logger.info("%s %s -> %d (%d bytes)", method, url, status, len(body))
+        return json.loads(body)
+    except HTTPError as e:
+        error_body = ""
+        try:
+            error_body = e.read().decode("utf-8")[:500]
+        except Exception:
+            pass
+        logger.error("%s %s -> HTTP %d: %s", method, url, e.code, error_body)
+        raise
+
+
+def get(url, headers=None, timeout=120):
+    """发送 GET 请求，返回 JSON。"""
+    return request(url, method="GET", headers=headers, timeout=timeout)
+
+
+def post(url, headers=None, json_body=None, timeout=120):
+    """发送 POST 请求，返回 JSON。"""
+    return request(url, method="POST", headers=headers, json_body=json_body, timeout=timeout)
+
+
+def put(url, headers=None, json_body=None, timeout=120):
+    """发送 PUT 请求，返回 JSON。"""
+    return request(url, method="PUT", headers=headers, json_body=json_body, timeout=timeout)

package/comate-engine/assets/skills/code-security-comate/scripts/parse_scan_result.py

@@ -0,0 +1,392 @@
+#!/usr/bin/env python3
+"""
+扫描结果解析与展示工具 - 解析 scan_result.json 文件，输出标准化漏洞报告。
+
+用法:
+    python3 parse_scan_result.py --scan-result <扫描结果文件路径> [--output-dir <输出目录>]
+
+输出:
+    1. 标准输出打印 Markdown 格式漏洞报告（供直接展示给用户）
+    2. 在输出目录生成 parsed_result.json，包含结构化漏洞数据（供后续修复脚本使用）
+
+parsed_result.json 格式:
+    {
+        "total": 10,
+        "common_count": 7,
+        "sensitive_count": 3,
+        "false_positive_count": 2,
+        "analyzing_count": 1,
+        "bundle_hash": "xxx",
+        "common_vuls": [ ... ],
+        "sensitive_vuls": [ ... ],
+        "false_positive_vuls": [ ... ],
+        "analyzing_vuls": [ ... ]
+    }
+
+每个漏洞条目包含:
+    {
+        "ruleID": "codescan_java_mybatis-java_sqli",
+        "name": "Sql 注入漏洞",
+        "description": "...",
+        "suggestion": "...",
+        "level": "ERROR",
+        "level_cn": "严重",
+        "file": "src/main/java/...",
+        "startLine": 12,
+        "endLine": 12,
+        "hash": "abc123...",
+        "is_sensitive": false,
+        "aiAnalysisStatus": 0,
+        "aiAnalysisStatusText": "无需分析",
+        "codeFlows": [ {"file": "...", "line": 63, "message": "..."}, ... ]
+    }
+
+aiAnalysisStatus 说明:
+    0 - 无需分析
+    1 - 分析中
+    2 - 真实漏洞
+    3 - 误报（不进入修复流程）
+"""
+
+import argparse
+from collections import OrderedDict
+import json
+import os
+import sys
+
+
+LEVEL_MAP = {
+    "ERROR": "严重",
+    "WARNING": "高危",
+    "NOTE": "中危",
+    "NONE": "低危",
+}
+
+# 等级排序优先级（越小越严重）
+LEVEL_PRIORITY = {
+    "ERROR": 0,
+    "WARNING": 1,
+    "NOTE": 2,
+    "NONE": 3,
+}
+
+# AI 分析状态映射
+AI_ANALYSIS_STATUS_MAP = {
+    0: "无需分析",
+    1: "分析中",
+    2: "真实漏洞",
+    3: "误报",
+}
+
+
+def parse_scan_result(scan_result):
+    # type: (dict) -> dict
+    """解析扫描结果，返回结构化漏洞数据。
+
+    aiAnalysisStatus=3 (误报) 和 aiAnalysisStatus=1 (分析中) 的漏洞会被单独归类，不进入修复流程。
+    """
+    data = scan_result.get("data", {})
+    runs = data.get("sarif", {}).get("runs", []) or data.get("runs", [])
+
+    # 提取 bundleHash（由 scan_vulnerability.py 写入顶层）
+    bundle_hash = scan_result.get("bundleHash", "")
+
+    # 建立 ruleID -> rule 映射
+    rule_map = {}
+    for run in runs:
+        tool = run.get("tool", {})
+        # 兼容 rules 在 tool 或 tool.driver 下
+        rules = tool.get("rules", []) or tool.get("driver", {}).get("rules", [])
+        for rule in rules:
+            rule_id = rule.get("id", "")
+            if rule_id:
+                rule_map[rule_id] = rule
+
+    # 解析漏洞结果
+    common_vuls = []
+    sensitive_vuls = []
+    false_positive_vuls = []  # 误报漏洞单独存放
+    analyzing_vuls = []  # AI 分析中的漏洞单独存放（不进入修复流程）
+
+    for run in runs:
+        results = run.get("results", []) or run.get("result", [])
+        for result in results:
+            rule_id = result.get("ruleID", "") or result.get("ruleId", "")
+            rule = rule_map.get(rule_id, {})
+
+            # 基础信息
+            name = rule.get("name", rule_id)
+            description = rule.get("description", "") or result.get("message", {}).get("text", "")
+            suggestion = rule.get("suggestion", "")
+            level_config = rule.get("defaultConfiguration", {})
+            level = level_config.get("level", "NONE") if level_config else "NONE"
+            level_cn = LEVEL_MAP.get(level, "低危")
+            vul_hash = result.get("properties", {}).get("hash", "")
+
+            # AI 分析状态
+            ai_status = result.get("properties", {}).get("aiAnalysisStatus", 0)
+            ai_status_text = AI_ANALYSIS_STATUS_MAP.get(ai_status, "未知")
+
+            # 位置信息
+            locations = result.get("locations", [])
+            file_uri = ""
+            start_line = 0
+            end_line = 0
+            if locations:
+                phys = locations[0].get("physicalLocation", {})
+                artifact = phys.get("artifactLocation", {})
+                file_uri = artifact.get("uri", "")
+                region = artifact.get("region", {})
+                start_line = region.get("startLine", 0)
+                end_line = region.get("endLine", start_line)
+
+            # 数据流信息
+            code_flows_raw = result.get("codeFlows", [])
+            code_flows = []
+            if code_flows_raw:
+                thread_flows = code_flows_raw[0].get("threadFlows", [])
+                if thread_flows:
+                    for loc_wrapper in thread_flows[0].get("locations", []):
+                        loc = loc_wrapper.get("location", {})
+                        loc_phys = loc.get("physicalLocation", {})
+                        loc_file = loc_phys.get("artifactLocation", {}).get("uri", "")
+                        loc_region = loc_phys.get("region", {})
+                        loc_line = loc_region.get("startLine", 0)
+                        loc_msg = loc.get("message", {}).get("text", "")
+                        code_flows.append({
+                            "file": loc_file,
+                            "line": loc_line,
+                            "message": loc_msg,
+                        })
+
+            is_sensitive = "sensitive" in rule_id.lower()
+
+            vul_entry = {
+                "ruleID": rule_id,
+                "name": name,
+                "description": description,
+                "suggestion": suggestion,
+                "level": level,
+                "level_cn": level_cn,
+                "file": file_uri,
+                "startLine": start_line,
+                "endLine": end_line,
+                "hash": vul_hash,
+                "is_sensitive": is_sensitive,
+                "aiAnalysisStatus": ai_status,
+                "aiAnalysisStatusText": ai_status_text,
+                "codeFlows": code_flows,
+            }
+
+            # 根据 AI 分析状态分类
+            if ai_status == 3:  # 误报
+                false_positive_vuls.append(vul_entry)
+            elif ai_status == 1:  # 分析中，不进入修复流程
+                analyzing_vuls.append(vul_entry)
+            elif is_sensitive:
+                sensitive_vuls.append(vul_entry)
+            else:
+                common_vuls.append(vul_entry)
+
+    # 按严重程度排序
+    def sort_key(v):
+        return (LEVEL_PRIORITY.get(v["level"], 99), v["file"], v["startLine"])
+
+    common_vuls.sort(key=sort_key)
+    sensitive_vuls.sort(key=sort_key)
+    false_positive_vuls.sort(key=sort_key)
+    analyzing_vuls.sort(key=sort_key)
+
+    return {
+        "total": len(common_vuls) + len(sensitive_vuls) + len(false_positive_vuls) + len(analyzing_vuls),
+        "common_count": len(common_vuls),
+        "sensitive_count": len(sensitive_vuls),
+        "false_positive_count": len(false_positive_vuls),
+        "analyzing_count": len(analyzing_vuls),
+        "bundle_hash": bundle_hash,
+        "common_vuls": common_vuls,
+        "sensitive_vuls": sensitive_vuls,
+        "false_positive_vuls": false_positive_vuls,
+        "analyzing_vuls": analyzing_vuls,
+    }
+
+
+def _make_file_link(file_path, line, project_dir=""):
+    # type: (str, int, str) -> str
+    """生成可点击的文件链接。"""
+    if not file_path:
+        return "未知位置"
+    basename = os.path.basename(file_path)
+    label = "{}:{}".format(basename, line)
+    if project_dir:
+        abs_path = os.path.join(project_dir, file_path)
+    else:
+        abs_path = file_path
+    return "[{}]({}#L{})".format(label, abs_path, line)
+
+
+def format_vul_report(vuls, title="漏洞报告", project_dir=""):
+    # type: (list, str, str) -> str
+    """将漏洞列表按类型分组格式化为 Markdown 报告。
+
+    同一 ruleID 的漏洞归为一组，描述/等级/修复建议只展示一次，
+    漏洞位置列表化展示，数据流用 <details> 折叠。
+    """
+    if not vuls:
+        return ""
+
+    # 按 ruleID 分组，保持原有排序
+    groups = OrderedDict()  # type: OrderedDict[str, list]
+    for vul in vuls:
+        rule_id = vul["ruleID"]
+        if rule_id not in groups:
+            groups[rule_id] = []
+        groups[rule_id].append(vul)
+
+    lines = ["**{}**\n".format(title)]
+
+    for group_idx, (rule_id, group_vuls) in enumerate(groups.items(), 1):
+        rep = group_vuls[0]  # 取第一个作为代表获取公共信息
+        count = len(group_vuls)
+        count_suffix = "（{} 处）".format(count) if count > 1 else ""
+
+        lines.append("{}. **{}**{}".format(group_idx, rep["name"], count_suffix))
+        if rep["description"]:
+            lines.append("- **漏洞描述**：{}".format(rep["description"]))
+        lines.append("- **漏洞等级**：{}".format(rep["level_cn"]))
+        if rep["suggestion"]:
+            lines.append("- **修复建议**：{}".format(rep["suggestion"]))
+
+        # 漏洞位置列表
+        lines.append("- **漏洞位置**：")
+        for vul in group_vuls:
+            loc_link = _make_file_link(vul["file"], vul["startLine"], project_dir)
+            lines.append("    - {}".format(loc_link))
+
+            # 数据流折叠展示
+            if vul["codeFlows"]:
+                lines.append("      <details><summary>数据流</summary>\n")
+                for hop_idx, hop in enumerate(vul["codeFlows"], 1):
+                    hop_link = _make_file_link(hop["file"], hop["line"], project_dir)
+                    hop_msg = hop.get("message", "")
+                    if hop_msg:
+                        lines.append("      {}. {} — {}".format(hop_idx, hop_link, hop_msg))
+                    else:
+                        lines.append("      {}. {}".format(hop_idx, hop_link))
+                lines.append("\n      </details>")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def format_full_report(parsed, project_dir=""):
+    # type: (dict, str) -> str
+    """生成完整的展示报告（包含各类漏洞）。"""
+    parts = []
+
+    common_count = parsed["common_count"]
+    sensitive_count = parsed["sensitive_count"]
+    false_positive_count = parsed.get("false_positive_count", 0)
+    analyzing_count = parsed.get("analyzing_count", 0)
+    real_vul_count = common_count + sensitive_count  # 需要处理的真实漏洞数
+
+    # 汇总信息
+    summary_parts = []
+    if real_vul_count > 0:
+        if common_count > 0 and sensitive_count > 0:
+            summary_parts.append("**{}** 个普通漏洞和 **{}** 个硬编码漏洞".format(common_count, sensitive_count))
+        elif common_count > 0:
+            summary_parts.append("**{}** 个普通漏洞".format(common_count))
+        elif sensitive_count > 0:
+            summary_parts.append("**{}** 个硬编码漏洞".format(sensitive_count))
+
+    if false_positive_count > 0:
+        summary_parts.append("**{}** 个误报（已由 AI 分析确认，无需修复）".format(false_positive_count))
+
+    if summary_parts:
+        parts.append("扫描发现 {}。\n".format("，".join(summary_parts)))
+    else:
+        if analyzing_count > 0:
+            parts.append("扫描完成，**{}** 个漏洞正在 AI 分析中，暂未发现已确认的漏洞。\n".format(analyzing_count))
+        else:
+            parts.append("扫描完成，未发现漏洞。\n")
+        return "\n".join(parts)
+
+    if analyzing_count > 0:
+        parts.append("另有 **{}** 个漏洞正在 AI 分析中，分析完成后可能会被判定为误报而排除，分析期间暂不处理。\n".format(analyzing_count))
+
+    # 只有误报没有真实漏洞的情况
+    if real_vul_count == 0 and false_positive_count > 0:
+        parts.append("所有检测到的漏洞均已被 AI 分析确认为误报，无需进行修复。\n")
+        if parsed.get("false_positive_vuls"):
+            parts.append(format_vul_report(parsed["false_positive_vuls"], "误报漏洞列表（仅供参考）", project_dir))
+        return "\n".join(parts)
+
+    if parsed["common_vuls"]:
+        parts.append(format_vul_report(parsed["common_vuls"], "普通漏洞报告", project_dir))
+
+    if parsed["sensitive_vuls"]:
+        parts.append(format_vul_report(parsed["sensitive_vuls"], "硬编码漏洞报告", project_dir))
+
+    # 误报漏洞折叠展示
+    if false_positive_count > 0:
+        parts.append("\n<details><summary>误报漏洞（{} 个，已由 AI 确认无需修复）</summary>\n".format(false_positive_count))
+        for vul in parsed.get("false_positive_vuls", []):
+            loc_link = _make_file_link(vul["file"], vul["startLine"], project_dir)
+            parts.append("- {} - {}".format(vul["name"], loc_link))
+        parts.append("\n</details>")
+
+    # AI 分析中漏洞折叠展示
+    if analyzing_count > 0:
+        parts.append("\n<details><summary>AI 分析中（{} 个，分析完成前暂不处理）</summary>\n".format(analyzing_count))
+        for vul in parsed.get("analyzing_vuls", []):
+            loc_link = _make_file_link(vul["file"], vul["startLine"], project_dir)
+            parts.append("- {} - {}".format(vul["name"], loc_link))
+        parts.append("\n</details>")
+
+    return "\n".join(parts)
+
+
+def main():
+    """
+    主函数入口，解析命令行参数并处理扫描结果
+
+    Args:
+        无（通过命令行参数 --scan-result 和 --output-dir 传入）
+
+    Returns:
+        无（结果直接输出到标准输出和文件）
+    """
+    parser = argparse.ArgumentParser(description="扫描结果解析与展示工具")
+    parser.add_argument("--scan-result", required=True, help="扫描结果文件路径 (scan_result.json)")
+    parser.add_argument("--output-dir", default=None, help="结构化结果输出目录，默认与输入文件同目录")
+    parser.add_argument("--project-dir", default="", help="项目根目录，用于生成可点击的绝对路径链接")
+    args = parser.parse_args()
+
+    # 读取扫描结果
+    try:
+        with open(args.scan_result, "r", encoding="utf-8") as f:
+            scan_result = json.load(f)
+    except Exception as e:
+        print("错误: 读取扫描结果失败 {}: {}".format(args.scan_result, e), file=sys.stderr)
+        sys.exit(1)
+
+    # 解析
+    parsed = parse_scan_result(scan_result)
+
+    # 输出 Markdown 报告到标准输出
+    report = format_full_report(parsed, args.project_dir)
+    print(report)
+
+    # 保存结构化 JSON
+    # 默认输出到输入文件同目录，自然跟随项目隔离
+    output_dir = args.output_dir or os.path.dirname(os.path.abspath(args.scan_result))
+    os.makedirs(output_dir, exist_ok=True)
+    output_file = os.path.join(output_dir, "parsed_result.json")
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(parsed, f, ensure_ascii=False, indent=2)
+    print(output_file, file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()