dsh-calibguard 0.2.5 → 0.3.1

package/engine/calibguard/preprocessor.py

@@ -0,0 +1,218 @@
+"""预处理模块：任意输入文件 → 统一文本。
+
+只按内容判定类型（魔数 + 解码尝试），不按扩展名。压缩包一律在内存中解包，
+禁止 extractall / extract 落盘（因此不需要 tarfile 的 filter 参数）。
+"""
+import gzip
+import io
+import json
+import logging
+import re
+import tarfile
+import zipfile
+
+from calibguard import constants as C
+from calibguard.schemas import PreprocessResult
+
+logger = logging.getLogger(__name__)
+
+# 控制字符模式（仅用于 text/binary 判定；不含 \t \n \r）
+_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")
+# 控制字符占比超过该值即判定为二进制（见 7.3 步骤 4）
+_CONTROL_CHAR_RATIO_MAX = 0.30
+# 二进制可打印串提取模式（最短连续长度取常量 PRINTABLE_MIN_RUN）
+_PRINTABLE_RE = re.compile(rb"[\x20-\x7e]{%d,}" % C.PRINTABLE_MIN_RUN)
+# 连续 3 个及以上空白字符（空格/制表符）压缩模式
+_BLANK_RUN_RE = re.compile(r"[ \t]{3,}")
+# 解包截断标记
+_ARCHIVE_TRUNCATED_MARK = "\n\n[ARCHIVE_PARTIALLY_EXTRACTED]"
+
+
+def preprocess_file(path: str) -> PreprocessResult:
+    """主流程：读入 → 判定压缩包 → 解包或解码 → JSON 规整 → 清洗。"""
+    data, read_truncated = _read_bytes(path)
+
+    # 步骤 2 + 步骤 3：压缩包走内存解包
+    if _detect_archive(data) is not None:
+        archive_text, parts, expand_truncated = _expand_archive(data, 0)
+        return PreprocessResult(
+            text=_clean_text(archive_text),
+            source_type="archive",
+            truncated=read_truncated or expand_truncated,
+            parts=parts,
+        )
+
+    # 步骤 4：非压缩包 → 解码或可打印串提取
+    text, source_type = _decode_text(data)
+
+    # 步骤 5：JSON 规整（仅 text 类；解析失败原样保留）
+    if source_type == "text":
+        stripped = text.strip()
+        if stripped.startswith("{") or stripped.startswith("["):
+            try:
+                parsed = json.loads(stripped)
+            except ValueError as exc:
+                logger.debug("JSON 解析失败，按原文保留: %s", exc)
+            else:
+                text = json.dumps(parsed, ensure_ascii=False, indent=2)
+
+    # 步骤 6 + 步骤 7
+    return PreprocessResult(
+        text=_clean_text(text),
+        source_type=source_type,
+        truncated=read_truncated,
+        parts=1,
+    )
+
+
+def _read_bytes(path: str) -> tuple[bytes, bool]:
+    """读入文件字节。(数据, 是否被读入截断)。不存在或不可读 → FileNotFoundError。"""
+    try:
+        with open(path, "rb") as handle:
+            data = handle.read(C.MAX_INPUT_BYTES + 1)
+    except FileNotFoundError:
+        raise
+    except OSError as exc:
+        # 不可读（权限等）按文档统一转为 FileNotFoundError
+        raise FileNotFoundError(f"文件不存在或不可读: {path}") from exc
+    if len(data) > C.MAX_INPUT_BYTES:
+        return data[:C.MAX_INPUT_BYTES], True
+    return data, False
+
+
+def _detect_archive(data: bytes) -> str | None:
+    """"zip" | "gzip" | "tar" | None（纯魔数判定）。"""
+    if data[:4] in (b"PK\x03\x04", b"PK\x05\x06"):
+        return "zip"
+    if data[:2] == b"\x1f\x8b":
+        return "gzip"
+    if data[257:262] == b"ustar":
+        return "tar"
+    return None
+
+
+def _expand_archive(data: bytes, depth: int) -> tuple[str, int, bool]:
+    """内存解包压缩包，返回 (拼接文本, 内部文件数, 是否被解包截断)。
+
+    只处理常规文件成员：zip 用 zf.read(info)，tar 用 tf.extractfile(m).read()；
+    目录、符号链接、硬链接、设备成员一律跳过（链接成员是 tar 的经典攻击面）。
+    """
+    archive_type = _detect_archive(data)
+    members: list[tuple[str, bytes]] = []
+    file_count = 0
+    total_bytes = 0
+    truncated = False
+
+    try:
+        if archive_type == "zip":
+            with zipfile.ZipFile(io.BytesIO(data)) as zf:
+                for info in zf.infolist():
+                    if info.is_dir():
+                        continue
+                    # 安全限制：读成员前先按其声明大小检查
+                    if (file_count + 1 > C.ARCHIVE_MAX_FILES
+                            or total_bytes + info.file_size > C.ARCHIVE_MAX_TOTAL_BYTES):
+                        truncated = True
+                        break
+                    blob = zf.read(info)
+                    members.append((info.filename, blob))
+                    file_count += 1
+                    total_bytes += len(blob)
+        elif archive_type == "tar":
+            with tarfile.open(fileobj=io.BytesIO(data), mode="r:*") as tf:
+                for member in tf:
+                    if not member.isreg():
+                        continue
+                    # 安全限制：读成员前先按其声明大小检查
+                    if (file_count + 1 > C.ARCHIVE_MAX_FILES
+                            or total_bytes + member.size > C.ARCHIVE_MAX_TOTAL_BYTES):
+                        truncated = True
+                        break
+                    extracted = tf.extractfile(member)
+                    if extracted is None:
+                        # 声明为常规文件却取不到内容：跳过该成员并记录
+                        logger.warning("tar 成员无法读取，已跳过: %s", member.name)
+                        continue
+                    blob = extracted.read()
+                    members.append((member.name, blob))
+                    file_count += 1
+                    total_bytes += len(blob)
+        elif archive_type == "gzip":
+            inner = gzip.decompress(data)
+            if _detect_archive(inner) == "tar":
+                # tar.gz 两步走：属同一压缩层级，深度不递增
+                return _expand_archive(inner, depth)
+            members.append(("", inner))
+            file_count = 1
+            total_bytes = len(inner)
+        else:
+            # 魔数判定为压缩包却无法识别类型：按二进制处理
+            return _extract_printable(data), 1, False
+    except Exception as exc:
+        # 损坏的压缩包 → 视为二进制，走可打印串提取（不吞异常：记录并明确返回）
+        logger.warning("压缩包解析失败，按二进制处理: %s: %s", type(exc).__name__, exc)
+        return _extract_printable(data), 1, False
+
+    # 第二遍：嵌套压缩包递归（深度 +1），其余解码为文本
+    collected: list[tuple[str, str]] = []
+    for name, blob in members:
+        if _detect_archive(blob) is not None:
+            if depth + 1 > C.ARCHIVE_MAX_DEPTH:
+                continue  # 超出递归深度上限，跳过该内部文件
+            inner_text, inner_parts, inner_truncated = _expand_archive(blob, depth + 1)
+            file_count += inner_parts
+            truncated = truncated or inner_truncated
+            collected.append((name, inner_text))
+        else:
+            member_text, _member_type = _decode_text(blob)
+            collected.append((name, member_text))
+
+    # 全部文本按文件名排序后用 "\n\n" 连接（不加任何标记）
+    collected.sort(key=lambda item: item[0])
+    joined = "\n\n".join(entry_text for _name, entry_text in collected)
+    if truncated:
+        joined += _ARCHIVE_TRUNCATED_MARK
+    return joined, file_count, truncated
+
+
+def _decode_text(data: bytes) -> tuple[str, str]:
+    """(文本, "text"|"binary")：先 utf-8 再 gbk，均严格模式。"""
+    text: str | None = None
+    for encoding in ("utf-8", "gbk"):
+        try:
+            text = data.decode(encoding)
+            break  # 解码成功，结束尝试
+        except UnicodeDecodeError as exc:
+            # 该编码不适用：记录原因后继续尝试下一种编码
+            logger.debug("按 %s 解码失败，尝试下一种编码: %s", encoding, exc)
+    if text is None:
+        # 两种解码都失败 → 二进制
+        return _extract_printable(data), "binary"
+    if text == "":
+        return text, "text"  # 空字符串直接判 text，不做除法
+    control_ratio = len(_CONTROL_CHARS_RE.findall(text)) / len(text)
+    if control_ratio > _CONTROL_CHAR_RATIO_MAX:
+        return _extract_printable(data), "binary"
+    return text, "text"
+
+
+def _extract_printable(data: bytes) -> str:
+    """提取二进制中的可打印 ASCII 串，各段用 "\\n" 连接。"""
+    runs = _PRINTABLE_RE.findall(data)
+    return b"\n".join(runs).decode("ascii")
+
+
+def _clean_text(text: str) -> str:
+    """a 删控制字符 → b 压缩连续空白 → c 单行截断 → d 总长截断。"""
+    text = re.sub(C.CONTROL_CHARS_PATTERN, "", text)
+    text = _BLANK_RUN_RE.sub(" ", text)
+    lines = text.split("\n")
+    clipped_lines: list[str] = []
+    for line in lines:
+        if len(line) > C.LINE_TRUNCATE_THRESHOLD:
+            line = line[:C.LINE_TRUNCATE_THRESHOLD] + "…[LINE_TRUNCATED]"
+        clipped_lines.append(line)
+    text = "\n".join(clipped_lines)
+    if len(text) > C.MAX_TEXT_CHARS:
+        text = text[:C.MAX_TEXT_CHARS] + "\n…[TEXT_TRUNCATED]"
+    return text

package/engine/calibguard/report.py

@@ -0,0 +1,132 @@
+"""审计报告模块：审计结果 dict → JSON 文件 + Markdown 文件。"""
+import json
+import os
+
+from calibguard.schemas import FeatureEvidence
+
+# 证据明细的七类标题（第八类"异常结构特征"由长行/随机标识符两行构成）
+_EVIDENCE_TITLES = [
+    ("【危险API调用】", "dangerous_apis"),
+    ("【网络行为】", "network_indicators"),
+    ("【凭据与敏感路径访问】", "credential_access"),
+    ("【编码与混淆】", "obfuscation_signals"),
+    ("【可疑文件操作】", "file_operations"),
+    ("【提示注入话术】", "injection_phrases"),
+    ("【高熵字符串】", "high_entropy_strings"),
+]
+# 证据全空时的固定文案
+_EMPTY_EVIDENCE_TEXT = "未发现明显可疑信号。"
+
+
+def write_reports(report: dict, output_dir: str) -> tuple[str, str]:
+    """生成 <output_dir>/<report_id>.json 与 <output_dir>/<report_id>.md，返回绝对路径。"""
+    os.makedirs(output_dir, exist_ok=True)
+    report_id = str(report["report_id"])
+    json_path = os.path.abspath(os.path.join(output_dir, report_id + ".json"))
+    md_path = os.path.abspath(os.path.join(output_dir, report_id + ".md"))
+    with open(json_path, "w", encoding="utf-8") as handle:
+        handle.write(json.dumps(report, ensure_ascii=False, indent=2))
+    with open(md_path, "w", encoding="utf-8") as handle:
+        handle.write(_to_markdown(report))
+    return json_path, md_path
+
+
+def _evidence_to_dict(ev: FeatureEvidence) -> dict:
+    """FeatureEvidence → 九个同名键的 dict。"""
+    return {
+        "dangerous_apis": list(ev.dangerous_apis),
+        "network_indicators": list(ev.network_indicators),
+        "credential_access": list(ev.credential_access),
+        "obfuscation_signals": list(ev.obfuscation_signals),
+        "file_operations": list(ev.file_operations),
+        "injection_phrases": list(ev.injection_phrases),
+        "high_entropy_strings": list(ev.high_entropy_strings),
+        "long_line_numbers": list(ev.long_line_numbers),
+        "random_identifier_count": ev.random_identifier_count,
+    }
+
+
+def _to_markdown(report: dict) -> str:
+    """按 13.3 固定模板渲染 Markdown 报告。"""
+    early_stopped = bool(report.get("early_stopped"))
+    if early_stopped:
+        early_stop_text = f"是，p={report.get('early_stop_probability')}"
+    else:
+        early_stop_text = "否"
+
+    lines: list[str] = [
+        "# CalibGuard 安全审计报告",
+        "",
+        f"- 报告编号：{report['report_id']}",
+        f"- 生成时间：{report['timestamp']}",
+        f"- 审计文件：{report['file_path']}",
+        f"- 文件 SHA256：{report['file_sha256']}",
+        f"- 文件大小：{report['file_size_bytes']} 字节",
+        f"- 来源类型：{report['source_type']}（内部文件数：{report['archive_parts']}）",
+        "",
+        "## 结论",
+        "",
+        "| 项目 | 结果 |",
+        "|---|---|",
+        f"| 原始 logit | {report['raw_logit']} |",
+        f"| 校准概率 | {report['calibrated_probability']} |",
+        f"| 校准方法 | {report['calibration_method']} |",
+        f"| 推理窗口数 | {report['windows_evaluated']} |",
+        f"| 滑窗早停 | {early_stop_text} |",
+        f"| 门控决策 | {report['gate_decision']} |",
+        f"| 最终决策 | **{report['final_decision']}** |",
+        f"| 需人工复核 | {report['manual_review_required']} |",
+        f"| 一致性告警 | {report['consistency_warning']} |",
+        "",
+        "## LLM 分析",
+        "",
+    ]
+
+    llm = report.get("llm")
+    if llm is None:
+        lines.append("（未提交云端分析）")
+    else:
+        lines.extend([
+            "| 字段 | 值 |",
+            "|---|---|",
+            f"| status | {llm.get('status')} |",
+            f"| risk | {llm.get('risk')} |",
+            f"| reason | {llm.get('reason')} |",
+            f"| action | {llm.get('action')} |",
+            f"| 重试次数 | {llm.get('retries_used')} |",
+        ])
+
+    lines.extend(["", "## 证据明细", ""])
+    lines.append(_render_evidence_section(report.get("evidence") or {}))
+
+    lines.extend(["", "## 错误信息", ""])
+    error = report.get("error")
+    lines.append(str(error) if error else "（无）")
+
+    return "\n".join(lines) + "\n"
+
+
+def _render_evidence_section(evidence: dict) -> str:
+    """按 8.4 的八类格式渲染证据明细；全空时返回固定文案。"""
+    blocks: list[str] = []
+    for title, key in _EVIDENCE_TITLES:
+        items = evidence.get(key) or []
+        if not items:
+            continue
+        block_lines = [title]
+        block_lines.extend(f"- {item}" for item in items)
+        blocks.append("\n".join(block_lines))
+
+    structural_lines: list[str] = []
+    long_line_numbers = evidence.get("long_line_numbers") or []
+    if long_line_numbers:
+        structural_lines.append(f"- 超长行行号: {long_line_numbers}")
+    random_identifier_count = evidence.get("random_identifier_count") or 0
+    if random_identifier_count > 0:
+        structural_lines.append(f"- 长随机标识符数量: {random_identifier_count}")
+    if structural_lines:
+        blocks.append("\n".join(["【异常结构特征】"] + structural_lines))
+
+    if not blocks:
+        return _EMPTY_EVIDENCE_TEXT
+    return "\n\n".join(blocks)

package/engine/calibguard/schemas.py

@@ -0,0 +1,53 @@
+"""CalibGuard 全局数据结构定义。本文件无任何业务逻辑。"""
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class GateDecision(str, Enum):
+    """门控路由决策。"""
+    ALLOW = "allow"          # 放行
+    BLOCK = "block"          # 拦截告警
+    LLM_REVIEW = "llm_review"  # 提交云端 LLM 分析
+
+
+@dataclass
+class PreprocessResult:
+    """预处理结果。"""
+    text: str                # 预处理并清洗后的统一文本（可能较长，token 截断在模型层进行）
+    source_type: str         # "text" | "archive" | "binary"
+    truncated: bool          # True 表示字节级读入截断(>1MB)或解包截断发生过
+    parts: int = 1           # 压缩包内部文件数；非压缩包恒为 1
+
+
+@dataclass
+class FeatureEvidence:
+    """特征提取产出的结构化证据。每个列表字段：去重、按首次出现顺序、最多 10 条。"""
+    dangerous_apis: list[str] = field(default_factory=list)      # 危险 API 调用
+    network_indicators: list[str] = field(default_factory=list)  # 网络行为
+    credential_access: list[str] = field(default_factory=list)   # 凭据与敏感路径访问
+    obfuscation_signals: list[str] = field(default_factory=list) # 编码与混淆
+    file_operations: list[str] = field(default_factory=list)     # 可疑文件操作
+    injection_phrases: list[str] = field(default_factory=list)   # 提示注入话术
+    high_entropy_strings: list[str] = field(default_factory=list)  # 高熵字符串（最多 10 条）
+    long_line_numbers: list[int] = field(default_factory=list)   # 超长行行号（1 起，最多 10 个）
+    random_identifier_count: int = 0                             # 长随机标识符总数（达到阈值才>0）
+
+
+@dataclass
+class LLMResult:
+    """云端 LLM 分析结果。"""
+    status: str               # "ok" | "invalid_output" | "api_error"
+    risk: str | None = None   # "high" | "medium" | "low"，失败时为 None
+    reason: str | None = None
+    action: str | None = None
+    consistency_warning: bool = False  # 本地概率高但 LLM 判 low 时为 True
+    retries_used: int = 0             # 实际发生重试的次数（0 或 1）
+    raw_response: str | None = None   # 最后一次 API 原始返回文本（调试用，报告必含）
+
+
+@dataclass
+class WindowInferenceStats:
+    """滑动窗口推理统计（v1.1 新增）。"""
+    windows_evaluated: int = 0            # 实际完成推理的窗口数
+    early_stopped: bool = False           # 是否触发早停
+    early_stop_probability: float | None = None  # 触发早停时的窗口校准概率

package/engine/models/calibration_params.json

@@ -0,0 +1 @@
+{"A": -1.0332483500324428, "B": 0.7344078473521912, "method": "platt", "n_samples": 25049, "fit_at": "2026-10-03T19:35:34.589148+08:00"}

package/engine/models/codebert_finetuned/config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "RobertaForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "transformers_version": "4.57.6",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50265
+}