dsh-calibguard 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +44 -27
- package/README.md +44 -26
- package/cordis.patch.yml +2 -1
- package/docs/configuration.md +2 -2
- package/docs/troubleshooting.md +2 -1
- package/engine/README.md +33 -0
- package/engine/audit_service.py +304 -0
- package/engine/calibguard/__init__.py +2 -0
- package/engine/calibguard/__main__.py +4 -0
- package/engine/calibguard/calibrator.py +78 -0
- package/engine/calibguard/cli.py +74 -0
- package/engine/calibguard/config.py +129 -0
- package/engine/calibguard/constants.py +117 -0
- package/engine/calibguard/feature_extractor.py +161 -0
- package/engine/calibguard/gate.py +17 -0
- package/engine/calibguard/llm_analyzer.py +171 -0
- package/engine/calibguard/model.py +79 -0
- package/engine/calibguard/pipeline.py +305 -0
- package/engine/calibguard/preprocessor.py +218 -0
- package/engine/calibguard/report.py +132 -0
- package/engine/calibguard/schemas.py +53 -0
- package/engine/models/calibration_params.json +1 -0
- package/engine/models/codebert_finetuned/config.json +28 -0
- package/engine/models/codebert_finetuned/merges.txt +50001 -0
- package/engine/models/codebert_finetuned/special_tokens_map.json +51 -0
- package/engine/models/codebert_finetuned/tokenizer.json +250371 -0
- package/engine/models/codebert_finetuned/tokenizer_config.json +58 -0
- package/engine/models/codebert_finetuned/vocab.json +1 -0
- package/engine/requirements-gui.txt +11 -0
- package/lib/bootstrap.js +4 -3
- package/lib/client.js +1 -1
- package/lib/config.js +34 -7
- package/lib/engine.js +1 -1
- package/lib/index.js +1 -1
- package/package.json +4 -3
- package/setup.mjs +278 -48
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"""流水线编排:串联 T4~T10,产出单文件完整报告 dict。"""
|
|
2
|
+
import datetime
|
|
3
|
+
import hashlib
|
|
4
|
+
import math
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
import uuid
|
|
8
|
+
|
|
9
|
+
from calibguard import constants as C
|
|
10
|
+
from calibguard.calibrator import PlattCalibrator
|
|
11
|
+
from calibguard.config import AppConfig
|
|
12
|
+
from calibguard.feature_extractor import evidence_to_prompt_text, extract_features
|
|
13
|
+
from calibguard.gate import route
|
|
14
|
+
from calibguard.llm_analyzer import LLMAnalyzer
|
|
15
|
+
from calibguard.model import CodeBERTClassifier
|
|
16
|
+
from calibguard.preprocessor import preprocess_file
|
|
17
|
+
from calibguard.report import _evidence_to_dict, write_reports
|
|
18
|
+
from calibguard.schemas import (
|
|
19
|
+
FeatureEvidence,
|
|
20
|
+
GateDecision,
|
|
21
|
+
LLMResult,
|
|
22
|
+
WindowInferenceStats,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class AuditPipeline:
|
|
27
|
+
"""单文件/目录审计流水线。"""
|
|
28
|
+
|
|
29
|
+
def __init__(self, cfg: AppConfig):
|
|
30
|
+
self.cfg = cfg
|
|
31
|
+
# 1. 模型(加载失败抛 RuntimeError,由 CLI 转退出码 4)
|
|
32
|
+
self.classifier = CodeBERTClassifier(
|
|
33
|
+
cfg.model_path, cfg.max_length,
|
|
34
|
+
window_stride=cfg.window_stride, window_batch_size=cfg.window_batch_size)
|
|
35
|
+
# 2. 校准器(calibration_method 一律取实例的 .method 属性)
|
|
36
|
+
if cfg.calibration_params_path == "":
|
|
37
|
+
self.calibrator = PlattCalibrator.identity()
|
|
38
|
+
elif os.path.isfile(cfg.calibration_params_path):
|
|
39
|
+
self.calibrator = PlattCalibrator.load(cfg.calibration_params_path)
|
|
40
|
+
else:
|
|
41
|
+
raise FileNotFoundError(
|
|
42
|
+
"校准参数文件缺失,请先运行 scripts/calibrate.py,"
|
|
43
|
+
"或在 config.yaml 中将 calibration.params_path 设为空字符串")
|
|
44
|
+
# 3. 云端 LLM 分析器(属性名必须是 llm)
|
|
45
|
+
self.llm = LLMAnalyzer(
|
|
46
|
+
api_key=cfg.llm_api_key, base_url=cfg.llm_base_url, model=cfg.llm_model,
|
|
47
|
+
temperature=cfg.llm_temperature, max_tokens=cfg.llm_max_tokens,
|
|
48
|
+
timeout=cfg.llm_timeout, evidence_max_chars=cfg.evidence_max_chars)
|
|
49
|
+
|
|
50
|
+
def audit_file(self, path: str) -> dict:
|
|
51
|
+
"""单文件 → 完整报告 dict。任何处理异常都降级为 manual_review,不向调用方抛出。"""
|
|
52
|
+
report = self._new_report(path)
|
|
53
|
+
try:
|
|
54
|
+
# 步骤 1:读前 1MB 字节,用于 sha256 与 size
|
|
55
|
+
with open(path, "rb") as handle:
|
|
56
|
+
head_bytes = handle.read(C.MAX_INPUT_BYTES)
|
|
57
|
+
report["file_sha256"] = hashlib.sha256(head_bytes).hexdigest()
|
|
58
|
+
report["file_size_bytes"] = len(head_bytes)
|
|
59
|
+
|
|
60
|
+
# 步骤 2:预处理
|
|
61
|
+
result = preprocess_file(path)
|
|
62
|
+
report["source_type"] = result.source_type
|
|
63
|
+
report["preprocess_truncated"] = result.truncated
|
|
64
|
+
report["archive_parts"] = result.parts
|
|
65
|
+
|
|
66
|
+
# 步骤 3:特征提取(全文,信息更全)
|
|
67
|
+
evidence = extract_features(result.text)
|
|
68
|
+
report["evidence"] = _evidence_to_dict(evidence)
|
|
69
|
+
|
|
70
|
+
# 步骤 4:滑窗推理与批间早停
|
|
71
|
+
agg_logit = -math.inf
|
|
72
|
+
stats = WindowInferenceStats()
|
|
73
|
+
for batch_logits in self.classifier.iter_window_batches(result.text):
|
|
74
|
+
stats.windows_evaluated += len(batch_logits)
|
|
75
|
+
batch_max = max(batch_logits)
|
|
76
|
+
agg_logit = max(agg_logit, batch_max)
|
|
77
|
+
# 批间早停:本批内任一窗口校准概率 > early_stop_threshold 即停
|
|
78
|
+
for window_logit in batch_logits:
|
|
79
|
+
if (self.calibrator.predict_proba(window_logit)
|
|
80
|
+
> self.cfg.early_stop_threshold):
|
|
81
|
+
stats.early_stopped = True
|
|
82
|
+
stats.early_stop_probability = self.calibrator.predict_proba(batch_max)
|
|
83
|
+
break
|
|
84
|
+
if stats.early_stopped:
|
|
85
|
+
# 生成器后续批次不再被消费,即"立即停止后续窗口"
|
|
86
|
+
break
|
|
87
|
+
raw_logit = float(agg_logit)
|
|
88
|
+
report["raw_logit"] = raw_logit
|
|
89
|
+
report["windows_evaluated"] = stats.windows_evaluated
|
|
90
|
+
report["early_stopped"] = stats.early_stopped
|
|
91
|
+
report["early_stop_probability"] = stats.early_stop_probability
|
|
92
|
+
|
|
93
|
+
# 步骤 5:校准概率(早停时 p = early_stop_probability > block_threshold)
|
|
94
|
+
probability = self.calibrator.predict_proba(raw_logit)
|
|
95
|
+
report["calibrated_probability"] = probability
|
|
96
|
+
|
|
97
|
+
# 步骤 6:门控路由
|
|
98
|
+
gate = route(probability, self.cfg.allow_threshold, self.cfg.block_threshold)
|
|
99
|
+
report["gate_decision"] = gate.value
|
|
100
|
+
|
|
101
|
+
# 步骤 7:仅门控为 LLM_REVIEW 时才上云
|
|
102
|
+
# 内容源:精炼证据 / 文件原文;投递模式:截断 / 滑窗 / 一次性全部上传
|
|
103
|
+
# 多段时逐段调用,按风险严重度取最严重的一段(high > medium > low)
|
|
104
|
+
llm_result: LLMResult | None = None
|
|
105
|
+
llm_payloads: list[str] = []
|
|
106
|
+
if gate == GateDecision.LLM_REVIEW:
|
|
107
|
+
if not self.cfg.llm_enabled:
|
|
108
|
+
# 用户关闭云端复核:灰区按本地模型自身能力 0.5 切(不改门控口径)
|
|
109
|
+
llm_result = LLMResult(status="disabled")
|
|
110
|
+
report["llm"] = {
|
|
111
|
+
"status": "disabled",
|
|
112
|
+
"risk": None,
|
|
113
|
+
"reason": None,
|
|
114
|
+
"action": None,
|
|
115
|
+
"retries_used": 0,
|
|
116
|
+
}
|
|
117
|
+
else:
|
|
118
|
+
llm_payloads = self._build_llm_payloads(evidence, result.text)
|
|
119
|
+
llm_result = self._review_with_llm(llm_payloads, probability)
|
|
120
|
+
report["llm"] = {
|
|
121
|
+
"status": llm_result.status,
|
|
122
|
+
"risk": llm_result.risk,
|
|
123
|
+
"reason": llm_result.reason,
|
|
124
|
+
"action": llm_result.action,
|
|
125
|
+
"retries_used": llm_result.retries_used,
|
|
126
|
+
}
|
|
127
|
+
report["consistency_warning"] = llm_result.consistency_warning
|
|
128
|
+
report["llm_calls"] = len(llm_payloads)
|
|
129
|
+
|
|
130
|
+
# 步骤 8:最终决策表
|
|
131
|
+
final_decision, manual_review_required = self._resolve_decision(
|
|
132
|
+
gate, llm_result, probability)
|
|
133
|
+
report["final_decision"] = final_decision
|
|
134
|
+
report["manual_review_required"] = manual_review_required
|
|
135
|
+
except Exception as exc:
|
|
136
|
+
# 安全降级:转人工 + 记录错误,继续处理下一个文件
|
|
137
|
+
print(f"[错误] 审计失败 {path}: {type(exc).__name__}: {exc}", file=sys.stderr)
|
|
138
|
+
report["final_decision"] = "manual_review"
|
|
139
|
+
report["manual_review_required"] = True
|
|
140
|
+
report["error"] = str(exc)
|
|
141
|
+
|
|
142
|
+
# 步骤 9:落盘并回填路径
|
|
143
|
+
try:
|
|
144
|
+
json_path, md_path = write_reports(report, self.cfg.report_output_dir)
|
|
145
|
+
except Exception as exc:
|
|
146
|
+
print(f"[错误] 报告写入失败 {path}: {type(exc).__name__}: {exc}", file=sys.stderr)
|
|
147
|
+
write_error = f"报告写入失败: {exc}"
|
|
148
|
+
report["error"] = (f"{report['error']} | {write_error}"
|
|
149
|
+
if report["error"] else write_error)
|
|
150
|
+
else:
|
|
151
|
+
report["report_json_path"] = json_path
|
|
152
|
+
report["report_md_path"] = md_path
|
|
153
|
+
return report
|
|
154
|
+
|
|
155
|
+
def audit_path(self, path: str) -> list[dict]:
|
|
156
|
+
"""文件或目录 → 报告列表。路径不存在抛 FileNotFoundError(CLI 转退出码 1)。"""
|
|
157
|
+
if not os.path.exists(path):
|
|
158
|
+
raise FileNotFoundError(f"路径不存在: {path}")
|
|
159
|
+
if os.path.isfile(path):
|
|
160
|
+
return [self.audit_file(path)]
|
|
161
|
+
reports: list[dict] = []
|
|
162
|
+
for file_path in self._collect_files(path):
|
|
163
|
+
try:
|
|
164
|
+
reports.append(self.audit_file(file_path))
|
|
165
|
+
except Exception as exc:
|
|
166
|
+
# 兜底:单文件失败不得中断整体
|
|
167
|
+
print(f"[错误] 审计失败 {file_path}: {type(exc).__name__}: {exc}",
|
|
168
|
+
file=sys.stderr)
|
|
169
|
+
fallback = self._new_report(file_path)
|
|
170
|
+
fallback["final_decision"] = "manual_review"
|
|
171
|
+
fallback["manual_review_required"] = True
|
|
172
|
+
fallback["error"] = str(exc)
|
|
173
|
+
reports.append(fallback)
|
|
174
|
+
return reports
|
|
175
|
+
|
|
176
|
+
def _collect_files(self, path: str) -> list[str]:
|
|
177
|
+
"""os.walk 递归收集常规文件(非符号链接),按路径排序,跳过输出目录。"""
|
|
178
|
+
output_dir_abs = os.path.abspath(self.cfg.report_output_dir)
|
|
179
|
+
collected: list[str] = []
|
|
180
|
+
for root, _dir_names, file_names in os.walk(path):
|
|
181
|
+
for file_name in file_names:
|
|
182
|
+
full_path = os.path.join(root, file_name)
|
|
183
|
+
if not os.path.isfile(full_path):
|
|
184
|
+
continue
|
|
185
|
+
if os.path.islink(full_path):
|
|
186
|
+
continue
|
|
187
|
+
file_abs = os.path.abspath(full_path)
|
|
188
|
+
if file_abs == output_dir_abs or file_abs.startswith(output_dir_abs + os.sep):
|
|
189
|
+
continue
|
|
190
|
+
collected.append(full_path)
|
|
191
|
+
collected.sort()
|
|
192
|
+
return collected
|
|
193
|
+
|
|
194
|
+
def _resolve_decision(self, gate: GateDecision, llm_result: LLMResult | None,
|
|
195
|
+
probability: float) -> tuple[str, bool]:
|
|
196
|
+
"""14.3 最终决策表(逐行照抄实现,禁止改动)。
|
|
197
|
+
|
|
198
|
+
唯一扩展(2026-10-05):用户显式关闭云端复核时,灰区按本地模型 0.5 划分
|
|
199
|
+
(p > 0.5 → block,否则 allow),不再转人工。
|
|
200
|
+
"""
|
|
201
|
+
if gate == GateDecision.ALLOW:
|
|
202
|
+
return "allow", False
|
|
203
|
+
if gate == GateDecision.BLOCK:
|
|
204
|
+
return "block", False
|
|
205
|
+
# gate == GateDecision.LLM_REVIEW
|
|
206
|
+
if llm_result is not None and llm_result.status == "disabled":
|
|
207
|
+
# 关闭云端复核:按模型自身能力在 0.5 处切
|
|
208
|
+
return ("block", False) if probability > 0.5 else ("allow", False)
|
|
209
|
+
if llm_result is None or llm_result.status != "ok":
|
|
210
|
+
# invalid_output / api_error
|
|
211
|
+
return "manual_review", True
|
|
212
|
+
if llm_result.risk == "high":
|
|
213
|
+
return "block", False
|
|
214
|
+
if llm_result.risk == "medium":
|
|
215
|
+
return "suspicious", bool(llm_result.consistency_warning)
|
|
216
|
+
if llm_result.risk == "low":
|
|
217
|
+
if llm_result.consistency_warning:
|
|
218
|
+
return "suspicious", True
|
|
219
|
+
return "allow", False
|
|
220
|
+
# risk 取值异常(理论上不可达)→ 铁律 5 安全降级为转人工
|
|
221
|
+
print(f"[错误] LLM 返回未知 risk={llm_result.risk!r},安全降级为转人工", file=sys.stderr)
|
|
222
|
+
return "manual_review", True
|
|
223
|
+
|
|
224
|
+
# ==================== 云端 LLM 复核投递(2026-10-05) ====================
|
|
225
|
+
|
|
226
|
+
def _build_llm_payloads(self, evidence: FeatureEvidence, text: str) -> list[str]:
|
|
227
|
+
"""按配置把待复核内容切成 1..N 段。
|
|
228
|
+
|
|
229
|
+
内容源(llm_content_source):evidence=精炼证据(沿用 evidence_max_chars 上限);
|
|
230
|
+
raw=预处理后的文件原文。投递模式(llm_delivery_mode):truncate=取前 N 字符;
|
|
231
|
+
window=按 window_chars/overlap 分片;all=整段一次性上传。
|
|
232
|
+
"""
|
|
233
|
+
if self.cfg.llm_content_source == "raw":
|
|
234
|
+
content = text or ""
|
|
235
|
+
else:
|
|
236
|
+
content = evidence_to_prompt_text(evidence, self.cfg.evidence_max_chars)
|
|
237
|
+
|
|
238
|
+
if self.cfg.llm_delivery_mode == "all":
|
|
239
|
+
return [content]
|
|
240
|
+
if self.cfg.llm_delivery_mode == "truncate":
|
|
241
|
+
return [content[:self.cfg.llm_truncate_chars]]
|
|
242
|
+
|
|
243
|
+
size = self.cfg.llm_window_chars
|
|
244
|
+
step = max(1, size - self.cfg.llm_window_overlap)
|
|
245
|
+
if len(content) <= size:
|
|
246
|
+
return [content]
|
|
247
|
+
chunks: list[str] = []
|
|
248
|
+
start = 0
|
|
249
|
+
while start < len(content):
|
|
250
|
+
chunks.append(content[start:start + size])
|
|
251
|
+
if start + size >= len(content):
|
|
252
|
+
break
|
|
253
|
+
start += step
|
|
254
|
+
return chunks
|
|
255
|
+
|
|
256
|
+
def _review_with_llm(self, payloads: list[str],
|
|
257
|
+
probability: float) -> LLMResult:
|
|
258
|
+
"""逐段调用云端 LLM,按风险严重度取最严重的一段(high > medium > low)。"""
|
|
259
|
+
results = [self.llm.analyze(payload, probability) for payload in payloads]
|
|
260
|
+
return self._merge_llm_results(results)
|
|
261
|
+
|
|
262
|
+
@staticmethod
|
|
263
|
+
def _merge_llm_results(results: list[LLMResult]) -> LLMResult:
|
|
264
|
+
"""合并多段 LLM 结果:有成功段则取最严重;全失败则按 invalid_output/api_error 归类。"""
|
|
265
|
+
ok_results = [item for item in results if item.status == "ok"]
|
|
266
|
+
if not ok_results:
|
|
267
|
+
if any(item.status == "invalid_output" for item in results):
|
|
268
|
+
return LLMResult(status="invalid_output",
|
|
269
|
+
retries_used=max(item.retries_used for item in results))
|
|
270
|
+
return LLMResult(status="api_error")
|
|
271
|
+
severity = {"high": 3, "medium": 2, "low": 1}
|
|
272
|
+
best = max(ok_results, key=lambda item: severity.get(item.risk, 0))
|
|
273
|
+
return LLMResult(
|
|
274
|
+
status="ok", risk=best.risk, reason=best.reason, action=best.action,
|
|
275
|
+
consistency_warning=any(item.consistency_warning for item in results),
|
|
276
|
+
retries_used=0)
|
|
277
|
+
|
|
278
|
+
def _new_report(self, path: str) -> dict:
|
|
279
|
+
"""报告骨架:含 13.2 规范的全部键(错误路径同样返回完整键集)。"""
|
|
280
|
+
return {
|
|
281
|
+
"report_id": uuid.uuid4().hex,
|
|
282
|
+
"timestamp": datetime.datetime.now().astimezone().isoformat(),
|
|
283
|
+
"file_path": os.path.abspath(path),
|
|
284
|
+
"file_sha256": "",
|
|
285
|
+
"file_size_bytes": 0,
|
|
286
|
+
"source_type": "",
|
|
287
|
+
"preprocess_truncated": False,
|
|
288
|
+
"archive_parts": 1,
|
|
289
|
+
"raw_logit": 0.0,
|
|
290
|
+
"calibrated_probability": 0.0,
|
|
291
|
+
"calibration_method": self.calibrator.method,
|
|
292
|
+
"windows_evaluated": 0,
|
|
293
|
+
"early_stopped": False,
|
|
294
|
+
"early_stop_probability": None,
|
|
295
|
+
"gate_decision": "",
|
|
296
|
+
"final_decision": "",
|
|
297
|
+
"manual_review_required": False,
|
|
298
|
+
"consistency_warning": False,
|
|
299
|
+
"llm": None,
|
|
300
|
+
"llm_calls": 0,
|
|
301
|
+
"evidence": _evidence_to_dict(FeatureEvidence()),
|
|
302
|
+
"report_json_path": "",
|
|
303
|
+
"report_md_path": "",
|
|
304
|
+
"error": None,
|
|
305
|
+
}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""预处理模块:任意输入文件 → 统一文本。
|
|
2
|
+
|
|
3
|
+
只按内容判定类型(魔数 + 解码尝试),不按扩展名。压缩包一律在内存中解包,
|
|
4
|
+
禁止 extractall / extract 落盘(因此不需要 tarfile 的 filter 参数)。
|
|
5
|
+
"""
|
|
6
|
+
import gzip
|
|
7
|
+
import io
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
import re
|
|
11
|
+
import tarfile
|
|
12
|
+
import zipfile
|
|
13
|
+
|
|
14
|
+
from calibguard import constants as C
|
|
15
|
+
from calibguard.schemas import PreprocessResult
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
# 控制字符模式(仅用于 text/binary 判定;不含 \t \n \r)
|
|
20
|
+
_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")
|
|
21
|
+
# 控制字符占比超过该值即判定为二进制(见 7.3 步骤 4)
|
|
22
|
+
_CONTROL_CHAR_RATIO_MAX = 0.30
|
|
23
|
+
# 二进制可打印串提取模式(最短连续长度取常量 PRINTABLE_MIN_RUN)
|
|
24
|
+
_PRINTABLE_RE = re.compile(rb"[\x20-\x7e]{%d,}" % C.PRINTABLE_MIN_RUN)
|
|
25
|
+
# 连续 3 个及以上空白字符(空格/制表符)压缩模式
|
|
26
|
+
_BLANK_RUN_RE = re.compile(r"[ \t]{3,}")
|
|
27
|
+
# 解包截断标记
|
|
28
|
+
_ARCHIVE_TRUNCATED_MARK = "\n\n[ARCHIVE_PARTIALLY_EXTRACTED]"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def preprocess_file(path: str) -> PreprocessResult:
|
|
32
|
+
"""主流程:读入 → 判定压缩包 → 解包或解码 → JSON 规整 → 清洗。"""
|
|
33
|
+
data, read_truncated = _read_bytes(path)
|
|
34
|
+
|
|
35
|
+
# 步骤 2 + 步骤 3:压缩包走内存解包
|
|
36
|
+
if _detect_archive(data) is not None:
|
|
37
|
+
archive_text, parts, expand_truncated = _expand_archive(data, 0)
|
|
38
|
+
return PreprocessResult(
|
|
39
|
+
text=_clean_text(archive_text),
|
|
40
|
+
source_type="archive",
|
|
41
|
+
truncated=read_truncated or expand_truncated,
|
|
42
|
+
parts=parts,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
# 步骤 4:非压缩包 → 解码或可打印串提取
|
|
46
|
+
text, source_type = _decode_text(data)
|
|
47
|
+
|
|
48
|
+
# 步骤 5:JSON 规整(仅 text 类;解析失败原样保留)
|
|
49
|
+
if source_type == "text":
|
|
50
|
+
stripped = text.strip()
|
|
51
|
+
if stripped.startswith("{") or stripped.startswith("["):
|
|
52
|
+
try:
|
|
53
|
+
parsed = json.loads(stripped)
|
|
54
|
+
except ValueError as exc:
|
|
55
|
+
logger.debug("JSON 解析失败,按原文保留: %s", exc)
|
|
56
|
+
else:
|
|
57
|
+
text = json.dumps(parsed, ensure_ascii=False, indent=2)
|
|
58
|
+
|
|
59
|
+
# 步骤 6 + 步骤 7
|
|
60
|
+
return PreprocessResult(
|
|
61
|
+
text=_clean_text(text),
|
|
62
|
+
source_type=source_type,
|
|
63
|
+
truncated=read_truncated,
|
|
64
|
+
parts=1,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _read_bytes(path: str) -> tuple[bytes, bool]:
|
|
69
|
+
"""读入文件字节。(数据, 是否被读入截断)。不存在或不可读 → FileNotFoundError。"""
|
|
70
|
+
try:
|
|
71
|
+
with open(path, "rb") as handle:
|
|
72
|
+
data = handle.read(C.MAX_INPUT_BYTES + 1)
|
|
73
|
+
except FileNotFoundError:
|
|
74
|
+
raise
|
|
75
|
+
except OSError as exc:
|
|
76
|
+
# 不可读(权限等)按文档统一转为 FileNotFoundError
|
|
77
|
+
raise FileNotFoundError(f"文件不存在或不可读: {path}") from exc
|
|
78
|
+
if len(data) > C.MAX_INPUT_BYTES:
|
|
79
|
+
return data[:C.MAX_INPUT_BYTES], True
|
|
80
|
+
return data, False
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _detect_archive(data: bytes) -> str | None:
|
|
84
|
+
""""zip" | "gzip" | "tar" | None(纯魔数判定)。"""
|
|
85
|
+
if data[:4] in (b"PK\x03\x04", b"PK\x05\x06"):
|
|
86
|
+
return "zip"
|
|
87
|
+
if data[:2] == b"\x1f\x8b":
|
|
88
|
+
return "gzip"
|
|
89
|
+
if data[257:262] == b"ustar":
|
|
90
|
+
return "tar"
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _expand_archive(data: bytes, depth: int) -> tuple[str, int, bool]:
|
|
95
|
+
"""内存解包压缩包,返回 (拼接文本, 内部文件数, 是否被解包截断)。
|
|
96
|
+
|
|
97
|
+
只处理常规文件成员:zip 用 zf.read(info),tar 用 tf.extractfile(m).read();
|
|
98
|
+
目录、符号链接、硬链接、设备成员一律跳过(链接成员是 tar 的经典攻击面)。
|
|
99
|
+
"""
|
|
100
|
+
archive_type = _detect_archive(data)
|
|
101
|
+
members: list[tuple[str, bytes]] = []
|
|
102
|
+
file_count = 0
|
|
103
|
+
total_bytes = 0
|
|
104
|
+
truncated = False
|
|
105
|
+
|
|
106
|
+
try:
|
|
107
|
+
if archive_type == "zip":
|
|
108
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
|
109
|
+
for info in zf.infolist():
|
|
110
|
+
if info.is_dir():
|
|
111
|
+
continue
|
|
112
|
+
# 安全限制:读成员前先按其声明大小检查
|
|
113
|
+
if (file_count + 1 > C.ARCHIVE_MAX_FILES
|
|
114
|
+
or total_bytes + info.file_size > C.ARCHIVE_MAX_TOTAL_BYTES):
|
|
115
|
+
truncated = True
|
|
116
|
+
break
|
|
117
|
+
blob = zf.read(info)
|
|
118
|
+
members.append((info.filename, blob))
|
|
119
|
+
file_count += 1
|
|
120
|
+
total_bytes += len(blob)
|
|
121
|
+
elif archive_type == "tar":
|
|
122
|
+
with tarfile.open(fileobj=io.BytesIO(data), mode="r:*") as tf:
|
|
123
|
+
for member in tf:
|
|
124
|
+
if not member.isreg():
|
|
125
|
+
continue
|
|
126
|
+
# 安全限制:读成员前先按其声明大小检查
|
|
127
|
+
if (file_count + 1 > C.ARCHIVE_MAX_FILES
|
|
128
|
+
or total_bytes + member.size > C.ARCHIVE_MAX_TOTAL_BYTES):
|
|
129
|
+
truncated = True
|
|
130
|
+
break
|
|
131
|
+
extracted = tf.extractfile(member)
|
|
132
|
+
if extracted is None:
|
|
133
|
+
# 声明为常规文件却取不到内容:跳过该成员并记录
|
|
134
|
+
logger.warning("tar 成员无法读取,已跳过: %s", member.name)
|
|
135
|
+
continue
|
|
136
|
+
blob = extracted.read()
|
|
137
|
+
members.append((member.name, blob))
|
|
138
|
+
file_count += 1
|
|
139
|
+
total_bytes += len(blob)
|
|
140
|
+
elif archive_type == "gzip":
|
|
141
|
+
inner = gzip.decompress(data)
|
|
142
|
+
if _detect_archive(inner) == "tar":
|
|
143
|
+
# tar.gz 两步走:属同一压缩层级,深度不递增
|
|
144
|
+
return _expand_archive(inner, depth)
|
|
145
|
+
members.append(("", inner))
|
|
146
|
+
file_count = 1
|
|
147
|
+
total_bytes = len(inner)
|
|
148
|
+
else:
|
|
149
|
+
# 魔数判定为压缩包却无法识别类型:按二进制处理
|
|
150
|
+
return _extract_printable(data), 1, False
|
|
151
|
+
except Exception as exc:
|
|
152
|
+
# 损坏的压缩包 → 视为二进制,走可打印串提取(不吞异常:记录并明确返回)
|
|
153
|
+
logger.warning("压缩包解析失败,按二进制处理: %s: %s", type(exc).__name__, exc)
|
|
154
|
+
return _extract_printable(data), 1, False
|
|
155
|
+
|
|
156
|
+
# 第二遍:嵌套压缩包递归(深度 +1),其余解码为文本
|
|
157
|
+
collected: list[tuple[str, str]] = []
|
|
158
|
+
for name, blob in members:
|
|
159
|
+
if _detect_archive(blob) is not None:
|
|
160
|
+
if depth + 1 > C.ARCHIVE_MAX_DEPTH:
|
|
161
|
+
continue # 超出递归深度上限,跳过该内部文件
|
|
162
|
+
inner_text, inner_parts, inner_truncated = _expand_archive(blob, depth + 1)
|
|
163
|
+
file_count += inner_parts
|
|
164
|
+
truncated = truncated or inner_truncated
|
|
165
|
+
collected.append((name, inner_text))
|
|
166
|
+
else:
|
|
167
|
+
member_text, _member_type = _decode_text(blob)
|
|
168
|
+
collected.append((name, member_text))
|
|
169
|
+
|
|
170
|
+
# 全部文本按文件名排序后用 "\n\n" 连接(不加任何标记)
|
|
171
|
+
collected.sort(key=lambda item: item[0])
|
|
172
|
+
joined = "\n\n".join(entry_text for _name, entry_text in collected)
|
|
173
|
+
if truncated:
|
|
174
|
+
joined += _ARCHIVE_TRUNCATED_MARK
|
|
175
|
+
return joined, file_count, truncated
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _decode_text(data: bytes) -> tuple[str, str]:
|
|
179
|
+
"""(文本, "text"|"binary"):先 utf-8 再 gbk,均严格模式。"""
|
|
180
|
+
text: str | None = None
|
|
181
|
+
for encoding in ("utf-8", "gbk"):
|
|
182
|
+
try:
|
|
183
|
+
text = data.decode(encoding)
|
|
184
|
+
break # 解码成功,结束尝试
|
|
185
|
+
except UnicodeDecodeError as exc:
|
|
186
|
+
# 该编码不适用:记录原因后继续尝试下一种编码
|
|
187
|
+
logger.debug("按 %s 解码失败,尝试下一种编码: %s", encoding, exc)
|
|
188
|
+
if text is None:
|
|
189
|
+
# 两种解码都失败 → 二进制
|
|
190
|
+
return _extract_printable(data), "binary"
|
|
191
|
+
if text == "":
|
|
192
|
+
return text, "text" # 空字符串直接判 text,不做除法
|
|
193
|
+
control_ratio = len(_CONTROL_CHARS_RE.findall(text)) / len(text)
|
|
194
|
+
if control_ratio > _CONTROL_CHAR_RATIO_MAX:
|
|
195
|
+
return _extract_printable(data), "binary"
|
|
196
|
+
return text, "text"
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _extract_printable(data: bytes) -> str:
|
|
200
|
+
"""提取二进制中的可打印 ASCII 串,各段用 "\\n" 连接。"""
|
|
201
|
+
runs = _PRINTABLE_RE.findall(data)
|
|
202
|
+
return b"\n".join(runs).decode("ascii")
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _clean_text(text: str) -> str:
|
|
206
|
+
"""a 删控制字符 → b 压缩连续空白 → c 单行截断 → d 总长截断。"""
|
|
207
|
+
text = re.sub(C.CONTROL_CHARS_PATTERN, "", text)
|
|
208
|
+
text = _BLANK_RUN_RE.sub(" ", text)
|
|
209
|
+
lines = text.split("\n")
|
|
210
|
+
clipped_lines: list[str] = []
|
|
211
|
+
for line in lines:
|
|
212
|
+
if len(line) > C.LINE_TRUNCATE_THRESHOLD:
|
|
213
|
+
line = line[:C.LINE_TRUNCATE_THRESHOLD] + "…[LINE_TRUNCATED]"
|
|
214
|
+
clipped_lines.append(line)
|
|
215
|
+
text = "\n".join(clipped_lines)
|
|
216
|
+
if len(text) > C.MAX_TEXT_CHARS:
|
|
217
|
+
text = text[:C.MAX_TEXT_CHARS] + "\n…[TEXT_TRUNCATED]"
|
|
218
|
+
return text
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""审计报告模块:审计结果 dict → JSON 文件 + Markdown 文件。"""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
from calibguard.schemas import FeatureEvidence
|
|
6
|
+
|
|
7
|
+
# 证据明细的七类标题(第八类"异常结构特征"由长行/随机标识符两行构成)
|
|
8
|
+
_EVIDENCE_TITLES = [
|
|
9
|
+
("【危险API调用】", "dangerous_apis"),
|
|
10
|
+
("【网络行为】", "network_indicators"),
|
|
11
|
+
("【凭据与敏感路径访问】", "credential_access"),
|
|
12
|
+
("【编码与混淆】", "obfuscation_signals"),
|
|
13
|
+
("【可疑文件操作】", "file_operations"),
|
|
14
|
+
("【提示注入话术】", "injection_phrases"),
|
|
15
|
+
("【高熵字符串】", "high_entropy_strings"),
|
|
16
|
+
]
|
|
17
|
+
# 证据全空时的固定文案
|
|
18
|
+
_EMPTY_EVIDENCE_TEXT = "未发现明显可疑信号。"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def write_reports(report: dict, output_dir: str) -> tuple[str, str]:
|
|
22
|
+
"""生成 <output_dir>/<report_id>.json 与 <output_dir>/<report_id>.md,返回绝对路径。"""
|
|
23
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
24
|
+
report_id = str(report["report_id"])
|
|
25
|
+
json_path = os.path.abspath(os.path.join(output_dir, report_id + ".json"))
|
|
26
|
+
md_path = os.path.abspath(os.path.join(output_dir, report_id + ".md"))
|
|
27
|
+
with open(json_path, "w", encoding="utf-8") as handle:
|
|
28
|
+
handle.write(json.dumps(report, ensure_ascii=False, indent=2))
|
|
29
|
+
with open(md_path, "w", encoding="utf-8") as handle:
|
|
30
|
+
handle.write(_to_markdown(report))
|
|
31
|
+
return json_path, md_path
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _evidence_to_dict(ev: FeatureEvidence) -> dict:
|
|
35
|
+
"""FeatureEvidence → 九个同名键的 dict。"""
|
|
36
|
+
return {
|
|
37
|
+
"dangerous_apis": list(ev.dangerous_apis),
|
|
38
|
+
"network_indicators": list(ev.network_indicators),
|
|
39
|
+
"credential_access": list(ev.credential_access),
|
|
40
|
+
"obfuscation_signals": list(ev.obfuscation_signals),
|
|
41
|
+
"file_operations": list(ev.file_operations),
|
|
42
|
+
"injection_phrases": list(ev.injection_phrases),
|
|
43
|
+
"high_entropy_strings": list(ev.high_entropy_strings),
|
|
44
|
+
"long_line_numbers": list(ev.long_line_numbers),
|
|
45
|
+
"random_identifier_count": ev.random_identifier_count,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _to_markdown(report: dict) -> str:
|
|
50
|
+
"""按 13.3 固定模板渲染 Markdown 报告。"""
|
|
51
|
+
early_stopped = bool(report.get("early_stopped"))
|
|
52
|
+
if early_stopped:
|
|
53
|
+
early_stop_text = f"是,p={report.get('early_stop_probability')}"
|
|
54
|
+
else:
|
|
55
|
+
early_stop_text = "否"
|
|
56
|
+
|
|
57
|
+
lines: list[str] = [
|
|
58
|
+
"# CalibGuard 安全审计报告",
|
|
59
|
+
"",
|
|
60
|
+
f"- 报告编号:{report['report_id']}",
|
|
61
|
+
f"- 生成时间:{report['timestamp']}",
|
|
62
|
+
f"- 审计文件:{report['file_path']}",
|
|
63
|
+
f"- 文件 SHA256:{report['file_sha256']}",
|
|
64
|
+
f"- 文件大小:{report['file_size_bytes']} 字节",
|
|
65
|
+
f"- 来源类型:{report['source_type']}(内部文件数:{report['archive_parts']})",
|
|
66
|
+
"",
|
|
67
|
+
"## 结论",
|
|
68
|
+
"",
|
|
69
|
+
"| 项目 | 结果 |",
|
|
70
|
+
"|---|---|",
|
|
71
|
+
f"| 原始 logit | {report['raw_logit']} |",
|
|
72
|
+
f"| 校准概率 | {report['calibrated_probability']} |",
|
|
73
|
+
f"| 校准方法 | {report['calibration_method']} |",
|
|
74
|
+
f"| 推理窗口数 | {report['windows_evaluated']} |",
|
|
75
|
+
f"| 滑窗早停 | {early_stop_text} |",
|
|
76
|
+
f"| 门控决策 | {report['gate_decision']} |",
|
|
77
|
+
f"| 最终决策 | **{report['final_decision']}** |",
|
|
78
|
+
f"| 需人工复核 | {report['manual_review_required']} |",
|
|
79
|
+
f"| 一致性告警 | {report['consistency_warning']} |",
|
|
80
|
+
"",
|
|
81
|
+
"## LLM 分析",
|
|
82
|
+
"",
|
|
83
|
+
]
|
|
84
|
+
|
|
85
|
+
llm = report.get("llm")
|
|
86
|
+
if llm is None:
|
|
87
|
+
lines.append("(未提交云端分析)")
|
|
88
|
+
else:
|
|
89
|
+
lines.extend([
|
|
90
|
+
"| 字段 | 值 |",
|
|
91
|
+
"|---|---|",
|
|
92
|
+
f"| status | {llm.get('status')} |",
|
|
93
|
+
f"| risk | {llm.get('risk')} |",
|
|
94
|
+
f"| reason | {llm.get('reason')} |",
|
|
95
|
+
f"| action | {llm.get('action')} |",
|
|
96
|
+
f"| 重试次数 | {llm.get('retries_used')} |",
|
|
97
|
+
])
|
|
98
|
+
|
|
99
|
+
lines.extend(["", "## 证据明细", ""])
|
|
100
|
+
lines.append(_render_evidence_section(report.get("evidence") or {}))
|
|
101
|
+
|
|
102
|
+
lines.extend(["", "## 错误信息", ""])
|
|
103
|
+
error = report.get("error")
|
|
104
|
+
lines.append(str(error) if error else "(无)")
|
|
105
|
+
|
|
106
|
+
return "\n".join(lines) + "\n"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _render_evidence_section(evidence: dict) -> str:
|
|
110
|
+
"""按 8.4 的八类格式渲染证据明细;全空时返回固定文案。"""
|
|
111
|
+
blocks: list[str] = []
|
|
112
|
+
for title, key in _EVIDENCE_TITLES:
|
|
113
|
+
items = evidence.get(key) or []
|
|
114
|
+
if not items:
|
|
115
|
+
continue
|
|
116
|
+
block_lines = [title]
|
|
117
|
+
block_lines.extend(f"- {item}" for item in items)
|
|
118
|
+
blocks.append("\n".join(block_lines))
|
|
119
|
+
|
|
120
|
+
structural_lines: list[str] = []
|
|
121
|
+
long_line_numbers = evidence.get("long_line_numbers") or []
|
|
122
|
+
if long_line_numbers:
|
|
123
|
+
structural_lines.append(f"- 超长行行号: {long_line_numbers}")
|
|
124
|
+
random_identifier_count = evidence.get("random_identifier_count") or 0
|
|
125
|
+
if random_identifier_count > 0:
|
|
126
|
+
structural_lines.append(f"- 长随机标识符数量: {random_identifier_count}")
|
|
127
|
+
if structural_lines:
|
|
128
|
+
blocks.append("\n".join(["【异常结构特征】"] + structural_lines))
|
|
129
|
+
|
|
130
|
+
if not blocks:
|
|
131
|
+
return _EMPTY_EVIDENCE_TEXT
|
|
132
|
+
return "\n\n".join(blocks)
|