dsh-calibguard 0.2.5 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +22 -16
- package/README.md +22 -15
- package/cordis.patch.yml +2 -1
- package/docs/configuration.md +2 -2
- package/docs/troubleshooting.md +2 -1
- package/engine/README.md +33 -0
- package/engine/audit_service.py +304 -0
- package/engine/calibguard/__init__.py +2 -0
- package/engine/calibguard/__main__.py +4 -0
- package/engine/calibguard/calibrator.py +78 -0
- package/engine/calibguard/cli.py +74 -0
- package/engine/calibguard/config.py +129 -0
- package/engine/calibguard/constants.py +117 -0
- package/engine/calibguard/feature_extractor.py +161 -0
- package/engine/calibguard/gate.py +17 -0
- package/engine/calibguard/llm_analyzer.py +171 -0
- package/engine/calibguard/model.py +79 -0
- package/engine/calibguard/pipeline.py +305 -0
- package/engine/calibguard/preprocessor.py +218 -0
- package/engine/calibguard/report.py +132 -0
- package/engine/calibguard/schemas.py +53 -0
- package/engine/models/calibration_params.json +1 -0
- package/engine/models/codebert_finetuned/config.json +28 -0
- package/engine/models/codebert_finetuned/merges.txt +50001 -0
- package/engine/models/codebert_finetuned/special_tokens_map.json +51 -0
- package/engine/models/codebert_finetuned/tokenizer.json +250371 -0
- package/engine/models/codebert_finetuned/tokenizer_config.json +58 -0
- package/engine/models/codebert_finetuned/vocab.json +1 -0
- package/engine/requirements-gui.txt +11 -0
- package/lib/bootstrap.js +4 -3
- package/lib/client.js +1 -1
- package/lib/config.js +34 -7
- package/lib/engine.js +1 -1
- package/lib/index.js +1 -1
- package/package.json +5 -3
- package/setup.mjs +291 -48
- package/bridge/__pycache__/calibguard_bridge.cpython-310.pyc +0 -0
package/README.en.md
CHANGED
|
@@ -53,15 +53,20 @@ file sha256, and the report path.
|
|
|
53
53
|
|
|
54
54
|
### Before you start: engine dependencies are prepared automatically (like CG_GUI's launcher)
|
|
55
55
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
56
|
+
The plugin **ships the engine code** (`engine/`: `calibguard/`, `audit_service.py`, tokenizer files and
|
|
57
|
+
calibration parameters copied from CG_GUI); only the **475 MB of model weights** and the Python dependencies
|
|
58
|
+
are missing. **You don't install them by hand.** After you install the plugin and restart DSH, if
|
|
59
|
+
`<engineRoot>/.venv` is missing the plugin automatically, in the background:
|
|
59
60
|
|
|
60
|
-
1.
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
3.
|
|
64
|
-
4.
|
|
61
|
+
1. copies the bundled `engine/` into `<engineRoot>` (default `~/.dsh/calibguard-engine`);
|
|
62
|
+
2. downloads the model weights from a **release asset of this repository** (~442 MB; GitHub is tried first,
|
|
63
|
+
then ghproxy-style mirrors; cached in `~/.dsh/calibguard-tools/model-cache`);
|
|
64
|
+
3. creates `.venv` (**no preinstalled Python required**: if no 64-bit 3.10–3.12 is found, a portable Python 3.10.8 is downloaded automatically, ~30 MB);
|
|
65
|
+
4. installs dependencies from `requirements-gui.txt` (1–3 GB, Tsinghua PyPI mirror by default);
|
|
66
|
+
5. becomes ready — tools and the panel just work.
|
|
67
|
+
|
|
68
|
+
> The CG release zip is **no longer downloaded from GitHub by default**; only an explicit
|
|
69
|
+
> `node setup.mjs --from <zip>` or `--url <url>` uses it (offline installs / pinning an engine version).
|
|
65
70
|
|
|
66
71
|
**Where to watch progress**: the panel shows a "First run: preparing engine and dependencies" card with
|
|
67
72
|
the current step and the tail of the installer output; host logs show `[calibguard]` lines. The audit
|
|
@@ -71,7 +76,7 @@ once (depends on network and pip mirror).
|
|
|
71
76
|
> You can also pre-warm, or install offline, by running the same installer manually (idempotent):
|
|
72
77
|
>
|
|
73
78
|
> ```powershell
|
|
74
|
-
> node setup.mjs #
|
|
79
|
+
> node setup.mjs # bundled engine → npm model shards → venv → deps → self-check
|
|
75
80
|
> node setup.mjs --from D:\CalibGuard-GUI.zip # use an already-downloaded release zip (offline)
|
|
76
81
|
> node setup.mjs --dest D:\CG_GUI # custom location (then configure engineRoot)
|
|
77
82
|
> ```
|
|
@@ -83,7 +88,7 @@ once (depends on network and pip mirror).
|
|
|
83
88
|
|
|
84
89
|
1. **DSH** (DeepSeek Harness) — verified on 0.1.5-rc.2;
|
|
85
90
|
2. **Python**: nothing to install — when no 64-bit 3.10–3.12 is available the installer downloads a portable Python automatically (cached in `~/.dsh/calibguard-tools`; disable with `--no-portable-python`);
|
|
86
|
-
3. **
|
|
91
|
+
3. **Engine** — bundled with the plugin; model weights are downloaded from npm mirrors automatically. You can also pass `--from <zip>` or point `engineRoot` at an existing `CG_GUI/`.
|
|
87
92
|
|
|
88
93
|
### Option 0 — desktop "Add plugin" dialog (easiest, recommended)
|
|
89
94
|
|
|
@@ -91,7 +96,7 @@ DSH desktop → Settings → **Add plugin**, paste any one of the following (res
|
|
|
91
96
|
|
|
92
97
|
| Paste | Example | Notes |
|
|
93
98
|
|---|---|---|
|
|
94
|
-
| **GitHub repository URL** (recommended) | `https://github.com/maomitgd2006/
|
|
99
|
+
| **GitHub repository URL** (recommended) | `https://github.com/maomitgd2006/dsh-CalibGuard` | Fetches the source and registers the bundle in the current profile |
|
|
95
100
|
| **Local directory path** | `<path to this package>` (e.g. `/opt/dsh-calibguard`) | For a local clone / extracted copy |
|
|
96
101
|
| **Package name** | `dsh-calibguard` | ✅ **Published on npm**; just enter the package name (installs npm `latest`) |
|
|
97
102
|
|
|
@@ -100,7 +105,7 @@ DSH desktop → Settings → **Add plugin**, paste any one of the following (res
|
|
|
100
105
|
>
|
|
101
106
|
> ```powershell
|
|
102
107
|
> dsh plugin --profile <profile> add dsh-calibguard # npm package (recommended)
|
|
103
|
-
> dsh plugin --profile <profile> add https://github.com/maomitgd2006/
|
|
108
|
+
> dsh plugin --profile <profile> add https://github.com/maomitgd2006/dsh-CalibGuard # GitHub source
|
|
104
109
|
> dsh plugin --profile <profile> add file:<absolute path to this package> # local directory
|
|
105
110
|
> ```
|
|
106
111
|
>
|
|
@@ -143,7 +148,7 @@ the optional `defineTool` / Config schema support is active (everything still wo
|
|
|
143
148
|
### Option C — release archive (offline / no npm)
|
|
144
149
|
|
|
145
150
|
Download `dsh-calibguard-<version>.zip` (~100 KB, no `test/`) from
|
|
146
|
-
[Releases](https://github.com/maomitgd2006/
|
|
151
|
+
[Releases](https://github.com/maomitgd2006/dsh-CalibGuard/releases/latest), extract it to a fixed directory,
|
|
147
152
|
then point the `name` in "Option B" at `<extracted dir>/lib/index.js`.
|
|
148
153
|
|
|
149
154
|
### Verify
|
|
@@ -235,7 +240,7 @@ Common keys of the loader row `config` (full table: [docs/configuration.md](docs
|
|
|
235
240
|
| Symptom | Fix |
|
|
236
241
|
|---|---|
|
|
237
242
|
| No sidebar icon / panel says `not found` | Client bundle and routes load at host startup: **restart DSH** |
|
|
238
|
-
| **First run takes 10–20 minutes** | Normal: the
|
|
243
|
+
| **First run takes 10–20 minutes** | Normal: the model (476 MB) is downloaded from npm mirrors and dependencies (1–3 GB) are installed once; the panel shows progress |
|
|
239
244
|
| First audit is slow | Normal: ~475 MB of weights are loaded once (then the process stays resident) |
|
|
240
245
|
| Big file takes tens of seconds | Normal: full-text sliding-window inference; see [docs/performance.md](docs/performance.md) |
|
|
241
246
|
| Gray zone always goes to manual review | No credential configured: set `credentialRef` (recommended) or configure it in DSH Models |
|
|
@@ -250,9 +255,10 @@ Full troubleshooting table: [docs/troubleshooting.md](docs/troubleshooting.md).
|
|
|
250
255
|
## Layout
|
|
251
256
|
|
|
252
257
|
```
|
|
253
|
-
|
|
258
|
+
dsh-CalibGuard/
|
|
254
259
|
├── lib/ Node side: plugin entry, engine manager, tools, routes, settings, browser panel
|
|
255
260
|
├── bridge/ Python bridge: persistent child process, line-delimited JSON protocol
|
|
261
|
+
├── engine/ Bundled engine core copied from CG_GUI (code / tokenizer / calibration, no weights)
|
|
256
262
|
├── cordis.patch.yml Bundle patch (inserts the loader row)
|
|
257
263
|
├── scripts/ link-dsh (dev-time junction), bridge-selftest (engine self-check)
|
|
258
264
|
├── test/ node --test (95 cases; auto-skip when the engine env is missing)
|
|
@@ -274,7 +280,7 @@ DSH_CG/
|
|
|
274
280
|
|
|
275
281
|
| Repository | Contents |
|
|
276
282
|
|---|---|
|
|
277
|
-
| **maomitgd2006/
|
|
283
|
+
| **maomitgd2006/dsh-CalibGuard** (this repo) | DSH plugin: tools / service / routes + browser panel + Python bridge |
|
|
278
284
|
| [P-436/CG](https://github.com/P-436/CG) | Engine and benchmark: `CG_GUI/` (desktop shell + engine + model), `CG_Bench/` (training/eval) |
|
|
279
285
|
|
|
280
286
|
## License
|
package/README.md
CHANGED
|
@@ -48,13 +48,19 @@
|
|
|
48
48
|
|
|
49
49
|
### 开始前:引擎依赖会自动准备(对齐 CG_GUI 的「首启装环境」)
|
|
50
50
|
|
|
51
|
-
|
|
52
|
-
|
|
51
|
+
本插件**自带引擎**(`engine/`:复制自 CG_GUI 的 `calibguard/` 包、`audit_service.py`、词表与校准参数),
|
|
52
|
+
只差 **475 MB 模型权重**和 Python 依赖。**不用手动装**——装好插件重启 DSH 后,
|
|
53
|
+
若检测到缺少 `<engineRoot>/.venv`,插件会在**后台自动**:
|
|
53
54
|
|
|
54
|
-
1.
|
|
55
|
-
2.
|
|
56
|
-
|
|
57
|
-
|
|
55
|
+
1. 把内置 `engine/` 复制到 `<engineRoot>`(默认 `~/.dsh/calibguard-engine`);
|
|
56
|
+
2. 从**本仓库 Release** 下载模型权重 `model.safetensors`(约 442 MB 压缩包,GitHub 直连失败会自动尝试 ghproxy 等镜像,
|
|
57
|
+
缓存于 `~/.dsh/calibguard-tools/model-cache`);
|
|
58
|
+
3. 创建 `.venv`(**不需要你预装 Python**:本机没有 64 位 3.10–3.12 时会自动下载便携 Python 3.10.8,约 30 MB);
|
|
59
|
+
4. 按 `requirements-gui.txt` 安装依赖(约 1–3 GB,默认清华 PyPI 镜像);
|
|
60
|
+
5. 就绪后工具/面板自动可用。
|
|
61
|
+
|
|
62
|
+
> 默认**不再**从 GitHub 下载 CG 发行包;只有显式 `node setup.mjs --from <zip>` 或 `--url <地址>`
|
|
63
|
+
> 才会走 CG 发行包,用于离线/指定引擎版本。
|
|
58
64
|
|
|
59
65
|
**进度在哪看**:面板顶部会出现「首次运行:正在自动准备引擎与依赖」卡片(含当前步骤与输出尾巴);
|
|
60
66
|
宿主日志里是 `[calibguard]` 开头的行。准备期间审计按钮不可用,完成后点「刷新状态」即可。
|
|
@@ -75,7 +81,7 @@
|
|
|
75
81
|
|
|
76
82
|
1. **DSH**(DeepSeek Harness)0.1.5-rc.2 实测可用;
|
|
77
83
|
2. **Python**:无需预装——没有 64 位 3.10–3.12 时安装器会自动下载便携版(缓存在 `~/.dsh/calibguard-tools`,可用 `--no-portable-python` 关闭);
|
|
78
|
-
3.
|
|
84
|
+
3. **引擎**:插件自带;模型权重会自动从 npm 镜像下载。也可用 `--from` 指定 CG 发行包,或把 `engineRoot` 指向已有 `CG_GUI/`。
|
|
79
85
|
|
|
80
86
|
### 方式 0:桌面端「添加插件」(最省事,推荐)
|
|
81
87
|
|
|
@@ -83,7 +89,7 @@ DSH 桌面版 → 设置 → **添加插件**,输入框里填以下任意一
|
|
|
83
89
|
|
|
84
90
|
| 填什么 | 示例 | 说明 |
|
|
85
91
|
|---|---|---|
|
|
86
|
-
| **GitHub 仓库地址**(推荐) | `https://github.com/maomitgd2006/
|
|
92
|
+
| **GitHub 仓库地址**(推荐) | `https://github.com/maomitgd2006/dsh-CalibGuard` | 自动拉取源码并登记到当前 profile 的 bundle |
|
|
87
93
|
| **本地目录路径** | `<你的插件目录>`(例:`D:\code\dsh-calibguard`) | 已 clone / 解压到本地时用 |
|
|
88
94
|
| **包名** | `dsh-calibguard` | ✅ **已发布到 npm**,直接输包名即可(自动装 npm 上的 latest) |
|
|
89
95
|
|
|
@@ -91,7 +97,7 @@ DSH 桌面版 → 设置 → **添加插件**,输入框里填以下任意一
|
|
|
91
97
|
>
|
|
92
98
|
> ```powershell
|
|
93
99
|
> dsh plugin --profile <profile> add dsh-calibguard # npm 包(推荐)
|
|
94
|
-
> dsh plugin --profile <profile> add https://github.com/maomitgd2006/
|
|
100
|
+
> dsh plugin --profile <profile> add https://github.com/maomitgd2006/dsh-CalibGuard # GitHub 源码
|
|
95
101
|
> dsh plugin --profile <profile> add file:<本包绝对路径> # 本地目录
|
|
96
102
|
> ```
|
|
97
103
|
>
|
|
@@ -133,7 +139,7 @@ dsh plugin --profile <profile> add dsh-calibguard
|
|
|
133
139
|
|
|
134
140
|
### 方式 C:发行包(离线 / 不想用 npm)
|
|
135
141
|
|
|
136
|
-
从 [Releases](https://github.com/maomitgd2006/
|
|
142
|
+
从 [Releases](https://github.com/maomitgd2006/dsh-CalibGuard/releases/latest) 下载
|
|
137
143
|
`dsh-calibguard-<版本>.zip`(约 100 KB,不含 `test/`),解压到一个固定目录,然后按「方式 B」
|
|
138
144
|
把 `name` 指向解压目录下的 `lib/index.js` 即可。
|
|
139
145
|
|
|
@@ -210,11 +216,11 @@ AI 能看到完整四档结论与证据;若你说「只审高危文件」,
|
|
|
210
216
|
| 键 | 默认 | 说明 |
|
|
211
217
|
|---|---|---|
|
|
212
218
|
| `enabled` | `true` | 关掉则什么都不注册(临时停用不必删行) |
|
|
213
|
-
| `engineRoot` | `''` |
|
|
219
|
+
| `engineRoot` | `''` | 引擎目录;留空自动推导:**npm/桌面端安装**时为 `~/.dsh/calibguard-engine`,源码仓库里为本包同级 `CG_GUI` |
|
|
214
220
|
| `device` | `auto` | 自动做「CUDA 版本 + 算力」双检,低算力显卡回落 CPU |
|
|
215
221
|
| `credentialRef` | `''` | 推荐的云端复核凭据引用名,如 `DEEPSEEK_API_KEY`(交给 DSH 凭据库解析) |
|
|
216
222
|
| `announceToModel` | `true` | 是否把「何时该用 CalibGuard」写进系统提示词 |
|
|
217
|
-
| `autoSetup` | `true` | 首次运行缺 `.venv`
|
|
223
|
+
| `autoSetup` | `true` | 首次运行缺 `.venv` 时**自动**准备引擎(内置代码 + npm 模型分片)、按需下载便携 Python、安装依赖;设 `false` 则只报错提示手动 `node setup.mjs` |
|
|
218
224
|
| `maxFiles` | `20` | 目录展开上限;显式给出/上传的文件不受限 |
|
|
219
225
|
|
|
220
226
|
---
|
|
@@ -224,7 +230,7 @@ AI 能看到完整四档结论与证据;若你说「只审高危文件」,
|
|
|
224
230
|
| 现象 | 处理 |
|
|
225
231
|
|---|---|
|
|
226
232
|
| 侧边栏没有图标 / 面板报 `not found` | 客户端与路由在宿主启动时加载:**重启 DSH** |
|
|
227
|
-
| **首次运行要等十几分钟** |
|
|
233
|
+
| **首次运行要等十几分钟** | 正常:首次下载模型(约 442 MB,自动镜像回退)并安装依赖(1–3 GB);面板顶部有进度卡片,只需一次 |
|
|
228
234
|
| 首次审计很慢 | 正常:要加载约 475 MB 权重(之后走常驻进程) |
|
|
229
235
|
| 大文件审计几十秒 | 正常:全文滑窗逐窗推理,见 [docs/performance.md](docs/performance.md) |
|
|
230
236
|
| 灰区全部转人工 | 没配凭据:设 `credentialRef`(推荐)或在 DSH Models 页配置 |
|
|
@@ -239,9 +245,10 @@ AI 能看到完整四档结论与证据;若你说「只审高危文件」,
|
|
|
239
245
|
## 项目结构
|
|
240
246
|
|
|
241
247
|
```
|
|
242
|
-
|
|
248
|
+
dsh-CalibGuard/
|
|
243
249
|
├── lib/ Node 侧:插件入口、引擎管理、工具、路由、共享设置、浏览器面板
|
|
244
250
|
├── bridge/ Python 桥接:常驻子进程,行式 JSON 协议,复用引擎
|
|
251
|
+
├── engine/ 从 CG_GUI 复制来的引擎核心(代码/词表/校准参数,不含 475 MB 权重)
|
|
245
252
|
├── cordis.patch.yml 组合包 patch(把插件行插进 profile 配置树)
|
|
246
253
|
├── scripts/ link-dsh(开发期挂包)、bridge-selftest(引擎自检)
|
|
247
254
|
├── test/ node --test(95 个用例;缺引擎环境时自动 skip)
|
|
@@ -263,7 +270,7 @@ DSH_CG/
|
|
|
263
270
|
|
|
264
271
|
| 仓库 | 内容 |
|
|
265
272
|
|---|---|
|
|
266
|
-
| **maomitgd2006/
|
|
273
|
+
| **maomitgd2006/dsh-CalibGuard**(本仓库) | DSH 插件:工具 / 服务 / 路由 + 浏览器面板 + Python 桥接 |
|
|
267
274
|
| [P-436/CG](https://github.com/P-436/CG) | 引擎与基准:`CG_GUI/`(桌面壳 + 引擎 + 模型)、`CG_Bench/`(训练/评测) |
|
|
268
275
|
|
|
269
276
|
## 许可证
|
package/cordis.patch.yml
CHANGED
|
@@ -16,7 +16,8 @@
|
|
|
16
16
|
config:
|
|
17
17
|
# 是否注册工具(false 时不注册任何东西,便于临时停用而不必删行)
|
|
18
18
|
enabled: true
|
|
19
|
-
#
|
|
19
|
+
# 引擎目录(含 calibguard/ 包与 models/ 的目录);留空=自动推导:npm/桌面端安装为 ~/.dsh/calibguard-engine,
|
|
20
|
+
# 源码仓库里为本包同级 CG_GUI
|
|
20
21
|
engineRoot: ''
|
|
21
22
|
# Python 解释器;留空=依次尝试 <engineRoot>/.venv、环境变量 DSH_CALIBGUARD_PYTHON、PATH
|
|
22
23
|
pythonPath: ''
|
package/docs/configuration.md
CHANGED
|
@@ -7,12 +7,12 @@
|
|
|
7
7
|
| 键 | 默认 | 说明 |
|
|
8
8
|
|---|---|---|
|
|
9
9
|
| `enabled` | `true` | 关掉则什么都不注册(临时停用不必删行) |
|
|
10
|
-
| `engineRoot` | `''` |
|
|
10
|
+
| `engineRoot` | `''` | 引擎目录;留空自动推导:**npm/桌面端安装**时为 `~/.dsh/calibguard-engine`,源码仓库里为本包同级 `CG_GUI` |
|
|
11
11
|
| `pythonPath` | `''` | Python 解释器;留空依次尝试 `<engineRoot>/.venv` → `DSH_CALIBGUARD_PYTHON` → PATH |
|
|
12
12
|
| `device` | `auto` | `auto` 做「CUDA 版本 + 算力」双检;`cpu` / `cuda` 为强制 |
|
|
13
13
|
| `useGuiSettings` | `true` | 复用 `CG_GUI/gui-settings.json`(与桌面壳共享阈值/云端配置) |
|
|
14
14
|
| `announceToModel` | `true` | 是否把「何时该用 CalibGuard」写进系统提示词 |
|
|
15
|
-
| `autoSetup` | `true` | 首次运行缺 `.venv`
|
|
15
|
+
| `autoSetup` | `true` | 首次运行缺 `.venv` 时**自动**准备引擎(复制内置 `engine/` + 从 npm 镜像下载模型分片约 476 MB,缓存 `~/.dsh/calibguard-tools/model-cache`)、按需下载便携 Python 3.10.8、安装依赖(1–3 GB);面板显示进度。设 `false` 则只报错并提示手动 `node setup.mjs` |
|
|
16
16
|
| `maxFiles` | `20` | **目录展开上限**(1–200):只限制"给一个目录时递归展开多少个文件";显式给出/上传的文件全部审计 |
|
|
17
17
|
| `auditTimeoutMs` | `600000` | 单次审计超时(毫秒) |
|
|
18
18
|
| `idleShutdownMs` | `600000` | 空闲多久回收桥接进程(`0`=常驻不回收) |
|
package/docs/troubleshooting.md
CHANGED
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
| 每份文件都 `manual_review` + `CUDA error: no kernel image` | 设备定档失效:确认 `device` 未被强制成 `cuda` |
|
|
10
10
|
| 灰区全部转人工、`llm.status=api_error` | 没配凭据(预期降级)→ 设 `credentialRef`(推荐)或在 DSH Models 页配置 |
|
|
11
11
|
| `凭据引用 X 未配置` 告警 | `credentialRef` 指向的引用在 DSH 凭据库里没有值;在 Models 页填 Key 或用 `llmApiKey` |
|
|
12
|
-
|
|
|
12
|
+
| **模型下载失败 / 想离线** | 手动下载 `calibguard-model-1.0.0.zip`(Release `model-v1.0.0`),解压出 `model.safetensors` 放到 `~/.dsh/calibguard-tools/model-cache/model.safetensors`(或 `<engineRoot>/models/codebert_finetuned/`),重跑安装即会校验 SHA 并跳过下载 |
|
|
13
|
+
| **首次运行卡在"正在准备环境"** | 正常:首次下载模型(约 442 MB,自动镜像回退)+ 装依赖(1–3 GB),约 10–20 分钟;面板顶部有进度与输出尾巴。若长时间无进度,可看 `~/.dsh/calibguard-tools/model-cache` 是否在增长 |
|
|
13
14
|
| 环境准备失败 | 面板会显示失败原因与最后几行输出;在本插件目录跑 `node setup.mjs` 看完整输出后重试(也支持 `--from` 离线包) |
|
|
14
15
|
| **本机没有 Python** | 会自动用 uv 下载便携 Python 3.10.8(约 30 MB,缓存 `~/.dsh/calibguard-tools`);若这一步失败会自动给出 python.org 下载链接,也可用 `--python` 指定解释器 |
|
|
15
16
|
| 大文件审计几十秒 | 正常:耗时 ≈ 窗口数 ÷ CPU 吞吐,见 [performance.md](performance.md) |
|
package/engine/README.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# engine/ —— 内置引擎核心(复制自 CG_GUI)
|
|
2
|
+
|
|
3
|
+
本目录是 [CG](https://github.com/P-436/CG) 仓库 `CG_GUI/` 里运行审计所需的**核心文件副本**,
|
|
4
|
+
让 `dsh-calibguard` 安装后不必再从 GitHub 下载 CG 发行包:
|
|
5
|
+
|
|
6
|
+
| 内容 | 说明 |
|
|
7
|
+
|---|---|
|
|
8
|
+
| `calibguard/` | 审计引擎 Python 包(未修改、原样复制) |
|
|
9
|
+
| `audit_service.py` | 桥接进程读取 GUI 设置时复用它的校验逻辑 |
|
|
10
|
+
| `requirements-gui.txt` | 依赖清单(安装到 `.venv`) |
|
|
11
|
+
| `models/calibration_params.json` | Platt 校准参数 |
|
|
12
|
+
| `models/codebert_finetuned/*` | 模型配置与分词器文件(**不含** `model.safetensors`) |
|
|
13
|
+
|
|
14
|
+
**模型权重(475 MB)不在本仓库的 Git 里**:它作为本仓库的 Release 资产分发
|
|
15
|
+
(`model-v1.0.0` 里的 `calibguard-model-1.0.0.zip`),首次运行时由 `setup.mjs` 下载、按 SHA-256
|
|
16
|
+
校验后释放为 `models/codebert_finetuned/model.safetensors`;已下载的会缓存在
|
|
17
|
+
`~/.dsh/calibguard-tools/model-cache`。
|
|
18
|
+
|
|
19
|
+
## 与 CG_GUI 同步
|
|
20
|
+
|
|
21
|
+
引擎代码更新时,把对应文件从 CG 的 `CG_GUI/` 重新复制过来即可(保持原样、不要改逻辑):
|
|
22
|
+
|
|
23
|
+
```powershell
|
|
24
|
+
# 在 CG 仓库根目录下执行(把 DSH_CG 换成你本地插件仓库目录)
|
|
25
|
+
Copy-Item CG_GUI\calibguard\*.py <插件仓库>\engine\calibguard\ -Force
|
|
26
|
+
Copy-Item CG_GUI\audit_service.py, CG_GUI\requirements-gui.txt <插件仓库>\engine\ -Force
|
|
27
|
+
Copy-Item CG_GUI\models\calibration_params.json <插件仓库>\engine\models\ -Force
|
|
28
|
+
Copy-Item CG_GUI\models\codebert_finetuned\*.json, CG_GUI\models\codebert_finetuned\merges.txt, `
|
|
29
|
+
CG_GUI\models\codebert_finetuned\vocab.json <插件仓库>\engine\models\codebert_finetuned\ -Force
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
如果模型权重本身变化,需要:① 重新压缩成 `calibguard-model-<版本>.zip` 发到本仓库 Release
|
|
33
|
+
(如 `model-v1.0.0`);② 更新 `setup.mjs` 里的 `MODEL_URL` 与 `MODEL_SHA256`。
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
"""审计任务封装(T-G2.5):把 calibguard.pipeline 包成“任务”模型供 API 层调用。
|
|
2
|
+
|
|
3
|
+
铁律:
|
|
4
|
+
- 后端审计单并发:_JOB_LOCK 一把全局锁,后来的任务排队等待,禁止多 worker 并行;
|
|
5
|
+
- 逐字复用引擎入口(load_config + AuditPipeline),禁止重写审计逻辑;
|
|
6
|
+
- 引擎无阶段回调钩子 → stage 只写 "model" → "report" 两档,禁止伪造细粒度进度;
|
|
7
|
+
- 单文件异常记账后继续下一个文件,异常文本必须能冒泡到 UI。
|
|
8
|
+
|
|
9
|
+
设置(gui-settings.json,2026-10-05 扩展):
|
|
10
|
+
- 是否启用云端 LLM 复核、API Key、Base URL、模型名;
|
|
11
|
+
- 灰区范围(allow/block 阈值,默认 0.15 / 0.85);
|
|
12
|
+
- LLM 送审内容源(精炼证据 / 文件原文)× 投递模式(截断 / 滑窗 / 一次性全部上传)
|
|
13
|
+
及其随附参数(截断字符数、滑窗字符数与重叠字符数)。
|
|
14
|
+
- 应用方式:直接改运行中 pipeline 的 cfg 字段 + 重建 LLM 客户端,**不重载模型**
|
|
15
|
+
(模型加载耗时数十秒,而上述设置都不影响模型权重与校准参数)。
|
|
16
|
+
"""
|
|
17
|
+
import dataclasses
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import threading
|
|
21
|
+
import uuid
|
|
22
|
+
|
|
23
|
+
from calibguard.config import LLM_CONTENT_SOURCES, LLM_DELIVERY_MODES, load_config, validate_config
|
|
24
|
+
from calibguard.llm_analyzer import LLMAnalyzer
|
|
25
|
+
from calibguard.pipeline import AuditPipeline
|
|
26
|
+
|
|
27
|
+
_ROOT = os.path.dirname(os.path.abspath(__file__))
|
|
28
|
+
SETTINGS_PATH = os.path.join(_ROOT, "gui-settings.json")
|
|
29
|
+
|
|
30
|
+
_JOB_LOCK = threading.Lock() # 全局单并发锁
|
|
31
|
+
_JOBS: dict[str, dict] = {} # job_id -> 状态
|
|
32
|
+
_PIPELINE: AuditPipeline | None = None # 引擎流水线单例(模型只加载一次)
|
|
33
|
+
_PENDING_SETTINGS_REFRESH = False # 审计运行期间被暂缓的设置应用请求
|
|
34
|
+
|
|
35
|
+
# 阶段取值封闭集合(引擎无回调钩子,实际只会出现 model / report 两档)
|
|
36
|
+
STAGE_MODEL = "model"
|
|
37
|
+
STAGE_REPORT = "report"
|
|
38
|
+
|
|
39
|
+
# ============================== 设置 ==============================
|
|
40
|
+
|
|
41
|
+
# 默认设置(gui-settings.json 缺键时用这些值;与引擎 AppConfig 默认值保持一致)
|
|
42
|
+
DEFAULT_SETTINGS: dict = {
|
|
43
|
+
"llm_enabled": True,
|
|
44
|
+
"deepseek_api_key": "",
|
|
45
|
+
"llm_base_url": "https://api.deepseek.com",
|
|
46
|
+
"llm_model": "deepseek-chat",
|
|
47
|
+
"allow_threshold": 0.15,
|
|
48
|
+
"block_threshold": 0.85,
|
|
49
|
+
"llm_content_source": "evidence",
|
|
50
|
+
"llm_delivery_mode": "truncate",
|
|
51
|
+
"llm_truncate_chars": 3000,
|
|
52
|
+
"llm_window_chars": 3000,
|
|
53
|
+
"llm_window_overlap": 500,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
# 设置键 → 引擎 AppConfig 字段(用于把设置写进运行中的 cfg)
|
|
57
|
+
_CFG_FIELDS = {
|
|
58
|
+
"llm_enabled": "llm_enabled",
|
|
59
|
+
"deepseek_api_key": "llm_api_key",
|
|
60
|
+
"llm_base_url": "llm_base_url",
|
|
61
|
+
"llm_model": "llm_model",
|
|
62
|
+
"allow_threshold": "allow_threshold",
|
|
63
|
+
"block_threshold": "block_threshold",
|
|
64
|
+
"llm_content_source": "llm_content_source",
|
|
65
|
+
"llm_delivery_mode": "llm_delivery_mode",
|
|
66
|
+
"llm_truncate_chars": "llm_truncate_chars",
|
|
67
|
+
"llm_window_chars": "llm_window_chars",
|
|
68
|
+
"llm_window_overlap": "llm_window_overlap",
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
_SETTINGS: dict = dict(DEFAULT_SETTINGS)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _coerce_settings(raw: dict) -> dict:
|
|
75
|
+
"""把外部输入(JSON)规范化为合法设置字典;任何非法项抛 ValueError(中文说明)。
|
|
76
|
+
|
|
77
|
+
只接受已知键;未知键忽略(向前兼容)。
|
|
78
|
+
"""
|
|
79
|
+
if not isinstance(raw, dict):
|
|
80
|
+
raise ValueError("设置必须是 JSON 对象")
|
|
81
|
+
merged = dict(DEFAULT_SETTINGS)
|
|
82
|
+
for key in DEFAULT_SETTINGS:
|
|
83
|
+
if key in raw:
|
|
84
|
+
merged[key] = raw[key]
|
|
85
|
+
|
|
86
|
+
if not isinstance(merged["llm_enabled"], bool):
|
|
87
|
+
raise ValueError("llm_enabled 必须是布尔值")
|
|
88
|
+
for key in ("deepseek_api_key", "llm_base_url", "llm_model"):
|
|
89
|
+
if not isinstance(merged[key], str):
|
|
90
|
+
raise ValueError(f"{key} 必须是字符串")
|
|
91
|
+
if merged["llm_enabled"] and not merged["llm_base_url"].strip():
|
|
92
|
+
raise ValueError("启用云端复核时 llm_base_url 不能为空")
|
|
93
|
+
if merged["llm_enabled"] and not merged["llm_model"].strip():
|
|
94
|
+
raise ValueError("启用云端复核时 llm_model 不能为空")
|
|
95
|
+
|
|
96
|
+
for key in ("allow_threshold", "block_threshold"):
|
|
97
|
+
try:
|
|
98
|
+
merged[key] = float(merged[key])
|
|
99
|
+
except (TypeError, ValueError) as exc:
|
|
100
|
+
raise ValueError(f"{key} 必须是 0~1 之间的数字") from exc
|
|
101
|
+
if not (0.0 < merged["allow_threshold"] < merged["block_threshold"] < 1.0):
|
|
102
|
+
raise ValueError(
|
|
103
|
+
f"灰区范围必须满足 0 < 下限({merged['allow_threshold']}) < "
|
|
104
|
+
f"上限({merged['block_threshold']}) < 1")
|
|
105
|
+
|
|
106
|
+
if merged["llm_content_source"] not in LLM_CONTENT_SOURCES:
|
|
107
|
+
raise ValueError(f"llm_content_source 必须是 {LLM_CONTENT_SOURCES} 之一")
|
|
108
|
+
if merged["llm_delivery_mode"] not in LLM_DELIVERY_MODES:
|
|
109
|
+
raise ValueError(f"llm_delivery_mode 必须是 {LLM_DELIVERY_MODES} 之一")
|
|
110
|
+
for key in ("llm_truncate_chars", "llm_window_chars", "llm_window_overlap"):
|
|
111
|
+
try:
|
|
112
|
+
merged[key] = int(merged[key])
|
|
113
|
+
except (TypeError, ValueError) as exc:
|
|
114
|
+
raise ValueError(f"{key} 必须是整数") from exc
|
|
115
|
+
if merged["llm_truncate_chars"] < 1:
|
|
116
|
+
raise ValueError("截断大小必须 ≥ 1 个字符")
|
|
117
|
+
if merged["llm_window_chars"] < 1:
|
|
118
|
+
raise ValueError("滑窗大小必须 ≥ 1 个字符")
|
|
119
|
+
if not (0 <= merged["llm_window_overlap"] < merged["llm_window_chars"]):
|
|
120
|
+
raise ValueError(
|
|
121
|
+
f"滑窗重叠必须满足 0 ≤ 重叠 < 滑窗大小({merged['llm_window_chars']})")
|
|
122
|
+
return merged
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def get_settings() -> dict:
|
|
126
|
+
"""返回当前生效设置的副本(供 GET /api/settings 与 UI 回填)。"""
|
|
127
|
+
return dict(_SETTINGS)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _apply_settings_to_cfg(cfg, settings: dict) -> None:
|
|
131
|
+
"""把设置写进 AppConfig 对应字段(键名映射见 _CFG_FIELDS)。"""
|
|
132
|
+
for key, field_name in _CFG_FIELDS.items():
|
|
133
|
+
setattr(cfg, field_name, settings[key])
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _build_llm(cfg):
|
|
137
|
+
"""按 cfg 构造云端 LLM 客户端(Key/BaseURL/模型名变更后重建)。"""
|
|
138
|
+
return LLMAnalyzer(
|
|
139
|
+
api_key=cfg.llm_api_key, base_url=cfg.llm_base_url, model=cfg.llm_model,
|
|
140
|
+
temperature=cfg.llm_temperature, max_tokens=cfg.llm_max_tokens,
|
|
141
|
+
timeout=cfg.llm_timeout, evidence_max_chars=cfg.evidence_max_chars)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _apply_settings_locked() -> None:
|
|
145
|
+
"""把当前设置应用到运行中的 pipeline(调用方必须已持有 _JOB_LOCK)。
|
|
146
|
+
|
|
147
|
+
只改 cfg 字段 + 重建 LLM 客户端,**不重载模型**:这样设置变更绝不会与正在进行的
|
|
148
|
+
审计并发改写状态(补丁 11 约束),也不付出模型加载代价。
|
|
149
|
+
"""
|
|
150
|
+
global _PENDING_SETTINGS_REFRESH
|
|
151
|
+
if _PIPELINE is None:
|
|
152
|
+
_PENDING_SETTINGS_REFRESH = False # 流水线未建:下次创建自然读新设置
|
|
153
|
+
return
|
|
154
|
+
_apply_settings_to_cfg(_PIPELINE.cfg, _SETTINGS)
|
|
155
|
+
_PIPELINE.llm = _build_llm(_PIPELINE.cfg)
|
|
156
|
+
_PENDING_SETTINGS_REFRESH = False
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _refresh_settings() -> bool:
|
|
160
|
+
"""在 _JOB_LOCK 保护下应用设置;审计运行中则暂缓(不阻塞等待)。
|
|
161
|
+
|
|
162
|
+
不阻塞的原因:设置请求若等锁,会在审计期间把 HTTP 请求挂住;补丁 11 要求
|
|
163
|
+
“审计运行中禁止触发重建,UI 提示当前任务完成后生效”,故记待办后立即返回。
|
|
164
|
+
第二次尝试用于消解与任务收尾的竞态(锁刚释放时仍可即时生效)。
|
|
165
|
+
"""
|
|
166
|
+
global _PENDING_SETTINGS_REFRESH
|
|
167
|
+
for _ in range(2):
|
|
168
|
+
if _JOB_LOCK.acquire(blocking=False):
|
|
169
|
+
try:
|
|
170
|
+
_apply_settings_locked()
|
|
171
|
+
return True
|
|
172
|
+
finally:
|
|
173
|
+
_JOB_LOCK.release()
|
|
174
|
+
_PENDING_SETTINGS_REFRESH = True # 有任务在跑:暂缓
|
|
175
|
+
return False
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def load_settings() -> None:
|
|
179
|
+
"""读 gui-settings.json,刷新内存设置,并把 Key 注入环境变量 DEEPSEEK_API_KEY。
|
|
180
|
+
|
|
181
|
+
引擎的 load_config 从环境变量读取 Key(主文档 6.2),本函数是 GUI 侧唯一注入点。
|
|
182
|
+
铁律:禁止读写 calibguard 的 config.yaml;禁止把 Key 写进任何日志。
|
|
183
|
+
"""
|
|
184
|
+
global _SETTINGS
|
|
185
|
+
if not os.path.isfile(SETTINGS_PATH):
|
|
186
|
+
return
|
|
187
|
+
with open(SETTINGS_PATH, "r", encoding="utf-8") as handle:
|
|
188
|
+
payload = json.load(handle)
|
|
189
|
+
if not isinstance(payload, dict):
|
|
190
|
+
raise ValueError(f"设置文件格式错误(应为 JSON 对象):{SETTINGS_PATH}")
|
|
191
|
+
_SETTINGS = _coerce_settings(payload)
|
|
192
|
+
_inject_api_key(_SETTINGS["deepseek_api_key"])
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _inject_api_key(key: str) -> None:
|
|
196
|
+
"""把 Key 注入(或清除)环境变量:留空即不使用云端(不留残余环境值)。"""
|
|
197
|
+
if key:
|
|
198
|
+
os.environ["DEEPSEEK_API_KEY"] = key
|
|
199
|
+
else:
|
|
200
|
+
os.environ.pop("DEEPSEEK_API_KEY", None)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def save_settings(payload: dict) -> bool:
|
|
204
|
+
"""合并并落盘设置(UTF-8 无 BOM),立即让配置生效。
|
|
205
|
+
|
|
206
|
+
支持部分更新:payload 中未出现的键保持当前值(旧版只传 deepseek_api_key 的调用
|
|
207
|
+
仍然可用)。返回设置是否已即时生效:False 表示当前有审计任务在跑(_JOB_LOCK 被
|
|
208
|
+
占用),应用已暂缓,等当前任务结束由 _run_job 落地(补丁 11 约束一)。
|
|
209
|
+
"""
|
|
210
|
+
global _SETTINGS
|
|
211
|
+
if not isinstance(payload, dict):
|
|
212
|
+
raise ValueError("设置必须是 JSON 对象")
|
|
213
|
+
merged = dict(_SETTINGS)
|
|
214
|
+
merged.update(payload)
|
|
215
|
+
settings = _coerce_settings(merged)
|
|
216
|
+
|
|
217
|
+
with open(SETTINGS_PATH, "w", encoding="utf-8") as handle:
|
|
218
|
+
json.dump(settings, handle, ensure_ascii=False, indent=2)
|
|
219
|
+
_SETTINGS = settings
|
|
220
|
+
_inject_api_key(settings["deepseek_api_key"])
|
|
221
|
+
return _refresh_settings()
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _get_pipeline() -> AuditPipeline:
|
|
225
|
+
"""引擎流水线单例:首次调用加载模型(可耗时数十秒),之后全程复用。"""
|
|
226
|
+
global _PIPELINE
|
|
227
|
+
if _PIPELINE is None:
|
|
228
|
+
# 铁律 7:禁止读写 calibguard 的 config.yaml → 传 None 走引擎内置默认值,
|
|
229
|
+
# 相对路径(models/、reports/)由 server 的 cwd(项目根)解析。
|
|
230
|
+
cfg = load_config()
|
|
231
|
+
_apply_settings_to_cfg(cfg, _SETTINGS)
|
|
232
|
+
error = validate_config(cfg)
|
|
233
|
+
if error:
|
|
234
|
+
raise ValueError(f"设置非法,无法初始化引擎:{error}")
|
|
235
|
+
_PIPELINE = AuditPipeline(cfg)
|
|
236
|
+
return _PIPELINE
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# ============================== 任务模型 ==============================
|
|
240
|
+
|
|
241
|
+
def create_job(paths: list[str]) -> str:
|
|
242
|
+
"""建 job(排队态),返回 job_id;执行交给 daemon 线程 + 全局单并发锁。"""
|
|
243
|
+
job_id = uuid.uuid4().hex[:12]
|
|
244
|
+
_JOBS[job_id] = {
|
|
245
|
+
"state": "queued",
|
|
246
|
+
"paths": list(paths),
|
|
247
|
+
"stage": "",
|
|
248
|
+
"done": 0,
|
|
249
|
+
"total": len(paths),
|
|
250
|
+
"results": [],
|
|
251
|
+
}
|
|
252
|
+
threading.Thread(target=_run_job, args=(job_id,), daemon=True).start()
|
|
253
|
+
return job_id
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def job_status(job_id: str) -> dict:
|
|
257
|
+
"""查询任务状态/进度/结果;未知 id 抛 KeyError(API 层转 404)。
|
|
258
|
+
|
|
259
|
+
返回浅拷贝:handler 线程序列化时后台线程仍在推进 job,避免“字典迭代期间改变大小”。
|
|
260
|
+
"""
|
|
261
|
+
return dict(_JOBS[job_id])
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _report_to_dict(report) -> dict:
|
|
265
|
+
"""引擎报告归一化:CGv0.2.2 的 audit_file 直接返回 dict,原样透传(不增删字段)。"""
|
|
266
|
+
if isinstance(report, dict):
|
|
267
|
+
return report
|
|
268
|
+
if hasattr(report, "to_dict"):
|
|
269
|
+
return report.to_dict()
|
|
270
|
+
if dataclasses.is_dataclass(report):
|
|
271
|
+
return dataclasses.asdict(report)
|
|
272
|
+
raise TypeError(f"引擎返回了无法序列化的报告对象:{type(report).__name__}")
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _run_job(job_id: str) -> None:
|
|
276
|
+
"""单并发执行任务:逐文件调用引擎,单文件异常记账后继续。"""
|
|
277
|
+
job = _JOBS[job_id]
|
|
278
|
+
with _JOB_LOCK: # 后来的任务在此排队等待
|
|
279
|
+
job["state"] = "running"
|
|
280
|
+
try:
|
|
281
|
+
pipeline = _get_pipeline()
|
|
282
|
+
except Exception as exc:
|
|
283
|
+
# 模型/校准参数/设置不可用属全局失败:显式暴露,绝不假成功
|
|
284
|
+
job["error"] = f"引擎初始化失败:{type(exc).__name__}: {exc}"
|
|
285
|
+
job["stage"] = STAGE_REPORT
|
|
286
|
+
for path in job["paths"]:
|
|
287
|
+
job["results"].append({"error": job["error"], "path": path})
|
|
288
|
+
job["done"] = job["total"]
|
|
289
|
+
job["state"] = "error"
|
|
290
|
+
else:
|
|
291
|
+
for path in job["paths"]:
|
|
292
|
+
job["stage"] = STAGE_MODEL # 引擎无回调钩子:粗粒度两档
|
|
293
|
+
try:
|
|
294
|
+
report = pipeline.audit_file(path)
|
|
295
|
+
job["results"].append(_report_to_dict(report))
|
|
296
|
+
except Exception as exc:
|
|
297
|
+
job["results"].append({"error": f"{type(exc).__name__}: {exc}", "path": path})
|
|
298
|
+
job["stage"] = STAGE_REPORT
|
|
299
|
+
job["done"] += 1
|
|
300
|
+
job["state"] = "done"
|
|
301
|
+
|
|
302
|
+
# 任务已结束(锁已释放):落地审计运行期间被暂缓的设置更新(补丁 11 约束一)
|
|
303
|
+
if _PENDING_SETTINGS_REFRESH:
|
|
304
|
+
_refresh_settings()
|