dsh-calibguard 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +44 -27
- package/README.md +44 -26
- package/cordis.patch.yml +2 -1
- package/docs/configuration.md +2 -2
- package/docs/troubleshooting.md +2 -1
- package/engine/README.md +33 -0
- package/engine/audit_service.py +304 -0
- package/engine/calibguard/__init__.py +2 -0
- package/engine/calibguard/__main__.py +4 -0
- package/engine/calibguard/calibrator.py +78 -0
- package/engine/calibguard/cli.py +74 -0
- package/engine/calibguard/config.py +129 -0
- package/engine/calibguard/constants.py +117 -0
- package/engine/calibguard/feature_extractor.py +161 -0
- package/engine/calibguard/gate.py +17 -0
- package/engine/calibguard/llm_analyzer.py +171 -0
- package/engine/calibguard/model.py +79 -0
- package/engine/calibguard/pipeline.py +305 -0
- package/engine/calibguard/preprocessor.py +218 -0
- package/engine/calibguard/report.py +132 -0
- package/engine/calibguard/schemas.py +53 -0
- package/engine/models/calibration_params.json +1 -0
- package/engine/models/codebert_finetuned/config.json +28 -0
- package/engine/models/codebert_finetuned/merges.txt +50001 -0
- package/engine/models/codebert_finetuned/special_tokens_map.json +51 -0
- package/engine/models/codebert_finetuned/tokenizer.json +250371 -0
- package/engine/models/codebert_finetuned/tokenizer_config.json +58 -0
- package/engine/models/codebert_finetuned/vocab.json +1 -0
- package/engine/requirements-gui.txt +11 -0
- package/lib/bootstrap.js +4 -3
- package/lib/client.js +1 -1
- package/lib/config.js +34 -7
- package/lib/engine.js +1 -1
- package/lib/index.js +1 -1
- package/package.json +4 -3
- package/setup.mjs +278 -48
package/lib/bootstrap.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* `<engineRoot>/.venv`,就在后台跑 `setup.mjs` 把「引擎 + venv + 依赖」补齐,
|
|
7
7
|
* 进度写进状态里供面板显示,DSH 启动不受阻塞。
|
|
8
8
|
*
|
|
9
|
-
* 为什么要异步:首次安装包含
|
|
9
|
+
* 为什么要异步:首次安装包含 476 MB 模型分片下载 + 1–3 GB 依赖安装(十几分钟),
|
|
10
10
|
* 不能卡住插件激活;工具调用在准备完成前会得到可读提示而不是超时。
|
|
11
11
|
*/
|
|
12
12
|
import { spawn } from 'node:child_process'
|
|
@@ -85,7 +85,7 @@ export class EnvironmentBootstrap {
|
|
|
85
85
|
|
|
86
86
|
this.state = 'running'
|
|
87
87
|
this.startedAt = Date.now()
|
|
88
|
-
this.message = '
|
|
88
|
+
this.message = '正在准备运行环境(首次运行会准备模型权重并安装 Python 依赖)…'
|
|
89
89
|
this.logger?.info?.(`[calibguard] ${this.message}`)
|
|
90
90
|
|
|
91
91
|
this.promise = new Promise((resolvePromise) => {
|
|
@@ -147,7 +147,8 @@ export class EnvironmentBootstrap {
|
|
|
147
147
|
} else {
|
|
148
148
|
this.state = 'failed'
|
|
149
149
|
this.error = `setup.mjs 退出码 ${code}`
|
|
150
|
-
|
|
150
|
+
const detail = this.tail.slice(-3).filter(Boolean).join(' | ')
|
|
151
|
+
this.message = `环境准备失败(${this.error}):${detail || '详见日志'}`
|
|
151
152
|
this.logger?.warn?.(`[calibguard] ${this.message}`)
|
|
152
153
|
}
|
|
153
154
|
resolvePromise(this.snapshot())
|
package/lib/client.js
CHANGED
|
@@ -623,7 +623,7 @@ window.__ModuleLoader__.load({
|
|
|
623
623
|
style: S.muted,
|
|
624
624
|
children: envFailed
|
|
625
625
|
? (envInfo.error || status.error || '详见日志与下方的安装输出')
|
|
626
|
-
: '
|
|
626
|
+
: '首次运行会下载模型权重(约 442 MB,自动镜像回退)并安装 Python 依赖(1–3 GB),可能需要 10–20 分钟。'
|
|
627
627
|
+ '完成后点「刷新状态」即可开始审计。进度也写在宿主日志的 [calibguard] 行。',
|
|
628
628
|
}),
|
|
629
629
|
envBusy && envInfo.message
|
package/lib/config.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* profile 的 node_modules 里正常安装,也能直接从仓库路径用 file:// 行加载。
|
|
7
7
|
*/
|
|
8
8
|
import { existsSync } from 'node:fs'
|
|
9
|
+
import { homedir } from 'node:os'
|
|
9
10
|
import { dirname, join, resolve } from 'node:path'
|
|
10
11
|
import { fileURLToPath } from 'node:url'
|
|
11
12
|
|
|
@@ -27,7 +28,7 @@ export const DEFAULT_CONFIG = Object.freeze({
|
|
|
27
28
|
device: 'auto',
|
|
28
29
|
useGuiSettings: true,
|
|
29
30
|
announceToModel: true,
|
|
30
|
-
/** 首次运行缺少 .venv
|
|
31
|
+
/** 首次运行缺少 .venv 时,是否自动准备引擎(内置代码 + npm 模型分片)并安装依赖。 */
|
|
31
32
|
autoSetup: true,
|
|
32
33
|
maxFiles: 20,
|
|
33
34
|
auditTimeoutMs: 600_000,
|
|
@@ -51,8 +52,18 @@ export const DEFAULT_CONFIG = Object.freeze({
|
|
|
51
52
|
llmDeliveryMode: '',
|
|
52
53
|
})
|
|
53
54
|
|
|
54
|
-
/**
|
|
55
|
+
/**
|
|
56
|
+
* 引擎默认目录:
|
|
57
|
+
* - 源码仓库里(本包不在 node_modules)→ 与本包同级的 CG_GUI;
|
|
58
|
+
* - 从 npm / GitHub 装进 node_modules → 放 DSH 主目录,避免被包管理器升级清掉。
|
|
59
|
+
*/
|
|
55
60
|
export function defaultEngineRoot() {
|
|
61
|
+
const installed = PACKAGE_ROOT.split(/[\\/]/).includes('node_modules')
|
|
62
|
+
if (installed) {
|
|
63
|
+
const fromEnv = process.env.DSH_HOME && process.env.DSH_HOME.trim()
|
|
64
|
+
const home = fromEnv ? resolve(fromEnv) : join(homedir(), '.dsh')
|
|
65
|
+
return join(home, 'calibguard-engine')
|
|
66
|
+
}
|
|
56
67
|
return resolve(PACKAGE_ROOT, '..', 'CG_GUI')
|
|
57
68
|
}
|
|
58
69
|
|
|
@@ -154,17 +165,33 @@ export function normalizeConfig(raw) {
|
|
|
154
165
|
)
|
|
155
166
|
}
|
|
156
167
|
|
|
157
|
-
const
|
|
158
|
-
|
|
168
|
+
const explicitRoot = out.engineRoot ? resolve(out.engineRoot) : ''
|
|
169
|
+
const engineRoot = explicitRoot || defaultEngineRoot()
|
|
170
|
+
const hasEnginePkg = existsSync(join(engineRoot, 'calibguard'))
|
|
171
|
+
// 显式配置的 engineRoot 写错要尽早报错;默认目录缺失则留给 autoSetup 下载补齐。
|
|
172
|
+
if (explicitRoot && !existsSync(engineRoot)) {
|
|
159
173
|
throw new Error(
|
|
160
174
|
`引擎目录不存在:${engineRoot}。请修正插件配置 engineRoot(应指向 CG_GUI 目录)。`,
|
|
161
175
|
)
|
|
162
176
|
}
|
|
163
|
-
if (!
|
|
177
|
+
if (explicitRoot && !hasEnginePkg) {
|
|
164
178
|
throw new Error(`引擎目录下找不到 calibguard 包:${engineRoot}(engineRoot 应指向 CG_GUI 目录)`)
|
|
165
179
|
}
|
|
180
|
+
if (!hasEnginePkg && !out.autoSetup) {
|
|
181
|
+
throw new Error(
|
|
182
|
+
`引擎目录还没准备好(${engineRoot}),且已关闭自动准备(config.autoSetup=false)。`
|
|
183
|
+
+ '请先运行 node setup.mjs,或把 engineRoot 指向已有的 CG_GUI 目录。',
|
|
184
|
+
)
|
|
185
|
+
}
|
|
166
186
|
|
|
167
|
-
return {
|
|
187
|
+
return {
|
|
188
|
+
...out,
|
|
189
|
+
engineRoot,
|
|
190
|
+
engineRootResolved: engineRoot,
|
|
191
|
+
packageRoot: PACKAGE_ROOT,
|
|
192
|
+
/** 引擎文件是否已就位(false = 等待 autoSetup / 用户手动准备)。 */
|
|
193
|
+
enginePresent: hasEnginePkg,
|
|
194
|
+
}
|
|
168
195
|
}
|
|
169
196
|
|
|
170
197
|
/**
|
|
@@ -211,7 +238,7 @@ export async function loadConfigSchema() {
|
|
|
211
238
|
.description('设备定档:auto 会按算力双检,低算力显卡自动回落 CPU'),
|
|
212
239
|
useGuiSettings: z.boolean().default(true).description('复用 CG_GUI/gui-settings.json 里的阈值与云端复核设置'),
|
|
213
240
|
announceToModel: z.boolean().default(true).description('是否把使用时机写入系统提示词'),
|
|
214
|
-
autoSetup: z.boolean().default(true).description('首次运行缺少 .venv
|
|
241
|
+
autoSetup: z.boolean().default(true).description('首次运行缺少 .venv 时是否自动准备引擎(内置代码 + npm 模型分片)并安装依赖'),
|
|
215
242
|
maxFiles: z.natural().default(20).description('目录展开上限(单次审计最多这些文件)'),
|
|
216
243
|
auditTimeoutMs: z.natural().default(600_000).description('单次审计超时(毫秒)'),
|
|
217
244
|
idleShutdownMs: z.natural().default(600_000).description('空闲多久后回收桥接进程(0=常驻不回收)'),
|
package/lib/engine.js
CHANGED
|
@@ -73,7 +73,7 @@ export class CalibGuardEngine {
|
|
|
73
73
|
this.lastCredential = { source: 'none', configured: false }
|
|
74
74
|
/**
|
|
75
75
|
* 首次运行自动准备环境(对齐 CG_GUI 的 launcher 首启装环境):
|
|
76
|
-
* 缺 `<engineRoot>/.venv` 时在后台跑 setup.mjs
|
|
76
|
+
* 缺 `<engineRoot>/.venv` 时在后台跑 setup.mjs(内置引擎 + npm 模型分片 → 建 venv → 装依赖)。
|
|
77
77
|
*/
|
|
78
78
|
this.bootstrap = new EnvironmentBootstrap({
|
|
79
79
|
engineRoot: config.engineRootResolved ?? config.engineRoot,
|
package/lib/index.js
CHANGED
|
@@ -82,7 +82,7 @@ function apply(ctx, rawConfig) {
|
|
|
82
82
|
logger.debug?.('[calibguard] 未找到 credentials 服务,credentialRef 将按同名环境变量解析')
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
-
//
|
|
85
|
+
// 首启自动装环境:不阻塞插件激活,后台准备引擎(内置代码 + npm 模型分片)/建 venv/装依赖。
|
|
86
86
|
try {
|
|
87
87
|
engine.warmup()
|
|
88
88
|
} catch (error) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-calibguard",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
@@ -16,9 +16,9 @@
|
|
|
16
16
|
"main": "lib/index.js",
|
|
17
17
|
"repository": {
|
|
18
18
|
"type": "git",
|
|
19
|
-
"url": "git+https://github.com/maomitgd2006/
|
|
19
|
+
"url": "git+https://github.com/maomitgd2006/dsh-CalibGuard.git"
|
|
20
20
|
},
|
|
21
|
-
"homepage": "https://github.com/maomitgd2006/
|
|
21
|
+
"homepage": "https://github.com/maomitgd2006/dsh-CalibGuard#readme",
|
|
22
22
|
"exports": {
|
|
23
23
|
".": "./lib/index.js",
|
|
24
24
|
"./client": "./lib/client.js",
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
"files": [
|
|
28
28
|
"lib",
|
|
29
29
|
"bridge",
|
|
30
|
+
"engine",
|
|
30
31
|
"cordis.patch.yml",
|
|
31
32
|
"docs",
|
|
32
33
|
"setup.mjs",
|
package/setup.mjs
CHANGED
|
@@ -18,7 +18,11 @@
|
|
|
18
18
|
*
|
|
19
19
|
* 无第三方依赖,只用 Node 内置模块;下载走 Node 20+ 的 fetch。
|
|
20
20
|
*/
|
|
21
|
-
import {
|
|
21
|
+
import {
|
|
22
|
+
chmodSync, copyFileSync, cpSync, createReadStream, createWriteStream, existsSync, mkdirSync,
|
|
23
|
+
readdirSync, renameSync, rmSync, statSync,
|
|
24
|
+
} from 'node:fs'
|
|
25
|
+
import { createHash } from 'node:crypto'
|
|
22
26
|
import { dirname, join, resolve } from 'node:path'
|
|
23
27
|
import { spawn, spawnSync } from 'node:child_process'
|
|
24
28
|
import { fileURLToPath } from 'node:url'
|
|
@@ -26,12 +30,145 @@ import { homedir } from 'node:os'
|
|
|
26
30
|
import { Readable } from 'node:stream'
|
|
27
31
|
|
|
28
32
|
const PKG_ROOT = dirname(fileURLToPath(import.meta.url))
|
|
29
|
-
const DEFAULT_DEST = resolve(PKG_ROOT, '..', 'CG_GUI')
|
|
30
|
-
const DEFAULT_URL = 'https://github.com/P-436/CG/releases/download/v1.0.0/CalibGuard-GUI.zip'
|
|
31
33
|
/** torch 官方轮子目前覆盖的 Python 版本上限(3.13/3.14 装不上 torch)。 */
|
|
32
34
|
const PY_MIN = 10
|
|
33
35
|
const PY_MAX = 12
|
|
34
36
|
|
|
37
|
+
/** 引擎下载的镜像前缀:GitHub 直连失败时依次重试。 */
|
|
38
|
+
const ENGINE_MIRRORS = ['https://ghproxy.cn/', 'https://gh-proxy.com/', 'https://ghfast.top/']
|
|
39
|
+
/** pip 索引:默认清华镜像,失败回退官方 PyPI。 */
|
|
40
|
+
const DEFAULT_PIP_INDEX = 'https://pypi.tuna.tsinghua.edu.cn/simple'
|
|
41
|
+
const OFFICIAL_PIP_INDEX = 'https://pypi.org/simple'
|
|
42
|
+
|
|
43
|
+
/** DSH 主目录。 */
|
|
44
|
+
function dshHomeDir() {
|
|
45
|
+
const fromEnv = process.env.DSH_HOME && process.env.DSH_HOME.trim()
|
|
46
|
+
return fromEnv ? resolve(fromEnv) : join(homedir(), '.dsh')
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* 默认引擎目录:
|
|
51
|
+
* - 源码仓库(本包不在 node_modules 里)→ 与本包同级的 CG_GUI;
|
|
52
|
+
* - 从 npm / GitHub 装进 node_modules → 放 DSH 主目录,避免被包管理器升级清掉。
|
|
53
|
+
*/
|
|
54
|
+
function defaultDest() {
|
|
55
|
+
const installed = PKG_ROOT.split(/[\\/]/).includes('node_modules')
|
|
56
|
+
return installed ? join(dshHomeDir(), 'calibguard-engine') : resolve(PKG_ROOT, '..', 'CG_GUI')
|
|
57
|
+
}
|
|
58
|
+
const DEFAULT_DEST = defaultDest()
|
|
59
|
+
|
|
60
|
+
/** 给 GitHub 直链生成「直连 + 镜像」候选列表。 */
|
|
61
|
+
function mirroredUrls(url) {
|
|
62
|
+
const extra = String(process.env.DSH_CALIBGUARD_ENGINE_MIRRORS || '')
|
|
63
|
+
.split(',')
|
|
64
|
+
.map((item) => item.trim())
|
|
65
|
+
.filter(Boolean)
|
|
66
|
+
const urls = [url]
|
|
67
|
+
for (const prefix of [...extra, ...ENGINE_MIRRORS]) {
|
|
68
|
+
const candidate = prefix + url
|
|
69
|
+
if (!urls.includes(candidate)) urls.push(candidate)
|
|
70
|
+
}
|
|
71
|
+
return urls
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** pip 索引候选(自定义 → 清华镜像 → 官方源)。 */
|
|
75
|
+
function pipIndexes() {
|
|
76
|
+
const custom = String(process.env.DSH_CALIBGUARD_PIP_INDEX || process.env.PIP_INDEX_URL || '').trim()
|
|
77
|
+
return [...new Set([custom, DEFAULT_PIP_INDEX, OFFICIAL_PIP_INDEX].filter(Boolean))]
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** 插件内置引擎目录:CG_GUI 核心文件(代码 / 词表 / 校准参数),不含 476 MB 模型权重。 */
|
|
81
|
+
const BUNDLED_ENGINE = join(PKG_ROOT, 'engine')
|
|
82
|
+
|
|
83
|
+
/** 模型权重下载地址:放在本仓库的 Release 资产上(GitHub 直连失败会自动尝试 ghproxy 等镜像)。 */
|
|
84
|
+
const MODEL_URL = 'https://github.com/maomitgd2006/dsh-CalibGuard/releases/download/model-v1.0.0/calibguard-model-1.0.0.zip'
|
|
85
|
+
|
|
86
|
+
/** 完整模型的 SHA-256(与 CG 发行包里的 model.safetensors 完全一致)。 */
|
|
87
|
+
const MODEL_SHA256 = '2f088a00c1a08c9331a58de2b9adc9bb7ace8f28a2491ee7b699f41c7b7efc65'
|
|
88
|
+
|
|
89
|
+
/** 文件 SHA-256(流式,避免大文件占内存)。 */
|
|
90
|
+
function sha256File(file) {
|
|
91
|
+
return new Promise((resolvePromise, rejectPromise) => {
|
|
92
|
+
const hash = createHash('sha256')
|
|
93
|
+
createReadStream(file)
|
|
94
|
+
.on('data', (chunk) => hash.update(chunk))
|
|
95
|
+
.on('error', rejectPromise)
|
|
96
|
+
.on('end', () => resolvePromise(hash.digest('hex')))
|
|
97
|
+
})
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* 准备模型权重:下载本仓库 Release 里的模型压缩包 → 校验 SHA-256 → 缓存到
|
|
102
|
+
* `~/.dsh/calibguard-tools/model-cache` → 复制进引擎目录。
|
|
103
|
+
* 支持断点续传、ghproxy 镜像;已缓存时秒级完成。
|
|
104
|
+
*/
|
|
105
|
+
async function fetchModelWeights(dest) {
|
|
106
|
+
const cacheDir = join(toolsDir(), 'model-cache')
|
|
107
|
+
mkdirSync(cacheDir, { recursive: true })
|
|
108
|
+
const cachedModel = join(cacheDir, 'model.safetensors')
|
|
109
|
+
const finalPath = join(dest, 'models', 'codebert_finetuned', 'model.safetensors')
|
|
110
|
+
|
|
111
|
+
let modelFile = ''
|
|
112
|
+
if (existsSync(cachedModel) && (await sha256File(cachedModel)) === MODEL_SHA256) {
|
|
113
|
+
log(`✓ 复用已缓存的模型:${cachedModel}`)
|
|
114
|
+
modelFile = cachedModel
|
|
115
|
+
} else {
|
|
116
|
+
const zipPath = join(cacheDir, 'calibguard-model-1.0.0.zip')
|
|
117
|
+
let lastError = null
|
|
118
|
+
for (const url of mirroredUrls(MODEL_URL)) {
|
|
119
|
+
try {
|
|
120
|
+
log(`下载模型权重(约 442 MB):${url}`)
|
|
121
|
+
await download(url, zipPath)
|
|
122
|
+
lastError = null
|
|
123
|
+
break
|
|
124
|
+
} catch (error) {
|
|
125
|
+
lastError = error
|
|
126
|
+
warn(`下载源失败:${String(error.message).split('\n')[0]}`)
|
|
127
|
+
deleteFile(`${zipPath}.part`)
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if (lastError) {
|
|
131
|
+
throw new SetupError(`模型下载失败:${lastError.message}\n`
|
|
132
|
+
+ `可以重试(支持断点续传),或手动下载后放到:${cachedModel}\n`
|
|
133
|
+
+ '(下载地址:https://github.com/maomitgd2006/dsh-CalibGuard/releases/tag/model-v1.0.0)')
|
|
134
|
+
}
|
|
135
|
+
const unpacked = join(cacheDir, 'unpacked')
|
|
136
|
+
rmSync(unpacked, { recursive: true, force: true })
|
|
137
|
+
mkdirSync(unpacked, { recursive: true })
|
|
138
|
+
extract(zipPath, unpacked)
|
|
139
|
+
const found = findBinary(unpacked, 'model.safetensors')
|
|
140
|
+
if (!found) throw new SetupError(`模型压缩包里没有 model.safetensors:${unpacked}`)
|
|
141
|
+
const actual = await sha256File(found)
|
|
142
|
+
if (actual !== MODEL_SHA256) throw new SetupError(`模型 SHA-256 不匹配:${actual}`)
|
|
143
|
+
deleteFile(cachedModel)
|
|
144
|
+
try {
|
|
145
|
+
renameSync(found, cachedModel)
|
|
146
|
+
} catch {
|
|
147
|
+
copyFileSync(found, cachedModel)
|
|
148
|
+
}
|
|
149
|
+
rmSync(unpacked, { recursive: true, force: true })
|
|
150
|
+
deleteFile(zipPath)
|
|
151
|
+
modelFile = cachedModel
|
|
152
|
+
log(`✓ 模型已缓存:${cachedModel}`)
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
log(`写入引擎目录:${finalPath}`)
|
|
156
|
+
mkdirSync(dirname(finalPath), { recursive: true })
|
|
157
|
+
deleteFile(finalPath)
|
|
158
|
+
copyFileSync(modelFile, finalPath)
|
|
159
|
+
const finalSha = await sha256File(finalPath)
|
|
160
|
+
if (finalSha !== MODEL_SHA256) {
|
|
161
|
+
deleteFile(finalPath)
|
|
162
|
+
throw new SetupError(`写入后模型校验失败(SHA-256 ${finalSha}),请重试。`)
|
|
163
|
+
}
|
|
164
|
+
log(`✓ 模型已就绪:${finalPath}(${bytes(statSync(finalPath).size)})`)
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** 删除文件(不存在也不报错)。 */
|
|
168
|
+
function deleteFile(file) {
|
|
169
|
+
rmSync(file, { force: true })
|
|
170
|
+
}
|
|
171
|
+
|
|
35
172
|
// ============================== 输出与进程 ==============================
|
|
36
173
|
|
|
37
174
|
const log = (...args) => console.log(...args)
|
|
@@ -78,7 +215,7 @@ function parseArgs(argv) {
|
|
|
78
215
|
const options = {
|
|
79
216
|
dest: DEFAULT_DEST,
|
|
80
217
|
from: '',
|
|
81
|
-
url:
|
|
218
|
+
url: '',
|
|
82
219
|
python: '',
|
|
83
220
|
skipDownload: false,
|
|
84
221
|
skipVenv: false,
|
|
@@ -123,8 +260,9 @@ function printHelp() {
|
|
|
123
260
|
|
|
124
261
|
选项:
|
|
125
262
|
--dest <dir> 引擎安装目录(默认:本包同级的 CG_GUI)
|
|
126
|
-
--from <zip>
|
|
127
|
-
--url <url>
|
|
263
|
+
--from <zip> 使用本地已下载的 CG 发行包(解压安装)
|
|
264
|
+
--url <url> 从指定地址下载 CG 发行包并解压安装
|
|
265
|
+
(默认不下载 CG 包:用插件内置引擎 + 从 npm 镜像取模型分片)
|
|
128
266
|
--python <exe> 指定 Python 解释器(需 3.10–3.12)
|
|
129
267
|
--skip-download 跳过下载(已有引擎或 --from 时)
|
|
130
268
|
--skip-venv 跳过创建虚拟环境
|
|
@@ -141,9 +279,16 @@ function printHelp() {
|
|
|
141
279
|
node setup.mjs --dest D:\\CG_GUI --skip-download
|
|
142
280
|
|
|
143
281
|
说明:
|
|
144
|
-
•
|
|
282
|
+
• 默认用插件内置的引擎(CG_GUI 核心代码,约 5 MB),模型权重(475 MB)分两片
|
|
283
|
+
从 npm 镜像下载(npmmirror 加速,每个分片 < 256 MB);也可用 --from 指定 CG 发行包;
|
|
284
|
+
• 依赖安装另需约 1–3 GB 磁盘;
|
|
145
285
|
• 本机没有 64 位 Python 3.10–3.12 时,会自动用 uv 下载便携 Python 3.10.8(约 30 MB);
|
|
146
|
-
•
|
|
286
|
+
• 下载默认走 GitHub 直连,失败会自动尝试 ghproxy / gh-proxy / ghfast 镜像;
|
|
287
|
+
可用 DSH_CALIBGUARD_ENGINE_MIRRORS 追加镜像前缀(逗号分隔);
|
|
288
|
+
• pip 默认用清华镜像(pypi.tuna.tsinghua.edu.cn),失败自动回退官方 PyPI;
|
|
289
|
+
可用 DSH_CALIBGUARD_PIP_INDEX 指定其它索引;
|
|
290
|
+
• 从 npm 安装时引擎默认放在 ~/.dsh/calibguard-engine(可用 --dest 或插件配置 engineRoot 指定);
|
|
291
|
+
• 默认用 CPU 版 torch(低算力显卡本来也会被定档到 CPU);
|
|
147
292
|
要用 GPU 请按 PyTorch 官网自行安装对应 CUDA 轮子。
|
|
148
293
|
`)
|
|
149
294
|
}
|
|
@@ -166,53 +311,90 @@ function describeEngine(status) {
|
|
|
166
311
|
|
|
167
312
|
// ============================== 下载 ==============================
|
|
168
313
|
|
|
314
|
+
/**
|
|
315
|
+
* 下载一个文件。
|
|
316
|
+
* - 断点续传:已有 `<file>.part` 时用 Range 续传(服务器不支持则从头下);
|
|
317
|
+
* - 空闲超时:120 秒收不到数据就中止,避免界面「一直卡在准备中」;
|
|
318
|
+
* - 失败时保留 .part,换源/重试可继续。
|
|
319
|
+
*/
|
|
169
320
|
async function download(url, file) {
|
|
170
321
|
log(`下载:${url}`)
|
|
171
322
|
mkdirSync(dirname(file), { recursive: true })
|
|
172
323
|
const temp = `${file}.part`
|
|
173
|
-
|
|
324
|
+
const resumed = existsSync(temp) ? statSync(temp).size : 0
|
|
325
|
+
if (resumed > 0) log(`发现未完成的下载,尝试续传(已有 ${bytes(resumed)})`)
|
|
326
|
+
|
|
327
|
+
const controller = new AbortController()
|
|
328
|
+
let idleTimer = null
|
|
329
|
+
const resetIdle = () => {
|
|
330
|
+
if (idleTimer) clearTimeout(idleTimer)
|
|
331
|
+
idleTimer = setTimeout(() => controller.abort(new Error('120 秒没有收到数据')), 120_000)
|
|
332
|
+
}
|
|
333
|
+
const clearIdle = () => { if (idleTimer) { clearTimeout(idleTimer); idleTimer = null } }
|
|
174
334
|
|
|
175
335
|
let response
|
|
176
336
|
try {
|
|
177
|
-
|
|
337
|
+
resetIdle()
|
|
338
|
+
response = await fetch(url, {
|
|
339
|
+
redirect: 'follow',
|
|
340
|
+
signal: controller.signal,
|
|
341
|
+
headers: resumed > 0 ? { Range: `bytes=${resumed}-` } : {},
|
|
342
|
+
})
|
|
178
343
|
} catch (error) {
|
|
344
|
+
clearIdle()
|
|
179
345
|
throw new SetupError(`下载失败(网络错误):${error.message}\n`
|
|
180
346
|
+ '可以手动下载后重试:node setup.mjs --from <下载到的 zip>')
|
|
181
347
|
}
|
|
348
|
+
if (response.status === 416 && resumed > 0) {
|
|
349
|
+
clearIdle()
|
|
350
|
+
rmSync(temp, { force: true })
|
|
351
|
+
throw new SetupError('续传位置无效(HTTP 416),已清除断点,请重试。')
|
|
352
|
+
}
|
|
182
353
|
if (!response.ok || !response.body) {
|
|
354
|
+
clearIdle()
|
|
183
355
|
throw new SetupError(`下载失败:HTTP ${response.status} ${response.statusText}\n`
|
|
184
356
|
+ '可以手动下载后重试:node setup.mjs --from <下载到的 zip>')
|
|
185
357
|
}
|
|
186
358
|
|
|
187
|
-
const
|
|
188
|
-
|
|
359
|
+
const partial = response.status === 206
|
|
360
|
+
const remaining = Number(response.headers.get('content-length') || 0)
|
|
361
|
+
const total = partial ? resumed + remaining : remaining
|
|
362
|
+
const append = partial && resumed > 0
|
|
363
|
+
log(`保存到:${file} 大小:${total ? bytes(total) : '未知'}${append ? '(续传)' : ''}`)
|
|
189
364
|
|
|
190
|
-
const sink = createWriteStream(temp)
|
|
365
|
+
const sink = createWriteStream(temp, { flags: append ? 'a' : 'w' })
|
|
191
366
|
const reader = Readable.fromWeb(response.body)
|
|
192
367
|
const startedAt = Date.now()
|
|
193
|
-
let received = 0
|
|
194
|
-
let lastPercent =
|
|
368
|
+
let received = append ? resumed : 0
|
|
369
|
+
let lastPercent = -1
|
|
370
|
+
let lastLogAt = 0
|
|
195
371
|
|
|
196
372
|
await new Promise((resolvePromise, rejectPromise) => {
|
|
197
373
|
reader.on('data', (chunk) => {
|
|
374
|
+
resetIdle()
|
|
198
375
|
received += chunk.length
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
lastPercent
|
|
203
|
-
|
|
204
|
-
|
|
376
|
+
const now = Date.now()
|
|
377
|
+
if (total) {
|
|
378
|
+
const percent = Math.floor((received / total) * 100)
|
|
379
|
+
if (percent >= lastPercent + 2) {
|
|
380
|
+
lastPercent = percent
|
|
381
|
+
const elapsed = Math.max(1, (now - startedAt) / 1000)
|
|
382
|
+
log(` ${percent}% ${bytes(received)} / ${bytes(total)} (${bytes((received - resumed) / elapsed)}/s)`)
|
|
383
|
+
}
|
|
384
|
+
} else if (now - lastLogAt > 5000) {
|
|
385
|
+
lastLogAt = now
|
|
386
|
+
const elapsed = Math.max(1, (now - startedAt) / 1000)
|
|
387
|
+
log(` ${bytes(received)} (${bytes((received - resumed) / elapsed)}/s)`)
|
|
205
388
|
}
|
|
206
389
|
})
|
|
207
|
-
reader.on('error', rejectPromise)
|
|
208
|
-
sink.on('error', rejectPromise)
|
|
209
|
-
sink.on('finish', resolvePromise)
|
|
390
|
+
reader.on('error', (error) => { clearIdle(); rejectPromise(error) })
|
|
391
|
+
sink.on('error', (error) => { clearIdle(); rejectPromise(error) })
|
|
392
|
+
sink.on('finish', () => { clearIdle(); resolvePromise() })
|
|
210
393
|
reader.pipe(sink)
|
|
211
394
|
})
|
|
212
395
|
|
|
213
396
|
if (total && received !== total) {
|
|
214
|
-
|
|
215
|
-
throw new SetupError(`下载不完整:${bytes(received)} / ${bytes(total)},请重试或用 --from 手动指定。`)
|
|
397
|
+
throw new SetupError(`下载不完整:${bytes(received)} / ${bytes(total)}(已保留断点,重试可续传)`)
|
|
216
398
|
}
|
|
217
399
|
rmSync(file, { force: true })
|
|
218
400
|
renameSync(temp, file)
|
|
@@ -355,10 +537,7 @@ async function ensureUv(dest) {
|
|
|
355
537
|
const dir = toolsDir()
|
|
356
538
|
mkdirSync(dir, { recursive: true })
|
|
357
539
|
const archive = join(dir, asset)
|
|
358
|
-
const urls =
|
|
359
|
-
`https://github.com/astral-sh/uv/releases/latest/download/${asset}`,
|
|
360
|
-
`https://ghproxy.cn/https://github.com/astral-sh/uv/releases/latest/download/${asset}`,
|
|
361
|
-
]
|
|
540
|
+
const urls = mirroredUrls(`https://github.com/astral-sh/uv/releases/latest/download/${asset}`)
|
|
362
541
|
let downloaded = false
|
|
363
542
|
for (const url of urls) {
|
|
364
543
|
try {
|
|
@@ -460,7 +639,7 @@ async function main() {
|
|
|
460
639
|
const { statfsSync } = await import('node:fs')
|
|
461
640
|
const free = Number(statfsSync(options.dest).bavail) * Number(statfsSync(options.dest).bsize)
|
|
462
641
|
log(`目标盘剩余空间:${bytes(free)}`)
|
|
463
|
-
if (free < 4 * 1024 ** 3) warn('剩余空间不足 4 GB
|
|
642
|
+
if (free < 4 * 1024 ** 3) warn('剩余空间不足 4 GB:模型(约 476 MB) + 依赖(1–3 GB) 可能不够。')
|
|
464
643
|
} catch { /* 平台不支持就跳过 */ }
|
|
465
644
|
|
|
466
645
|
// ---------- 1. 引擎 ----------
|
|
@@ -482,14 +661,6 @@ async function main() {
|
|
|
482
661
|
log('\n跳过下载(--skip-download):直接使用现有目录')
|
|
483
662
|
} else {
|
|
484
663
|
step('准备引擎')
|
|
485
|
-
let zipFile = options.from
|
|
486
|
-
if (!zipFile) {
|
|
487
|
-
downloadedZip = join(dirname(options.dest), `CalibGuard-GUI-${Date.now()}.zip`)
|
|
488
|
-
await download(options.url, downloadedZip)
|
|
489
|
-
zipFile = downloadedZip
|
|
490
|
-
} else {
|
|
491
|
-
log(`使用本地压缩包:${zipFile}(${bytes(statSync(zipFile).size)})`)
|
|
492
|
-
}
|
|
493
664
|
if (options.force && existsSync(options.dest)) {
|
|
494
665
|
warn(`--force:清空已有目录 ${options.dest}`)
|
|
495
666
|
rmSync(options.dest, { recursive: true, force: true })
|
|
@@ -498,19 +669,64 @@ async function main() {
|
|
|
498
669
|
if (readdirSync(options.dest).length > 0 && !options.force) {
|
|
499
670
|
warn('目标目录非空:将覆盖同名文件(如需全新安装请加 --force)')
|
|
500
671
|
}
|
|
501
|
-
|
|
672
|
+
|
|
673
|
+
if (options.from || options.url) {
|
|
674
|
+
// 显式指定了 CG 发行包(--from 本地 zip / --url 下载 zip)
|
|
675
|
+
let zipFile = options.from
|
|
676
|
+
if (!zipFile) {
|
|
677
|
+
downloadedZip = join(dirname(options.dest), `CalibGuard-GUI-${Date.now()}.zip`)
|
|
678
|
+
const candidates = mirroredUrls(options.url)
|
|
679
|
+
let lastError = null
|
|
680
|
+
for (const candidate of candidates) {
|
|
681
|
+
try {
|
|
682
|
+
await download(candidate, downloadedZip)
|
|
683
|
+
lastError = null
|
|
684
|
+
break
|
|
685
|
+
} catch (error) {
|
|
686
|
+
lastError = error
|
|
687
|
+
warn(`下载源失败:${String(error.message).split('\n')[0]}`)
|
|
688
|
+
rmSync(`${downloadedZip}.part`, { force: true })
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
if (lastError) {
|
|
692
|
+
fail(`引擎下载失败:${lastError.message}\n`
|
|
693
|
+
+ '可手动下载后重试:node setup.mjs --from <CalibGuard-GUI.zip>\n'
|
|
694
|
+
+ '或在插件配置里把 engineRoot 指向已有的 CG_GUI 目录。')
|
|
695
|
+
}
|
|
696
|
+
zipFile = downloadedZip
|
|
697
|
+
} else {
|
|
698
|
+
log(`使用本地压缩包:${zipFile}(${bytes(statSync(zipFile).size)})`)
|
|
699
|
+
}
|
|
700
|
+
extract(zipFile, options.dest)
|
|
701
|
+
if (downloadedZip) {
|
|
702
|
+
rmSync(downloadedZip, { force: true })
|
|
703
|
+
log('已清理下载的临时压缩包')
|
|
704
|
+
}
|
|
705
|
+
} else {
|
|
706
|
+
// 默认:内置引擎(CG_GUI 核心文件)+ npm 分片下载模型
|
|
707
|
+
if (!existsSync(BUNDLED_ENGINE)) {
|
|
708
|
+
fail(`安装包缺少内置引擎目录:${BUNDLED_ENGINE}\n`
|
|
709
|
+
+ '(安装包可能不完整;可用 --from 指定 CG 发行包,或在插件配置里把 engineRoot 指向已有 CG_GUI)')
|
|
710
|
+
}
|
|
711
|
+
log('复制内置引擎(CG_GUI 核心:代码 / 词表 / 校准参数)…')
|
|
712
|
+
cpSync(BUNDLED_ENGINE, options.dest, { recursive: true, force: true })
|
|
713
|
+
const modelPath = join(options.dest, 'models', 'codebert_finetuned', 'model.safetensors')
|
|
714
|
+
if (existsSync(modelPath) && (await sha256File(modelPath)) === MODEL_SHA256) {
|
|
715
|
+
log('✓ 模型权重已存在,跳过下载')
|
|
716
|
+
} else {
|
|
717
|
+
step('下载模型权重(约 442 MB,GitHub 失败自动换镜像)')
|
|
718
|
+
await fetchModelWeights(options.dest)
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
|
|
502
722
|
const after = engineStatus(options.dest)
|
|
503
723
|
if (!after.hasEngine) {
|
|
504
|
-
fail(
|
|
505
|
-
+ '
|
|
724
|
+
fail(`引擎准备完成,但 ${options.dest} 里没有 calibguard/。\n`
|
|
725
|
+
+ '请确认安装包完整,或用 --from 指定 CG 的发行包 CalibGuard-GUI.zip。')
|
|
506
726
|
}
|
|
507
727
|
if (!after.hasModel || !after.hasCalibration) {
|
|
508
728
|
warn('没找到模型权重或校准参数(models/):引擎能启动,但审计会失败,请确认下载完整。')
|
|
509
729
|
}
|
|
510
|
-
if (downloadedZip) {
|
|
511
|
-
rmSync(downloadedZip, { force: true })
|
|
512
|
-
log('已清理下载的临时压缩包')
|
|
513
|
-
}
|
|
514
730
|
}
|
|
515
731
|
|
|
516
732
|
const status = engineStatus(options.dest)
|
|
@@ -560,8 +776,22 @@ async function main() {
|
|
|
560
776
|
if (!existsSync(python)) fail(`找不到虚拟环境解释器:${python}`)
|
|
561
777
|
const requirements = join(options.dest, 'requirements-gui.txt')
|
|
562
778
|
if (!existsSync(requirements)) fail(`找不到依赖清单:${requirements}`)
|
|
563
|
-
|
|
564
|
-
|
|
779
|
+
const installWithMirrors = async (args) => {
|
|
780
|
+
let lastError = null
|
|
781
|
+
for (const index of pipIndexes()) {
|
|
782
|
+
log(`pip 源:${index}`)
|
|
783
|
+
try {
|
|
784
|
+
await runLive(python, ['-m', 'pip', 'install', '--timeout', '180', '--retries', '5', '-i', index, ...args])
|
|
785
|
+
return
|
|
786
|
+
} catch (error) {
|
|
787
|
+
lastError = error
|
|
788
|
+
warn(`pip 安装失败(${index}):${error.message}`)
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
throw lastError ?? new SetupError('pip 安装失败')
|
|
792
|
+
}
|
|
793
|
+
await installWithMirrors(['--upgrade', 'pip', '--quiet'])
|
|
794
|
+
await installWithMirrors(['-r', requirements])
|
|
565
795
|
log('✓ 依赖安装完成')
|
|
566
796
|
}
|
|
567
797
|
|