dsh-data-domain-knowledge 0.1.0-alpha.13 → 0.1.0-alpha.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -56
- package/cordis.patch.yml +36 -129
- package/eval/benchmark-extraction-format.mjs +20 -0
- package/lib/client.js +3262 -4142
- package/lib/doc-page.js +1 -1
- package/lib/pages.js +1147 -364
- package/lib/types/axis-registry.d.ts +8 -1
- package/lib/types/axis-registry.d.ts.map +1 -1
- package/lib/types/axis-registry.js +62 -7
- package/lib/types/axis-registry.js.map +1 -1
- package/lib/types/convert/host.d.ts.map +1 -1
- package/lib/types/convert/host.js +3 -0
- package/lib/types/convert/host.js.map +1 -1
- package/lib/types/convert/ingest.d.ts +9 -0
- package/lib/types/convert/ingest.d.ts.map +1 -1
- package/lib/types/convert/ingest.js +68 -9
- package/lib/types/convert/ingest.js.map +1 -1
- package/lib/types/diagnosis/cards.d.ts +7 -3
- package/lib/types/diagnosis/cards.d.ts.map +1 -1
- package/lib/types/diagnosis/cards.js +13 -8
- package/lib/types/diagnosis/cards.js.map +1 -1
- package/lib/types/diagnosis/run.d.ts +4 -13
- package/lib/types/diagnosis/run.d.ts.map +1 -1
- package/lib/types/diagnosis/run.js +26 -16
- package/lib/types/diagnosis/run.js.map +1 -1
- package/lib/types/diagnosis/schema.js +2 -2
- package/lib/types/diagnosis/schema.js.map +1 -1
- package/lib/types/diagnosis/seed-source.d.ts +1 -1
- package/lib/types/diagnosis/seed-source.d.ts.map +1 -1
- package/lib/types/diagnosis/seed-source.js +5 -4
- package/lib/types/diagnosis/seed-source.js.map +1 -1
- package/lib/types/diagnosis/seed.d.ts +4 -4
- package/lib/types/diagnosis/seed.d.ts.map +1 -1
- package/lib/types/diagnosis/seed.js +6 -8
- package/lib/types/diagnosis/seed.js.map +1 -1
- package/lib/types/diagnosis/store.d.ts +18 -17
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +94 -66
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/diagnosis/types.d.ts +10 -16
- package/lib/types/diagnosis/types.d.ts.map +1 -1
- package/lib/types/diagnosis/types.js.map +1 -1
- package/lib/types/environment/dsh-python.d.ts +77 -0
- package/lib/types/environment/dsh-python.d.ts.map +1 -0
- package/lib/types/environment/dsh-python.js +139 -0
- package/lib/types/environment/dsh-python.js.map +1 -0
- package/lib/types/environment/install.d.ts +9 -1
- package/lib/types/environment/install.d.ts.map +1 -1
- package/lib/types/environment/install.js +149 -24
- package/lib/types/environment/install.js.map +1 -1
- package/lib/types/environment/model-catalog.d.ts +34 -2
- package/lib/types/environment/model-catalog.d.ts.map +1 -1
- package/lib/types/environment/model-catalog.js +40 -2
- package/lib/types/environment/model-catalog.js.map +1 -1
- package/lib/types/environment/service.d.ts +13 -3
- package/lib/types/environment/service.d.ts.map +1 -1
- package/lib/types/environment/service.js +127 -23
- package/lib/types/environment/service.js.map +1 -1
- package/lib/types/environment/types.d.ts +9 -0
- package/lib/types/environment/types.d.ts.map +1 -1
- package/lib/types/events.d.ts +2 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js +13 -0
- package/lib/types/events.js.map +1 -1
- package/lib/types/extract-contract.d.ts +1 -1
- package/lib/types/extract-contract.d.ts.map +1 -1
- package/lib/types/extract-contract.js +34 -31
- package/lib/types/extract-contract.js.map +1 -1
- package/lib/types/extract-model.d.ts +2 -0
- package/lib/types/extract-model.d.ts.map +1 -1
- package/lib/types/extract-model.js.map +1 -1
- package/lib/types/extract-table-contract.d.ts +3 -0
- package/lib/types/extract-table-contract.d.ts.map +1 -0
- package/lib/types/extract-table-contract.js +30 -0
- package/lib/types/extract-table-contract.js.map +1 -0
- package/lib/types/extract-table.d.ts +8 -0
- package/lib/types/extract-table.d.ts.map +1 -0
- package/lib/types/extract-table.js +151 -0
- package/lib/types/extract-table.js.map +1 -0
- package/lib/types/extraction-runs.d.ts +30 -6
- package/lib/types/extraction-runs.d.ts.map +1 -1
- package/lib/types/extraction-runs.js +69 -8
- package/lib/types/extraction-runs.js.map +1 -1
- package/lib/types/fs-atomic.d.ts +0 -7
- package/lib/types/fs-atomic.d.ts.map +1 -1
- package/lib/types/fs-atomic.js +17 -2
- package/lib/types/fs-atomic.js.map +1 -1
- package/lib/types/images.d.ts +1 -1
- package/lib/types/images.d.ts.map +1 -1
- package/lib/types/images.js +2 -2
- package/lib/types/images.js.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +232 -50
- package/lib/types/index.js.map +1 -1
- package/lib/types/intent/scope.d.ts.map +1 -1
- package/lib/types/intent/scope.js +1 -0
- package/lib/types/intent/scope.js.map +1 -1
- package/lib/types/intent/tools.d.ts +19 -0
- package/lib/types/intent/tools.d.ts.map +1 -1
- package/lib/types/intent/tools.js +37 -1
- package/lib/types/intent/tools.js.map +1 -1
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +5 -16
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/knowledge.d.ts +3 -0
- package/lib/types/knowledge.d.ts.map +1 -1
- package/lib/types/knowledge.js +21 -5
- package/lib/types/knowledge.js.map +1 -1
- package/lib/types/materialize-snapshot.d.ts +49 -0
- package/lib/types/materialize-snapshot.d.ts.map +1 -0
- package/lib/types/materialize-snapshot.js +2 -0
- package/lib/types/materialize-snapshot.js.map +1 -0
- package/lib/types/model-text.d.ts +1 -1
- package/lib/types/model-text.d.ts.map +1 -1
- package/lib/types/model-text.js +5 -4
- package/lib/types/model-text.js.map +1 -1
- package/lib/types/oplog.d.ts.map +1 -1
- package/lib/types/oplog.js +7 -0
- package/lib/types/oplog.js.map +1 -1
- package/lib/types/pipeline/engine.d.ts +60 -0
- package/lib/types/pipeline/engine.d.ts.map +1 -0
- package/lib/types/pipeline/engine.js +338 -0
- package/lib/types/pipeline/engine.js.map +1 -0
- package/lib/types/pipeline/store.d.ts +73 -0
- package/lib/types/pipeline/store.d.ts.map +1 -0
- package/lib/types/pipeline/store.js +143 -0
- package/lib/types/pipeline/store.js.map +1 -0
- package/lib/types/pipeline/types.d.ts +101 -0
- package/lib/types/pipeline/types.d.ts.map +1 -0
- package/lib/types/pipeline/types.js +5 -0
- package/lib/types/pipeline/types.js.map +1 -0
- package/lib/types/present.d.ts +2 -0
- package/lib/types/present.d.ts.map +1 -1
- package/lib/types/present.js +9 -4
- package/lib/types/present.js.map +1 -1
- package/lib/types/retention.d.ts +39 -0
- package/lib/types/retention.d.ts.map +1 -0
- package/lib/types/retention.js +42 -0
- package/lib/types/retention.js.map +1 -0
- package/lib/types/runtime.d.ts +115 -5
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +862 -132
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/source-evidence.d.ts +9 -1
- package/lib/types/source-evidence.d.ts.map +1 -1
- package/lib/types/source-evidence.js +13 -5
- package/lib/types/source-evidence.js.map +1 -1
- package/lib/types/store/axis-references.js +3 -3
- package/lib/types/store/axis-references.js.map +1 -1
- package/lib/types/store/derived.d.ts +1 -1
- package/lib/types/store/derived.d.ts.map +1 -1
- package/lib/types/store/derived.js +41 -7
- package/lib/types/store/derived.js.map +1 -1
- package/lib/types/store/knowledge-db.d.ts +18 -0
- package/lib/types/store/knowledge-db.d.ts.map +1 -1
- package/lib/types/store/knowledge-db.js +111 -6
- package/lib/types/store/knowledge-db.js.map +1 -1
- package/lib/types/store/migrate.js +1 -1
- package/lib/types/store/migrate.js.map +1 -1
- package/lib/types/trajectory-types.d.ts +172 -0
- package/lib/types/trajectory-types.d.ts.map +1 -0
- package/lib/types/trajectory-types.js +36 -0
- package/lib/types/trajectory-types.js.map +1 -0
- package/lib/types/types.d.ts +3 -4
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js.map +1 -1
- package/lib/types/wiki-finalize.d.ts +16 -0
- package/lib/types/wiki-finalize.d.ts.map +1 -1
- package/lib/types/wiki-finalize.js +26 -0
- package/lib/types/wiki-finalize.js.map +1 -1
- package/lib/types/wiki-materialize.d.ts.map +1 -1
- package/lib/types/wiki-materialize.js +3 -15
- package/lib/types/wiki-materialize.js.map +1 -1
- package/lib/types/wiki-trial-types.d.ts +3 -0
- package/lib/types/wiki-trial-types.d.ts.map +1 -1
- package/lib/types/wiki-trial.d.ts +2 -0
- package/lib/types/wiki-trial.d.ts.map +1 -1
- package/lib/types/wiki-trial.js +31 -7
- package/lib/types/wiki-trial.js.map +1 -1
- package/package.json +6 -3
- package/preset/tita-data-agent/agent.cordis.yml +36 -129
- package/src/axis-registry.ts +57 -9
- package/src/client/AttributionEventGuide.tsx +10 -9
- package/src/client/CompileDialogs.tsx +10 -2
- package/src/client/EnvironmentSetup.tsx +76 -16
- package/src/client/ExpansionGraph.tsx +13 -7
- package/src/client/ExtractionTasksPage.tsx +543 -156
- package/src/client/ImportFilesDialog.tsx +2 -13
- package/src/client/IntentTab.tsx +3 -1
- package/src/client/KnowledgeBaseTab.tsx +139 -122
- package/src/client/KnowledgeLayerSection.tsx +19 -2
- package/src/client/KnowledgePanel.tsx +17 -3
- package/src/client/ListPager.tsx +15 -5
- package/src/client/MaterializeTrajectoryView.tsx +124 -0
- package/src/client/NodeSettings.tsx +90 -25
- package/src/client/NormalizeTrajectoryView.tsx +136 -0
- package/src/client/PipelineNotice.tsx +26 -0
- package/src/client/RecallPanel.tsx +55 -25
- package/src/client/TabGuide.tsx +4 -4
- package/src/client/TrialReportView.tsx +61 -16
- package/src/client/TypeDefinitionDrawer.tsx +33 -9
- package/src/client/WikiCardPreview.tsx +43 -10
- package/src/client/WikiFlowPanel.tsx +4 -4
- package/src/client/architecture.tsx +1 -1
- package/src/client/attribution/AttributionTab.tsx +37 -36
- package/src/client/attribution/CaseView.tsx +3 -1
- package/src/client/attribution/EventView.tsx +14 -4
- package/src/client/attribution/RunView.tsx +2 -2
- package/src/client/attribution/ViewSwitch.tsx +20 -5
- package/src/client/attribution/graph-options.ts +38 -31
- package/src/client/attribution/model.ts +16 -9
- package/src/client/graph/palette.ts +165 -0
- package/src/client/pages.css +224 -28
- package/src/client/pages.tsx +33 -11
- package/src/client/styles.css +61 -6
- package/src/client/wiki-trial.ts +30 -1
- package/src/convert/host.ts +3 -0
- package/src/convert/ingest.ts +73 -9
- package/src/diagnosis/cards.ts +13 -8
- package/src/diagnosis/run.ts +27 -17
- package/src/diagnosis/schema.ts +2 -2
- package/src/diagnosis/seed-source.ts +5 -4
- package/src/diagnosis/seed.ts +6 -8
- package/src/diagnosis/store.ts +99 -69
- package/src/diagnosis/types.ts +10 -16
- package/src/environment/dsh-python.ts +156 -0
- package/src/environment/install.ts +134 -19
- package/src/environment/model-catalog.ts +45 -3
- package/src/environment/service.ts +111 -23
- package/src/environment/types.ts +6 -0
- package/src/events.ts +12 -0
- package/src/extract-contract.ts +6 -3
- package/src/extract-model.ts +2 -0
- package/src/extract-table-contract.ts +29 -0
- package/src/extract-table.ts +126 -0
- package/src/extraction-runs.ts +70 -11
- package/src/fs-atomic.ts +19 -2
- package/src/images.ts +2 -2
- package/src/index.ts +207 -72
- package/src/intent/scope.ts +1 -0
- package/src/intent/tools.ts +38 -1
- package/src/knowledge-bases.ts +4 -12
- package/src/knowledge.ts +24 -4
- package/src/materialize-snapshot.ts +46 -0
- package/src/model-text.ts +5 -3
- package/src/oplog.ts +7 -0
- package/src/pipeline/engine.ts +312 -0
- package/src/pipeline/store.ts +147 -0
- package/src/pipeline/types.ts +92 -0
- package/src/present.ts +10 -4
- package/src/retention.ts +76 -0
- package/src/runtime.ts +853 -131
- package/src/source-evidence.ts +12 -5
- package/src/store/axis-references.ts +3 -3
- package/src/store/derived.ts +40 -7
- package/src/store/knowledge-db.ts +111 -5
- package/src/store/migrate.ts +1 -1
- package/src/trajectory-types.ts +142 -0
- package/src/types.ts +3 -4
- package/src/wiki-finalize.ts +28 -0
- package/src/wiki-materialize.ts +3 -14
- package/src/wiki-trial-types.ts +3 -0
- package/src/wiki-trial.ts +26 -7
- package/src/client/convert/ConvertPanel.tsx +0 -583
- package/src/client/convert/styles.css +0 -138
- package/src/client/convert/styles.ts +0 -49
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash, randomUUID } from 'node:crypto'
|
|
2
2
|
import { existsSync } from 'node:fs'
|
|
3
|
-
import { chmod, copyFile, mkdir, open, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
|
3
|
+
import { chmod, copyFile, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
4
4
|
import { homedir } from 'node:os'
|
|
5
5
|
import { dirname, join } from 'node:path'
|
|
6
6
|
import { fileURLToPath } from 'node:url'
|
|
@@ -8,11 +8,19 @@ import { modelsForSelection, UV_VERSION } from './manifest.js'
|
|
|
8
8
|
import { DEFAULT_MODEL_SELECTION, type ModelSelection } from './model-catalog.js'
|
|
9
9
|
import { writeModelSelection } from './selection.js'
|
|
10
10
|
import { runProcess, type ProcessOptions } from './process.js'
|
|
11
|
+
import { hasHardenedRuntime, needsManagedInterpreter, pickBasePython, pythonSourceLabel } from './dsh-python.js'
|
|
11
12
|
|
|
12
13
|
export const requirementsFile = fileURLToPath(new URL('../../../python/requirements.txt', import.meta.url))
|
|
13
14
|
export const setupScript = fileURLToPath(new URL('../../../python/prepare-models.py', import.meta.url))
|
|
14
15
|
export function managedPython(root: string): string { return process.platform === 'win32' ? join(root, 'python', 'Scripts', 'python.exe') : join(root, 'python', 'bin', 'python') }
|
|
15
16
|
|
|
17
|
+
/** 只问次版本:判断的是「能不能装这套依赖」,不是补丁号。 */
|
|
18
|
+
const PYTHON_VERSION_SCRIPT = 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'
|
|
19
|
+
/** 推理依赖支持的解释器区间(torch 的 wheel 覆盖到 3.13)。 */
|
|
20
|
+
const SUPPORTED_PYTHON = /^3\.(10|11|12|13)$/
|
|
21
|
+
/** 走 uv 管理解释器时请求的版本:与桌面端自带的那份对齐,避免两套行为。 */
|
|
22
|
+
const MANAGED_PYTHON = '3.12'
|
|
23
|
+
|
|
16
24
|
async function download(url: string, signal: AbortSignal): Promise<Buffer> {
|
|
17
25
|
const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(120_000)]) })
|
|
18
26
|
if (!response.ok) throw new Error(`下载失败(HTTP ${response.status}):${url}`)
|
|
@@ -64,49 +72,153 @@ async function ensureUv(root: string, signal: AbortSignal, log: (line: string) =
|
|
|
64
72
|
} finally { await rm(scratch, { recursive: true, force: true }) }
|
|
65
73
|
}
|
|
66
74
|
|
|
75
|
+
/**
|
|
76
|
+
* 一把「陈旧」锁可以被接管的时限。
|
|
77
|
+
*
|
|
78
|
+
* 单步安装超时是 30 分钟,整轮远小于这个上限,所以 6 小时之后还在的锁一定是崩溃残留。
|
|
79
|
+
* 有了它,「持锁进程已死但 PID 被系统复用」才不会让用户永久卡在「另一个进程正在准备环境」上。
|
|
80
|
+
*/
|
|
81
|
+
const STALE_LOCK_MS = 6 * 60 * 60 * 1000
|
|
82
|
+
|
|
67
83
|
async function installLock(root: string): Promise<() => Promise<void>> {
|
|
68
84
|
const path = join(root, 'environment-install.lock')
|
|
69
|
-
|
|
85
|
+
/** 这一把锁的归属凭据。释放时靠它判断「这份文件还是不是我的」。 */
|
|
86
|
+
const token = randomUUID()
|
|
87
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
70
88
|
try {
|
|
71
89
|
const file = await open(path, 'wx')
|
|
72
|
-
try { await file.writeFile(JSON.stringify({ pid: process.pid, at: new Date().toISOString() })) }
|
|
90
|
+
try { await file.writeFile(JSON.stringify({ token, pid: process.pid, at: new Date().toISOString() })) }
|
|
73
91
|
catch (cause) { await rm(path, { force: true }); throw cause }
|
|
74
92
|
finally { await file.close() }
|
|
75
|
-
return () =>
|
|
93
|
+
return async () => {
|
|
94
|
+
/**
|
|
95
|
+
* 只删**自己那一把**。判据是 token,不是「路径上有没有文件」:锁可能已被另一个
|
|
96
|
+
* 进程按陈旧接管过,那时这份文件是它的 —— 删掉就等于让两个安装并发写同一个 venv。
|
|
97
|
+
*/
|
|
98
|
+
try {
|
|
99
|
+
const current = JSON.parse(await readFile(path, 'utf8')) as { token?: unknown }
|
|
100
|
+
if (current.token === token) await rm(path, { force: true })
|
|
101
|
+
} catch { /* 已被接管或已不存在:不是自己的锁就不动 */ }
|
|
102
|
+
}
|
|
76
103
|
} catch (error) {
|
|
77
104
|
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error
|
|
78
|
-
let pid
|
|
79
|
-
try {
|
|
80
|
-
if (
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
105
|
+
let holder: { token?: unknown; pid?: unknown; at?: unknown } | undefined
|
|
106
|
+
try { holder = JSON.parse(await readFile(path, 'utf8')) as { token?: unknown; pid?: unknown; at?: unknown } } catch { holder = undefined }
|
|
107
|
+
if (holder === undefined) {
|
|
108
|
+
/**
|
|
109
|
+
* 解析不了(0 字节、半截 JSON、磁盘满)也要能自愈。
|
|
110
|
+
*
|
|
111
|
+
* 锁是「`open('wx')` 建文件 → 再写内容」两步,中间被杀就会留下这种文件;
|
|
112
|
+
* 原来这里直接抛「安装锁异常」,于是**每一次**安装尝试都失败,只能人工删锁。
|
|
113
|
+
* 判据换成文件的 mtime:与「持锁进程还活着」同一套时限,超时就接管。
|
|
114
|
+
*/
|
|
115
|
+
const info = await stat(path).catch(() => undefined)
|
|
116
|
+
const aged = info !== undefined && Date.now() - info.mtimeMs > STALE_LOCK_MS
|
|
117
|
+
if (!aged) throw new Error('安装锁异常,请检查 resource/environment-install.lock')
|
|
84
118
|
await rm(path, { force: true })
|
|
119
|
+
continue
|
|
120
|
+
}
|
|
121
|
+
const pid = Number(holder.pid)
|
|
122
|
+
const at = Date.parse(String(holder.at ?? ''))
|
|
123
|
+
const aged = Number.isFinite(at) && Date.now() - at > STALE_LOCK_MS
|
|
124
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
125
|
+
// 内容不成立:没到期就如实报出来让人看一眼,到期了才当陈旧处理。
|
|
126
|
+
if (!aged) throw new Error('安装锁异常,请检查 resource/environment-install.lock')
|
|
127
|
+
} else {
|
|
128
|
+
let dead = false
|
|
129
|
+
try { process.kill(pid, 0) } catch (cause) { dead = (cause as NodeJS.ErrnoException).code === 'ESRCH' }
|
|
130
|
+
// `dead`(进程真的没了)或 `aged`(号码可能已被别人复用)任一成立都算陈旧。
|
|
131
|
+
if (!dead && !aged) throw new Error('另一个进程正在准备环境,请等待完成后重试')
|
|
85
132
|
}
|
|
133
|
+
/**
|
|
134
|
+
* 接管之前**再读一次**并确认还是刚才那一把。读完到删之间仍有一个极窄的窗口,
|
|
135
|
+
* 但「先删后判」会让两个进程同时接管同一把锁 —— 那才是真正的并发安装。
|
|
136
|
+
*/
|
|
137
|
+
try {
|
|
138
|
+
const again = JSON.parse(await readFile(path, 'utf8')) as { token?: unknown }
|
|
139
|
+
if (again.token !== holder.token) continue
|
|
140
|
+
} catch { continue }
|
|
141
|
+
await rm(path, { force: true })
|
|
86
142
|
}
|
|
87
143
|
}
|
|
88
144
|
throw new Error('无法获取环境安装锁')
|
|
89
145
|
}
|
|
90
146
|
|
|
91
|
-
|
|
147
|
+
/**
|
|
148
|
+
* 准备这个库的推理运行时。
|
|
149
|
+
*
|
|
150
|
+
* `basePython` 省略时按 `pickBasePython()` 找:`DSH_PYTHON` → DSH 自带运行时 → PATH 上的
|
|
151
|
+
* `python3`。**默认值不能再写死 `python3`**:macOS 命令行工具自带的是 3.9,而桌面端明明把
|
|
152
|
+
* 3.12 打在了自己的运行时里 —— 写死默认值会让「一键准备」在任何没配 `DSH_PYTHON` 的机器上
|
|
153
|
+
* 第一步就失败(见 `dsh-python.ts` 的注释)。
|
|
154
|
+
*/
|
|
155
|
+
export async function installRequiredEnvironment(root: string, signal: AbortSignal, stage: (label: string) => void, log: (line: string) => void, runner: typeof runProcess = runProcess, selection: ModelSelection = DEFAULT_MODEL_SELECTION, basePython?: string, hardened: (path: string) => Promise<boolean> = path => hasHardenedRuntime(path, runner)): Promise<void> {
|
|
92
156
|
await mkdir(root, { recursive: true })
|
|
157
|
+
const picked = basePython === undefined ? pickBasePython() : { path: basePython, source: 'override' as const }
|
|
93
158
|
const unlock = await installLock(root)
|
|
94
|
-
const env = {
|
|
159
|
+
const env = {
|
|
160
|
+
...process.env,
|
|
161
|
+
UV_CACHE_DIR: join(root, 'cache', 'uv'),
|
|
162
|
+
/**
|
|
163
|
+
* uv 自己管理的解释器也落在资源目录里:它和依赖、模型一样属于「这个 Agent 的家当」,
|
|
164
|
+
* 备份 / 迁移 / 删除该一起走,而不是散到 `~/.local/share/uv/python`。
|
|
165
|
+
*/
|
|
166
|
+
UV_PYTHON_INSTALL_DIR: join(root, 'tools', 'python'),
|
|
167
|
+
HF_HOME: join(root, 'cache', 'huggingface'),
|
|
168
|
+
PIP_DISABLE_PIP_VERSION_CHECK: '1',
|
|
169
|
+
}
|
|
95
170
|
const run = (command: string, args: string[], extra: ProcessOptions = {}): Promise<string> => runner(command, args, { env, signal, timeoutMs: 30 * 60_000, onLine: log, ...extra })
|
|
96
171
|
try {
|
|
97
172
|
stage('准备模型运行时')
|
|
98
173
|
const uv = await ensureUv(root, signal, log, runner)
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
174
|
+
log(`基础 Python:${picked.path}(${pythonSourceLabel(picked.source)})`)
|
|
175
|
+
/**
|
|
176
|
+
* 自带运行时那份 Python 带 hardened runtime,**装不了 torch**(见 `hasHardenedRuntime`):
|
|
177
|
+
* 库校验只喂它自己 Team ID 签的 dylib,而 pip 上的原生轮子是 ad-hoc 签名的。
|
|
178
|
+
* 这时改用 uv 管的独立 CPython(不带 hardened runtime),装到 `<resource>/tools/python`。
|
|
179
|
+
*
|
|
180
|
+
* 显式给了 `DSH_PYTHON` 就不改道 —— 那是用户的选择,只把风险写进日志。
|
|
181
|
+
*/
|
|
182
|
+
const isHardened = await hardened(picked.path)
|
|
183
|
+
const managed = needsManagedInterpreter(picked, isHardened)
|
|
184
|
+
if (isHardened && picked.source === 'override') {
|
|
185
|
+
log(`注意:${picked.path} 带 hardened runtime,第三方原生扩展(torch 等)可能加载不了。`)
|
|
186
|
+
}
|
|
187
|
+
let baseVersion: string | undefined
|
|
188
|
+
if (managed) {
|
|
189
|
+
log('基础 Python:由 uv 管理的独立 CPython 3.12(自带的那份带 hardened runtime,加载不了第三方原生扩展)')
|
|
190
|
+
} else {
|
|
191
|
+
log(`基础 Python:${picked.path}(${pythonSourceLabel(picked.source)})`)
|
|
192
|
+
const versionOutput = await run(picked.path, ['-c', PYTHON_VERSION_SCRIPT], { timeoutMs: 15_000 })
|
|
193
|
+
baseVersion = versionOutput.trim().split(/\r?\n/).at(-1)
|
|
194
|
+
if (!baseVersion || !SUPPORTED_PYTHON.test(baseVersion)) {
|
|
195
|
+
/**
|
|
196
|
+
* 这句要能直接照着做。原来只说「版本不受支持」,而人看到的是自己机器上
|
|
197
|
+
* `python3 --version` 报 3.9 —— 会以为插件在挑刺,不知道桌面端本身带了 3.12。
|
|
198
|
+
*/
|
|
199
|
+
throw new Error([
|
|
200
|
+
`DSH Python 版本不受支持:${baseVersion || '无法读取'}(${picked.path},来自${pythonSourceLabel(picked.source)});需要 Python 3.10–3.13。`,
|
|
201
|
+
picked.source === 'path'
|
|
202
|
+
? '没有找到 DSH 自带的 Python(桌面端运行时里应有一份 3.12)。请安装 Python 3.10–3.13,再用环境变量 DSH_PYTHON 指向它。'
|
|
203
|
+
: `请把 DSH_PYTHON 指向一个 3.10–3.13 的解释器后重试。`,
|
|
204
|
+
].join(' '))
|
|
205
|
+
}
|
|
103
206
|
}
|
|
104
207
|
const python = managedPython(root)
|
|
105
208
|
let compatible = false
|
|
106
209
|
if (existsSync(python)) {
|
|
107
210
|
try {
|
|
108
|
-
const
|
|
109
|
-
|
|
211
|
+
const current = (await run(python, ['-c', PYTHON_VERSION_SCRIPT], { timeoutMs: 15_000 })).trim().split(/\r?\n/).at(-1)
|
|
212
|
+
// 管好的解释器只要求落在支持区间(它的次版本会随 uv 走),其余必须与基础解释器同一个次版本。
|
|
213
|
+
const versionOk = managed ? SUPPORTED_PYTHON.test(current ?? '') : current === baseVersion
|
|
214
|
+
/**
|
|
215
|
+
* 还要**真的**确认这个环境不带 hardened runtime。
|
|
216
|
+
*
|
|
217
|
+
* 只比版本的话,一个用自带 Python 建出来的 3.12 环境会被判成「兼容」,于是「修复」
|
|
218
|
+
* 不重建 —— 而它正是装不了 torch 的那一个。`codesign` 跟着符号链接走,
|
|
219
|
+
* 所以这里量的是它最终指向的那个解释器。
|
|
220
|
+
*/
|
|
221
|
+
compatible = versionOk && (!managed || !(await hardened(python)))
|
|
110
222
|
} catch { if (signal.aborted) throw new Error('ENVIRONMENT_CANCELLED') }
|
|
111
223
|
}
|
|
112
224
|
if (!compatible) {
|
|
@@ -115,7 +227,10 @@ export async function installRequiredEnvironment(root: string, signal: AbortSign
|
|
|
115
227
|
await rename(join(root, 'python'), backup)
|
|
116
228
|
log(`原模型运行环境不可用,已保留备份:${backup}`)
|
|
117
229
|
}
|
|
118
|
-
await run(uv, ['venv',
|
|
230
|
+
await run(uv, ['venv',
|
|
231
|
+
...(managed ? ['--python', MANAGED_PYTHON, '--python-preference', 'only-managed'] : ['--python', picked.path, '--python-preference', 'only-system']),
|
|
232
|
+
join(root, 'python')])
|
|
233
|
+
if (managed) log(`基础 Python:${(await run(python, ['-c', PYTHON_VERSION_SCRIPT], { timeoutMs: 15_000 })).trim()}(uv 管理的独立 CPython)`)
|
|
119
234
|
}
|
|
120
235
|
stage('安装推理依赖')
|
|
121
236
|
await run(uv, ['pip', 'install', '--reinstall', '--python', python, '-r', requirementsFile])
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
/**
|
|
1
|
+
/**
|
|
2
|
+
* Models supported by the local embedding and reranking workers. Revisions are pinned in manifest.ts.
|
|
3
|
+
*
|
|
4
|
+
* 这一份是**纯数据**:客户端也要 import 它(模型选项、下载量的文案),所以这里不能出现
|
|
5
|
+
* `node:*`。磁盘上有没有那份模型由服务端判(`modelOptionsView(root)`)。
|
|
6
|
+
*/
|
|
2
7
|
export const EMBEDDING_MODELS = [
|
|
3
8
|
{
|
|
4
9
|
id: 'bge-small-zh-v1.5', repo: 'BAAI/bge-small-zh-v1.5', directory: 'bge-small-zh-v1.5',
|
|
@@ -65,6 +70,11 @@ export interface ModelOptionView {
|
|
|
65
70
|
download: string
|
|
66
71
|
description: string
|
|
67
72
|
modelCardUrl?: string
|
|
73
|
+
/**
|
|
74
|
+
* 这份模型在资源目录里是否**完整**(有 config、有权重、有分词器词表)。
|
|
75
|
+
* 省略 = 服务端没给资源目录,不知道 —— 不知道就不显示徽标,不猜。
|
|
76
|
+
*/
|
|
77
|
+
installed?: boolean
|
|
68
78
|
}
|
|
69
79
|
export const DEFAULT_MODEL_SELECTION: ModelSelection = {
|
|
70
80
|
embedding: 'bge-small-zh-v1.5',
|
|
@@ -92,10 +102,42 @@ export function rerankModel(id: RerankModelId) {
|
|
|
92
102
|
return RERANK_MODELS.find(model => model.id === id)!
|
|
93
103
|
}
|
|
94
104
|
|
|
95
|
-
|
|
96
|
-
|
|
105
|
+
/**
|
|
106
|
+
* 把「约 1.1 GB」这样的文案换回字节数。
|
|
107
|
+
*
|
|
108
|
+
* 判据只留一份:**目录里写的就是那句文案**(顺序与大小都在那里定),再写一份数字字段迟早
|
|
109
|
+
* 会和它对不上。格式是我们自己定的(`约 <数字> MB|GB`),认不出来就返回 undefined ——
|
|
110
|
+
* 算不出总数时宁可不显示总数,也不编一个。
|
|
111
|
+
*/
|
|
112
|
+
export function downloadBytes(text: string): number | undefined {
|
|
113
|
+
const matched = /约\s*([\d.]+)\s*(MB|GB)/i.exec(text)
|
|
114
|
+
if (!matched) return undefined
|
|
115
|
+
const value = Number(matched[1])
|
|
116
|
+
if (!Number.isFinite(value)) return undefined
|
|
117
|
+
return Math.round(value * (matched[2].toUpperCase() === 'GB' ? 1024 ** 3 : 1024 ** 2))
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** 字节数 → 「约 1.2 GB / 约 96 MB」。用于把几个模型加起来报一个总数。 */
|
|
121
|
+
export function formatBytes(bytes: number): string {
|
|
122
|
+
if (bytes >= 1024 ** 3) return `约 ${(bytes / 1024 ** 3).toFixed(1)} GB`
|
|
123
|
+
return `约 ${Math.max(1, Math.round(bytes / 1024 ** 2))} MB`
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* 模型选项。
|
|
128
|
+
*
|
|
129
|
+
* `installed` 是一个**回调**而不是路径:这一份是纯数据模块(客户端也 import 它),
|
|
130
|
+
* 而「磁盘上有没有、完不完整」要用 `node:fs` 判。服务端把判据传进来,顺便也就把它
|
|
131
|
+
* 与自检那一栏的「模型结构」检查绑成了**同一个函数** —— 否则会出现「选项说已下载、
|
|
132
|
+
* 自检说文件不全」这种同一件事两种说法。
|
|
133
|
+
*
|
|
134
|
+
* 不传回调就省略 `installed`:不知道就别猜,客户端也就不显示徽标。
|
|
135
|
+
*/
|
|
136
|
+
export function modelOptionsView(installed?: (directory: string) => boolean) {
|
|
137
|
+
const present = <T extends { id: string; label: string; download: string; description: string; repo: string; directory: string }>(model: T) => ({
|
|
97
138
|
id: model.id, label: model.label, download: model.download, description: model.description,
|
|
98
139
|
modelCardUrl: `https://huggingface.co/${model.repo}`,
|
|
140
|
+
...(installed === undefined ? {} : { installed: installed(model.directory) }),
|
|
99
141
|
...('dimension' in model ? { dimension: model.dimension as number } : {}),
|
|
100
142
|
})
|
|
101
143
|
return {
|
|
@@ -1,28 +1,59 @@
|
|
|
1
1
|
import { createHash, randomUUID } from 'node:crypto'
|
|
2
|
-
import { readdirSync, statSync } from 'node:fs'
|
|
2
|
+
import { readFileSync, readdirSync, statSync } from 'node:fs'
|
|
3
3
|
import { mkdir, writeFile } from 'node:fs/promises'
|
|
4
|
-
import { dirname, isAbsolute, join } from 'node:path'
|
|
4
|
+
import { basename, dirname, isAbsolute, join, resolve } from 'node:path'
|
|
5
5
|
import { embeddingFingerprint } from '../compiler/embedding-identity.js'
|
|
6
6
|
import type { RuntimeKnowledgeConfig } from '../types.js'
|
|
7
|
+
import { hasHardenedRuntime } from './dsh-python.js'
|
|
7
8
|
import { installRequiredEnvironment, requirementsFile } from './install.js'
|
|
8
|
-
import { modelOptionsView, parseModelSelection, type ModelSelection } from './model-catalog.js'
|
|
9
|
+
import { EMBEDDING_MODELS, modelOptionsView, parseModelSelection, RERANK_MODELS, type ModelSelection } from './model-catalog.js'
|
|
9
10
|
import { ENVIRONMENT_PROBE_VERSION } from './manifest.js'
|
|
10
11
|
import { runProcess } from './process.js'
|
|
11
12
|
import { readModelSelection, writeModelSelection } from './selection.js'
|
|
12
|
-
import type { EnvironmentItem, EnvironmentItemId, EnvironmentReport, EnvironmentView } from './types.js'
|
|
13
|
+
import type { EnvironmentCurrentModel, EnvironmentItem, EnvironmentItemId, EnvironmentReport, EnvironmentView } from './types.js'
|
|
13
14
|
|
|
14
15
|
export interface EnvironmentOptions {
|
|
15
16
|
resourceRoot: string
|
|
16
17
|
snapshot: () => { config: RuntimeKnowledgeConfig; paths: { python: string; embeddingModel: string; rerankModel: string } }
|
|
17
18
|
probe: (kind: 'embedding' | 'rerank') => Promise<{ ok: boolean; message: string }>
|
|
18
19
|
reload: () => void
|
|
20
|
+
activateSelection?: () => Promise<() => Promise<void>>
|
|
19
21
|
run?: typeof runProcess
|
|
20
22
|
install?: typeof installRequiredEnvironment
|
|
21
23
|
}
|
|
22
24
|
|
|
23
25
|
const labels = { embedding: '向量模型', rerank: '重排模型' }
|
|
26
|
+
const CHECK_FILE = 'environment-last-check.json'
|
|
27
|
+
type ModelCatalogEntry = { id: string; label: string; directory: string; repo: string }
|
|
28
|
+
function currentModel(root: string, path: string, catalog: readonly ModelCatalogEntry[], disabled: boolean, provider?: string): EnvironmentCurrentModel {
|
|
29
|
+
if (disabled) return { state: 'disabled', label: '未启用' }
|
|
30
|
+
if (provider) return { state: 'active', label: `${provider} · ${basename(path) || path || '未指定模型'}` }
|
|
31
|
+
const option = catalog.find(model => isAbsolute(path)
|
|
32
|
+
? resolve(path) === resolve(join(root, 'models', model.directory))
|
|
33
|
+
: path === model.repo)
|
|
34
|
+
if (option) return { state: 'active', id: option.id, label: option.label }
|
|
35
|
+
const name = isAbsolute(path) ? basename(path) : basename(path.replaceAll('\\', '/'))
|
|
36
|
+
return { state: 'active', label: `自定义模型 · ${name || '未指定模型'}` }
|
|
37
|
+
}
|
|
24
38
|
function blankReport(): EnvironmentReport {
|
|
25
|
-
return { state: 'unchecked', required: true, items: (['embedding', 'rerank'] as const).map(id => ({ id, label: labels[id], state: 'unchecked', message: '
|
|
39
|
+
return { state: 'unchecked', required: true, items: (['embedding', 'rerank'] as const).map(id => ({ id, label: labels[id], state: 'unchecked', message: '尚未完成模型推理自检' })) }
|
|
40
|
+
}
|
|
41
|
+
function readCachedReport(resourceRoot: string, fingerprint: string): EnvironmentReport | undefined {
|
|
42
|
+
try {
|
|
43
|
+
const cached = JSON.parse(readFileSync(join(resourceRoot, CHECK_FILE), 'utf8')) as Record<string, unknown>
|
|
44
|
+
if (cached.required !== true || cached.fingerprint !== fingerprint || typeof cached.checkedAt !== 'string'
|
|
45
|
+
|| !Number.isFinite(Date.parse(cached.checkedAt)) || !Array.isArray(cached.items)) return undefined
|
|
46
|
+
const items: EnvironmentReport['items'] = []
|
|
47
|
+
for (const id of ['embedding', 'rerank'] as const) {
|
|
48
|
+
const item = cached.items.find((value): value is Record<string, unknown> => Boolean(value && typeof value === 'object' && (value as Record<string, unknown>).id === id))
|
|
49
|
+
if (!item || item.label !== labels[id] || typeof item.message !== 'string'
|
|
50
|
+
|| !['unchecked', 'ready', 'missing', 'failed'].includes(String(item.state))) return undefined
|
|
51
|
+
items.push({ id, label: labels[id], state: item.state as EnvironmentReport['items'][number]['state'], message: item.message,
|
|
52
|
+
...(typeof item.path === 'string' ? { path: item.path } : {}) })
|
|
53
|
+
}
|
|
54
|
+
return { state: items.every(item => item.state === 'ready') ? 'ready' : 'blocked', required: true,
|
|
55
|
+
checkedAt: cached.checkedAt, fingerprint, items }
|
|
56
|
+
} catch { return undefined }
|
|
26
57
|
}
|
|
27
58
|
function stamp(path: string): unknown { try { const value = statSync(path); return [value.size, value.mtimeMs] } catch { return null } }
|
|
28
59
|
function sitePackagesPath(pythonPath: string): string {
|
|
@@ -52,14 +83,16 @@ export function modelFilesProblem(path: string): string | undefined {
|
|
|
52
83
|
|
|
53
84
|
/** Preparation and readiness are shared by all bases in one deployment. */
|
|
54
85
|
export class KnowledgeEnvironment {
|
|
55
|
-
private report
|
|
86
|
+
private report: EnvironmentReport
|
|
56
87
|
private job?: EnvironmentView['job']
|
|
57
88
|
private checkPending?: Promise<EnvironmentReport>
|
|
58
89
|
private work?: Promise<void>
|
|
59
90
|
private controller?: AbortController
|
|
60
91
|
private checkController?: AbortController
|
|
61
92
|
private disposed = false
|
|
62
|
-
constructor(private readonly options: EnvironmentOptions) {
|
|
93
|
+
constructor(private readonly options: EnvironmentOptions) {
|
|
94
|
+
this.report = readCachedReport(options.resourceRoot, this.fingerprint()) ?? blankReport()
|
|
95
|
+
}
|
|
63
96
|
|
|
64
97
|
private fingerprint(): string {
|
|
65
98
|
const { config, paths } = this.options.snapshot()
|
|
@@ -73,24 +106,39 @@ export class KnowledgeEnvironment {
|
|
|
73
106
|
}
|
|
74
107
|
|
|
75
108
|
view(): EnvironmentView {
|
|
76
|
-
if (!this.checkPending && this.report.
|
|
109
|
+
if (!this.checkPending && this.report.fingerprint && this.report.fingerprint !== this.fingerprint()) {
|
|
110
|
+
if (this.report.state === 'ready') this.report.state = 'unchecked'
|
|
111
|
+
this.report.items = this.report.items.map(item => item.state === 'ready' ? { ...item, state: 'unchecked', message: '自检后配置或资源发生变化,请重新自检' } : item)
|
|
112
|
+
}
|
|
113
|
+
const { config, paths } = this.options.snapshot()
|
|
77
114
|
const selection = readModelSelection(this.options.resourceRoot)
|
|
78
|
-
|
|
115
|
+
const current = {
|
|
116
|
+
embedding: currentModel(this.options.resourceRoot, paths.embeddingModel, EMBEDDING_MODELS, config.embeddingProvider === 'none', config.embeddingProvider === 'ollama' ? 'Ollama' : undefined),
|
|
117
|
+
rerank: currentModel(this.options.resourceRoot, paths.rerankModel, RERANK_MODELS, config.rerankProvider === 'none'),
|
|
118
|
+
}
|
|
119
|
+
return structuredClone({
|
|
120
|
+
report: this.report, selection, current,
|
|
121
|
+
// 「已下载 / 需下载」的判据与自检的「模型结构」那一项是同一个函数。
|
|
122
|
+
models: modelOptionsView(directory => modelFilesProblem(join(this.options.resourceRoot, 'models', directory)) === undefined),
|
|
123
|
+
...(this.job ? { job: this.job } : {}),
|
|
124
|
+
})
|
|
79
125
|
}
|
|
80
126
|
invalidate(): void { if (!this.checkPending) this.report = blankReport() }
|
|
81
127
|
|
|
82
|
-
async requireReady(): Promise<void> {
|
|
128
|
+
async requireReady(kind?: EnvironmentItemId): Promise<void> {
|
|
83
129
|
if (this.disposed) throw new Error('环境服务已关闭')
|
|
84
130
|
if (this.job?.state === 'running') throw new Error('环境正在准备或自检,请等待完成后再编译和召回')
|
|
85
131
|
const view = this.view()
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
132
|
+
// A passing result stays valid while the model, provider, runtime, and device fingerprint stays the same.
|
|
133
|
+
// The operation itself still reports inference failures, and the user can always request a fresh probe.
|
|
134
|
+
const alreadyReady = kind ? view.report.items.find(item => item.id === kind)?.state === 'ready' : view.report.state === 'ready'
|
|
135
|
+
const report = alreadyReady ? view.report : await this.check(true, kind)
|
|
136
|
+
if (kind ? report.items.find(item => item.id === kind)?.state !== 'ready' : report.state !== 'ready') throw new Error(`环境未就绪:${report.items.filter(item => (!kind || item.id === kind) && item.state !== 'ready').map(item => `${item.label}:${item.message}`).join(';')}。请在 Tita 设置中完成环境准备`)
|
|
89
137
|
}
|
|
90
138
|
|
|
91
|
-
check(deep = true): Promise<EnvironmentReport> {
|
|
92
|
-
if (this.checkPending) return this.checkPending
|
|
93
|
-
const pending = this.performCheck(deep)
|
|
139
|
+
check(deep = true, kind?: EnvironmentItemId): Promise<EnvironmentReport> {
|
|
140
|
+
if (this.checkPending) return this.checkPending.then(report => !kind || report.items.find(item => item.id === kind)?.state === 'ready' ? report : this.check(deep, kind))
|
|
141
|
+
const pending = this.performCheck(deep, kind)
|
|
94
142
|
this.checkPending = pending
|
|
95
143
|
void pending.finally(() => { if (this.checkPending === pending) this.checkPending = undefined }).catch(() => undefined)
|
|
96
144
|
return pending
|
|
@@ -100,23 +148,38 @@ export class KnowledgeEnvironment {
|
|
|
100
148
|
this.report.items = this.report.items.map(item => item.id === id ? { ...item, ...patch } : item)
|
|
101
149
|
}
|
|
102
150
|
|
|
103
|
-
private async performCheck(deep: boolean): Promise<EnvironmentReport> {
|
|
151
|
+
private async performCheck(deep: boolean, kind?: EnvironmentItemId): Promise<EnvironmentReport> {
|
|
104
152
|
const checkController = new AbortController()
|
|
105
153
|
this.checkController = checkController
|
|
106
154
|
const signal = this.controller ? AbortSignal.any([this.controller.signal, checkController.signal]) : checkController.signal
|
|
107
155
|
const fingerprint = this.fingerprint()
|
|
108
156
|
const { config, paths } = this.options.snapshot()
|
|
109
|
-
this.report = blankReport()
|
|
157
|
+
if (this.report.fingerprint !== fingerprint) this.report = blankReport()
|
|
158
|
+
/**
|
|
159
|
+
* 解释器**结构上**装不了原生扩展时,先判失败,不白跑一次模型加载。
|
|
160
|
+
*
|
|
161
|
+
* 这一条是 2026-10-18 补的:那之前自检会照常 `import torch`,然后把 macOS 的
|
|
162
|
+
* `dlopen ... different Team IDs` 整段异常当成失败消息贴给用户 —— 一屏报文,
|
|
163
|
+
* 但真正该说的是「这个 Python 带签名保护,点修复换一个」。一次 `codesign` 就能判,
|
|
164
|
+
* 不必先加载几百 MB 的模型。
|
|
165
|
+
*/
|
|
166
|
+
const interpreter = deep ? await this.interpreterProblem(paths.python) : undefined
|
|
110
167
|
this.report.state = 'checking'
|
|
111
|
-
for (const id of ['embedding', 'rerank'] as const) {
|
|
168
|
+
for (const id of kind ? [kind] : ['embedding', 'rerank'] as const) {
|
|
112
169
|
if (signal.aborted || this.disposed) break
|
|
113
170
|
const path = id === 'embedding' ? paths.embeddingModel : paths.rerankModel
|
|
171
|
+
const cached = this.report.fingerprint === fingerprint ? this.report.items.find(item => item.id === id) : undefined
|
|
114
172
|
this.item(id, { path })
|
|
115
173
|
const disabled = id === 'embedding' ? config.embeddingProvider === 'none' : config.rerankProvider === 'none'
|
|
116
174
|
const problem = disabled ? '此能力是必需项,请启用模型'
|
|
117
175
|
: id === 'embedding' && config.embeddingProvider === 'ollama' ? undefined : modelFilesProblem(path)
|
|
118
176
|
if (problem) { this.item(id, { state: 'missing', message: problem }); continue }
|
|
119
|
-
if (
|
|
177
|
+
if (interpreter) { this.item(id, { state: 'failed', message: interpreter }); continue }
|
|
178
|
+
if (!deep) {
|
|
179
|
+
if (cached && cached.state !== 'unchecked' && cached.state !== 'checking') this.item(id, { state: cached.state, message: cached.message })
|
|
180
|
+
else this.item(id, { state: 'unchecked', message: '资源已找到,尚未完成实际推理自检' })
|
|
181
|
+
continue
|
|
182
|
+
}
|
|
120
183
|
this.item(id, { state: 'checking', message: '加载模型并运行推理自检,首次加载可能较慢' })
|
|
121
184
|
try {
|
|
122
185
|
const result = await this.options.probe(id)
|
|
@@ -131,12 +194,34 @@ export class KnowledgeEnvironment {
|
|
|
131
194
|
? { ...item, state: 'unchecked', message: '自检已停止,请重新运行' }
|
|
132
195
|
: item)
|
|
133
196
|
this.report.state = this.report.items.every(item => item.state === 'ready') ? 'ready' : 'blocked'
|
|
134
|
-
this.report.checkedAt = new Date().toISOString()
|
|
197
|
+
if (deep) this.report.checkedAt = new Date().toISOString()
|
|
135
198
|
this.report.fingerprint = fingerprint
|
|
136
199
|
if (this.checkController === checkController) this.checkController = undefined
|
|
200
|
+
if (deep && !signal.aborted && !this.disposed) {
|
|
201
|
+
try {
|
|
202
|
+
await mkdir(this.options.resourceRoot, { recursive: true })
|
|
203
|
+
await writeFile(join(this.options.resourceRoot, CHECK_FILE), JSON.stringify(this.report, null, 2))
|
|
204
|
+
} catch { /* The current process can use the result; a later restart may need to probe again. */ }
|
|
205
|
+
}
|
|
137
206
|
return structuredClone(this.report)
|
|
138
207
|
}
|
|
139
208
|
|
|
209
|
+
/**
|
|
210
|
+
* 这个解释器有没有「加载不了第三方原生扩展」的结构性问题。
|
|
211
|
+
*
|
|
212
|
+
* 目前只有一种:macOS 上带 hardened runtime(`codesign` 的 flags 里有 `runtime`)——
|
|
213
|
+
* 库校验只喂它自己 Team ID 签的 dylib,而 pip 上的 torch 等轮子是 ad-hoc 签名的。
|
|
214
|
+
* 判据来自 `dsh-python.ts`,安装那条路用的是同一个函数:**自检与安装必须看同一件事**,
|
|
215
|
+
* 否则会出现「自检说没问题、装的时候换了解释器」这种两边不对账的情况。
|
|
216
|
+
*/
|
|
217
|
+
private async interpreterProblem(python: string): Promise<string | undefined> {
|
|
218
|
+
if (!python) return undefined
|
|
219
|
+
const hardened = await hasHardenedRuntime(python, this.options.run ?? runProcess)
|
|
220
|
+
if (!hardened) return undefined
|
|
221
|
+
return '这个环境的 Python 带 macOS 签名保护(hardened runtime),加载不了 pip 安装的原生扩展;'
|
|
222
|
+
+ '点「一键安装 / 修复」会用不带签名保护的解释器重建环境。'
|
|
223
|
+
}
|
|
224
|
+
|
|
140
225
|
start(action: 'check' | 'install', requestedSelection?: unknown): EnvironmentView {
|
|
141
226
|
if (this.disposed) throw new Error('环境服务已关闭')
|
|
142
227
|
if (this.job?.state === 'running') return this.view()
|
|
@@ -151,6 +236,7 @@ export class KnowledgeEnvironment {
|
|
|
151
236
|
const job = this.job
|
|
152
237
|
const log = (line: string): void => { job.logs = [...job.logs, line.slice(-1500)].slice(-12) }
|
|
153
238
|
const work = async (): Promise<void> => {
|
|
239
|
+
let rollbackSettings: (() => Promise<void>) | undefined
|
|
154
240
|
try {
|
|
155
241
|
// A first-use guard can already be checking. Do not install into that probe.
|
|
156
242
|
if (this.checkPending) await this.checkPending
|
|
@@ -164,16 +250,18 @@ export class KnowledgeEnvironment {
|
|
|
164
250
|
const id = label.includes('向量') ? 'embedding' : label.includes('重排') ? 'rerank' : undefined
|
|
165
251
|
if (id) this.item(id, { state: 'checking', message: label })
|
|
166
252
|
}, log, this.options.run ?? runProcess, selection)
|
|
253
|
+
rollbackSettings = await this.options.activateSelection?.()
|
|
167
254
|
this.options.reload()
|
|
168
255
|
} else this.options.reload()
|
|
169
256
|
job.stage = '运行实际推理自检'
|
|
170
257
|
const report = await this.check(true)
|
|
171
258
|
if (controller.signal.aborted) throw new Error('ENVIRONMENT_CANCELLED')
|
|
172
259
|
if (report.state !== 'ready') throw new Error(report.items.filter(item => item.state !== 'ready').map(item => `${item.label}:${item.message}`).join(';'))
|
|
173
|
-
await mkdir(this.options.resourceRoot, { recursive: true })
|
|
174
|
-
await writeFile(join(this.options.resourceRoot, 'environment-last-check.json'), JSON.stringify(report, null, 2))
|
|
175
260
|
job.state = 'succeeded'; job.stage = '环境已就绪'
|
|
176
261
|
} catch (cause) {
|
|
262
|
+
if (rollbackSettings) {
|
|
263
|
+
try { await rollbackSettings() } catch (rollbackCause) { log(`恢复模型设置失败:${rollbackCause instanceof Error ? rollbackCause.message : String(rollbackCause)}`) }
|
|
264
|
+
}
|
|
177
265
|
if (action === 'install') {
|
|
178
266
|
const activeSelection = readModelSelection(this.options.resourceRoot)
|
|
179
267
|
if (activeSelection.embedding !== previousSelection.embedding || activeSelection.rerank !== previousSelection.rerank) {
|
package/src/environment/types.ts
CHANGED
|
@@ -17,6 +17,11 @@ export interface EnvironmentReport {
|
|
|
17
17
|
fingerprint?: string
|
|
18
18
|
items: EnvironmentItem[]
|
|
19
19
|
}
|
|
20
|
+
export interface EnvironmentCurrentModel {
|
|
21
|
+
id?: string
|
|
22
|
+
label: string
|
|
23
|
+
state: 'active' | 'disabled'
|
|
24
|
+
}
|
|
20
25
|
export interface EnvironmentJob {
|
|
21
26
|
id: string
|
|
22
27
|
state: 'running' | 'succeeded' | 'failed' | 'cancelled'
|
|
@@ -30,5 +35,6 @@ export interface EnvironmentView {
|
|
|
30
35
|
report: EnvironmentReport
|
|
31
36
|
job?: EnvironmentJob
|
|
32
37
|
selection: ModelSelection
|
|
38
|
+
current: { embedding: EnvironmentCurrentModel; rerank: EnvironmentCurrentModel }
|
|
33
39
|
models: { embedding: ModelOptionView[]; rerank: ModelOptionView[] }
|
|
34
40
|
}
|
package/src/events.ts
CHANGED
|
@@ -46,6 +46,18 @@ export interface EventEntity {
|
|
|
46
46
|
updatedAt: string
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
/** Remove the legacy generated Event list from a WikiCard body. References live in card_event. */
|
|
50
|
+
export function stripEventReferenceSection(content: string): string {
|
|
51
|
+
const lines = content.split('\n')
|
|
52
|
+
for (let start = lines.length - 1; start >= 0; start -= 1) {
|
|
53
|
+
if (!/^##\s+关联\s*Events?\s*$|^##\s+关联事件\s*$/.test((lines[start] ?? '').trim())) continue
|
|
54
|
+
let end = start + 1
|
|
55
|
+
while (end < lines.length && !/^#{1,2}\s+/.test(lines[end] ?? '')) end += 1
|
|
56
|
+
lines.splice(start, end - start)
|
|
57
|
+
}
|
|
58
|
+
return lines.join('\n').trim()
|
|
59
|
+
}
|
|
60
|
+
|
|
49
61
|
export function compositeEventIdentity(occurredAt: string | null, coordinate: CardCoordinate, symptom: string | null): string {
|
|
50
62
|
const payload = [occurredAt, coordinate.objectId, coordinate.actionId, coordinate.sceneId, normalizeKnowledge(symptom ?? '')]
|
|
51
63
|
return 'composite:' + createHash('sha256').update(JSON.stringify(payload)).digest('hex').slice(0, 24)
|
package/src/extract-contract.ts
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
import { AXIS_CONTRACT_LINES } from './axis-contract.js'
|
|
16
|
+
import { TABLE_OUTPUT_CONTRACT } from './extract-table-contract.js'
|
|
16
17
|
|
|
17
18
|
/**
|
|
18
19
|
* 把骨架允许的小节拼成提示词里那一条。
|
|
@@ -37,14 +38,15 @@ function sectionRule(sections: string[] | undefined): string[] {
|
|
|
37
38
|
* 编译链路用 `buildTrialSystem(profileSections)`;试跑面板用默认实例(不下发小节清单),
|
|
38
39
|
* 因为那一层没有「本库骨架」这个概念。
|
|
39
40
|
*/
|
|
40
|
-
export function buildTrialSystem(sections?: string[]): string {
|
|
41
|
-
return trialSystemBody(sectionRule(sections))
|
|
41
|
+
export function buildTrialSystem(sections?: string[], format: 'table' | 'json' = 'table'): string {
|
|
42
|
+
return trialSystemBody(sectionRule(sections), format)
|
|
42
43
|
}
|
|
43
44
|
|
|
44
|
-
const trialSystemBody = (positionRule: string[]): string => [
|
|
45
|
+
const trialSystemBody = (positionRule: string[], format: 'table' | 'json'): string => [
|
|
45
46
|
'你在把一份业务文档拆成知识库要用的内容。知识以 Knowledge 为单位:',
|
|
46
47
|
'一条可独立检索、验证或执行的最小知识单元。',
|
|
47
48
|
'',
|
|
49
|
+
...(format === 'table' ? TABLE_OUTPUT_CONTRACT : [
|
|
48
50
|
'只回一个 JSON 对象,不要散文,不要代码围栏。',
|
|
49
51
|
'',
|
|
50
52
|
'输出形状:',
|
|
@@ -73,6 +75,7 @@ const trialSystemBody = (positionRule: string[]): string => [
|
|
|
73
75
|
' "dropped": [ { "quote": "原文片段", "why": "为什么丢掉" } ]',
|
|
74
76
|
'}',
|
|
75
77
|
'',
|
|
78
|
+
]),
|
|
76
79
|
'# 知识:一条只讲一个单元',
|
|
77
80
|
'- text 是一条能脱离上下文独立阅读的最小知识单元。事实要能判断真假,操作步骤要能被执行。',
|
|
78
81
|
'- 一段原文里有几个独立单元就写几条。不要压成一条,也不要拆碎。',
|
package/src/extract-model.ts
CHANGED
|
@@ -65,6 +65,8 @@ export type ModelCall = (request: {
|
|
|
65
65
|
sessionId?: string
|
|
66
66
|
/** 可选观察本次请求的 provider 用量;普通调用方可以忽略。 */
|
|
67
67
|
onUsage?: (usage: TrialTokenUsage) => void
|
|
68
|
+
/** Public response text only; reasoning chunks are not exposed. */
|
|
69
|
+
onDelta?: (text: string) => void
|
|
68
70
|
}) => Promise<string>
|
|
69
71
|
|
|
70
72
|
/** 归并条目:与规则抽取的 `NormalizedCandidates['merges']` 保持同一形状。 */
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** Kept free of server dependencies so the architecture page shows the real contract. */
|
|
2
|
+
export const TABLE_OUTPUT_CONTRACT = [
|
|
3
|
+
'只返回 TOON 固定列表格,不要 Markdown 围栏或说明。format 必须是 knowledge-table-v1。',
|
|
4
|
+
'每个数组头的 [N] 是本次实际行数;每行必须与 {列名} 的列数和顺序一致。空表写 名称[0]:。',
|
|
5
|
+
'逗号分列。字符串包含逗号、冒号、换行或首尾空格时用双引号包裹;内部双引号写 \\",换行写 \\n,反斜杠写 \\\\。',
|
|
6
|
+
'缺失值写 null。字符串列表 heading_path 和 missing 写为 JSON 数组字符串。',
|
|
7
|
+
'evidence 表每个引文只写一次。其它表的 evidence 列写出处 ID,多个 ID 用空格分隔。所有 ID 必须存在。',
|
|
8
|
+
'以下是列定义和示例;示例内容必须换成当前文档的数据:',
|
|
9
|
+
'format: knowledge-table-v1',
|
|
10
|
+
'subject: 业务主体',
|
|
11
|
+
'knowledge[1]{id,text,object,action,scene,section,order,evidence}:',
|
|
12
|
+
' k1,一条最小知识单元,对象名,动作名,场景名,字段,1,q1',
|
|
13
|
+
'events[1]{id,external_id,title,summary,object,action,scene,occurred_at,symptom,root_cause,root_cause_status,outcome,missing,evidence}:',
|
|
14
|
+
' e1,null,一次具体发生,原文明确的概述,对象名,动作名,场景名,null,null,null,null,null,"[]",q1',
|
|
15
|
+
'timeline[1]{event_id,order,type,description,result,at,evidence}:',
|
|
16
|
+
' e1,1,observation,原文记录的现象,null,null,q1',
|
|
17
|
+
'objects[1]{mention,definition,evidence,existing,parent}:',
|
|
18
|
+
' 对象名,对象的边界和含义,q1,null,null',
|
|
19
|
+
'actions[1]{mention,definition,evidence,existing,parent}:',
|
|
20
|
+
' 动作名,动作的含义,q1,null,null',
|
|
21
|
+
'scenes[1]{mention,definition,evidence,existing,parent}:',
|
|
22
|
+
' 场景名,场景的含义,q1,null,null',
|
|
23
|
+
'evidence[1]{id,quote,heading_path}:',
|
|
24
|
+
' q1,原文逐字引文,"[]"',
|
|
25
|
+
'dropped[0]:',
|
|
26
|
+
'timeline.type 只能是 observation/check/action/result/conclusion;root_cause_status 只能是 confirmed/suspected/null。',
|
|
27
|
+
'dropped 非空时列为 {quote,why}。对象 parent 只填原文有证据的父对象;existing 填已有实体名。',
|
|
28
|
+
'',
|
|
29
|
+
]
|