@kenz1117/dsh-engram 0.13.10 → 0.13.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.en.md +2 -2
- package/README.md +4 -4
- package/i18n/ru.txt +3 -0
- package/lib/client.js +1 -1
- package/lib/client.js.map +1 -1
- package/lib/engine/node_modules/onnxruntime-common/index.mjs +3 -0
- package/lib/engine/node_modules/onnxruntime-common/package.json +10 -0
- package/lib/engine/node_modules/onnxruntime-web/package.json +11 -0
- package/lib/engine/node_modules/onnxruntime-web/webgpu.mjs +3 -0
- package/lib/engine/ort-wasm-simd-threaded.jsep.mjs +108 -0
- package/lib/engine/ort-wasm-simd-threaded.jsep.wasm +0 -0
- package/lib/engine/ort.bundle.min.mjs +2954 -0
- package/lib/engine/transformers.web.js +32562 -0
- package/lib/index.js +437 -99
- package/lib/local.js +1459 -0
- package/package.json +11 -10
- package/scripts/backfill-embeddings.mjs +14 -16
- package/scripts/copy-engine.mjs +102 -0
- package/scripts/engine-smoke.mjs +250 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kenz1117/dsh-engram",
|
|
3
3
|
"description": "Local-first cross-session long-term memory for DeepSeek Harness: dual-scope SQLite stores, hybrid FTS5 + local-vector retrieval with RRF, an evidence gate, provenance audit and SM-2 spaced repetition. 23 narrow tools, zero external services, zero API keys.",
|
|
4
|
-
"version": "0.13.
|
|
4
|
+
"version": "0.13.11",
|
|
5
5
|
"icon": "icon.svg",
|
|
6
6
|
"logo": "icon-256.svg",
|
|
7
7
|
"publishConfig": {
|
|
@@ -54,7 +54,9 @@
|
|
|
54
54
|
},
|
|
55
55
|
"scripts": {
|
|
56
56
|
"prepare": "node scripts/link-peers.mjs && node scripts/ensure-build.mjs",
|
|
57
|
-
"bundle": "tsdown",
|
|
57
|
+
"bundle": "node scripts/copy-engine.mjs && tsdown",
|
|
58
|
+
"copy-engine": "node scripts/copy-engine.mjs",
|
|
59
|
+
"prepack": "node scripts/copy-engine.mjs",
|
|
58
60
|
"watch": "tsdown --watch",
|
|
59
61
|
"test": "vitest run",
|
|
60
62
|
"lint": "biome check src tests",
|
|
@@ -123,20 +125,19 @@
|
|
|
123
125
|
"optional": true
|
|
124
126
|
}
|
|
125
127
|
},
|
|
126
|
-
"optionalDependencies": {
|
|
127
|
-
"@huggingface/transformers": "^4.3.2"
|
|
128
|
-
},
|
|
129
128
|
"devDependencies": {
|
|
130
|
-
"@biomejs/biome": "^2.6.
|
|
129
|
+
"@biomejs/biome": "^2.6.5",
|
|
130
|
+
"@huggingface/transformers": "4.3.2",
|
|
131
131
|
"@types/node": "^26.6.6",
|
|
132
132
|
"@types/react": "^18.3.12",
|
|
133
133
|
"@types/react-dom": "^18.3.1",
|
|
134
|
-
"@vitest/coverage-v8": "^5.0
|
|
134
|
+
"@vitest/coverage-v8": "^5.1.0",
|
|
135
135
|
"lightningcss": "^1.32.0",
|
|
136
|
-
"
|
|
136
|
+
"onnxruntime-web": "1.31.0-dev.20260914-8d85527a0",
|
|
137
|
+
"tsdown": "^0.23.5",
|
|
137
138
|
"typescript": "^7.0.2",
|
|
138
|
-
"vite": "^8.3.
|
|
139
|
-
"vitest": "^5.0
|
|
139
|
+
"vite": "^8.3.6",
|
|
140
|
+
"vitest": "^5.1.0"
|
|
140
141
|
},
|
|
141
142
|
"packageManager": "pnpm@10.15.0"
|
|
142
143
|
}
|
|
@@ -177,29 +177,27 @@ function findDatabases(dbDir) {
|
|
|
177
177
|
}
|
|
178
178
|
|
|
179
179
|
/**
|
|
180
|
-
*
|
|
180
|
+
* 建本地嵌入器:随包 WASM 引擎 + q8 量化权重,与插件默认路径一致。
|
|
181
181
|
* @param {{model: string, pooling: 'mean' | 'cls', cacheDir: string}} options 嵌入参数。
|
|
182
182
|
* @returns {Promise<(texts: string[]) => Promise<Float32Array[]>>} 批量取向量。
|
|
183
183
|
*/
|
|
184
184
|
async function createEmbedder(options) {
|
|
185
|
-
// 动态 import:--help
|
|
186
|
-
//
|
|
187
|
-
//
|
|
188
|
-
|
|
185
|
+
// 动态 import:--help 与参数错误不加载引擎运行时。
|
|
186
|
+
// 0.13.11 引擎出树:装载器随包在 lib/local.js(插件入口的导出面是宿主契约,
|
|
187
|
+
// 嵌入器不在其上)。引擎目录缺失/不完整时 createLocalEmbedder 抛
|
|
188
|
+
// EMBEDDER_DOWNLOAD_FAILED(人话文案指引用户重装插件);这里只补构建产物
|
|
189
|
+
// 缺失(源码安装没跑 pnpm bundle)的情形。
|
|
190
|
+
let embedder
|
|
189
191
|
try {
|
|
190
|
-
|
|
192
|
+
({ createLocalEmbedder } = await import(new URL('../lib/local.js', import.meta.url)))
|
|
191
193
|
} catch {
|
|
192
|
-
throw new Error(
|
|
193
|
-
}
|
|
194
|
-
const { env, pipeline } = transformers
|
|
195
|
-
env.cacheDir = options.cacheDir
|
|
196
|
-
// 下载端点可经 HF_ENDPOINT 覆盖(镜像);模型已在缓存时不会访问网络。
|
|
197
|
-
if (process.env.HF_ENDPOINT) env.remoteHost = process.env.HF_ENDPOINT
|
|
198
|
-
const extractor = await pipeline('feature-extraction', options.model, { dtype: 'q8' })
|
|
199
|
-
return async texts => {
|
|
200
|
-
const output = await extractor([...texts], { pooling: options.pooling, normalize: true })
|
|
201
|
-
return output.tolist().map(list => new Float32Array(list))
|
|
194
|
+
throw new Error('嵌入器装载器 lib/local.js 缺失——从源码安装请先运行 pnpm bundle,或直接重装插件')
|
|
202
195
|
}
|
|
196
|
+
return await createLocalEmbedder(options.cacheDir, process.env.HF_ENDPOINT, {
|
|
197
|
+
id: options.model,
|
|
198
|
+
pooling: options.pooling,
|
|
199
|
+
dtype: 'q8',
|
|
200
|
+
})
|
|
203
201
|
}
|
|
204
202
|
|
|
205
203
|
/**
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* 引擎出树(0.13.11):把嵌入引擎从安装树(optionalDependencies,原生件解包约
|
|
4
|
+
* 300MB,弱网机器 pnpm store 永远填不满,市场更新被限时杀掉,issue #50/#54)
|
|
5
|
+
* 搬进随包分发的 lib/engine/——纯 WASM 技术路线,包里不再有任何原生依赖。
|
|
6
|
+
*
|
|
7
|
+
* 四个文件,全部来自 devDependencies(只在构建机存在,用户安装树里没有):
|
|
8
|
+
* 1. @huggingface/transformers/dist/transformers.web.js
|
|
9
|
+
* transformers.js 的 web 构建(ESM,~1MB)。Node 下它的 ONNX 后端是空壳
|
|
10
|
+
* (node:fs 被构建 stub 掉),必须靠 2 预注入的 ORT 才能推理。
|
|
11
|
+
* 2. onnxruntime-web/dist/ort.bundle.min.mjs
|
|
12
|
+
* 自含 wasm 后端的 ORT 构建(~400KB)。spike 实测必须从绝对路径 import
|
|
13
|
+
* (不能用包名:exports map 会解析到 onnxruntime-node 原生构建),
|
|
14
|
+
* 经 globalThis[Symbol.for('onnxruntime')] 预注入给 1 用。
|
|
15
|
+
* 3. onnxruntime-web/dist/ort-wasm-simd-threaded.jsep.wasm
|
|
16
|
+
* wasm 二进制(~28MB),readFileSync 注入 ORT.env.wasm.wasmBinary,
|
|
17
|
+
* 不走网络加载。
|
|
18
|
+
* 4. onnxruntime-web/dist/ort-wasm-simd-threaded.jsep.mjs
|
|
19
|
+
* emscripten 工厂(~47KB),pathToFileURL 动态 import。
|
|
20
|
+
* 版本钉死关系:onnxruntime-web 必须与 transformers 4.3.2 内部钉死的
|
|
21
|
+
* 1.31.0-dev.20260914-8d85527a0 完全一致,否则工厂与 wasm 二进制不配套。
|
|
22
|
+
*
|
|
23
|
+
* 幂等:每次先删旧 lib/engine/ 再整体拷贝。路径全部经 node:path / node:url
|
|
24
|
+
* 拼装(Windows 开发机也要能跑)。devDependencies 缺失(如 GitHub 源码安装
|
|
25
|
+
* 现场构建但依赖不全)时报错退出——发布一个没有引擎的包比安装失败更糟。
|
|
26
|
+
*
|
|
27
|
+
* 另生成一个解析 shim(lib/engine/node_modules/onnxruntime-web/):transformers
|
|
28
|
+
* 的 web 构建顶部有 `import * as ONNX_WEB from "onnxruntime-web/webgpu"`——
|
|
29
|
+
* 包名Specifier 必须可解析,模块才能被 import;注入模式下 ONNX_WEB 不会被
|
|
30
|
+
* 消费(globalThis[Symbol.for('onnxruntime')] 优先),但解析这关绕不过。
|
|
31
|
+
* shim 把该 Specifier 指回本目录的 ort.bundle.min.mjs(同一次 ESM 求值,
|
|
32
|
+
* 零额外体积)。
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
36
|
+
import { basename, dirname, join, resolve } from 'node:path'
|
|
37
|
+
import { fileURLToPath } from 'node:url'
|
|
38
|
+
|
|
39
|
+
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
|
40
|
+
const engineDir = join(repoRoot, 'lib', 'engine')
|
|
41
|
+
|
|
42
|
+
/** 源文件(包内路径)→ 落地文件名。顺序即上文的 1→4。 */
|
|
43
|
+
const ENGINE_FILES = [
|
|
44
|
+
['@huggingface/transformers/dist/transformers.web.js', 'transformers.web.js'],
|
|
45
|
+
['onnxruntime-web/dist/ort.bundle.min.mjs', 'ort.bundle.min.mjs'],
|
|
46
|
+
['onnxruntime-web/dist/ort-wasm-simd-threaded.jsep.wasm', 'ort-wasm-simd-threaded.jsep.wasm'],
|
|
47
|
+
['onnxruntime-web/dist/ort-wasm-simd-threaded.jsep.mjs', 'ort-wasm-simd-threaded.jsep.mjs'],
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
const sources = ENGINE_FILES.map(([from, to]) => ({ from: join(repoRoot, 'node_modules', from), to }))
|
|
51
|
+
|
|
52
|
+
const missing = sources.filter(source => !existsSync(source.from))
|
|
53
|
+
if (missing.length > 0) {
|
|
54
|
+
console.error(`[copy-engine] devDependencies 里没有嵌入引擎的源文件(先 pnpm install):`)
|
|
55
|
+
for (const source of missing) console.error(` 缺 ${source.from}`)
|
|
56
|
+
process.exit(1)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
rmSync(engineDir, { recursive: true, force: true })
|
|
60
|
+
// mkdir 递归建 lib/engine/(lib/ 本身可能还不存在——bundle 前先跑本脚本)。
|
|
61
|
+
mkdirSync(engineDir, { recursive: true })
|
|
62
|
+
|
|
63
|
+
for (const source of sources) {
|
|
64
|
+
const target = join(engineDir, source.to)
|
|
65
|
+
copyFileSync(source.from, target)
|
|
66
|
+
console.log(`[copy-engine] ${basename(source.from)}: ${statSync(target).size.toLocaleString('en-US')} bytes <- ${source.from}`)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// —— 解析 shim:包名 Specifier → 本目录的 ORT bundle ——
|
|
70
|
+
// transformers web 构建里有真实静态 import:"onnxruntime-web/webgpu"(浏览器
|
|
71
|
+
// 分支的后端绑定,注入模式下不被消费)与 "onnxruntime-common"(Tensor,运行时
|
|
72
|
+
// 真用)。Specifier 必须可解析模块才能 import;两个 shim 都指回本目录的
|
|
73
|
+
// ort.bundle.min.mjs——URL 相同,ESM 缓存保证与注入用的是同一个模块实例。
|
|
74
|
+
// 版本号取自 devDependencies 里真实的 onnxruntime-web,保持同步不手写。
|
|
75
|
+
const ortVersion = JSON.parse(readFileSync(join(repoRoot, 'node_modules', 'onnxruntime-web', 'package.json'), 'utf8')).version
|
|
76
|
+
const SHIM_PACKAGES = [
|
|
77
|
+
['onnxruntime-web', './webgpu.mjs', { '.': './webgpu.mjs', './webgpu': './webgpu.mjs' }],
|
|
78
|
+
['onnxruntime-common', './index.mjs', { '.': './index.mjs' }],
|
|
79
|
+
]
|
|
80
|
+
for (const [name, entry, exports] of SHIM_PACKAGES) {
|
|
81
|
+
const shimDir = join(engineDir, 'node_modules', name)
|
|
82
|
+
mkdirSync(shimDir, { recursive: true })
|
|
83
|
+
writeFileSync(
|
|
84
|
+
join(shimDir, 'package.json'),
|
|
85
|
+
JSON.stringify({
|
|
86
|
+
name,
|
|
87
|
+
version: ortVersion,
|
|
88
|
+
description: `Resolution shim generated by scripts/copy-engine.mjs; re-exports the bundled wasm ORT (${ortVersion}).`,
|
|
89
|
+
type: 'module',
|
|
90
|
+
main: entry,
|
|
91
|
+
exports,
|
|
92
|
+
}, null, 2) + '\n',
|
|
93
|
+
)
|
|
94
|
+
writeFileSync(
|
|
95
|
+
join(shimDir, entry.slice(2)),
|
|
96
|
+
`// scripts/copy-engine.mjs 生成的解析 shim:Specifier 必须可解析(transformers\n`
|
|
97
|
+
+ `// web 构建的静态 import);指回同一份 ORT bundle(同 URL = 同一模块实例)。\n`
|
|
98
|
+
+ `export * from "../../ort.bundle.min.mjs";\n`,
|
|
99
|
+
)
|
|
100
|
+
console.log(`[copy-engine] resolution shim ${name}@${ortVersion} -> ${shimDir}`)
|
|
101
|
+
}
|
|
102
|
+
console.log(`[copy-engine] ok: ${sources.length} files + ${SHIM_PACKAGES.length} shims in ${engineDir}`)
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* 引擎烟测(0.13.11「引擎出树」的发布面回归):真实走完
|
|
4
|
+
* build 产物 → npm pack → 临时目录安装 → 从安装形态的 lib/index.js import
|
|
5
|
+
* createLocalEmbedder → 真实下载模型并 embed 两条中文文本。
|
|
6
|
+
*
|
|
7
|
+
* 它专查「打包后的路径推导」这类源码形态测不到的问题:
|
|
8
|
+
* - 引擎四文件在 tarball 里、解包后从 lib/engine/ 装载(spike 的四步注入);
|
|
9
|
+
* - resolveEngine 不靠 DSH_ENGRAM_ENGINE_DIR,纯 import.meta.url 推导;
|
|
10
|
+
* - fs customCache shim 把模型缓存落盘(下一轮烟测可复用端点测得的热身差异)。
|
|
11
|
+
*
|
|
12
|
+
* 用法:
|
|
13
|
+
* node scripts/engine-smoke.mjs [--work-dir <dir>] [--tarball <path.tgz>]
|
|
14
|
+
*
|
|
15
|
+
* --work-dir 工作目录(装 tarball、模型缓存);默认 <系统临时目录>/dsh-engine-smoke。
|
|
16
|
+
* 每次运行先整目录清空——模型每次冷下载是有意的:烟测要测的正是
|
|
17
|
+
* 「全新安装 + 首次下载」这条最坏路径。
|
|
18
|
+
* --tarball 跳过打包直接用现成 tarball(CI 里 bundle 与 pack 分步时用)。
|
|
19
|
+
* --keep-cache 跨运行保留模型缓存(model-cache 目录不随清空丢失):本地反复
|
|
20
|
+
* 重跑时不吃模型端点的网络抖动;CI 不用(CI 要测的正是冷路径)。
|
|
21
|
+
*
|
|
22
|
+
* 退出码非 0 即烟测失败(维度不对 / 模长偏离 / 引擎装载失败)。CI 用法见
|
|
23
|
+
* .github/workflows/engine-smoke.yml(windows-latest 主 + ubuntu-latest 对照,
|
|
24
|
+
* 模型端点优先 https://hf-mirror.com/,不可达回退 huggingface.co)。
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { execSync } from 'node:child_process'
|
|
28
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
29
|
+
import { tmpdir } from 'node:os'
|
|
30
|
+
import { basename, dirname, join, resolve } from 'node:path'
|
|
31
|
+
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
32
|
+
import { performance } from 'node:perf_hooks'
|
|
33
|
+
|
|
34
|
+
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
|
35
|
+
|
|
36
|
+
/** 烟测对象模型:zh 默认模型(registry/models.json 的 zh 绑定),512 维。 */
|
|
37
|
+
const MODEL_ID = 'Xenova/bge-small-zh-v1.5'
|
|
38
|
+
const EXPECTED_DIMENSIONS = 512
|
|
39
|
+
/** 优先端点与回退端点:弱网/受限网络环境里镜像通常更可达(issue #50/#54)。 */
|
|
40
|
+
const HOSTS = ['https://hf-mirror.com/', 'https://huggingface.co/']
|
|
41
|
+
/** 单阶段看门狗:模型下载 + 首次装载在慢机上也很少超过 10 分钟。 */
|
|
42
|
+
const PHASE_TIMEOUT_MS = 10 * 60 * 1000
|
|
43
|
+
|
|
44
|
+
const argv = process.argv.slice(2)
|
|
45
|
+
function argValue(name) {
|
|
46
|
+
const index = argv.indexOf(name)
|
|
47
|
+
return index === -1 ? undefined : argv[index + 1]
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const workDir = argValue('--work-dir') ?? join(tmpdir(), 'dsh-engine-smoke')
|
|
51
|
+
const tarballArg = argValue('--tarball')
|
|
52
|
+
const keepCache = argv.includes('--keep-cache')
|
|
53
|
+
|
|
54
|
+
const fail = message => {
|
|
55
|
+
console.error(`[engine-smoke] FAIL: ${message}`)
|
|
56
|
+
process.exit(1)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const withTimeout = (promise, tag) =>
|
|
60
|
+
Promise.race([
|
|
61
|
+
promise,
|
|
62
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`${tag} 超时(${PHASE_TIMEOUT_MS}ms)`)), PHASE_TIMEOUT_MS)),
|
|
63
|
+
])
|
|
64
|
+
|
|
65
|
+
const seconds = ms => `${(ms / 1000).toFixed(1)}s`
|
|
66
|
+
const megabytes = bytes => `${(bytes / 1048576).toFixed(1)}MB`
|
|
67
|
+
|
|
68
|
+
console.log(`[engine-smoke] node ${process.version} | ${process.platform} ${process.arch}`)
|
|
69
|
+
console.log(`[engine-smoke] work dir: ${workDir}`)
|
|
70
|
+
|
|
71
|
+
// 每次都从零开始:全新安装 + 冷模型缓存,正是弱网机器市场更新的那条路径。
|
|
72
|
+
// --keep-cache:模型缓存目录先挪出去,清空后放回——本地重跑不再吃端点抖动。
|
|
73
|
+
// 挪动用 fs.renameSync(同盘原子),不用 shell mv(Windows 兼容纪律)。
|
|
74
|
+
const modelCacheToKeep = keepCache ? join(workDir, 'model-cache') : undefined
|
|
75
|
+
if (modelCacheToKeep !== undefined && existsSync(modelCacheToKeep)) {
|
|
76
|
+
// 保留目录必须是 workDir 的兄弟:放里面会被下一步的整目录清空连带删掉。
|
|
77
|
+
const preserved = `${workDir}.model-cache-keep`
|
|
78
|
+
rmSync(preserved, { recursive: true, force: true })
|
|
79
|
+
renameSync(modelCacheToKeep, preserved)
|
|
80
|
+
rmSync(workDir, { recursive: true, force: true })
|
|
81
|
+
mkdirSync(workDir, { recursive: true })
|
|
82
|
+
mkdirSync(dirname(modelCacheToKeep), { recursive: true })
|
|
83
|
+
renameSync(preserved, modelCacheToKeep)
|
|
84
|
+
} else {
|
|
85
|
+
rmSync(workDir, { recursive: true, force: true })
|
|
86
|
+
mkdirSync(workDir, { recursive: true })
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// —— 1. 打包(或用现成 tarball)——
|
|
90
|
+
let tarball = tarballArg !== undefined ? resolve(tarballArg) : undefined
|
|
91
|
+
if (tarball === undefined) {
|
|
92
|
+
console.log('[engine-smoke] npm pack ...')
|
|
93
|
+
execSync(`npm pack "${repoRoot}" --pack-destination .`, { cwd: workDir, stdio: ['ignore', 'ignore', 'inherit'] })
|
|
94
|
+
const packed = readdirSync(workDir).find(name => name.endsWith('.tgz'))
|
|
95
|
+
if (packed === undefined) fail('npm pack 没有产出 tarball')
|
|
96
|
+
tarball = join(workDir, packed)
|
|
97
|
+
}
|
|
98
|
+
const tarballBytes = statSync(tarball).size
|
|
99
|
+
console.log(`[engine-smoke] tarball: ${basename(tarball)} (${megabytes(tarballBytes)})`)
|
|
100
|
+
|
|
101
|
+
// —— 2. 临时目录安装(等价于用户 `npm install <tarball>`)——
|
|
102
|
+
const installDir = join(workDir, 'install')
|
|
103
|
+
mkdirSync(installDir, { recursive: true })
|
|
104
|
+
writeFileSync(join(installDir, 'package.json'), JSON.stringify({ name: 'engine-smoke-fixture', private: true }, null, 2))
|
|
105
|
+
console.log('[engine-smoke] npm install <tarball> ...')
|
|
106
|
+
execSync(`npm install "${tarball}" --no-audit --no-fund`, {
|
|
107
|
+
cwd: installDir,
|
|
108
|
+
stdio: ['ignore', 'ignore', 'inherit'],
|
|
109
|
+
env: { ...process.env, // 安装时不要被开发机的引擎目录环境变量干扰
|
|
110
|
+
DSH_ENGRAM_ENGINE_DIR: '' },
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
// —— 3. 核对随包引擎四文件——
|
|
114
|
+
const installedPkg = join(installDir, 'node_modules', '@kenz1117', 'dsh-engram')
|
|
115
|
+
const engineDir = join(installedPkg, 'lib', 'engine')
|
|
116
|
+
const expectedFiles = [
|
|
117
|
+
'transformers.web.js',
|
|
118
|
+
'ort.bundle.min.mjs',
|
|
119
|
+
'ort-wasm-simd-threaded.jsep.mjs',
|
|
120
|
+
'ort-wasm-simd-threaded.jsep.wasm',
|
|
121
|
+
join('node_modules', 'onnxruntime-web', 'package.json'),
|
|
122
|
+
join('node_modules', 'onnxruntime-common', 'package.json'),
|
|
123
|
+
]
|
|
124
|
+
for (const name of expectedFiles) {
|
|
125
|
+
const file = join(engineDir, name)
|
|
126
|
+
if (!existsSync(file)) fail(`安装形态缺引擎文件 ${name}(lib/engine 不完整)`)
|
|
127
|
+
console.log(`[engine-smoke] lib/engine/${name}: ${statSync(file).size.toLocaleString('en-US')} bytes`)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// —— 4. 给 fixture 供 peer(模拟 dsh profile 的角色)——
|
|
131
|
+
// 发布形态里 @deepseek-ai/* 是 peerDependencies,由宿主 profile 的 node_modules
|
|
132
|
+
// 提供(ESM 沿安装树向上解析)。裸 fixture 里没有宿主,这里从 bundle 的 import
|
|
133
|
+
// 语句机械生成同名的桩包:任意属性/调用/构造都返回自身代理,恰好够模块求值用。
|
|
134
|
+
// 生成是机械的:bundle 的 peer 面变了,桩跟着变,不需要维护清单。
|
|
135
|
+
function installStubPeers(bundleFile, intoDir) {
|
|
136
|
+
const source = readFileSync(bundleFile, 'utf8')
|
|
137
|
+
const imports = new Map() // 包名 → 导入名集合
|
|
138
|
+
const pattern = /(?:import|export)\s+(?:([A-Za-z_$][\w$]*)\s*,?\s*)?(?:\*\s+as\s+[A-Za-z_$][\w$]*\s*,?\s*)?(?:\{([^}]*)\})?\s*from\s*"(@deepseek-ai\/[^"]+)"/g
|
|
139
|
+
for (const [, def, brace, pkg] of source.matchAll(pattern)) {
|
|
140
|
+
if (!imports.has(pkg)) imports.set(pkg, new Set())
|
|
141
|
+
if (def !== undefined) imports.get(pkg).add(def)
|
|
142
|
+
if (brace !== undefined) {
|
|
143
|
+
for (const part of brace.split(',')) {
|
|
144
|
+
const name = part.split(' as ')[0]?.trim()
|
|
145
|
+
if (name !== '') imports.get(pkg).add(name)
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (imports.size === 0) fail('bundle 里没有找到任何 @deepseek-ai/* 静态导入——桩无从生成')
|
|
150
|
+
for (const [pkg, names] of imports) {
|
|
151
|
+
const pkgDir = join(intoDir, pkg)
|
|
152
|
+
mkdirSync(pkgDir, { recursive: true })
|
|
153
|
+
writeFileSync(join(pkgDir, 'package.json'), JSON.stringify({ name: pkg, version: '0.0.0-peer-stub', type: 'module', main: 'index.js', exports: './index.js' }))
|
|
154
|
+
const lines = [
|
|
155
|
+
'// 机械生成的桩(scripts/engine-smoke.mjs):任何属性、调用、构造都返回自身。',
|
|
156
|
+
'const stub = new Proxy(function () {}, {',
|
|
157
|
+
' get: (_target, prop) => (prop === Symbol.toPrimitive ? () => "" : stub),',
|
|
158
|
+
' apply: () => stub,',
|
|
159
|
+
' construct: () => stub,',
|
|
160
|
+
'})',
|
|
161
|
+
...[...names].map(name => `export const ${name} = stub`),
|
|
162
|
+
'export default stub',
|
|
163
|
+
]
|
|
164
|
+
writeFileSync(join(pkgDir, 'index.js'), `${lines.join('\n')}\n`)
|
|
165
|
+
console.log(`[engine-smoke] peer 桩 ${pkg}: { ${[...names].join(', ')} }`)
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
installStubPeers(join(installedPkg, 'lib', 'index.js'), join(installDir, 'node_modules'))
|
|
169
|
+
|
|
170
|
+
// —— 5. 端点探测:镜像优先,回退默认——
|
|
171
|
+
async function probeHost(host) {
|
|
172
|
+
const controller = new AbortController()
|
|
173
|
+
const timer = setTimeout(() => controller.abort(), 10_000)
|
|
174
|
+
try {
|
|
175
|
+
const response = await fetch(`${host}${MODEL_ID}/resolve/main/config.json`, {
|
|
176
|
+
method: 'HEAD', redirect: 'follow', signal: controller.signal,
|
|
177
|
+
})
|
|
178
|
+
return response.ok
|
|
179
|
+
} catch {
|
|
180
|
+
return false
|
|
181
|
+
} finally {
|
|
182
|
+
clearTimeout(timer)
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
let hosts = [...HOSTS]
|
|
187
|
+
const reachable = await probeHost(hosts[0])
|
|
188
|
+
if (!reachable) {
|
|
189
|
+
console.log(`[engine-smoke] ${hosts[0]} 不可达(10s 探测),回退默认端点`)
|
|
190
|
+
hosts = [hosts[1], hosts[0]]
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// —— 6. 从安装形态 import(真实验证打包后的路径推导)——
|
|
194
|
+
// 嵌入器入口是 lib/local.js(见 tsdown.config.ts):插件入口 lib/index.js 的
|
|
195
|
+
// 导出面是宿主契约(name/apply/inject…),createLocalEmbedder 不在其上。
|
|
196
|
+
// 两个入口的引擎路径推导同构——import.meta.url 都锚在 lib/<x>.js,引擎都在
|
|
197
|
+
// 同级 lib/engine/;lib/index.js 的模块图由 peer 桩加载(第 4 步已验)。
|
|
198
|
+
// 只删不设:DSH_ENGRAM_ENGINE_DIR 若在环境里,这次烟测就白做了。
|
|
199
|
+
delete process.env.DSH_ENGRAM_ENGINE_DIR
|
|
200
|
+
const { createLocalEmbedder } = await import(pathToFileURL(join(installedPkg, 'lib', 'local.js')))
|
|
201
|
+
|
|
202
|
+
const modelCacheDir = join(workDir, 'model-cache')
|
|
203
|
+
const texts = ['今天天气很好,阳光明媚,适合出门散步', '正弦函数的导数是余弦函数,这在微积分里是基本公式']
|
|
204
|
+
|
|
205
|
+
let lastError = undefined
|
|
206
|
+
let embedder
|
|
207
|
+
let loadMs = 0
|
|
208
|
+
for (const host of hosts) {
|
|
209
|
+
const started = performance.now()
|
|
210
|
+
try {
|
|
211
|
+
console.log(`[engine-smoke] createLocalEmbedder(remoteHost=${host},模型 ${MODEL_ID} q8,冷缓存)...`)
|
|
212
|
+
embedder = await withTimeout(
|
|
213
|
+
createLocalEmbedder(modelCacheDir, host, { id: MODEL_ID, pooling: 'mean', dtype: 'q8' }),
|
|
214
|
+
'模型下载+装载',
|
|
215
|
+
)
|
|
216
|
+
loadMs = performance.now() - started
|
|
217
|
+
console.log(`[engine-smoke] 模型就绪:${seconds(loadMs)}(含下载;热身 2 次在内)`)
|
|
218
|
+
break
|
|
219
|
+
} catch (error) {
|
|
220
|
+
lastError = error
|
|
221
|
+
const cause = error instanceof Error ? error.cause : undefined
|
|
222
|
+
console.log(`[engine-smoke] ${host} 装载失败:${error instanceof Error ? error.message.split('\n')[0] : error}`
|
|
223
|
+
+ (cause instanceof Error ? `;cause: ${cause.message.split('\n')[0]}` : ''))
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (embedder === undefined) fail(`两个端点都装不出嵌入器;最后一次错误:${lastError instanceof Error ? lastError.message : lastError}`)
|
|
227
|
+
|
|
228
|
+
// —— 7. 真实 embed:维度与归一化断言——
|
|
229
|
+
let vectors
|
|
230
|
+
try {
|
|
231
|
+
const started = performance.now()
|
|
232
|
+
vectors = await withTimeout(embedder.embed(texts), 'embed')
|
|
233
|
+
const embedMs = performance.now() - started
|
|
234
|
+
console.log(`[engine-smoke] embed(${texts.length} 条): ${seconds(embedMs)}`)
|
|
235
|
+
if (vectors.length !== texts.length) fail(`返回 ${vectors.length} 条向量,期望 ${texts.length} 条`)
|
|
236
|
+
for (const [index, vector] of vectors.entries()) {
|
|
237
|
+
if (!(vector instanceof Float32Array)) fail(`第 ${index} 条不是 Float32Array`)
|
|
238
|
+
if (vector.length !== EXPECTED_DIMENSIONS) fail(`第 ${index} 条 ${vector.length} 维,期望 ${EXPECTED_DIMENSIONS} 维`)
|
|
239
|
+
let sum = 0
|
|
240
|
+
for (const value of vector) sum += value * value
|
|
241
|
+
const norm = Math.sqrt(sum)
|
|
242
|
+
if (Math.abs(norm - 1) > 1e-2) fail(`第 ${index} 条模长 ${norm},偏离 1 超过 1e-2`)
|
|
243
|
+
console.log(`[engine-smoke] vector[${index}]: dims=${vector.length}, norm=${norm.toFixed(6)}`)
|
|
244
|
+
}
|
|
245
|
+
} finally {
|
|
246
|
+
await embedder.close()
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// 两条断言(512 维、模长≈1)已足够说明引擎、模型与缓存链路全通。
|
|
250
|
+
console.log('[engine-smoke] OK:随包引擎在安装形态下完成真实推理')
|