@modusensus/dsh-mneme 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +727 -636
- package/bin/cli.mjs +0 -0
- package/bin/dsh-mneme-mcp.mjs +510 -0
- package/cordis.patch.yml +21 -21
- package/lib/api-standalone.js +506 -316
- package/lib/api.js +1403 -1316
- package/lib/bootstrap.js +211 -0
- package/lib/client.js +4657 -4366
- package/lib/commands.js +95 -95
- package/lib/config.js +615 -510
- package/lib/document.js +221 -0
- package/lib/dream/clustering.js +118 -118
- package/lib/dream/decisions.js +658 -658
- package/lib/dream/narratives.js +44 -0
- package/lib/dream/sleep.js +702 -700
- package/lib/dream.js +147 -16
- package/lib/embedding.js +177 -177
- package/lib/entities/extractor.js +249 -249
- package/lib/guide.js +44 -0
- package/lib/heat.js +144 -136
- package/lib/hot-memory.js +46 -46
- package/lib/index.js +515 -515
- package/lib/inject.js +375 -295
- package/lib/lang.js +208 -179
- package/lib/local-embedder.js +320 -320
- package/lib/mirror.js +198 -198
- package/lib/organize.js +450 -0
- package/lib/quality-filter.js +126 -126
- package/lib/recall-stats.js +125 -0
- package/lib/reranker.js +269 -269
- package/lib/runtime/adopt.js +176 -176
- package/lib/runtime/closure.js +172 -172
- package/lib/runtime/download.js +247 -247
- package/lib/runtime/layout.js +265 -235
- package/lib/runtime/loader.js +252 -249
- package/lib/runtime/model-fetch.js +361 -361
- package/lib/runtime/provision.js +237 -237
- package/lib/runtime/tarball.js +130 -130
- package/lib/runtime/verify.js +204 -191
- package/lib/scope.js +118 -118
- package/lib/search/adaptive.js +60 -22
- package/lib/search/bm25.js +96 -96
- package/lib/service.js +2324 -2087
- package/lib/settings.js +459 -420
- package/lib/store.js +2549 -2367
- package/lib/summarize.js +820 -449
- package/lib/tools.js +702 -574
- package/lib/vector-index.js +110 -110
- package/lib/version-check.js +122 -122
- package/package.json +109 -108
- package/runtime-manifest.json +526 -596
- package/scripts/benchmark-embed.js +201 -201
- package/scripts/benchmark-recall.js +199 -199
- package/scripts/benchmark-rerank.js +166 -166
- package/scripts/build-runtime-manifest.mjs +224 -217
- package/scripts/check-sync.js +42 -42
- package/scripts/e2e-dsh.js +220 -220
- package/scripts/mneme-runtime.mjs +277 -277
- package/scripts/stress-dsh.js +255 -255
- package/scripts/sync-lib.js +47 -47
- package/scripts/sync-test-badge.mjs +67 -61
- package/src/api-standalone.js +506 -316
- package/src/api.js +1403 -1316
- package/src/bootstrap.js +211 -0
- package/src/commands.js +95 -95
- package/src/config.js +615 -510
- package/src/document.js +221 -0
- package/src/dream/clustering.js +118 -118
- package/src/dream/decisions.js +658 -658
- package/src/dream/narratives.js +44 -0
- package/src/dream/sleep.js +702 -700
- package/src/dream.js +147 -16
- package/src/embedding.js +177 -177
- package/src/entities/extractor.js +249 -249
- package/src/guide.js +44 -0
- package/src/heat.js +144 -136
- package/src/hot-memory.js +46 -46
- package/src/index.js +515 -515
- package/src/inject.js +375 -295
- package/src/lang.js +208 -179
- package/src/local-embedder.js +320 -320
- package/src/mirror.js +198 -198
- package/src/organize.js +450 -0
- package/src/quality-filter.js +126 -126
- package/src/recall-stats.js +125 -0
- package/src/reranker.js +269 -269
- package/src/runtime/adopt.js +176 -176
- package/src/runtime/closure.js +172 -172
- package/src/runtime/download.js +247 -247
- package/src/runtime/layout.js +265 -235
- package/src/runtime/loader.js +252 -249
- package/src/runtime/model-fetch.js +361 -361
- package/src/runtime/provision.js +237 -237
- package/src/runtime/tarball.js +130 -130
- package/src/runtime/verify.js +204 -191
- package/src/scope.js +118 -118
- package/src/search/adaptive.js +60 -22
- package/src/search/bm25.js +96 -96
- package/src/service.js +2324 -2087
- package/src/settings.js +459 -420
- package/src/store.js +2549 -2367
- package/src/summarize.js +820 -449
- package/src/tools.js +702 -574
- package/src/vector-index.js +110 -110
- package/src/version-check.js +122 -122
- package/test/adaptive-inject-budget.test.js +85 -0
- package/test/api-routes-gaps.test.js +218 -218
- package/test/api.test.js +1640 -1482
- package/test/audit.test.js +448 -448
- package/test/benchmark.test.js +94 -94
- package/test/bootstrap.test.js +220 -0
- package/test/client.test.js +860 -736
- package/test/clustering.test.js +100 -100
- package/test/commands.test.js +106 -106
- package/test/config.test.js +66 -66
- package/test/conflict-freeze.test.js +300 -300
- package/test/document.test.js +523 -0
- package/test/dream-archive-guardrail.test.js +116 -116
- package/test/dream-candidate-hybrid.test.js +281 -281
- package/test/dream-coverage-degrade.test.js +86 -86
- package/test/dream.test.js +1308 -1254
- package/test/embedding-ready.test.js +196 -196
- package/test/entities.test.js +547 -547
- package/test/entity-recall.test.js +179 -0
- package/test/epistemic.test.js +298 -298
- package/test/fnew-0112.test.js +311 -311
- package/test/fnew-03.test.js +422 -422
- package/test/graph-api.test.js +175 -175
- package/test/heat.test.js +175 -148
- package/test/helpers/dream-mock.js +82 -82
- package/test/helpers/peer-worker.mjs +25 -25
- package/test/helpers/runtime-source.js +60 -60
- package/test/helpers/tar-builder.js +80 -80
- package/test/hot-memory.test.js +145 -145
- package/test/inject-content-cap.test.js +91 -0
- package/test/inject-first-round-bm25.test.js +89 -89
- package/test/inject-heat.test.js +61 -0
- package/test/inject-pools.test.js +170 -0
- package/test/inject-recall.test.js +105 -0
- package/test/inject-rotation.test.js +161 -161
- package/test/inject.test.js +283 -249
- package/test/lang.test.js +87 -87
- package/test/lib-smoke.test.js +96 -96
- package/test/llm-audit.test.js +462 -309
- package/test/llm-message-source.test.js +177 -177
- package/test/local-embedder.test.js +249 -249
- package/test/mcp-stdio.test.js +385 -0
- package/test/mirror-dirty.test.js +424 -424
- package/test/mirror-edit-digest.test.js +187 -187
- package/test/mirror-generation.test.js +499 -499
- package/test/mirror.test.js +249 -249
- package/test/narrative-bars.test.js +162 -0
- package/test/organize.test.js +418 -0
- package/test/peer-blockers.test.js +193 -193
- package/test/policy-epoch.test.js +259 -259
- package/test/quality-filter.test.js +186 -186
- package/test/reasoning-effort.test.js +746 -746
- package/test/recall-evals.test.js +235 -235
- package/test/recall-layer.test.js +326 -326
- package/test/recall-stats.test.js +123 -0
- package/test/receipt-chain.test.js +451 -451
- package/test/reflection.test.js +226 -226
- package/test/reindex-backfill.test.js +62 -62
- package/test/reranker-single-logit.test.js +73 -73
- package/test/reranker.test.js +197 -197
- package/test/resident-status.test.js +83 -0
- package/test/runtime-adopt.test.js +206 -206
- package/test/runtime-cli.test.js +89 -89
- package/test/runtime-closure.test.js +193 -193
- package/test/runtime-download.test.js +302 -302
- package/test/runtime-layout.test.js +163 -163
- package/test/runtime-loader.test.js +255 -237
- package/test/runtime-manifest.test.js +159 -158
- package/test/runtime-model-fetch.test.js +348 -348
- package/test/runtime-provision.test.js +101 -101
- package/test/runtime-tarball.test.js +81 -81
- package/test/runtime-verify.test.js +279 -214
- package/test/scope-candidates.test.js +239 -239
- package/test/scope-explicit.test.js +452 -452
- package/test/scope-retrieval.test.js +197 -197
- package/test/scope-storage.test.js +255 -255
- package/test/scope-strict.test.js +231 -231
- package/test/search-fusion.test.js +90 -90
- package/test/semantic.test.js +124 -124
- package/test/service-search.test.js +174 -174
- package/test/service.test.js +405 -405
- package/test/settings.test.js +261 -261
- package/test/skip-invalid-restore.test.js +262 -262
- package/test/sleep-action-set.test.js +422 -422
- package/test/sleep-heat.test.js +126 -125
- package/test/sleep-scheduler-rearm.test.js +89 -89
- package/test/sleep.test.js +410 -373
- package/test/standalone-api.test.js +544 -359
- package/test/store-vector-cache.test.js +98 -98
- package/test/store.test.js +396 -383
- package/test/stress.test.js +209 -209
- package/test/summarize.test.js +1288 -575
- package/test/tools-dto-schema-matrix.test.js +325 -0
- package/test/tools-memory-get-schema.test.js +65 -65
- package/test/tools.test.js +374 -341
- package/test/updated-at-semantics.test.js +113 -113
- package/test/vector-index.test.js +221 -221
- package/test/version-check.test.js +152 -152
- package/test/webServer-optional.test.js +82 -82
|
@@ -1,217 +1,224 @@
|
|
|
1
|
-
// 生成 runtime-manifest.json(issue #131 / PR-C)
|
|
2
|
-
//
|
|
3
|
-
// **平台无关**是这份清单的核心设计:数据源是 `package-lock.json` 而不是「本机装好的那棵树」。
|
|
4
|
-
// npm 的 lockfile 里包含**所有平台**的包条目(各自的 os / cpu 与 integrity),所以一份清单就能
|
|
5
|
-
// 覆盖 win32 / darwin / linux × x64 / arm64 —— 下载时再按平台过滤(`matchesPlatform`)。
|
|
6
|
-
//
|
|
7
|
-
// 为什么不做成「按平台各存一份、谁在哪个系统上生成」:那样 macOS 用户在一台没人跑过脚本的机器上
|
|
8
|
-
// 会看到「清单里没有 darwin-arm64 的条目」而直接卡住。清单是随包发布的静态数据,它必须一次覆盖
|
|
9
|
-
// 所有平台,而不是依赖某个贡献者手上有对应的机器。
|
|
10
|
-
//
|
|
11
|
-
// 哈希取自 lockfile 的 integrity(npm 自己算的),所以维护成本是「改依赖后重跑一次脚本」。
|
|
12
|
-
//
|
|
13
|
-
// 用法:
|
|
14
|
-
// node scripts/build-runtime-manifest.mjs # 生成/更新
|
|
15
|
-
// node scripts/build-runtime-manifest.mjs --check # 只校验现有文件与 lockfile 一致(CI 用)
|
|
16
|
-
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
17
|
-
import { join } from "node:path";
|
|
18
|
-
import { fileURLToPath } from "node:url";
|
|
19
|
-
import { matchesPlatform } from "../src/runtime/closure.js";
|
|
20
|
-
|
|
21
|
-
const ROOT = fileURLToPath(new URL("..", import.meta.url));
|
|
22
|
-
const MANIFEST = join(ROOT, "runtime-manifest.json");
|
|
23
|
-
const MANIFEST_VERSION = 2;
|
|
24
|
-
const ENTRY = "@huggingface/transformers";
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* 证据:`@huggingface/transformers` 的 Node 构建(dist/transformers.node.mjs)顶层只 import
|
|
28
|
-
* onnxruntime-node 与 sharp,从不 import onnxruntime-web(把一个空的 onnxruntime-web 打桩进去,
|
|
29
|
-
* 嵌入结果逐字节相同:dims=[2,512]、L2 范数 1.0000、cos 0.3053)。而它解包后 127.7MB,占整个
|
|
30
|
-
* 闭包约三分之一。所以清单里剔除它,风险由 verify 的真实推理兜底。
|
|
31
|
-
*/
|
|
32
|
-
const EXCLUDED = {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
export function
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
return
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
/**
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
const
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
seen.
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
name
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
//
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
process.
|
|
217
|
-
|
|
1
|
+
// 生成 runtime-manifest.json(issue #131 / PR-C)
|
|
2
|
+
//
|
|
3
|
+
// **平台无关**是这份清单的核心设计:数据源是 `package-lock.json` 而不是「本机装好的那棵树」。
|
|
4
|
+
// npm 的 lockfile 里包含**所有平台**的包条目(各自的 os / cpu 与 integrity),所以一份清单就能
|
|
5
|
+
// 覆盖 win32 / darwin / linux × x64 / arm64 —— 下载时再按平台过滤(`matchesPlatform`)。
|
|
6
|
+
//
|
|
7
|
+
// 为什么不做成「按平台各存一份、谁在哪个系统上生成」:那样 macOS 用户在一台没人跑过脚本的机器上
|
|
8
|
+
// 会看到「清单里没有 darwin-arm64 的条目」而直接卡住。清单是随包发布的静态数据,它必须一次覆盖
|
|
9
|
+
// 所有平台,而不是依赖某个贡献者手上有对应的机器。
|
|
10
|
+
//
|
|
11
|
+
// 哈希取自 lockfile 的 integrity(npm 自己算的),所以维护成本是「改依赖后重跑一次脚本」。
|
|
12
|
+
//
|
|
13
|
+
// 用法:
|
|
14
|
+
// node scripts/build-runtime-manifest.mjs # 生成/更新
|
|
15
|
+
// node scripts/build-runtime-manifest.mjs --check # 只校验现有文件与 lockfile 一致(CI 用)
|
|
16
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
import { fileURLToPath } from "node:url";
|
|
19
|
+
import { matchesPlatform } from "../src/runtime/closure.js";
|
|
20
|
+
|
|
21
|
+
const ROOT = fileURLToPath(new URL("..", import.meta.url));
|
|
22
|
+
const MANIFEST = join(ROOT, "runtime-manifest.json");
|
|
23
|
+
const MANIFEST_VERSION = 2;
|
|
24
|
+
const ENTRY = "@huggingface/transformers";
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* 证据:`@huggingface/transformers` 的 Node 构建(dist/transformers.node.mjs)顶层只 import
|
|
28
|
+
* onnxruntime-node 与 sharp,从不 import onnxruntime-web(把一个空的 onnxruntime-web 打桩进去,
|
|
29
|
+
* 嵌入结果逐字节相同:dims=[2,512]、L2 范数 1.0000、cos 0.3053)。而它解包后 127.7MB,占整个
|
|
30
|
+
* 闭包约三分之一。所以清单里剔除它,风险由 verify 的真实推理兜底。
|
|
31
|
+
*/
|
|
32
|
+
const EXCLUDED = {
|
|
33
|
+
"onnxruntime-web": "Node 构建从不 import 它(打桩验证:嵌入结果逐字节相同)",
|
|
34
|
+
// sharp 0.35 起把 WASM 回退包装进 optionalDependencies(freebsd/webcontainers 场景),
|
|
35
|
+
// 六个覆盖平台在 Node 里从不 import;wasm32 无 os 约束,会混进每个平台的 natives 断言。
|
|
36
|
+
"@img/sharp-freebsd-wasm32": "FreeBSD 专用 WASM 回退,覆盖平台之外",
|
|
37
|
+
"@img/sharp-webcontainers-wasm32": "WebContainers 专用 WASM 回退,覆盖平台之外",
|
|
38
|
+
"@img/sharp-wasm32": "无 os 约束的 WASM 回退,Node 构建从不 import",
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* 被剔除的包,以及**它子树下的一切**。
|
|
43
|
+
*
|
|
44
|
+
* 只按包名过滤会漏掉嵌套条目:闭包是按源布局镜像的,所以存在
|
|
45
|
+
* `onnxruntime-web/node_modules/onnxruntime-common` 这种位于被剔除包内部的包 ——
|
|
46
|
+
* 那个位置根本不会被取回来,留在清单里就是让下载器去填一个不存在的坑。
|
|
47
|
+
* 同名但位于别处(例如顶层 onnxruntime-common)不受影响。
|
|
48
|
+
*/
|
|
49
|
+
export function isExcluded(pkg) {
|
|
50
|
+
return Object.keys(EXCLUDED).some(
|
|
51
|
+
(name) => pkg.name === name || pkg.rel === name || pkg.rel.startsWith(`${name}/`)
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** lockfile 的路径 → 包名(最后一段 `node_modules/` 之后的部分)。 */
|
|
56
|
+
export function nameOfPath(path) {
|
|
57
|
+
const marker = path.lastIndexOf("node_modules/");
|
|
58
|
+
return marker === -1 ? path : path.slice(marker + "node_modules/".length);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** lockfile 的路径 → payload 里的相对路径(去掉开头的 `node_modules/`)。 */
|
|
62
|
+
export function relOfPath(path) {
|
|
63
|
+
return path.startsWith("node_modules/") ? path.slice("node_modules/".length) : path;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* 按 Node 的解析语义列出候选路径:从引用者自身的 `node_modules` 逐级向上,最后兜底到根部。
|
|
68
|
+
*
|
|
69
|
+
* 例:引用者在 `node_modules/a/node_modules/b`、依赖 `x` →
|
|
70
|
+
* `…/b/node_modules/x` → `node_modules/a/node_modules/x` → `node_modules/x`
|
|
71
|
+
* @param {string} fromPath - 引用者的 lockfile 路径。
|
|
72
|
+
* @param {string} name - 依赖名。
|
|
73
|
+
* @returns {string[]} 候选路径,由近到远。
|
|
74
|
+
*/
|
|
75
|
+
export function resolveCandidates(fromPath, name) {
|
|
76
|
+
const out = [`${fromPath}/node_modules/${name}`];
|
|
77
|
+
let cursor = fromPath;
|
|
78
|
+
while (cursor.includes("/node_modules/")) {
|
|
79
|
+
cursor = cursor.slice(0, cursor.lastIndexOf("/node_modules/"));
|
|
80
|
+
out.push(cursor === "" ? `node_modules/${name}` : `${cursor}/node_modules/${name}`);
|
|
81
|
+
}
|
|
82
|
+
if (!out.includes(`node_modules/${name}`)) out.push(`node_modules/${name}`);
|
|
83
|
+
return out;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** registry 的 tarball 地址:lockfile 记了 resolved 就用它(与 integrity 同源),否则按 npm 约定推。 */
|
|
87
|
+
function tarballUrl(name, version, resolved) {
|
|
88
|
+
if (typeof resolved === "string" && resolved.startsWith("http")) return resolved;
|
|
89
|
+
const base = name.startsWith("@") ? name.slice(name.indexOf("/") + 1) : name;
|
|
90
|
+
return `https://registry.npmjs.org/${name}/-/${base}-${version}.tgz`;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* 从 lockfile 走一遍闭包。
|
|
95
|
+
* @param {object} lock - package-lock.json 的内容。
|
|
96
|
+
* @returns {{packages: object[], missing: string[]}} 包列表(按 rel 排序)与解析不到的依赖。
|
|
97
|
+
*/
|
|
98
|
+
export function closureFromLockfile(lock) {
|
|
99
|
+
const all = lock.packages ?? {};
|
|
100
|
+
const known = new Set(
|
|
101
|
+
Object.entries(all)
|
|
102
|
+
.filter(([path, meta]) => path.includes("node_modules/") && meta?.version !== undefined)
|
|
103
|
+
.map(([path]) => path)
|
|
104
|
+
);
|
|
105
|
+
const seen = new Map();
|
|
106
|
+
const missing = [];
|
|
107
|
+
const queue = [`node_modules/${ENTRY}`];
|
|
108
|
+
|
|
109
|
+
while (queue.length > 0) {
|
|
110
|
+
const path = queue.shift();
|
|
111
|
+
if (seen.has(path)) continue;
|
|
112
|
+
const meta = all[path];
|
|
113
|
+
if (meta?.version === undefined) {
|
|
114
|
+
missing.push(path);
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
if (isExcluded({ name: nameOfPath(path), rel: relOfPath(path) })) continue;
|
|
118
|
+
seen.set(path, meta);
|
|
119
|
+
|
|
120
|
+
const deps = { ...(meta.dependencies ?? {}), ...(meta.optionalDependencies ?? {}) };
|
|
121
|
+
for (const name of Object.keys(deps)) {
|
|
122
|
+
const resolved = resolveCandidates(path, name).find((option) => known.has(option));
|
|
123
|
+
if (resolved === undefined) missing.push(`${name}(被 ${relOfPath(path)} 需要)`);
|
|
124
|
+
else queue.push(resolved);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const packages = [...seen.entries()]
|
|
129
|
+
.map(([path, meta]) => ({
|
|
130
|
+
name: nameOfPath(path),
|
|
131
|
+
version: meta.version,
|
|
132
|
+
rel: relOfPath(path),
|
|
133
|
+
integrity: meta.integrity ?? null,
|
|
134
|
+
tarball: tarballUrl(nameOfPath(path), meta.version, meta.resolved),
|
|
135
|
+
// 平台约束照抄 lockfile:下载器按它过滤。没有这三个字段 = 与平台无关。
|
|
136
|
+
...(Array.isArray(meta.os) ? { os: meta.os } : {}),
|
|
137
|
+
...(Array.isArray(meta.cpu) ? { cpu: meta.cpu } : {}),
|
|
138
|
+
// libc 也带上:linux-x64 的 glibc 机器靠它剔掉两套 musl 变体(十几 MB)。
|
|
139
|
+
// 下载器只在**确认**了本机 libc 时才据此过滤,拿不到证据就不看这个字段。
|
|
140
|
+
...(Array.isArray(meta.libc) ? { libc: meta.libc } : {})
|
|
141
|
+
}))
|
|
142
|
+
.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
|
|
143
|
+
|
|
144
|
+
return { packages, missing };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** 某个平台实际需要取哪些包(与下载器用同一条判据)。 */
|
|
148
|
+
export function packagesForPlatform(manifest, platform, arch, libc = null) {
|
|
149
|
+
return manifest.packages.filter((pkg) => matchesPlatform(pkg, platform, arch, libc));
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** 生成一份**平台无关**的清单。 */
|
|
153
|
+
export function build() {
|
|
154
|
+
const lock = JSON.parse(readFileSync(join(ROOT, "package-lock.json"), "utf8"));
|
|
155
|
+
const { packages, missing } = closureFromLockfile(lock);
|
|
156
|
+
// 解析不到的依赖必须让生成**失败**:带缺口的清单会让用户在下载时才失败,而且「少了一个包」
|
|
157
|
+
// 这种缺口结构检查看不出来(它只点三个必需包)。生成期的数据问题就该在生成期炸掉 ——
|
|
158
|
+
// 这也是我第一版真正踩过的:被剔除包的孤儿依赖留在了清单里,而生成器一声不吭。
|
|
159
|
+
if (missing.length > 0) {
|
|
160
|
+
throw new Error(`这些依赖在 package-lock.json 里解析不到:${missing.slice(0, 10).join("、")}`);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const entry = packages.find((pkg) => pkg.rel === ENTRY);
|
|
164
|
+
if (entry === undefined) throw new Error(`闭包入口没解析出来:${missing.join("; ") || "lockfile 里没有它"}`);
|
|
165
|
+
// 查不到 sha512 就不能写进清单:那等于让下载器「先信任再校验」,是假校验。
|
|
166
|
+
const withoutIntegrity = packages.filter(
|
|
167
|
+
(pkg) => typeof pkg.integrity !== "string" || !pkg.integrity.startsWith("sha512-")
|
|
168
|
+
);
|
|
169
|
+
if (withoutIntegrity.length > 0) {
|
|
170
|
+
throw new Error(`这些包在 package-lock.json 里没有 sha512 integrity:${withoutIntegrity.map((p) => p.rel).join("、")}`);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
return {
|
|
174
|
+
manifestVersion: MANIFEST_VERSION,
|
|
175
|
+
entry: ENTRY,
|
|
176
|
+
transformersVersion: entry.version,
|
|
177
|
+
excluded: EXCLUDED,
|
|
178
|
+
// 平台无关:下载时按 os/cpu/libc 过滤(payloadId 也由下载器按本机平台算)。
|
|
179
|
+
packages
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** 应当被覆盖的平台(自检与 CI 断言用)。 */
|
|
184
|
+
export const COVERED_PLATFORMS = [
|
|
185
|
+
["win32", "x64"],
|
|
186
|
+
["win32", "arm64"],
|
|
187
|
+
["darwin", "x64"],
|
|
188
|
+
["darwin", "arm64"],
|
|
189
|
+
["linux", "x64"],
|
|
190
|
+
["linux", "arm64"]
|
|
191
|
+
];
|
|
192
|
+
|
|
193
|
+
/** 命令行入口。`--check` 只校验,不写文件。 */
|
|
194
|
+
export function main(args = process.argv.slice(2)) {
|
|
195
|
+
const check = args.includes("--check");
|
|
196
|
+
const built = build();
|
|
197
|
+
const coverage = COVERED_PLATFORMS.map(([platform, arch]) => {
|
|
198
|
+
const set = packagesForPlatform(built, platform, arch);
|
|
199
|
+
const hasEntry = set.some((pkg) => pkg.rel === ENTRY);
|
|
200
|
+
const hasNative = set.some((pkg) => pkg.rel === "onnxruntime-node");
|
|
201
|
+
// linux 再报一次「确认 glibc 时」的包数:下载器会剔掉另一套 libc 的变体,这行让
|
|
202
|
+
// 「剔掉了多少」在 --check 里看得见,而不是只能翻清单。
|
|
203
|
+
const glibc = platform === "linux" ? `;确认 glibc 时 ${packagesForPlatform(built, platform, arch, "glibc").length} 个` : "";
|
|
204
|
+
return `${platform}-${arch}: ${set.length} 个包${glibc}${hasEntry && hasNative ? "" : "(缺入口或原生依赖!)"}`;
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
if (check) {
|
|
208
|
+
const existing = existsSync(MANIFEST) ? readFileSync(MANIFEST, "utf8") : "";
|
|
209
|
+
const same = existing === `${JSON.stringify(built, null, 2)}\n`;
|
|
210
|
+
process.stdout.write(`${same ? "✓" : "✗"} 清单与 package-lock.json ${same ? "一致" : "不一致,请重跑生成"}\n`);
|
|
211
|
+
for (const line of coverage) process.stdout.write(` ${line}\n`);
|
|
212
|
+
return same ? 0 : 1;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
writeFileSync(MANIFEST, `${JSON.stringify(built, null, 2)}\n`, "utf8");
|
|
216
|
+
process.stdout.write(`写入 ${MANIFEST}\n共 ${built.packages.length} 个包(已剔除 ${Object.keys(EXCLUDED).join("、")} 及其子树)\n`);
|
|
217
|
+
for (const line of coverage) process.stdout.write(` ${line}\n`);
|
|
218
|
+
return 0;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// 只有被直接执行时才干活:被 import 时(测试里)不该顺手改写清单文件。
|
|
222
|
+
const invokedDirectly =
|
|
223
|
+
process.argv[1] != null && import.meta.url.endsWith(process.argv[1].replace(/\\/g, "/").split("/").pop() ?? "");
|
|
224
|
+
if (invokedDirectly) process.exitCode = main();
|
package/scripts/check-sync.js
CHANGED
|
@@ -1,42 +1,42 @@
|
|
|
1
|
-
// 发布前校验 src/ 与 lib/ 一致性。
|
|
2
|
-
//
|
|
3
|
-
// npm 包实际加载 lib/(package main → lib/index.js),而 root 层 `npm publish`
|
|
4
|
-
// 不触发 dsh-mneme/ 的 prepack → sync 同步。历史教训:PR #60 只改了 src 忘了同步
|
|
5
|
-
// lib,v0.7.8 发出去的 npm 包跑的还是旧代码(issue #65)。此脚本让这类漂移在发布时
|
|
6
|
-
// 直接 fail,而不是带着旧产物上线。
|
|
7
|
-
//
|
|
8
|
-
// 用法:node scripts/check-sync.js (root package.json 的 prepack 钩子自动调用)
|
|
9
|
-
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
10
|
-
import { join, relative } from "node:path";
|
|
11
|
-
import { fileURLToPath } from "node:url";
|
|
12
|
-
|
|
13
|
-
const root = join(fileURLToPath(new URL("..", import.meta.url))); // dsh-mneme/
|
|
14
|
-
const srcDir = join(root, "src");
|
|
15
|
-
const libDir = join(root, "lib");
|
|
16
|
-
|
|
17
|
-
function walk(dir) {
|
|
18
|
-
const out = [];
|
|
19
|
-
for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
20
|
-
const full = join(dir, entry.name);
|
|
21
|
-
if (entry.isDirectory()) out.push(...walk(full));
|
|
22
|
-
else if (entry.isFile()) out.push(full);
|
|
23
|
-
}
|
|
24
|
-
return out;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
const bad = [];
|
|
28
|
-
for (const file of walk(srcDir)) {
|
|
29
|
-
const rel = relative(srcDir, file);
|
|
30
|
-
const dest = join(libDir, rel);
|
|
31
|
-
if (!existsSync(dest)) {
|
|
32
|
-
bad.push(`missing lib/${rel}`);
|
|
33
|
-
} else if (!readFileSync(file).equals(readFileSync(dest))) {
|
|
34
|
-
bad.push(`differ lib/${rel}`);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
if (bad.length) {
|
|
38
|
-
console.error(`✗ src/ 与 lib/ 不一致(${bad.length} 处)——npm 包实际加载 lib/,请先 npm run sync 并提交:`);
|
|
39
|
-
for (const line of bad) console.error(` ${line}`);
|
|
40
|
-
process.exit(1);
|
|
41
|
-
}
|
|
42
|
-
console.log(`✓ src/ 与 lib/ 一致(${walk(srcDir).length} 个文件)`);
|
|
1
|
+
// 发布前校验 src/ 与 lib/ 一致性。
|
|
2
|
+
//
|
|
3
|
+
// npm 包实际加载 lib/(package main → lib/index.js),而 root 层 `npm publish`
|
|
4
|
+
// 不触发 dsh-mneme/ 的 prepack → sync 同步。历史教训:PR #60 只改了 src 忘了同步
|
|
5
|
+
// lib,v0.7.8 发出去的 npm 包跑的还是旧代码(issue #65)。此脚本让这类漂移在发布时
|
|
6
|
+
// 直接 fail,而不是带着旧产物上线。
|
|
7
|
+
//
|
|
8
|
+
// 用法:node scripts/check-sync.js (root package.json 的 prepack 钩子自动调用)
|
|
9
|
+
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
10
|
+
import { join, relative } from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
|
|
13
|
+
const root = join(fileURLToPath(new URL("..", import.meta.url))); // dsh-mneme/
|
|
14
|
+
const srcDir = join(root, "src");
|
|
15
|
+
const libDir = join(root, "lib");
|
|
16
|
+
|
|
17
|
+
function walk(dir) {
|
|
18
|
+
const out = [];
|
|
19
|
+
for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
20
|
+
const full = join(dir, entry.name);
|
|
21
|
+
if (entry.isDirectory()) out.push(...walk(full));
|
|
22
|
+
else if (entry.isFile()) out.push(full);
|
|
23
|
+
}
|
|
24
|
+
return out;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const bad = [];
|
|
28
|
+
for (const file of walk(srcDir)) {
|
|
29
|
+
const rel = relative(srcDir, file);
|
|
30
|
+
const dest = join(libDir, rel);
|
|
31
|
+
if (!existsSync(dest)) {
|
|
32
|
+
bad.push(`missing lib/${rel}`);
|
|
33
|
+
} else if (!readFileSync(file).equals(readFileSync(dest))) {
|
|
34
|
+
bad.push(`differ lib/${rel}`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
if (bad.length) {
|
|
38
|
+
console.error(`✗ src/ 与 lib/ 不一致(${bad.length} 处)——npm 包实际加载 lib/,请先 npm run sync 并提交:`);
|
|
39
|
+
for (const line of bad) console.error(` ${line}`);
|
|
40
|
+
process.exit(1);
|
|
41
|
+
}
|
|
42
|
+
console.log(`✓ src/ 与 lib/ 一致(${walk(srcDir).length} 个文件)`);
|