@modusensus/dsh-mneme 0.8.3 → 0.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +727 -636
  3. package/bin/cli.mjs +0 -0
  4. package/bin/dsh-mneme-mcp.mjs +510 -0
  5. package/cordis.patch.yml +21 -21
  6. package/lib/api-standalone.js +506 -316
  7. package/lib/api.js +1403 -1316
  8. package/lib/bootstrap.js +211 -0
  9. package/lib/client.js +4657 -4366
  10. package/lib/commands.js +95 -95
  11. package/lib/config.js +615 -510
  12. package/lib/document.js +221 -0
  13. package/lib/dream/clustering.js +118 -118
  14. package/lib/dream/decisions.js +658 -658
  15. package/lib/dream/narratives.js +44 -0
  16. package/lib/dream/sleep.js +702 -700
  17. package/lib/dream.js +147 -16
  18. package/lib/embedding.js +177 -177
  19. package/lib/entities/extractor.js +249 -249
  20. package/lib/guide.js +44 -0
  21. package/lib/heat.js +144 -136
  22. package/lib/hot-memory.js +46 -46
  23. package/lib/index.js +515 -515
  24. package/lib/inject.js +375 -295
  25. package/lib/lang.js +208 -179
  26. package/lib/local-embedder.js +320 -320
  27. package/lib/mirror.js +198 -198
  28. package/lib/organize.js +450 -0
  29. package/lib/quality-filter.js +126 -126
  30. package/lib/recall-stats.js +125 -0
  31. package/lib/reranker.js +269 -269
  32. package/lib/runtime/adopt.js +176 -176
  33. package/lib/runtime/closure.js +172 -172
  34. package/lib/runtime/download.js +247 -247
  35. package/lib/runtime/layout.js +265 -235
  36. package/lib/runtime/loader.js +252 -249
  37. package/lib/runtime/model-fetch.js +361 -361
  38. package/lib/runtime/provision.js +237 -237
  39. package/lib/runtime/tarball.js +130 -130
  40. package/lib/runtime/verify.js +204 -191
  41. package/lib/scope.js +118 -118
  42. package/lib/search/adaptive.js +60 -22
  43. package/lib/search/bm25.js +96 -96
  44. package/lib/service.js +2324 -2087
  45. package/lib/settings.js +459 -420
  46. package/lib/store.js +2549 -2367
  47. package/lib/summarize.js +820 -449
  48. package/lib/tools.js +702 -574
  49. package/lib/vector-index.js +110 -110
  50. package/lib/version-check.js +122 -122
  51. package/package.json +109 -108
  52. package/runtime-manifest.json +526 -596
  53. package/scripts/benchmark-embed.js +201 -201
  54. package/scripts/benchmark-recall.js +199 -199
  55. package/scripts/benchmark-rerank.js +166 -166
  56. package/scripts/build-runtime-manifest.mjs +224 -217
  57. package/scripts/check-sync.js +42 -42
  58. package/scripts/e2e-dsh.js +220 -220
  59. package/scripts/mneme-runtime.mjs +277 -277
  60. package/scripts/stress-dsh.js +255 -255
  61. package/scripts/sync-lib.js +47 -47
  62. package/scripts/sync-test-badge.mjs +67 -61
  63. package/src/api-standalone.js +506 -316
  64. package/src/api.js +1403 -1316
  65. package/src/bootstrap.js +211 -0
  66. package/src/commands.js +95 -95
  67. package/src/config.js +615 -510
  68. package/src/document.js +221 -0
  69. package/src/dream/clustering.js +118 -118
  70. package/src/dream/decisions.js +658 -658
  71. package/src/dream/narratives.js +44 -0
  72. package/src/dream/sleep.js +702 -700
  73. package/src/dream.js +147 -16
  74. package/src/embedding.js +177 -177
  75. package/src/entities/extractor.js +249 -249
  76. package/src/guide.js +44 -0
  77. package/src/heat.js +144 -136
  78. package/src/hot-memory.js +46 -46
  79. package/src/index.js +515 -515
  80. package/src/inject.js +375 -295
  81. package/src/lang.js +208 -179
  82. package/src/local-embedder.js +320 -320
  83. package/src/mirror.js +198 -198
  84. package/src/organize.js +450 -0
  85. package/src/quality-filter.js +126 -126
  86. package/src/recall-stats.js +125 -0
  87. package/src/reranker.js +269 -269
  88. package/src/runtime/adopt.js +176 -176
  89. package/src/runtime/closure.js +172 -172
  90. package/src/runtime/download.js +247 -247
  91. package/src/runtime/layout.js +265 -235
  92. package/src/runtime/loader.js +252 -249
  93. package/src/runtime/model-fetch.js +361 -361
  94. package/src/runtime/provision.js +237 -237
  95. package/src/runtime/tarball.js +130 -130
  96. package/src/runtime/verify.js +204 -191
  97. package/src/scope.js +118 -118
  98. package/src/search/adaptive.js +60 -22
  99. package/src/search/bm25.js +96 -96
  100. package/src/service.js +2324 -2087
  101. package/src/settings.js +459 -420
  102. package/src/store.js +2549 -2367
  103. package/src/summarize.js +820 -449
  104. package/src/tools.js +702 -574
  105. package/src/vector-index.js +110 -110
  106. package/src/version-check.js +122 -122
  107. package/test/adaptive-inject-budget.test.js +85 -0
  108. package/test/api-routes-gaps.test.js +218 -218
  109. package/test/api.test.js +1640 -1482
  110. package/test/audit.test.js +448 -448
  111. package/test/benchmark.test.js +94 -94
  112. package/test/bootstrap.test.js +220 -0
  113. package/test/client.test.js +860 -736
  114. package/test/clustering.test.js +100 -100
  115. package/test/commands.test.js +106 -106
  116. package/test/config.test.js +66 -66
  117. package/test/conflict-freeze.test.js +300 -300
  118. package/test/document.test.js +523 -0
  119. package/test/dream-archive-guardrail.test.js +116 -116
  120. package/test/dream-candidate-hybrid.test.js +281 -281
  121. package/test/dream-coverage-degrade.test.js +86 -86
  122. package/test/dream.test.js +1308 -1254
  123. package/test/embedding-ready.test.js +196 -196
  124. package/test/entities.test.js +547 -547
  125. package/test/entity-recall.test.js +179 -0
  126. package/test/epistemic.test.js +298 -298
  127. package/test/fnew-0112.test.js +311 -311
  128. package/test/fnew-03.test.js +422 -422
  129. package/test/graph-api.test.js +175 -175
  130. package/test/heat.test.js +175 -148
  131. package/test/helpers/dream-mock.js +82 -82
  132. package/test/helpers/peer-worker.mjs +25 -25
  133. package/test/helpers/runtime-source.js +60 -60
  134. package/test/helpers/tar-builder.js +80 -80
  135. package/test/hot-memory.test.js +145 -145
  136. package/test/inject-content-cap.test.js +91 -0
  137. package/test/inject-first-round-bm25.test.js +89 -89
  138. package/test/inject-heat.test.js +61 -0
  139. package/test/inject-pools.test.js +170 -0
  140. package/test/inject-recall.test.js +105 -0
  141. package/test/inject-rotation.test.js +161 -161
  142. package/test/inject.test.js +283 -249
  143. package/test/lang.test.js +87 -87
  144. package/test/lib-smoke.test.js +96 -96
  145. package/test/llm-audit.test.js +462 -309
  146. package/test/llm-message-source.test.js +177 -177
  147. package/test/local-embedder.test.js +249 -249
  148. package/test/mcp-stdio.test.js +385 -0
  149. package/test/mirror-dirty.test.js +424 -424
  150. package/test/mirror-edit-digest.test.js +187 -187
  151. package/test/mirror-generation.test.js +499 -499
  152. package/test/mirror.test.js +249 -249
  153. package/test/narrative-bars.test.js +162 -0
  154. package/test/organize.test.js +418 -0
  155. package/test/peer-blockers.test.js +193 -193
  156. package/test/policy-epoch.test.js +259 -259
  157. package/test/quality-filter.test.js +186 -186
  158. package/test/reasoning-effort.test.js +746 -746
  159. package/test/recall-evals.test.js +235 -235
  160. package/test/recall-layer.test.js +326 -326
  161. package/test/recall-stats.test.js +123 -0
  162. package/test/receipt-chain.test.js +451 -451
  163. package/test/reflection.test.js +226 -226
  164. package/test/reindex-backfill.test.js +62 -62
  165. package/test/reranker-single-logit.test.js +73 -73
  166. package/test/reranker.test.js +197 -197
  167. package/test/resident-status.test.js +83 -0
  168. package/test/runtime-adopt.test.js +206 -206
  169. package/test/runtime-cli.test.js +89 -89
  170. package/test/runtime-closure.test.js +193 -193
  171. package/test/runtime-download.test.js +302 -302
  172. package/test/runtime-layout.test.js +163 -163
  173. package/test/runtime-loader.test.js +255 -237
  174. package/test/runtime-manifest.test.js +159 -158
  175. package/test/runtime-model-fetch.test.js +348 -348
  176. package/test/runtime-provision.test.js +101 -101
  177. package/test/runtime-tarball.test.js +81 -81
  178. package/test/runtime-verify.test.js +279 -214
  179. package/test/scope-candidates.test.js +239 -239
  180. package/test/scope-explicit.test.js +452 -452
  181. package/test/scope-retrieval.test.js +197 -197
  182. package/test/scope-storage.test.js +255 -255
  183. package/test/scope-strict.test.js +231 -231
  184. package/test/search-fusion.test.js +90 -90
  185. package/test/semantic.test.js +124 -124
  186. package/test/service-search.test.js +174 -174
  187. package/test/service.test.js +405 -405
  188. package/test/settings.test.js +261 -261
  189. package/test/skip-invalid-restore.test.js +262 -262
  190. package/test/sleep-action-set.test.js +422 -422
  191. package/test/sleep-heat.test.js +126 -125
  192. package/test/sleep-scheduler-rearm.test.js +89 -89
  193. package/test/sleep.test.js +410 -373
  194. package/test/standalone-api.test.js +544 -359
  195. package/test/store-vector-cache.test.js +98 -98
  196. package/test/store.test.js +396 -383
  197. package/test/stress.test.js +209 -209
  198. package/test/summarize.test.js +1288 -575
  199. package/test/tools-dto-schema-matrix.test.js +325 -0
  200. package/test/tools-memory-get-schema.test.js +65 -65
  201. package/test/tools.test.js +374 -341
  202. package/test/updated-at-semantics.test.js +113 -113
  203. package/test/vector-index.test.js +221 -221
  204. package/test/version-check.test.js +152 -152
  205. package/test/webServer-optional.test.js +82 -82
@@ -1,217 +1,224 @@
1
- // 生成 runtime-manifest.json(issue #131 / PR-C)
2
- //
3
- // **平台无关**是这份清单的核心设计:数据源是 `package-lock.json` 而不是「本机装好的那棵树」。
4
- // npm 的 lockfile 里包含**所有平台**的包条目(各自的 os / cpu 与 integrity),所以一份清单就能
5
- // 覆盖 win32 / darwin / linux × x64 / arm64 —— 下载时再按平台过滤(`matchesPlatform`)。
6
- //
7
- // 为什么不做成「按平台各存一份、谁在哪个系统上生成」:那样 macOS 用户在一台没人跑过脚本的机器上
8
- // 会看到「清单里没有 darwin-arm64 的条目」而直接卡住。清单是随包发布的静态数据,它必须一次覆盖
9
- // 所有平台,而不是依赖某个贡献者手上有对应的机器。
10
- //
11
- // 哈希取自 lockfile 的 integrity(npm 自己算的),所以维护成本是「改依赖后重跑一次脚本」。
12
- //
13
- // 用法:
14
- // node scripts/build-runtime-manifest.mjs # 生成/更新
15
- // node scripts/build-runtime-manifest.mjs --check # 只校验现有文件与 lockfile 一致(CI 用)
16
- import { existsSync, readFileSync, writeFileSync } from "node:fs";
17
- import { join } from "node:path";
18
- import { fileURLToPath } from "node:url";
19
- import { matchesPlatform } from "../src/runtime/closure.js";
20
-
21
- const ROOT = fileURLToPath(new URL("..", import.meta.url));
22
- const MANIFEST = join(ROOT, "runtime-manifest.json");
23
- const MANIFEST_VERSION = 2;
24
- const ENTRY = "@huggingface/transformers";
25
-
26
- /**
27
- * 证据:`@huggingface/transformers` 的 Node 构建(dist/transformers.node.mjs)顶层只 import
28
- * onnxruntime-node 与 sharp,从不 import onnxruntime-web(把一个空的 onnxruntime-web 打桩进去,
29
- * 嵌入结果逐字节相同:dims=[2,512]、L2 范数 1.0000、cos 0.3053)。而它解包后 127.7MB,占整个
30
- * 闭包约三分之一。所以清单里剔除它,风险由 verify 的真实推理兜底。
31
- */
32
- const EXCLUDED = { "onnxruntime-web": "Node 构建从不 import 它(打桩验证:嵌入结果逐字节相同)" };
33
-
34
- /**
35
- * 被剔除的包,以及**它子树下的一切**。
36
- *
37
- * 只按包名过滤会漏掉嵌套条目:闭包是按源布局镜像的,所以存在
38
- * `onnxruntime-web/node_modules/onnxruntime-common` 这种位于被剔除包内部的包 ——
39
- * 那个位置根本不会被取回来,留在清单里就是让下载器去填一个不存在的坑。
40
- * 同名但位于别处(例如顶层 onnxruntime-common)不受影响。
41
- */
42
- export function isExcluded(pkg) {
43
- return Object.keys(EXCLUDED).some(
44
- (name) => pkg.name === name || pkg.rel === name || pkg.rel.startsWith(`${name}/`)
45
- );
46
- }
47
-
48
- /** lockfile 的路径 → 包名(最后一段 `node_modules/` 之后的部分)。 */
49
- export function nameOfPath(path) {
50
- const marker = path.lastIndexOf("node_modules/");
51
- return marker === -1 ? path : path.slice(marker + "node_modules/".length);
52
- }
53
-
54
- /** lockfile 的路径 → payload 里的相对路径(去掉开头的 `node_modules/`)。 */
55
- export function relOfPath(path) {
56
- return path.startsWith("node_modules/") ? path.slice("node_modules/".length) : path;
57
- }
58
-
59
- /**
60
- * 按 Node 的解析语义列出候选路径:从引用者自身的 `node_modules` 逐级向上,最后兜底到根部。
61
- *
62
- * 例:引用者在 `node_modules/a/node_modules/b`、依赖 `x` →
63
- * `…/b/node_modules/x` → `node_modules/a/node_modules/x` → `node_modules/x`
64
- * @param {string} fromPath - 引用者的 lockfile 路径。
65
- * @param {string} name - 依赖名。
66
- * @returns {string[]} 候选路径,由近到远。
67
- */
68
- export function resolveCandidates(fromPath, name) {
69
- const out = [`${fromPath}/node_modules/${name}`];
70
- let cursor = fromPath;
71
- while (cursor.includes("/node_modules/")) {
72
- cursor = cursor.slice(0, cursor.lastIndexOf("/node_modules/"));
73
- out.push(cursor === "" ? `node_modules/${name}` : `${cursor}/node_modules/${name}`);
74
- }
75
- if (!out.includes(`node_modules/${name}`)) out.push(`node_modules/${name}`);
76
- return out;
77
- }
78
-
79
- /** registry 的 tarball 地址:lockfile 记了 resolved 就用它(与 integrity 同源),否则按 npm 约定推。 */
80
- function tarballUrl(name, version, resolved) {
81
- if (typeof resolved === "string" && resolved.startsWith("http")) return resolved;
82
- const base = name.startsWith("@") ? name.slice(name.indexOf("/") + 1) : name;
83
- return `https://registry.npmjs.org/${name}/-/${base}-${version}.tgz`;
84
- }
85
-
86
- /**
87
- * 从 lockfile 走一遍闭包。
88
- * @param {object} lock - package-lock.json 的内容。
89
- * @returns {{packages: object[], missing: string[]}} 包列表(按 rel 排序)与解析不到的依赖。
90
- */
91
- export function closureFromLockfile(lock) {
92
- const all = lock.packages ?? {};
93
- const known = new Set(
94
- Object.entries(all)
95
- .filter(([path, meta]) => path.includes("node_modules/") && meta?.version !== undefined)
96
- .map(([path]) => path)
97
- );
98
- const seen = new Map();
99
- const missing = [];
100
- const queue = [`node_modules/${ENTRY}`];
101
-
102
- while (queue.length > 0) {
103
- const path = queue.shift();
104
- if (seen.has(path)) continue;
105
- const meta = all[path];
106
- if (meta?.version === undefined) {
107
- missing.push(path);
108
- continue;
109
- }
110
- if (isExcluded({ name: nameOfPath(path), rel: relOfPath(path) })) continue;
111
- seen.set(path, meta);
112
-
113
- const deps = { ...(meta.dependencies ?? {}), ...(meta.optionalDependencies ?? {}) };
114
- for (const name of Object.keys(deps)) {
115
- const resolved = resolveCandidates(path, name).find((option) => known.has(option));
116
- if (resolved === undefined) missing.push(`${name}(被 ${relOfPath(path)} 需要)`);
117
- else queue.push(resolved);
118
- }
119
- }
120
-
121
- const packages = [...seen.entries()]
122
- .map(([path, meta]) => ({
123
- name: nameOfPath(path),
124
- version: meta.version,
125
- rel: relOfPath(path),
126
- integrity: meta.integrity ?? null,
127
- tarball: tarballUrl(nameOfPath(path), meta.version, meta.resolved),
128
- // 平台约束照抄 lockfile:下载器按它过滤。没有这三个字段 = 与平台无关。
129
- ...(Array.isArray(meta.os) ? { os: meta.os } : {}),
130
- ...(Array.isArray(meta.cpu) ? { cpu: meta.cpu } : {}),
131
- // libc 也带上:linux-x64 的 glibc 机器靠它剔掉两套 musl 变体(十几 MB)。
132
- // 下载器只在**确认**了本机 libc 时才据此过滤,拿不到证据就不看这个字段。
133
- ...(Array.isArray(meta.libc) ? { libc: meta.libc } : {})
134
- }))
135
- .sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
136
-
137
- return { packages, missing };
138
- }
139
-
140
- /** 某个平台实际需要取哪些包(与下载器用同一条判据)。 */
141
- export function packagesForPlatform(manifest, platform, arch, libc = null) {
142
- return manifest.packages.filter((pkg) => matchesPlatform(pkg, platform, arch, libc));
143
- }
144
-
145
- /** 生成一份**平台无关**的清单。 */
146
- export function build() {
147
- const lock = JSON.parse(readFileSync(join(ROOT, "package-lock.json"), "utf8"));
148
- const { packages, missing } = closureFromLockfile(lock);
149
- // 解析不到的依赖必须让生成**失败**:带缺口的清单会让用户在下载时才失败,而且「少了一个包」
150
- // 这种缺口结构检查看不出来(它只点三个必需包)。生成期的数据问题就该在生成期炸掉 ——
151
- // 这也是我第一版真正踩过的:被剔除包的孤儿依赖留在了清单里,而生成器一声不吭。
152
- if (missing.length > 0) {
153
- throw new Error(`这些依赖在 package-lock.json 里解析不到:${missing.slice(0, 10).join("、")}`);
154
- }
155
-
156
- const entry = packages.find((pkg) => pkg.rel === ENTRY);
157
- if (entry === undefined) throw new Error(`闭包入口没解析出来:${missing.join("; ") || "lockfile 里没有它"}`);
158
- // 查不到 sha512 就不能写进清单:那等于让下载器「先信任再校验」,是假校验。
159
- const withoutIntegrity = packages.filter(
160
- (pkg) => typeof pkg.integrity !== "string" || !pkg.integrity.startsWith("sha512-")
161
- );
162
- if (withoutIntegrity.length > 0) {
163
- throw new Error(`这些包在 package-lock.json 里没有 sha512 integrity:${withoutIntegrity.map((p) => p.rel).join("、")}`);
164
- }
165
-
166
- return {
167
- manifestVersion: MANIFEST_VERSION,
168
- entry: ENTRY,
169
- transformersVersion: entry.version,
170
- excluded: EXCLUDED,
171
- // 平台无关:下载时按 os/cpu/libc 过滤(payloadId 也由下载器按本机平台算)。
172
- packages
173
- };
174
- }
175
-
176
- /** 应当被覆盖的平台(自检与 CI 断言用)。 */
177
- export const COVERED_PLATFORMS = [
178
- ["win32", "x64"],
179
- ["win32", "arm64"],
180
- ["darwin", "x64"],
181
- ["darwin", "arm64"],
182
- ["linux", "x64"],
183
- ["linux", "arm64"]
184
- ];
185
-
186
- /** 命令行入口。`--check` 只校验,不写文件。 */
187
- export function main(args = process.argv.slice(2)) {
188
- const check = args.includes("--check");
189
- const built = build();
190
- const coverage = COVERED_PLATFORMS.map(([platform, arch]) => {
191
- const set = packagesForPlatform(built, platform, arch);
192
- const hasEntry = set.some((pkg) => pkg.rel === ENTRY);
193
- const hasNative = set.some((pkg) => pkg.rel === "onnxruntime-node");
194
- // linux 再报一次「确认 glibc 时」的包数:下载器会剔掉另一套 libc 的变体,这行让
195
- // 「剔掉了多少」在 --check 里看得见,而不是只能翻清单。
196
- const glibc = platform === "linux" ? `;确认 glibc 时 ${packagesForPlatform(built, platform, arch, "glibc").length} 个` : "";
197
- return `${platform}-${arch}: ${set.length} 个包${glibc}${hasEntry && hasNative ? "" : "(缺入口或原生依赖!)"}`;
198
- });
199
-
200
- if (check) {
201
- const existing = existsSync(MANIFEST) ? readFileSync(MANIFEST, "utf8") : "";
202
- const same = existing === `${JSON.stringify(built, null, 2)}\n`;
203
- process.stdout.write(`${same ? "✓" : "✗"} 清单与 package-lock.json ${same ? "一致" : "不一致,请重跑生成"}\n`);
204
- for (const line of coverage) process.stdout.write(` ${line}\n`);
205
- return same ? 0 : 1;
206
- }
207
-
208
- writeFileSync(MANIFEST, `${JSON.stringify(built, null, 2)}\n`, "utf8");
209
- process.stdout.write(`写入 ${MANIFEST}\n共 ${built.packages.length} 个包(已剔除 ${Object.keys(EXCLUDED).join("、")} 及其子树)\n`);
210
- for (const line of coverage) process.stdout.write(` ${line}\n`);
211
- return 0;
212
- }
213
-
214
- // 只有被直接执行时才干活:被 import 时(测试里)不该顺手改写清单文件。
215
- const invokedDirectly =
216
- process.argv[1] != null && import.meta.url.endsWith(process.argv[1].replace(/\\/g, "/").split("/").pop() ?? "");
217
- if (invokedDirectly) process.exitCode = main();
1
+ // 生成 runtime-manifest.json(issue #131 / PR-C)
2
+ //
3
+ // **平台无关**是这份清单的核心设计:数据源是 `package-lock.json` 而不是「本机装好的那棵树」。
4
+ // npm 的 lockfile 里包含**所有平台**的包条目(各自的 os / cpu 与 integrity),所以一份清单就能
5
+ // 覆盖 win32 / darwin / linux × x64 / arm64 —— 下载时再按平台过滤(`matchesPlatform`)。
6
+ //
7
+ // 为什么不做成「按平台各存一份、谁在哪个系统上生成」:那样 macOS 用户在一台没人跑过脚本的机器上
8
+ // 会看到「清单里没有 darwin-arm64 的条目」而直接卡住。清单是随包发布的静态数据,它必须一次覆盖
9
+ // 所有平台,而不是依赖某个贡献者手上有对应的机器。
10
+ //
11
+ // 哈希取自 lockfile 的 integrity(npm 自己算的),所以维护成本是「改依赖后重跑一次脚本」。
12
+ //
13
+ // 用法:
14
+ // node scripts/build-runtime-manifest.mjs # 生成/更新
15
+ // node scripts/build-runtime-manifest.mjs --check # 只校验现有文件与 lockfile 一致(CI 用)
16
+ import { existsSync, readFileSync, writeFileSync } from "node:fs";
17
+ import { join } from "node:path";
18
+ import { fileURLToPath } from "node:url";
19
+ import { matchesPlatform } from "../src/runtime/closure.js";
20
+
21
+ const ROOT = fileURLToPath(new URL("..", import.meta.url));
22
+ const MANIFEST = join(ROOT, "runtime-manifest.json");
23
+ const MANIFEST_VERSION = 2;
24
+ const ENTRY = "@huggingface/transformers";
25
+
26
+ /**
27
+ * 证据:`@huggingface/transformers` 的 Node 构建(dist/transformers.node.mjs)顶层只 import
28
+ * onnxruntime-node 与 sharp,从不 import onnxruntime-web(把一个空的 onnxruntime-web 打桩进去,
29
+ * 嵌入结果逐字节相同:dims=[2,512]、L2 范数 1.0000、cos 0.3053)。而它解包后 127.7MB,占整个
30
+ * 闭包约三分之一。所以清单里剔除它,风险由 verify 的真实推理兜底。
31
+ */
32
+ const EXCLUDED = {
33
+ "onnxruntime-web": "Node 构建从不 import 它(打桩验证:嵌入结果逐字节相同)",
34
+ // sharp 0.35 起把 WASM 回退包装进 optionalDependencies(freebsd/webcontainers 场景),
35
+ // 六个覆盖平台在 Node 里从不 import;wasm32 无 os 约束,会混进每个平台的 natives 断言。
36
+ "@img/sharp-freebsd-wasm32": "FreeBSD 专用 WASM 回退,覆盖平台之外",
37
+ "@img/sharp-webcontainers-wasm32": "WebContainers 专用 WASM 回退,覆盖平台之外",
38
+ "@img/sharp-wasm32": "无 os 约束的 WASM 回退,Node 构建从不 import",
39
+ };
40
+
41
+ /**
42
+ * 被剔除的包,以及**它子树下的一切**。
43
+ *
44
+ * 只按包名过滤会漏掉嵌套条目:闭包是按源布局镜像的,所以存在
45
+ * `onnxruntime-web/node_modules/onnxruntime-common` 这种位于被剔除包内部的包 ——
46
+ * 那个位置根本不会被取回来,留在清单里就是让下载器去填一个不存在的坑。
47
+ * 同名但位于别处(例如顶层 onnxruntime-common)不受影响。
48
+ */
49
+ export function isExcluded(pkg) {
50
+ return Object.keys(EXCLUDED).some(
51
+ (name) => pkg.name === name || pkg.rel === name || pkg.rel.startsWith(`${name}/`)
52
+ );
53
+ }
54
+
55
+ /** lockfile 的路径 → 包名(最后一段 `node_modules/` 之后的部分)。 */
56
+ export function nameOfPath(path) {
57
+ const marker = path.lastIndexOf("node_modules/");
58
+ return marker === -1 ? path : path.slice(marker + "node_modules/".length);
59
+ }
60
+
61
+ /** lockfile 的路径 → payload 里的相对路径(去掉开头的 `node_modules/`)。 */
62
+ export function relOfPath(path) {
63
+ return path.startsWith("node_modules/") ? path.slice("node_modules/".length) : path;
64
+ }
65
+
66
+ /**
67
+ * 按 Node 的解析语义列出候选路径:从引用者自身的 `node_modules` 逐级向上,最后兜底到根部。
68
+ *
69
+ * 例:引用者在 `node_modules/a/node_modules/b`、依赖 `x` →
70
+ * `…/b/node_modules/x` → `node_modules/a/node_modules/x` → `node_modules/x`
71
+ * @param {string} fromPath - 引用者的 lockfile 路径。
72
+ * @param {string} name - 依赖名。
73
+ * @returns {string[]} 候选路径,由近到远。
74
+ */
75
+ export function resolveCandidates(fromPath, name) {
76
+ const out = [`${fromPath}/node_modules/${name}`];
77
+ let cursor = fromPath;
78
+ while (cursor.includes("/node_modules/")) {
79
+ cursor = cursor.slice(0, cursor.lastIndexOf("/node_modules/"));
80
+ out.push(cursor === "" ? `node_modules/${name}` : `${cursor}/node_modules/${name}`);
81
+ }
82
+ if (!out.includes(`node_modules/${name}`)) out.push(`node_modules/${name}`);
83
+ return out;
84
+ }
85
+
86
+ /** registry 的 tarball 地址:lockfile 记了 resolved 就用它(与 integrity 同源),否则按 npm 约定推。 */
87
+ function tarballUrl(name, version, resolved) {
88
+ if (typeof resolved === "string" && resolved.startsWith("http")) return resolved;
89
+ const base = name.startsWith("@") ? name.slice(name.indexOf("/") + 1) : name;
90
+ return `https://registry.npmjs.org/${name}/-/${base}-${version}.tgz`;
91
+ }
92
+
93
+ /**
94
+ * 从 lockfile 走一遍闭包。
95
+ * @param {object} lock - package-lock.json 的内容。
96
+ * @returns {{packages: object[], missing: string[]}} 包列表(按 rel 排序)与解析不到的依赖。
97
+ */
98
+ export function closureFromLockfile(lock) {
99
+ const all = lock.packages ?? {};
100
+ const known = new Set(
101
+ Object.entries(all)
102
+ .filter(([path, meta]) => path.includes("node_modules/") && meta?.version !== undefined)
103
+ .map(([path]) => path)
104
+ );
105
+ const seen = new Map();
106
+ const missing = [];
107
+ const queue = [`node_modules/${ENTRY}`];
108
+
109
+ while (queue.length > 0) {
110
+ const path = queue.shift();
111
+ if (seen.has(path)) continue;
112
+ const meta = all[path];
113
+ if (meta?.version === undefined) {
114
+ missing.push(path);
115
+ continue;
116
+ }
117
+ if (isExcluded({ name: nameOfPath(path), rel: relOfPath(path) })) continue;
118
+ seen.set(path, meta);
119
+
120
+ const deps = { ...(meta.dependencies ?? {}), ...(meta.optionalDependencies ?? {}) };
121
+ for (const name of Object.keys(deps)) {
122
+ const resolved = resolveCandidates(path, name).find((option) => known.has(option));
123
+ if (resolved === undefined) missing.push(`${name}(被 ${relOfPath(path)} 需要)`);
124
+ else queue.push(resolved);
125
+ }
126
+ }
127
+
128
+ const packages = [...seen.entries()]
129
+ .map(([path, meta]) => ({
130
+ name: nameOfPath(path),
131
+ version: meta.version,
132
+ rel: relOfPath(path),
133
+ integrity: meta.integrity ?? null,
134
+ tarball: tarballUrl(nameOfPath(path), meta.version, meta.resolved),
135
+ // 平台约束照抄 lockfile:下载器按它过滤。没有这三个字段 = 与平台无关。
136
+ ...(Array.isArray(meta.os) ? { os: meta.os } : {}),
137
+ ...(Array.isArray(meta.cpu) ? { cpu: meta.cpu } : {}),
138
+ // libc 也带上:linux-x64 的 glibc 机器靠它剔掉两套 musl 变体(十几 MB)。
139
+ // 下载器只在**确认**了本机 libc 时才据此过滤,拿不到证据就不看这个字段。
140
+ ...(Array.isArray(meta.libc) ? { libc: meta.libc } : {})
141
+ }))
142
+ .sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
143
+
144
+ return { packages, missing };
145
+ }
146
+
147
+ /** 某个平台实际需要取哪些包(与下载器用同一条判据)。 */
148
+ export function packagesForPlatform(manifest, platform, arch, libc = null) {
149
+ return manifest.packages.filter((pkg) => matchesPlatform(pkg, platform, arch, libc));
150
+ }
151
+
152
+ /** 生成一份**平台无关**的清单。 */
153
+ export function build() {
154
+ const lock = JSON.parse(readFileSync(join(ROOT, "package-lock.json"), "utf8"));
155
+ const { packages, missing } = closureFromLockfile(lock);
156
+ // 解析不到的依赖必须让生成**失败**:带缺口的清单会让用户在下载时才失败,而且「少了一个包」
157
+ // 这种缺口结构检查看不出来(它只点三个必需包)。生成期的数据问题就该在生成期炸掉 ——
158
+ // 这也是我第一版真正踩过的:被剔除包的孤儿依赖留在了清单里,而生成器一声不吭。
159
+ if (missing.length > 0) {
160
+ throw new Error(`这些依赖在 package-lock.json 里解析不到:${missing.slice(0, 10).join("、")}`);
161
+ }
162
+
163
+ const entry = packages.find((pkg) => pkg.rel === ENTRY);
164
+ if (entry === undefined) throw new Error(`闭包入口没解析出来:${missing.join("; ") || "lockfile 里没有它"}`);
165
+ // 查不到 sha512 就不能写进清单:那等于让下载器「先信任再校验」,是假校验。
166
+ const withoutIntegrity = packages.filter(
167
+ (pkg) => typeof pkg.integrity !== "string" || !pkg.integrity.startsWith("sha512-")
168
+ );
169
+ if (withoutIntegrity.length > 0) {
170
+ throw new Error(`这些包在 package-lock.json 里没有 sha512 integrity:${withoutIntegrity.map((p) => p.rel).join("、")}`);
171
+ }
172
+
173
+ return {
174
+ manifestVersion: MANIFEST_VERSION,
175
+ entry: ENTRY,
176
+ transformersVersion: entry.version,
177
+ excluded: EXCLUDED,
178
+ // 平台无关:下载时按 os/cpu/libc 过滤(payloadId 也由下载器按本机平台算)。
179
+ packages
180
+ };
181
+ }
182
+
183
+ /** 应当被覆盖的平台(自检与 CI 断言用)。 */
184
+ export const COVERED_PLATFORMS = [
185
+ ["win32", "x64"],
186
+ ["win32", "arm64"],
187
+ ["darwin", "x64"],
188
+ ["darwin", "arm64"],
189
+ ["linux", "x64"],
190
+ ["linux", "arm64"]
191
+ ];
192
+
193
+ /** 命令行入口。`--check` 只校验,不写文件。 */
194
+ export function main(args = process.argv.slice(2)) {
195
+ const check = args.includes("--check");
196
+ const built = build();
197
+ const coverage = COVERED_PLATFORMS.map(([platform, arch]) => {
198
+ const set = packagesForPlatform(built, platform, arch);
199
+ const hasEntry = set.some((pkg) => pkg.rel === ENTRY);
200
+ const hasNative = set.some((pkg) => pkg.rel === "onnxruntime-node");
201
+ // linux 再报一次「确认 glibc 时」的包数:下载器会剔掉另一套 libc 的变体,这行让
202
+ // 「剔掉了多少」在 --check 里看得见,而不是只能翻清单。
203
+ const glibc = platform === "linux" ? `;确认 glibc 时 ${packagesForPlatform(built, platform, arch, "glibc").length} 个` : "";
204
+ return `${platform}-${arch}: ${set.length} 个包${glibc}${hasEntry && hasNative ? "" : "(缺入口或原生依赖!)"}`;
205
+ });
206
+
207
+ if (check) {
208
+ const existing = existsSync(MANIFEST) ? readFileSync(MANIFEST, "utf8") : "";
209
+ const same = existing === `${JSON.stringify(built, null, 2)}\n`;
210
+ process.stdout.write(`${same ? "✓" : "✗"} 清单与 package-lock.json ${same ? "一致" : "不一致,请重跑生成"}\n`);
211
+ for (const line of coverage) process.stdout.write(` ${line}\n`);
212
+ return same ? 0 : 1;
213
+ }
214
+
215
+ writeFileSync(MANIFEST, `${JSON.stringify(built, null, 2)}\n`, "utf8");
216
+ process.stdout.write(`写入 ${MANIFEST}\n共 ${built.packages.length} 个包(已剔除 ${Object.keys(EXCLUDED).join("、")} 及其子树)\n`);
217
+ for (const line of coverage) process.stdout.write(` ${line}\n`);
218
+ return 0;
219
+ }
220
+
221
+ // 只有被直接执行时才干活:被 import 时(测试里)不该顺手改写清单文件。
222
+ const invokedDirectly =
223
+ process.argv[1] != null && import.meta.url.endsWith(process.argv[1].replace(/\\/g, "/").split("/").pop() ?? "");
224
+ if (invokedDirectly) process.exitCode = main();
@@ -1,42 +1,42 @@
1
- // 发布前校验 src/ 与 lib/ 一致性。
2
- //
3
- // npm 包实际加载 lib/(package main → lib/index.js),而 root 层 `npm publish`
4
- // 不触发 dsh-mneme/ 的 prepack → sync 同步。历史教训:PR #60 只改了 src 忘了同步
5
- // lib,v0.7.8 发出去的 npm 包跑的还是旧代码(issue #65)。此脚本让这类漂移在发布时
6
- // 直接 fail,而不是带着旧产物上线。
7
- //
8
- // 用法:node scripts/check-sync.js (root package.json 的 prepack 钩子自动调用)
9
- import { existsSync, readFileSync, readdirSync } from "node:fs";
10
- import { join, relative } from "node:path";
11
- import { fileURLToPath } from "node:url";
12
-
13
- const root = join(fileURLToPath(new URL("..", import.meta.url))); // dsh-mneme/
14
- const srcDir = join(root, "src");
15
- const libDir = join(root, "lib");
16
-
17
- function walk(dir) {
18
- const out = [];
19
- for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
20
- const full = join(dir, entry.name);
21
- if (entry.isDirectory()) out.push(...walk(full));
22
- else if (entry.isFile()) out.push(full);
23
- }
24
- return out;
25
- }
26
-
27
- const bad = [];
28
- for (const file of walk(srcDir)) {
29
- const rel = relative(srcDir, file);
30
- const dest = join(libDir, rel);
31
- if (!existsSync(dest)) {
32
- bad.push(`missing lib/${rel}`);
33
- } else if (!readFileSync(file).equals(readFileSync(dest))) {
34
- bad.push(`differ lib/${rel}`);
35
- }
36
- }
37
- if (bad.length) {
38
- console.error(`✗ src/ 与 lib/ 不一致(${bad.length} 处)——npm 包实际加载 lib/,请先 npm run sync 并提交:`);
39
- for (const line of bad) console.error(` ${line}`);
40
- process.exit(1);
41
- }
42
- console.log(`✓ src/ 与 lib/ 一致(${walk(srcDir).length} 个文件)`);
1
+ // 发布前校验 src/ 与 lib/ 一致性。
2
+ //
3
+ // npm 包实际加载 lib/(package main → lib/index.js),而 root 层 `npm publish`
4
+ // 不触发 dsh-mneme/ 的 prepack → sync 同步。历史教训:PR #60 只改了 src 忘了同步
5
+ // lib,v0.7.8 发出去的 npm 包跑的还是旧代码(issue #65)。此脚本让这类漂移在发布时
6
+ // 直接 fail,而不是带着旧产物上线。
7
+ //
8
+ // 用法:node scripts/check-sync.js (root package.json 的 prepack 钩子自动调用)
9
+ import { existsSync, readFileSync, readdirSync } from "node:fs";
10
+ import { join, relative } from "node:path";
11
+ import { fileURLToPath } from "node:url";
12
+
13
+ const root = join(fileURLToPath(new URL("..", import.meta.url))); // dsh-mneme/
14
+ const srcDir = join(root, "src");
15
+ const libDir = join(root, "lib");
16
+
17
+ function walk(dir) {
18
+ const out = [];
19
+ for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
20
+ const full = join(dir, entry.name);
21
+ if (entry.isDirectory()) out.push(...walk(full));
22
+ else if (entry.isFile()) out.push(full);
23
+ }
24
+ return out;
25
+ }
26
+
27
+ const bad = [];
28
+ for (const file of walk(srcDir)) {
29
+ const rel = relative(srcDir, file);
30
+ const dest = join(libDir, rel);
31
+ if (!existsSync(dest)) {
32
+ bad.push(`missing lib/${rel}`);
33
+ } else if (!readFileSync(file).equals(readFileSync(dest))) {
34
+ bad.push(`differ lib/${rel}`);
35
+ }
36
+ }
37
+ if (bad.length) {
38
+ console.error(`✗ src/ 与 lib/ 不一致(${bad.length} 处)——npm 包实际加载 lib/,请先 npm run sync 并提交:`);
39
+ for (const line of bad) console.error(` ${line}`);
40
+ process.exit(1);
41
+ }
42
+ console.log(`✓ src/ 与 lib/ 一致(${walk(srcDir).length} 个文件)`);