modelstage 0.0.0-stage → 0.5.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +274 -2
- package/THIRD_PARTY_NOTICES.md +90 -0
- package/bin/modelstage.mjs +137 -0
- package/config/diag-upload.example.json +7 -0
- package/config/models/live2d.assetbundls.json +1 -0
- package/config/models/live2d.azurlane.json +1 -0
- package/config/models/live2d.azurlane_bundle.json +1 -0
- package/config/models/live2d.azurlane_spine.json +1 -0
- package/config/models/live2d.bd2.json +1 -0
- package/config/models/live2d.bluearchive.json +1 -0
- package/config/models/live2d.live2d.json +1 -0
- package/config/models/live2d.nikke.json +1 -0
- package/config/models/live2d.nikke_2.json +1 -0
- package/config/models/live2d.npc.json +1 -0
- package/config/models/spine.assetbundls.json +1 -0
- package/config/models/spine.azurlane.json +1 -0
- package/config/models/spine.azurlane_bundle.json +1 -0
- package/config/models/spine.azurlane_spine.json +1 -0
- package/config/models/spine.bd2.json +1 -0
- package/config/models/spine.bluearchive.json +1 -0
- package/config/models/spine.live2d.json +1 -0
- package/config/models/spine.nikke.json +1 -0
- package/config/models/spine.nikke_2.json +1 -0
- package/config/models/spine.npc.json +1 -0
- package/config/names/NIKKE.json +1058 -0
- package/config/names/README.md +47 -0
- package/config/names//345/244/251/345/221/275/344/271/213/345/255/220.json +42897 -0
- package/config/names//346/243/225/350/211/262/345/260/230/345/237/2032.json +4016 -0
- package/config/names//347/242/247/350/223/235/350/210/252/347/272/277.json +38342 -0
- package/config/plugins.json +13 -0
- package/config/resources.example.json +40 -0
- package/package.json +63 -4
- package/plugins/unity-live2d-bundle/README.md +98 -0
- package/plugins/unity-live2d-bundle/index.js +891 -0
- package/plugins/unity-live2d-bundle/lib/anim-to-motion.js +402 -0
- package/plugins/unity-live2d-bundle/lib/bundle-probe.js +244 -0
- package/plugins/unity-live2d-bundle/lib/bytes.js +189 -0
- package/plugins/unity-live2d-bundle/lib/common-strings.js +46 -0
- package/plugins/unity-live2d-bundle/lib/dxt.js +197 -0
- package/plugins/unity-live2d-bundle/lib/errors.js +71 -0
- package/plugins/unity-live2d-bundle/lib/etc2.js +371 -0
- package/plugins/unity-live2d-bundle/lib/extract.js +242 -0
- package/plugins/unity-live2d-bundle/lib/lzma-d-min.LICENSE.txt +21 -0
- package/plugins/unity-live2d-bundle/lib/lzma-d-min.js +34 -0
- package/plugins/unity-live2d-bundle/lib/model3.js +137 -0
- package/plugins/unity-live2d-bundle/lib/png.js +424 -0
- package/plugins/unity-live2d-bundle/lib/serialized.js +233 -0
- package/plugins/unity-live2d-bundle/lib/texdecode.js +190 -0
- package/plugins/unity-live2d-bundle/lib/typetree.js +215 -0
- package/plugins/unity-live2d-bundle/lib/unityfs.js +396 -0
- package/plugins/unity-live2d-bundle/plugin.json +81 -0
- package/plugins/unity-live2d-bundle/profiles/unityfs-live2d.json +26 -0
- package/plugins/unity-spine-bundle/index.js +535 -0
- package/plugins/unity-spine-bundle/lib/bytes.js +189 -0
- package/plugins/unity-spine-bundle/lib/common-strings.js +46 -0
- package/plugins/unity-spine-bundle/lib/dxt.js +197 -0
- package/plugins/unity-spine-bundle/lib/errors.js +79 -0
- package/plugins/unity-spine-bundle/lib/etc2.js +371 -0
- package/plugins/unity-spine-bundle/lib/extract.js +242 -0
- package/plugins/unity-spine-bundle/lib/lzma-d-min.LICENSE.txt +21 -0
- package/plugins/unity-spine-bundle/lib/lzma-d-min.js +34 -0
- package/plugins/unity-spine-bundle/lib/png.js +424 -0
- package/plugins/unity-spine-bundle/lib/serialized.js +233 -0
- package/plugins/unity-spine-bundle/lib/texdecode.js +190 -0
- package/plugins/unity-spine-bundle/lib/typetree.js +215 -0
- package/plugins/unity-spine-bundle/lib/unityfs.js +396 -0
- package/plugins/unity-spine-bundle/plugin.json +75 -0
- package/src/server/README.md +69 -0
- package/src/server/assets/browse.mjs +56 -0
- package/src/server/assets/bundle-probe.mjs +94 -0
- package/src/server/assets/probe.mjs +184 -0
- package/src/server/core/config.mjs +173 -0
- package/src/server/core/env-num.mjs +60 -0
- package/src/server/core/env.mjs +49 -0
- package/src/server/core/fs-portable.mjs +37 -0
- package/src/server/core/log.mjs +107 -0
- package/src/server/core/paths.mjs +44 -0
- package/src/server/data/catalog.mjs +1091 -0
- package/src/server/data/entry-states.mjs +499 -0
- package/src/server/data/live2d.mjs +272 -0
- package/src/server/data/manifest-cache.mjs +58 -0
- package/src/server/data/manifest-stamp.mjs +135 -0
- package/src/server/data/model-meta.mjs +114 -0
- package/src/server/data/names.mjs +106 -0
- package/src/server/data/shape-probe-worker.mjs +33 -0
- package/src/server/data/shape-probe.mjs +626 -0
- package/src/server/data/spine.mjs +336 -0
- package/src/server/data/tree.mjs +126 -0
- package/src/server/http/atomic-json.mjs +145 -0
- package/src/server/http/background-solids.mjs +270 -0
- package/src/server/http/backgrounds.mjs +388 -0
- package/src/server/http/diag-upload.mjs +264 -0
- package/src/server/http/diag.mjs +173 -0
- package/src/server/http/ready.mjs +167 -0
- package/src/server/http/static.mjs +312 -0
- package/src/server/http/write.mjs +654 -0
- package/src/server/index.mjs +114 -0
- package/src/server/plugins/builtin-providers.json +30 -0
- package/src/server/plugins/registry.mjs +650 -0
- package/src/server/plugins/slot-catalog.mjs +286 -0
- package/src/server/routes/background-solids.mjs +56 -0
- package/src/server/routes/backgrounds.mjs +77 -0
- package/src/server/routes/config.mjs +38 -0
- package/src/server/routes/data.mjs +313 -0
- package/src/server/routes/diag.mjs +37 -0
- package/src/server/routes/entries.mjs +170 -0
- package/src/server/routes/ready.mjs +122 -0
- package/src/server/serve.mjs +396 -0
- package/src/server/unpack/capacity.mjs +217 -0
- package/src/server/unpack/mem-probe.mjs +27 -0
- package/src/server/unpack/plugin-runner.mjs +529 -0
- package/src/server/unpack/plugin-store.mjs +736 -0
- package/src/server/unpack/probe-store.mjs +73 -0
- package/src/server/unpack/routes.mjs +359 -0
- package/src/server/unpack/runtime/exec.mjs +558 -0
- package/src/server/unpack/runtime/index.mjs +141 -0
- package/src/server/unpack/runtime/node-vm.mjs +111 -0
- package/src/server/unpack/unpack-worker.mjs +105 -0
- package/src/server/unpack/worker-client.mjs +184 -0
- package/src/server/unpack/write-result.mjs +84 -0
- package/src/shared/bundle-shape.mjs +83 -0
- package/src/shared/entry-reasons.mjs +119 -0
- package/src/shared/names.mjs +45 -0
- package/src/shared/type-labels.mjs +31 -0
- package/src/web/README.md +193 -0
- package/src/web/app/anim-list.js +222 -0
- package/src/web/app/background-solids.js +150 -0
- package/src/web/app/backgrounds.js +152 -0
- package/src/web/app/camera-log.js +28 -0
- package/src/web/app/context.js +72 -0
- package/src/web/app/dir-picker.js +242 -0
- package/src/web/app/entry-report.js +71 -0
- package/src/web/app/memory-bars.js +91 -0
- package/src/web/app/memory-budget.js +228 -0
- package/src/web/app/memory-guard.js +204 -0
- package/src/web/app/model-actions.js +682 -0
- package/src/web/app/renderer-setting.js +70 -0
- package/src/web/app/root-manager.js +254 -0
- package/src/web/app/status.js +138 -0
- package/src/web/app/topbar-actions.js +357 -0
- package/src/web/app/transport-actions.js +297 -0
- package/src/web/app/ui-state.js +559 -0
- package/src/web/app/url-params.js +171 -0
- package/src/web/app/view-settings.js +186 -0
- package/src/web/components.css +524 -0
- package/src/web/debug/diag-hud.js +95 -0
- package/src/web/debug/drag.js +369 -0
- package/src/web/debug/log.js +320 -0
- package/src/web/debug/model-cmp.js +292 -0
- package/src/web/debug/panel.css +474 -0
- package/src/web/debug/panel.js +712 -0
- package/src/web/debug/race-probe.js +82 -0
- package/src/web/debug/upload.js +130 -0
- package/src/web/engine/adapters/atlas.js +47 -0
- package/src/web/engine/adapters/index.js +22 -0
- package/src/web/engine/adapters/renderer.js +223 -0
- package/src/web/engine/adapters/skeleton.js +105 -0
- package/src/web/engine/adapters/texture.js +64 -0
- package/src/web/engine/animation.js +181 -0
- package/src/web/engine/appearance.js +136 -0
- package/src/web/engine/camera-adapter.js +212 -0
- package/src/web/engine/effect-chain.js +138 -0
- package/src/web/engine/engine-state.js +120 -0
- package/src/web/engine/filters.js +224 -0
- package/src/web/engine/fit.js +197 -0
- package/src/web/engine/frame-gate.js +33 -0
- package/src/web/engine/gl-driver.js +56 -0
- package/src/web/engine/gl.js +77 -0
- package/src/web/engine/index.js +20 -0
- package/src/web/engine/loader.js +55 -0
- package/src/web/engine/model-info.js +26 -0
- package/src/web/engine/pipeline.js +200 -0
- package/src/web/engine/player.js +427 -0
- package/src/web/engine/pma/blend.js +76 -0
- package/src/web/engine/pma/detect.js +118 -0
- package/src/web/engine/pma/index.js +8 -0
- package/src/web/engine/pma/resolve.js +112 -0
- package/src/web/engine/pma/upload.js +127 -0
- package/src/web/engine/pma.js +5 -0
- package/src/web/engine/render-frame.js +93 -0
- package/src/web/engine/runtime-capabilities.js +209 -0
- package/src/web/engine/runtime.js +37 -0
- package/src/web/engine/skins.js +165 -0
- package/src/web/engine/textures.js +174 -0
- package/src/web/index.html +117 -0
- package/src/web/infra/api.js +531 -0
- package/src/web/infra/diag.js +509 -0
- package/src/web/infra/document.js +269 -0
- package/src/web/infra/dom.js +31 -0
- package/src/web/infra/events.mjs +27 -0
- package/src/web/infra/log.js +153 -0
- package/src/web/infra/mem-stats.js +112 -0
- package/src/web/infra/plugin-host-registration.mjs +285 -0
- package/src/web/infra/plugin-host-report.mjs +32 -0
- package/src/web/infra/plugin-host-unload.mjs +79 -0
- package/src/web/infra/plugin-host-unpack.mjs +899 -0
- package/src/web/infra/plugin-host-warmup.mjs +180 -0
- package/src/web/infra/plugin-host.js +628 -0
- package/src/web/infra/ready.js +461 -0
- package/src/web/infra/schedule.js +51 -0
- package/src/web/infra/shortcuts.js +129 -0
- package/src/web/infra/slot-registry.js +402 -0
- package/src/web/infra/vfs.js +247 -0
- package/src/web/live2d/bridge.js +400 -0
- package/src/web/live2d/model.js +321 -0
- package/src/web/live2d/overlay.js +142 -0
- package/src/web/live2d/pixi-app.js +111 -0
- package/src/web/live2d/playback-probe.js +92 -0
- package/src/web/live2d/probe-geometry.js +329 -0
- package/src/web/live2d/probe.js +119 -0
- package/src/web/live2d/release.js +66 -0
- package/src/web/live2d/runtime-adapter.js +289 -0
- package/src/web/live2d/stage-ctx.js +152 -0
- package/src/web/live2d/stage.js +204 -0
- package/src/web/live2d/vfs-settings.js +174 -0
- package/src/web/live2d/view.js +89 -0
- package/src/web/main.js +629 -0
- package/src/web/panel/background-list.js +373 -0
- package/src/web/panel/bg-source.js +291 -0
- package/src/web/panel/color-picker.css +199 -0
- package/src/web/panel/color-picker.js +278 -0
- package/src/web/panel/dialog.css +116 -0
- package/src/web/panel/dialog.js +173 -0
- package/src/web/panel/dom.js +35 -0
- package/src/web/panel/icons.js +58 -0
- package/src/web/panel/memory-bar.js +195 -0
- package/src/web/panel/panel-settings.js +1263 -0
- package/src/web/panel/panel.css +1304 -0
- package/src/web/panel/render-choice.js +104 -0
- package/src/web/panel/schema.js +784 -0
- package/src/web/panel/settings-io.js +291 -0
- package/src/web/panel/shell-controls.js +274 -0
- package/src/web/panel/slider.js +210 -0
- package/src/web/panel/toolbar-schema.js +111 -0
- package/src/web/panel/toolbar.js +195 -0
- package/src/web/panel/transport-picker.js +375 -0
- package/src/web/panel/transport.js +321 -0
- package/src/web/panel/view.js +553 -0
- package/src/web/plugin/unpack-worker.js +185 -0
- package/src/web/render/camera/constants.js +60 -0
- package/src/web/render/camera/index.js +123 -0
- package/src/web/render/camera/spine.js +124 -0
- package/src/web/render/capture.js +30 -0
- package/src/web/render/interaction.js +527 -0
- package/src/web/render/live2d.js +627 -0
- package/src/web/render/playback.js +47 -0
- package/src/web/render/reset-view.js +29 -0
- package/src/web/render/spine.js +386 -0
- package/src/web/render/surface-switch.js +176 -0
- package/src/web/render/surface.js +571 -0
- package/src/web/render/surfaces.js +70 -0
- package/src/web/roster/filter.js +31 -0
- package/src/web/roster/list.js +276 -0
- package/src/web/roster/persist.js +92 -0
- package/src/web/roster/reveal.js +77 -0
- package/src/web/roster/rows.js +248 -0
- package/src/web/roster/snapshot.js +39 -0
- package/src/web/roster/type-filter.js +97 -0
- package/src/web/roster/view.js +761 -0
- package/src/web/spine/canvas-api-groups.js +267 -0
- package/src/web/spine/canvas-api.js +97 -0
- package/src/web/spine/canvas-document.js +80 -0
- package/src/web/spine/canvas-engine.js +193 -0
- package/src/web/spine/canvas-entry.js +28 -0
- package/src/web/spine/canvas-pixels.js +36 -0
- package/src/web/spine/canvas-runtime.js +127 -0
- package/src/web/spine/canvas-view.js +108 -0
- package/src/web/spine/canvas.js +7 -0
- package/src/web/spine/runtime-hint.js +73 -0
- package/src/web/spine/runtimes.js +212 -0
- package/src/web/style.css +493 -0
- package/tool/lib/bundle-probe-node.mjs +121 -0
- package/tool/lib/paths.mjs +20 -0
- package/tool/lib/runtime-guides.mjs +65 -0
- package/tool/lib/runtime-package.mjs +403 -0
- package/tool/lib/spine-meta.mjs +227 -0
- package/tool/lib/spine-motions.mjs +104 -0
- package/tool/parse-jsonc.mjs +86 -0
- package/tool/resolve-live2d-runtimes.mjs +276 -0
- package/tool/spine-motion-rules.json +101 -0
|
@@ -0,0 +1,626 @@
|
|
|
1
|
+
// data/shape-probe.mjs —— 后台形态探测(契约《条目状态与展示归属》§八)
|
|
2
|
+
//
|
|
3
|
+
// 职责:把"这个容器包到底能不能装"从【点击才知道】变成【不必点击也有结论】。
|
|
4
|
+
// S4 要求"未点击的美术依赖包该行已带 ⚠",而 catalog 的同步路径**禁止**解析包体
|
|
5
|
+
// (同根数百包会把毫秒级拖成分钟级),所以这件事只能由这里在后台补。
|
|
6
|
+
//
|
|
7
|
+
// 三条硬约束(写成代码约束,不是靠自觉):
|
|
8
|
+
// · INV-D7 事件驱动,不是常驻轮询:只有 kick() 被调用时才跑;没有待探测项就**立刻返回**,
|
|
9
|
+
// 连计数都不加(S10 的"启动次数 = 0"就是靠这条成立的)。
|
|
10
|
+
// · 跑完即停:一次 kick 处理一批(maxPerRun)就退出,不留任何定时器;剩余条目留在队列里,
|
|
11
|
+
// 等下一次触发(下一次 /api/catalog)继续。
|
|
12
|
+
// · 不抢装载资源:同一个进程内**串行**、包里逐个读、包间限速(delayMs),
|
|
13
|
+
// 并且单包超过 maxBytes 直接跳过(44MB 级的包解析峰值更高,不能和用户在点的那一发抢内存)。
|
|
14
|
+
//
|
|
15
|
+
// 写回只填 support(putProbe),绝不覆盖 loadResult —— 那是当事端上报的(INV-D1)。
|
|
16
|
+
|
|
17
|
+
import fs from 'node:fs';
|
|
18
|
+
import os from 'node:os';
|
|
19
|
+
import path from 'node:path';
|
|
20
|
+
import { Worker } from 'node:worker_threads';
|
|
21
|
+
|
|
22
|
+
// 与 /api/unpack 的判型**同一份实现**(契约 §八"判定复用"):
|
|
23
|
+
// 本文件在 src/server/data/ ⇒ 项目根要上溯三层
|
|
24
|
+
import { probeBundleBytes } from '../../../tool/lib/bundle-probe-node.mjs';
|
|
25
|
+
import { providerFingerprintOf, assetFingerprintOf } from './entry-states.mjs';
|
|
26
|
+
|
|
27
|
+
/** 包间停顿:给装载/解包让路(这些包解析是纯 CPU + IO,连跑会顶满一个核) */
|
|
28
|
+
const DEFAULT_DELAY_MS = 250;
|
|
29
|
+
/** 单次 kick 最多处理多少个包(跑完即停;余量留给下次触发)。
|
|
30
|
+
* 取小值是为了把"一轮"压到几十秒内 —— 探测是后台补充信息,不该长时间占着 IO。 */
|
|
31
|
+
const DEFAULT_MAX_PER_RUN = 8;
|
|
32
|
+
/** 单包上限:超过就不探(读进内存 + 解析峰值太高,见文件头"不抢装载资源") */
|
|
33
|
+
const DEFAULT_MAX_BYTES = 96 * 1024 * 1024;
|
|
34
|
+
/**
|
|
35
|
+
* 距上一次请求多久算"真空闲"(毫秒),只用于**批次开始**;可用 MODELSTAGE_PROBE_IDLE_GAP_MS 覆盖。
|
|
36
|
+
*
|
|
37
|
+
* 为什么批次开始要保守(2s):解析已移入 worker(CPU 不占主线程),但它读 40MB 包时
|
|
38
|
+
* 会与 catalog 的 CIFS 扫描抢 IO —— 实测门控太松时 catalog P95 会 +58ms。
|
|
39
|
+
* 批内则改用宽松判据(只要求"没有请求在飞"),否则密集浏览下永远跑不完一批。
|
|
40
|
+
*/
|
|
41
|
+
const DEFAULT_IDLE_GAP_MS = 2000;
|
|
42
|
+
/** 一批最多为"等空闲"耗多久;超时就把剩余条目放回队列,等下次触发(毫秒) */
|
|
43
|
+
const DEFAULT_MAX_WAIT_MS = 20000;
|
|
44
|
+
/** 等空闲时的轮询步长:越短越容易抢到刚出现的空隙,代价是空转次数更多 */
|
|
45
|
+
const WAIT_STEP_MS = 200;
|
|
46
|
+
/**
|
|
47
|
+
* 内存判据的缺省值(都可用 env 覆盖,见 index.mjs 的接线)。
|
|
48
|
+
* BASELINE:探测之外系统还要留的余量(别把可用内存吃到 0)
|
|
49
|
+
* FACTOR :包体积 → 预估占用 的倍数。**不能按 1:1 估**:解包产物(纹理/RGBA 缓冲)
|
|
50
|
+
* 通常比包本身大好几倍,按 1 估等于默认"装得下",那这道门就白加了。
|
|
51
|
+
*/
|
|
52
|
+
const DEFAULT_MEM_BASELINE_BYTES = 256 * 1024 * 1024;
|
|
53
|
+
const DEFAULT_MEM_FACTOR = 3;
|
|
54
|
+
|
|
55
|
+
const sleep = (ms) => new Promise((r) => { const t = setTimeout(r, ms); if (typeof t.unref === 'function') t.unref(); });
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* 字节数 → 可读 MB(日志用;拿不到值给 '?')。
|
|
59
|
+
* @param {number|null|undefined} n
|
|
60
|
+
* @returns {string}
|
|
61
|
+
*/
|
|
62
|
+
function fmtMb(n) {
|
|
63
|
+
const v = Number(n);
|
|
64
|
+
return Number.isFinite(v) && v >= 0 ? `${Math.round(v / 1024 / 1024)}MB` : '?';
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* 读"可用内存"(字节)。**优先 MemAvailable,不是 MemFree**:
|
|
69
|
+
* MemFree 不含可回收的页缓存,在 NAS 上常年只有几十 MB,按它判会把探测永久锁死。
|
|
70
|
+
* 取不到就返回 null —— 调用方按"放行"处理(fail-open:宁可偶尔多探一个包,也不要因为读不到 /proc 就停摆)。
|
|
71
|
+
* @returns {number|null}
|
|
72
|
+
*/
|
|
73
|
+
function readMemAvailable() {
|
|
74
|
+
return readMemAvailableInfo().bytes;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* 同上,但连"这个数是怎么来的"一起返回(排障要能分辨 cgroup 与宿主口径)。
|
|
79
|
+
* @returns {{bytes: number|null, source: 'cgroup-v2'|'MemAvailable'|'freemem'|'none'}}
|
|
80
|
+
*/
|
|
81
|
+
function readMemAvailableInfo() {
|
|
82
|
+
const cg = readCgroupAvailable();
|
|
83
|
+
if (cg && Number.isFinite(cg.bytes) && cg.bytes > 0) return { bytes: cg.bytes, source: 'cgroup-v2' };
|
|
84
|
+
try {
|
|
85
|
+
const txt = fs.readFileSync('/proc/meminfo', 'utf8');
|
|
86
|
+
const m = /^MemAvailable:\s+(\d+)\s+kB/m.exec(txt);
|
|
87
|
+
if (m) {
|
|
88
|
+
const n = Number(m[1]) * 1024;
|
|
89
|
+
if (Number.isFinite(n) && n > 0) return { bytes: n, source: 'MemAvailable' };
|
|
90
|
+
}
|
|
91
|
+
} catch { /* 非 Linux / 容器里没有 procfs:走回落 */ }
|
|
92
|
+
try {
|
|
93
|
+
const n = os.freemem();
|
|
94
|
+
if (Number.isFinite(n) && n > 0) return { bytes: n, source: 'freemem' };
|
|
95
|
+
} catch { /* 再兜底 */ }
|
|
96
|
+
return { bytes: null, source: 'none' };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** cgroup v2 的 memory.max:字符串 "max" = 不限额 */
|
|
100
|
+
const CGROUP_UNLIMITED = 'max';
|
|
101
|
+
/** 缺省挂载点(cgroup v2 unified) */
|
|
102
|
+
const DEFAULT_CGROUP_MAX_PATH = '/sys/fs/cgroup/memory.max';
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* 读 cgroup v2 的"可用内存" = memory.max − memory.current。
|
|
106
|
+
*
|
|
107
|
+
* 为什么要优先它:/proc/meminfo 的 MemAvailable 是**宿主口径**。
|
|
108
|
+
* 容器里限额 1GB、宿主还有 30GB 时,按 MemAvailable 判会得出"想探多少探多少",
|
|
109
|
+
* 实际一探就被 OOM-kill。cgroup 才是本进程真正能用的上限。
|
|
110
|
+
*
|
|
111
|
+
* 取不到 / 不限额 / 值不合法 ⇒ 返回 null(调用方回落 /proc/meminfo,fail-open 不变)。
|
|
112
|
+
* 路径可用 MODELSTAGE_CGROUP_MEMORY_MAX_PATH 覆盖(非标准挂载 + 测试注入都用它);
|
|
113
|
+
* memory.current 取同目录下的同名文件。
|
|
114
|
+
* @returns {{bytes: number, max: number, current: number, source: string}|null}
|
|
115
|
+
*/
|
|
116
|
+
function readCgroupAvailable() {
|
|
117
|
+
const maxPath = String(process.env.MODELSTAGE_CGROUP_MEMORY_MAX_PATH || '').trim() || DEFAULT_CGROUP_MAX_PATH;
|
|
118
|
+
try {
|
|
119
|
+
if (!fs.existsSync(maxPath)) return null;
|
|
120
|
+
const maxRaw = String(fs.readFileSync(maxPath, 'utf8')).trim();
|
|
121
|
+
if (!maxRaw || maxRaw === CGROUP_UNLIMITED) return null; // 不限额 ⇒ 交给宿主口径
|
|
122
|
+
const max = Number(maxRaw);
|
|
123
|
+
if (!Number.isFinite(max) || max <= 0) return null;
|
|
124
|
+
const curPath = path.join(path.dirname(maxPath), 'memory.current');
|
|
125
|
+
let current = 0;
|
|
126
|
+
try {
|
|
127
|
+
const curRaw = String(fs.readFileSync(curPath, 'utf8')).trim();
|
|
128
|
+
const n = Number(curRaw);
|
|
129
|
+
if (Number.isFinite(n) && n >= 0) current = n;
|
|
130
|
+
} catch { /* 没有 memory.current:按 0 算(保守方向:可用=上限) */ }
|
|
131
|
+
const bytes = max - current;
|
|
132
|
+
if (!Number.isFinite(bytes) || bytes <= 0) return null;
|
|
133
|
+
return { bytes, max, current, source: 'cgroup-v2' };
|
|
134
|
+
} catch {
|
|
135
|
+
return null;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* 造一个后台形态探测器。
|
|
141
|
+
*
|
|
142
|
+
* @param {{
|
|
143
|
+
* catalog: {takeProbeKeys: Function, requeueProbeKey: Function},
|
|
144
|
+
* entryStates: {putProbe: Function},
|
|
145
|
+
* providers: () => Array<{id: string, version?: unknown}>,
|
|
146
|
+
* readConfig: () => Array<{id: string, path: string}>,
|
|
147
|
+
* log?: Function,
|
|
148
|
+
* delayMs?: number, maxPerRun?: number, maxBytes?: number,
|
|
149
|
+
* }} opts
|
|
150
|
+
*/
|
|
151
|
+
export { readMemAvailable, readCgroupAvailable };
|
|
152
|
+
|
|
153
|
+
export function createShapeProbe(opts = {}) {
|
|
154
|
+
const { catalog, entryStates } = opts;
|
|
155
|
+
const providers = typeof opts.providers === 'function' ? opts.providers : () => [];
|
|
156
|
+
const readConfig = typeof opts.readConfig === 'function' ? opts.readConfig : () => [];
|
|
157
|
+
const log = typeof opts.log === 'function' ? opts.log : () => {};
|
|
158
|
+
const delayMs = Number.isFinite(opts.delayMs) ? Number(opts.delayMs) : DEFAULT_DELAY_MS;
|
|
159
|
+
const maxPerRun = Number.isFinite(opts.maxPerRun) ? Number(opts.maxPerRun) : DEFAULT_MAX_PER_RUN;
|
|
160
|
+
const maxBytes = Number.isFinite(opts.maxBytes) ? Number(opts.maxBytes) : DEFAULT_MAX_BYTES;
|
|
161
|
+
/**
|
|
162
|
+
* 内存判据("该不该探"的第二道门):每次**逐包**取一次可用内存。
|
|
163
|
+
* readLimits 由装配层注入(每批重读 env)⇒ 阈值可调而不用改代码;
|
|
164
|
+
* availableBytes 可注入,测试里就能精确造"内存不够/够/拿不到"三种情形。
|
|
165
|
+
*/
|
|
166
|
+
const readLimits = typeof opts.readLimits === 'function'
|
|
167
|
+
? opts.readLimits
|
|
168
|
+
: () => ({ baselineBytes: DEFAULT_MEM_BASELINE_BYTES, factor: DEFAULT_MEM_FACTOR });
|
|
169
|
+
const injectedAvailable = typeof opts.availableBytes === 'function';
|
|
170
|
+
const availableBytes = injectedAvailable ? opts.availableBytes : readMemAvailable;
|
|
171
|
+
// 判型函数可注入:默认用与 /api/unpack 同一份实现;测试里换成假实现就能在
|
|
172
|
+
// 不造真实 UnityFS 包的前提下验证"写回 unsupported / unknown"的分支(契约 §八)。
|
|
173
|
+
const probeBytes = typeof opts.probeBytes === 'function' ? opts.probeBytes : probeBundleBytes;
|
|
174
|
+
// 请求活跃度探针(由 serve.mjs 注入):{ idle(), msSinceLastRequest() }。
|
|
175
|
+
// 没有它时退化成"永远空闲"(单测/脚本里就是这么用的),行为与从前一致。
|
|
176
|
+
const activity = opts.activity && typeof opts.activity === 'object' ? opts.activity : null;
|
|
177
|
+
const idleGapMs = Number.isFinite(opts.idleGapMs) ? Number(opts.idleGapMs) : DEFAULT_IDLE_GAP_MS;
|
|
178
|
+
const maxWaitMs = Number.isFinite(opts.maxWaitMs) ? Number(opts.maxWaitMs) : DEFAULT_MAX_WAIT_MS;
|
|
179
|
+
// 解析放 worker(第 3 步):只对"真实解析"生效。测试注入了假判型函数时直接同步跑,
|
|
180
|
+
// 不起线程(既省资源,也让用例保持确定性)。
|
|
181
|
+
const useWorker = opts.worker !== false && typeof opts.probeBytes !== 'function';
|
|
182
|
+
/** worker 句柄:按批创建/销毁(跑完即停,不留常驻线程) */
|
|
183
|
+
let worker = null;
|
|
184
|
+
/** worker 起不来(环境不允许)时置位:之后一律回落同步解析,不再反复尝试 */
|
|
185
|
+
let workerBroken = false;
|
|
186
|
+
let workerSeq = 0;
|
|
187
|
+
/** 待回包的请求:id → {resolve, reject} */
|
|
188
|
+
const workerPending = new Map();
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* 计数(S10 的佐证:指纹没变时 runs 必须一直是 0)。
|
|
192
|
+
*
|
|
193
|
+
* 后四个字段是给"将来面板显示进度"预留的(本次不做 UI):
|
|
194
|
+
* lastTrigger 本轮是谁触发的:catalog(清单重建)/ fingerprint(指纹变了)/ manual(手动)
|
|
195
|
+
* lastBatchMs 上一轮耗时(ms)
|
|
196
|
+
* lastQueueAfter 上一轮跑完后仍排队的条目数(>0 说明还要等下次触发)
|
|
197
|
+
* 字段只增不改 ⇒ GET /api/entries/probe 的形状对既有消费方保持兼容。
|
|
198
|
+
*/
|
|
199
|
+
const stats = {
|
|
200
|
+
runs: 0, probed: 0, unsupported: 0, unknown: 0, skipped: 0,
|
|
201
|
+
lastRunAt: null, lastRootId: null,
|
|
202
|
+
lastTrigger: null, lastBatchMs: null, lastQueueAfter: null,
|
|
203
|
+
// 空闲门控的观测值(本轮为"不抢主线程"新增):让路次数 / 累计等待毫秒
|
|
204
|
+
deferred: 0, waitedMs: 0,
|
|
205
|
+
// 超过单包上限而跳过的包数(保持 unknown,留待点击验证)
|
|
206
|
+
skippedBig: 0,
|
|
207
|
+
// 因**可用内存不足**而跳过的包数(与 skippedBig 分开:一个是"太大",一个是"装不下")
|
|
208
|
+
skippedMemory: 0,
|
|
209
|
+
// 最近一次内存判据看到的可用内存(字节;null = 拿不到,fail-open 放行)
|
|
210
|
+
lastMemAvailable: null,
|
|
211
|
+
// 这个数从哪来:cgroup-v2(容器限额)/ MemAvailable(宿主口径)/ freemem / injected(测试)
|
|
212
|
+
lastMemSource: null,
|
|
213
|
+
// 最近一次"因内存不足而跳过"的判定明细(available/need/baseline/size/factor)
|
|
214
|
+
lastMemDetail: null,
|
|
215
|
+
};
|
|
216
|
+
/** 进行中的那一轮(同一时刻只允许一轮:串行是硬约束,多个 kick 合并成一个) */
|
|
217
|
+
let running = null;
|
|
218
|
+
/** 触发来源白名单(三选一,别混成一条 —— 排查时要能分清"谁让它跑的") */
|
|
219
|
+
const TRIGGERS = new Set(['catalog', 'fingerprint', 'manual']);
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* 取(必要时创建)解析 worker。
|
|
223
|
+
* unref() 是必须的:worker 不该阻止进程退出(否则 Ctrl-C 停服务时要多等一轮)。
|
|
224
|
+
* @returns {Worker|null} null = 不可用(调用方回落同步解析)
|
|
225
|
+
*/
|
|
226
|
+
function startWorker() {
|
|
227
|
+
if (!useWorker || workerBroken) return null;
|
|
228
|
+
if (worker) return worker;
|
|
229
|
+
try {
|
|
230
|
+
worker = new Worker(new URL('./shape-probe-worker.mjs', import.meta.url));
|
|
231
|
+
if (typeof worker.unref === 'function') worker.unref();
|
|
232
|
+
worker.on('message', (m) => {
|
|
233
|
+
const p = workerPending.get(m && m.id);
|
|
234
|
+
if (!p) return;
|
|
235
|
+
workerPending.delete(m.id);
|
|
236
|
+
if (m.ok) p.resolve(m);
|
|
237
|
+
else p.reject(new Error(m.error || 'worker 解析失败'));
|
|
238
|
+
});
|
|
239
|
+
worker.on('error', (e) => {
|
|
240
|
+
workerBroken = true; // 一次失败就永久回落,避免每包都踩一次
|
|
241
|
+
for (const p of workerPending.values()) p.reject(e);
|
|
242
|
+
workerPending.clear();
|
|
243
|
+
});
|
|
244
|
+
worker.on('exit', () => { worker = null; });
|
|
245
|
+
} catch {
|
|
246
|
+
workerBroken = true;
|
|
247
|
+
worker = null;
|
|
248
|
+
}
|
|
249
|
+
return worker;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** 收掉 worker(一批结束就销毁:不留常驻线程,也就没有"后台偷偷占 CPU") */
|
|
253
|
+
async function stopWorker() {
|
|
254
|
+
if (!worker) return;
|
|
255
|
+
const w = worker;
|
|
256
|
+
worker = null;
|
|
257
|
+
for (const p of workerPending.values()) p.reject(new Error('worker 已停止'));
|
|
258
|
+
workerPending.clear();
|
|
259
|
+
try { await w.terminate(); } catch { /* 已经退出,忽略 */ }
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* 在 worker 里探一个包的形态(主线程只等消息,不做 CPU)。
|
|
264
|
+
* @param {string} abs 绝对路径
|
|
265
|
+
* @returns {Promise<{assetDependency: boolean, dependencyReason: string|null}>}
|
|
266
|
+
*/
|
|
267
|
+
function probeInWorker(abs) {
|
|
268
|
+
const w = startWorker();
|
|
269
|
+
if (!w) return Promise.reject(new Error('worker 不可用'));
|
|
270
|
+
const id = (workerSeq += 1);
|
|
271
|
+
return new Promise((resolve, reject) => {
|
|
272
|
+
workerPending.set(id, { resolve, reject });
|
|
273
|
+
try { w.postMessage({ id, abs }); } catch (e) { workerPending.delete(id); reject(e); }
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* 同步兜底:worker 不可用时在主线程解析(旧行为)。
|
|
279
|
+
* @param {string} abs 绝对路径
|
|
280
|
+
* @returns {{assetDependency: boolean, dependencyReason: string|null}}
|
|
281
|
+
*/
|
|
282
|
+
function probeInline(abs) {
|
|
283
|
+
const shape = probeBytes(new Uint8Array(fs.readFileSync(abs)));
|
|
284
|
+
return {
|
|
285
|
+
assetDependency: !!(shape && shape.assetDependency),
|
|
286
|
+
dependencyReason: (shape && shape.dependencyReason) || null,
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* 批内判据(比 isIdleNow 宽):**当前没有请求在飞**就继续。
|
|
292
|
+
*
|
|
293
|
+
* 为什么不要求同样的静默间隔:一批 8 个包,若每个包前都等一次完整静默,
|
|
294
|
+
* 遇到"用户每隔几秒点一下"的节奏就永远跑不完一批(实测吞吐掉到旧行为的 40%)。
|
|
295
|
+
* 而"有请求在飞时绝不动手"已经足以避免把 CPU/IO 从用户手里抢走。
|
|
296
|
+
* @returns {boolean}
|
|
297
|
+
*/
|
|
298
|
+
function noRequestInFlight() {
|
|
299
|
+
if (!activity) return true;
|
|
300
|
+
try {
|
|
301
|
+
return typeof activity.idle === 'function' ? !!activity.idle() : true;
|
|
302
|
+
} catch {
|
|
303
|
+
return true;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* 此刻是否"真空闲":没有短请求在飞,且距上次请求已过 idleGapMs。
|
|
309
|
+
*
|
|
310
|
+
* 为什么要求"再等一会儿"而不只是"当前空闲":请求之间常只隔几十毫秒,
|
|
311
|
+
* 解析一开始就正好撞上新请求 —— 那正是用户感到"某个请求莫名慢半秒"的来源。
|
|
312
|
+
* @returns {boolean}
|
|
313
|
+
*/
|
|
314
|
+
function isIdleNow() {
|
|
315
|
+
if (!activity) return true;
|
|
316
|
+
try {
|
|
317
|
+
const idle = typeof activity.idle === 'function' ? !!activity.idle() : true;
|
|
318
|
+
const gap = typeof activity.msSinceLastRequest === 'function'
|
|
319
|
+
? Number(activity.msSinceLastRequest()) : idleGapMs;
|
|
320
|
+
return idle && Number.isFinite(gap) && gap >= idleGapMs;
|
|
321
|
+
} catch {
|
|
322
|
+
// 探针本身出错不该让探测停摆(旁路原则)
|
|
323
|
+
return true;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* 等到"真空闲",最多等 budgetMs。
|
|
329
|
+
* @param {number} budgetMs 预算(ms)
|
|
330
|
+
* @returns {Promise<boolean>} true = 已空闲;false = 预算耗尽(调用方应让路,别硬跑)
|
|
331
|
+
*/
|
|
332
|
+
async function waitUntilIdle(budgetMs) {
|
|
333
|
+
if (!activity) return true;
|
|
334
|
+
const deadline = Date.now() + budgetMs;
|
|
335
|
+
while (!isIdleNow()) {
|
|
336
|
+
if (Date.now() >= deadline) return false;
|
|
337
|
+
await sleep(WAIT_STEP_MS);
|
|
338
|
+
stats.waitedMs += WAIT_STEP_MS;
|
|
339
|
+
}
|
|
340
|
+
return true;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/**
|
|
344
|
+
* 触发一次探测(不阻塞调用方:catalog 的读路径只负责"喊一声")。
|
|
345
|
+
* @param {string} [rootId] 触发来源的根(仅用于日志/计数,队列本身跨根共享)
|
|
346
|
+
* @param {'catalog'|'fingerprint'|'manual'} [trigger] 触发来源;非法值按 catalog 处理
|
|
347
|
+
* @returns {Promise<{probed: number}>|null} 已在跑时返回那一轮的 promise
|
|
348
|
+
*/
|
|
349
|
+
function kick(rootId, trigger = 'catalog') {
|
|
350
|
+
if (!catalog || !entryStates) return null;
|
|
351
|
+
if (running) return running; // 合并触发:事件驱动,不叠加并发
|
|
352
|
+
const why = TRIGGERS.has(trigger) ? trigger : 'catalog';
|
|
353
|
+
running = runOnce(rootId, why).catch((e) => {
|
|
354
|
+
// 旁路:探测失败绝不能影响任何主流程(INV-D3 同源思想)
|
|
355
|
+
log('warn', '后台形态探测异常(不影响装载)', { err: (e && e.message) || String(e) });
|
|
356
|
+
return { probed: 0 };
|
|
357
|
+
}).finally(() => { running = null; });
|
|
358
|
+
return running;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* 一轮:取一批 → 逐个判 → 写回 → 停(并留下三条可读日志:启动 / 完成 / 仍有剩余)。
|
|
363
|
+
* @param {string} [rootId] 触发来源的根(只用于日志与统计,队列本身跨根共享)
|
|
364
|
+
* @param {'catalog'|'fingerprint'|'manual'} trigger 触发来源(三选一,排查时要能分清)
|
|
365
|
+
*/
|
|
366
|
+
async function runOnce(rootId, trigger) {
|
|
367
|
+
// 先看有没有活(纯内存读,零代价):没活就立刻返回。
|
|
368
|
+
// 顺序很重要 —— 先等空闲再发现"其实没东西可探",会让 running 白占住 maxWaitMs,
|
|
369
|
+
// 期间所有新的触发都被合并掉(实测:空队列也会显示 running=true 二十秒)。
|
|
370
|
+
const hasWork = typeof catalog.probeQueueSize === 'function' ? catalog.probeQueueSize() > 0 : true;
|
|
371
|
+
if (!hasWork) return { probed: 0 };
|
|
372
|
+
// 空闲门控(第 1 步):忙的时候**连队列都不取** —— 取走再放回是白折腾,
|
|
373
|
+
// 而且"取了不放"会让条目在下次触发前处于"既不排队也不探"的夹缝里。
|
|
374
|
+
if (activity && !(await waitUntilIdle(maxWaitMs))) {
|
|
375
|
+
stats.deferred += 1;
|
|
376
|
+
log('info', `后台形态探测让路 · 等待 ${maxWaitMs}ms 仍有请求在忙,本轮不启动`, {
|
|
377
|
+
trigger, deferred: stats.deferred, waitedMs: stats.waitedMs,
|
|
378
|
+
});
|
|
379
|
+
return { probed: 0 };
|
|
380
|
+
}
|
|
381
|
+
const keys = catalog.takeProbeKeys(maxPerRun);
|
|
382
|
+
// 零待探测 ⇒ **完全不启动**:不读盘、不加计数、不写日志噪音(S10)
|
|
383
|
+
if (!keys.length) return { probed: 0 };
|
|
384
|
+
|
|
385
|
+
const t0 = Date.now();
|
|
386
|
+
// 取走这一批之后还剩多少("剩余待探"的口径:排队中,不含本轮这 N 个)
|
|
387
|
+
const queueAfterTake = typeof catalog.probeQueueSize === 'function' ? catalog.probeQueueSize() : null;
|
|
388
|
+
stats.runs += 1;
|
|
389
|
+
stats.lastRunAt = t0;
|
|
390
|
+
stats.lastRootId = rootId || null;
|
|
391
|
+
stats.lastTrigger = trigger;
|
|
392
|
+
// ① 启动:触发来源 + 本轮批量 + 排队剩多少(三者分开写,别混成一句)
|
|
393
|
+
log('info', `后台形态探测启动 · 触发=${trigger} · 本轮 ${keys.length} 个包 · 排队剩 ${queueAfterTake ?? '?'}`, {
|
|
394
|
+
rootId: rootId || null, trigger, runs: stats.runs, batch: keys.length, queue: queueAfterTake,
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
const fp = providerFingerprintOf(providers());
|
|
398
|
+
const roots = new Map();
|
|
399
|
+
for (const r of readConfig() || []) {
|
|
400
|
+
if (r && r.id && r.path) roots.set(String(r.id), String(r.path));
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
const out = await probeBatch(keys, providerFingerprintOf(providers()), rootPathMap(), trigger);
|
|
404
|
+
|
|
405
|
+
// 计数在**一处**累加(probeOne 只回报结论种类,不自己改全局计数 —— 免得两处各加一遍)
|
|
406
|
+
stats.probed += out.probed;
|
|
407
|
+
stats.unsupported += out.unsupported;
|
|
408
|
+
stats.unknown += out.unknown;
|
|
409
|
+
const queueLeft = queueSize();
|
|
410
|
+
stats.lastBatchMs = Date.now() - t0;
|
|
411
|
+
stats.lastQueueAfter = queueLeft;
|
|
412
|
+
|
|
413
|
+
// ② 完成:本轮结果分类 + 耗时 + 剩余(让路时额外说明"放回了几个")
|
|
414
|
+
log('info', `后台形态探测完成 · 本轮 ${out.probed}/${keys.length} · unsupported ${out.unsupported} · unknown ${out.unknown} · 跳过 ${stats.skipped} · 内存跳过 ${out.memSkipped || 0} · 排队剩 ${queueLeft ?? '?'} · 耗时 ${stats.lastBatchMs}ms`
|
|
415
|
+
+ (out.yielded ? ` · 让路放回 ${out.yielded} 个` : ''), {
|
|
416
|
+
trigger, probed: out.probed, unsupported: out.unsupported, unknown: out.unknown,
|
|
417
|
+
queue: queueLeft, ms: stats.lastBatchMs, yielded: out.yielded,
|
|
418
|
+
});
|
|
419
|
+
// ③ 仍有剩余:明确说明"为什么停了"(跑完即停是设计,不是卡住)
|
|
420
|
+
if (queueLeft) {
|
|
421
|
+
log('info', `后台形态探测暂停 · 排队还剩 ${queueLeft} 个(跑完即停,等下次触发继续)`, { trigger, queue: queueLeft });
|
|
422
|
+
}
|
|
423
|
+
return { probed: out.probed };
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/** 当前待探测队列深度(拿不到给 null,日志显示 '?') */
|
|
427
|
+
function queueSize() {
|
|
428
|
+
return typeof catalog.probeQueueSize === 'function' ? catalog.probeQueueSize() : null;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** rootId → 绝对路径(每批取一次;别塞进逐包循环里) */
|
|
432
|
+
function rootPathMap() {
|
|
433
|
+
const m = new Map();
|
|
434
|
+
for (const r of readConfig() || []) {
|
|
435
|
+
if (r && r.id && r.path) m.set(String(r.id), String(r.path));
|
|
436
|
+
}
|
|
437
|
+
return m;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* 解析一批:逐个探 → 累计本轮分类;遇到"有请求在飞"就把剩余放回队列并收手。
|
|
442
|
+
*
|
|
443
|
+
* 为什么单独成函数:runOnce 只管门控/取批/收尾日志,解析循环独立出来
|
|
444
|
+
* 才读得动(也才不会被"函数过长"卡住)。
|
|
445
|
+
* @param {string[]} keys 本批条目
|
|
446
|
+
* @param {string} fp 当前提供者指纹
|
|
447
|
+
* @param {Map<string,string>} roots rootId → 绝对路径
|
|
448
|
+
* @param {'catalog'|'fingerprint'|'manual'} trigger 触发来源(只进日志)
|
|
449
|
+
* @returns {Promise<{probed: number, unsupported: number, unknown: number, yielded: number}>}
|
|
450
|
+
*/
|
|
451
|
+
async function probeBatch(keys, fp, roots, trigger) {
|
|
452
|
+
let probed = 0;
|
|
453
|
+
let unsupported = 0;
|
|
454
|
+
let unknown = 0;
|
|
455
|
+
let yielded = 0;
|
|
456
|
+
let memSkipped = 0; // 本批因内存不足跳过的条数
|
|
457
|
+
let memLogged = false; // 本批是否已记过"内存不足"(每批只记一次,避免刷屏)
|
|
458
|
+
for (let i = 0; i < keys.length; i += 1) {
|
|
459
|
+
const key = keys[i];
|
|
460
|
+
// 批内也要让路:一批 8 个包跑十几秒,中途用户来请求了就该停手(已探的结论已写回,不浪费)。
|
|
461
|
+
// 判据用"无请求在飞"(不含静默间隔)—— 否则密集浏览下永远跑不完一批。
|
|
462
|
+
if (activity && !noRequestInFlight()) {
|
|
463
|
+
const rest = keys.slice(i);
|
|
464
|
+
for (const k of rest) catalog.requeueProbeKey(k);
|
|
465
|
+
yielded = rest.length;
|
|
466
|
+
log('info', `后台形态探测让路 · 有请求在忙,剩余 ${rest.length} 个包放回队列`, {
|
|
467
|
+
trigger, yielded, waitedMs: stats.waitedMs,
|
|
468
|
+
});
|
|
469
|
+
break;
|
|
470
|
+
}
|
|
471
|
+
try {
|
|
472
|
+
const kind = await probeOne(key, fp, roots);
|
|
473
|
+
if (kind === 'unsupported') { probed += 1; unsupported += 1; }
|
|
474
|
+
else if (kind === 'unknown') { probed += 1; unknown += 1; }
|
|
475
|
+
else if (kind === 'memory') {
|
|
476
|
+
// 内存不够 ⇒ 这一批里**只记一条**(同类跳过会连续发生,逐条记就是刷屏)
|
|
477
|
+
memSkipped += 1;
|
|
478
|
+
if (!memLogged) {
|
|
479
|
+
memLogged = true;
|
|
480
|
+
const d = stats.lastMemDetail || {};
|
|
481
|
+
log('info', `形态探测跳过(可用内存不足,保持 unknown)· available ${fmtMb(d.available)} · 本包需 ≥ ${fmtMb(d.need)}(基线 ${fmtMb(d.baseline)} + 包 ${fmtMb(d.size)} × 系数 ${d.factor ?? '?'})`, {
|
|
482
|
+
available: d.available ?? null, need: d.need ?? null, baseline: d.baseline ?? null, size: d.size ?? null, factor: d.factor ?? null,
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
} catch (e) {
|
|
487
|
+
// 单个包出错不能带走整轮:记日志、把 key 放回队列(下一轮再试)
|
|
488
|
+
stats.skipped += 1;
|
|
489
|
+
catalog.requeueProbeKey(key);
|
|
490
|
+
log('warn', '形态探测失败(该条保持原结论)', { key, err: (e && e.message) || String(e) });
|
|
491
|
+
}
|
|
492
|
+
if (delayMs > 0) await sleep(delayMs); // 包间限速:不顶满 CPU/IO
|
|
493
|
+
}
|
|
494
|
+
return { probed, unsupported, unknown, yielded, memSkipped };
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* 超过单包上限的处理:记 unknown(**不是** unsupported)+ 单独计数 + 一条 info。
|
|
499
|
+
* unknown 是合法状态:点击装载时仍会得到真实结论(探测不是必须给答案)。
|
|
500
|
+
* @param {string} key 条目键
|
|
501
|
+
* @param {number} size 包体积
|
|
502
|
+
* @param {string|null} fp 当前提供者指纹
|
|
503
|
+
*/
|
|
504
|
+
function skipTooBig(key, size, fp) {
|
|
505
|
+
entryStates.putProbe(key, { result: 'unknown', message: null }, { providerFingerprint: fp, assetFingerprint: assetFingerprintOf(size) });
|
|
506
|
+
stats.skippedBig += 1;
|
|
507
|
+
log('info', `形态探测跳过(包太大,留待点击验证)· ${fmtMb(size)} > 上限 ${fmtMb(maxBytes)}`, { key, size, maxBytes });
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/**
|
|
511
|
+
* 内存判据:可用 ≥ 基线余量 + 包体积 × 系数。
|
|
512
|
+
*
|
|
513
|
+
* 取不到可用内存时**放行**(fail-open):读不到 /proc 或 os.freemem 异常的环境里,
|
|
514
|
+
* 宁可能探就探,也不要因为"量不到"而让探测永久停摆(那是把排查工具弄成哑巴)。
|
|
515
|
+
* @param {number} size 包体积(字节)
|
|
516
|
+
* @returns {{blocked: boolean, detail: null | {available: number, need: number, baseline: number, size: number, factor: number}}}
|
|
517
|
+
*/
|
|
518
|
+
function checkMemoryFor(size) {
|
|
519
|
+
const avail = availableBytes();
|
|
520
|
+
stats.lastMemAvailable = avail === null ? null : Number(avail);
|
|
521
|
+
// 来源能分辨"cgroup 限额"与"宿主 MemAvailable",否则同一个数字两种口径无法排障
|
|
522
|
+
stats.lastMemSource = injectedAvailable ? 'injected' : readMemAvailableInfo().source;
|
|
523
|
+
if (avail === null) return { blocked: false, detail: null };
|
|
524
|
+
const lim = readLimits() || {};
|
|
525
|
+
const baseline = Number.isFinite(lim.baselineBytes) ? Number(lim.baselineBytes) : DEFAULT_MEM_BASELINE_BYTES;
|
|
526
|
+
const factor = Number.isFinite(lim.factor) ? Number(lim.factor) : DEFAULT_MEM_FACTOR;
|
|
527
|
+
const need = Math.ceil(baseline + size * factor);
|
|
528
|
+
if (Number(avail) < need) return { blocked: true, detail: { available: Number(avail), need, baseline, size, factor } };
|
|
529
|
+
return { blocked: false, detail: null };
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/**
|
|
533
|
+
* 探一个包并写回结论(读盘+解析在 worker,worker 不可用时同步回落)。
|
|
534
|
+
* @param {string} key `${rootId}/${rel}`
|
|
535
|
+
* @param {string|null} fp 当前提供者指纹(写回时一起落库,否则下次读取又判过期 —— 白探)
|
|
536
|
+
* @param {Map<string,string>} roots rootId → 绝对路径
|
|
537
|
+
* @returns {Promise<'unsupported'|'unknown'|null>} 结论种类;null = 没探(文件不在/键不合法)
|
|
538
|
+
*/
|
|
539
|
+
async function probeOne(key, fp, roots) {
|
|
540
|
+
const s = String(key || '');
|
|
541
|
+
const i = s.indexOf('/');
|
|
542
|
+
if (i <= 0) return null;
|
|
543
|
+
const rootId = s.slice(0, i);
|
|
544
|
+
const rel = s.slice(i + 1);
|
|
545
|
+
const base = roots.get(rootId);
|
|
546
|
+
if (!base || !rel) return null;
|
|
547
|
+
|
|
548
|
+
const abs = path.join(base, rel);
|
|
549
|
+
let size = null;
|
|
550
|
+
try {
|
|
551
|
+
const st = await fs.promises.stat(abs);
|
|
552
|
+
if (!st.isFile()) return null;
|
|
553
|
+
size = st.size;
|
|
554
|
+
} catch {
|
|
555
|
+
// 文件不在了(被移走/改名):不写结论,也不重排队 —— 下次扫描它就不在列表里了
|
|
556
|
+
return null;
|
|
557
|
+
}
|
|
558
|
+
// 太大的包不探(内存/阻塞风险):跳过也要落一条"探过没结论",否则每次启动都会重试它
|
|
559
|
+
if (size > maxBytes) { skipTooBig(s, size, fp); return 'unknown'; }
|
|
560
|
+
|
|
561
|
+
// ── 第二道门:可用内存(不是包大小)────────────────────────────────
|
|
562
|
+
// 为什么需要:包不大但机器装不下时(实测可用内存被压到 283MB),
|
|
563
|
+
// 解析 20MB 包仍会把进程拖进 swap。判据 = 可用 ≥ 基线余量 + 本包预估占用。
|
|
564
|
+
const mem = checkMemoryFor(size);
|
|
565
|
+
if (mem.blocked) {
|
|
566
|
+
// 内存不够 ⇒ 记 unknown(**不是** unsupported:这是"没探",不是"装不动")
|
|
567
|
+
entryStates.putProbe(s, { result: 'unknown', message: null }, { providerFingerprint: fp, assetFingerprint: assetFingerprintOf(size) });
|
|
568
|
+
stats.skippedMemory += 1;
|
|
569
|
+
stats.lastMemDetail = mem.detail; // 批内日志要能打出 available / need / 系数
|
|
570
|
+
return 'memory';
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
// 解析:优先走 worker(主线程不参与 CPU)—— 这是"不阻塞事件循环"的关键一步。
|
|
574
|
+
// worker 起不来(受限环境)或中途报错时回落同步解析:探测可以慢,但结论不能丢。
|
|
575
|
+
let shape = null;
|
|
576
|
+
if (useWorker && !workerBroken) {
|
|
577
|
+
try {
|
|
578
|
+
shape = await probeInWorker(abs);
|
|
579
|
+
} catch {
|
|
580
|
+
shape = probeInline(abs);
|
|
581
|
+
}
|
|
582
|
+
} else {
|
|
583
|
+
shape = probeInline(abs);
|
|
584
|
+
}
|
|
585
|
+
const assetFingerprint = assetFingerprintOf(size);
|
|
586
|
+
|
|
587
|
+
if (shape && shape.assetDependency) {
|
|
588
|
+
// 判据与文案都来自 src/shared/bundle-shape.mjs(与点击装载失败同源)⇒ S4 的"文案同源"
|
|
589
|
+
entryStates.putProbe(s, {
|
|
590
|
+
result: 'unsupported',
|
|
591
|
+
reason: 'asset-dependency',
|
|
592
|
+
message: shape.dependencyReason,
|
|
593
|
+
}, { providerFingerprint: fp, assetFingerprint });
|
|
594
|
+
return 'unsupported';
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
// 探不出/正常包 ⇒ **保持 unknown,不写 unsupported**(契约 §八"不猜内容")。
|
|
598
|
+
// 但必须落一条 result:'unknown':它既是"探过了"的凭据(S10 零开销),
|
|
599
|
+
// 也保证下一次读取不再把它当"没结论"重复排队。
|
|
600
|
+
entryStates.putProbe(s, { result: 'unknown', message: null }, { providerFingerprint: fp, assetFingerprint });
|
|
601
|
+
return 'unknown';
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
return {
|
|
605
|
+
kick,
|
|
606
|
+
/** 计数快照(S10 验收:指纹未变时 runs 必须是 0) */
|
|
607
|
+
stats: () => ({ ...stats }),
|
|
608
|
+
/** 是否有一轮在跑(诊断用) */
|
|
609
|
+
isRunning: () => !!running,
|
|
610
|
+
/**
|
|
611
|
+
* 关闭常驻的解析线程 —— **持有者必须调用**。
|
|
612
|
+
*
|
|
613
|
+
* 为什么必须有它:worker 为了"跨批复用"(否则每批重建解析沙箱,一批从 4.5s 涨到 76s)
|
|
614
|
+
* 会留在进程里。`unref()` 只能让它不单独撑住进程,**挡不住** `worker.on('message')`
|
|
615
|
+
* 这类监听把事件循环 hold 住 —— 实测表现:6 条用例全过、`node --test` 却永不退出
|
|
616
|
+
* (被超时杀掉,退出码 124)。
|
|
617
|
+
*
|
|
618
|
+
* 三个必须调用的地方(漏一处就留悬挂线程):
|
|
619
|
+
* · 服务进程收尾:SIGINT/SIGTERM(serve.mjs 的 registerShutdown)
|
|
620
|
+
* · 测试用例收尾:finally / t.after
|
|
621
|
+
* · 临时脚本用完
|
|
622
|
+
* @returns {Promise<void>}
|
|
623
|
+
*/
|
|
624
|
+
close: async () => { await stopWorker(); },
|
|
625
|
+
};
|
|
626
|
+
}
|