@camstack/system 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +40 -23
- package/dist/addon-runner.mjs +20 -4
- package/dist/addon-utils.d.ts +20 -0
- package/dist/addon-utils.js +11 -0
- package/dist/addon-utils.mjs +3 -0
- package/dist/builtins/device-manager/device-manager.addon.js +8 -8
- package/dist/builtins/device-manager/device-manager.addon.mjs +8 -8
- package/dist/builtins/native-metrics/native-metrics.addon.d.ts +8 -0
- package/dist/builtins/native-metrics/native-metrics.addon.js +50 -3
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +50 -3
- package/dist/builtins/platform-probe/index.js +27 -139
- package/dist/builtins/platform-probe/index.mjs +28 -140
- package/dist/builtins/platform-probe/platform-scorer.d.ts +17 -10
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +2 -2
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +2 -2
- package/dist/custom-action-registry-BEXwC-oo.mjs +38 -0
- package/dist/custom-action-registry-vLYEFTtv.js +43 -0
- package/dist/index.js +129 -779
- package/dist/index.mjs +100 -750
- package/dist/kernel/config-manager.d.ts +4 -4
- package/dist/kernel/fs-utils.d.ts +16 -6
- package/dist/kernel/index.d.ts +1 -1
- package/dist/kernel/moleculer/device-cap-proxy.d.ts +2 -1
- package/dist/kernel/moleculer/readiness-context.d.ts +2 -1
- package/dist/kernel/transport/child-cap-protocol.d.ts +10 -0
- package/dist/{manifest-python-deps-B4BmMoGT.js → manifest-python-deps-BWURo7dc.js} +62 -88
- package/dist/{manifest-python-deps-CXbKrOdk.mjs → manifest-python-deps-BcrTzHH_.mjs} +55 -75
- package/dist/model-download-service-C7AjBsX9.mjs +668 -0
- package/dist/model-download-service-JtVQtbb6.js +752 -0
- package/dist/process/resource-monitor.d.ts +9 -0
- package/dist/{resource-monitor-ClDGFyf6.mjs → resource-monitor-BkP504Vq.mjs} +20 -1
- package/dist/{resource-monitor-IIEanuJt.js → resource-monitor-DNNomR-i.js} +21 -1
- package/package.json +6 -1
|
@@ -0,0 +1,752 @@
|
|
|
1
|
+
const require_chunk = require("./chunk-Cek0wNdY.js");
|
|
2
|
+
let node_http = require("node:http");
|
|
3
|
+
let node_fs = require("node:fs");
|
|
4
|
+
node_fs = require_chunk.__toESM(node_fs);
|
|
5
|
+
let node_path = require("node:path");
|
|
6
|
+
node_path = require_chunk.__toESM(node_path);
|
|
7
|
+
let node_crypto = require("node:crypto");
|
|
8
|
+
//#region src/http/authenticated-file-server.ts
|
|
9
|
+
/**
|
|
10
|
+
* Authenticated data-plane file-server — a shared host primitive for addons that
|
|
11
|
+
* must serve files (media segments, scrub-frame stores, exported clips) directly
|
|
12
|
+
* to browsers/players, node-direct (NOT through the hub tRPC, per the recording
|
|
13
|
+
* design's "no media bytes traverse the hub").
|
|
14
|
+
*
|
|
15
|
+
* It bundles the three things every such server needs so addons don't re-roll
|
|
16
|
+
* them: CORS (the data plane is cross-origin from the admin-ui — a native
|
|
17
|
+
* `<video src>` is exempt but hls.js / WebCodecs fetch via XHR and are blocked
|
|
18
|
+
* without it), HTTP `Range` (seeking / partial fetch), and a SCOPED TOKEN check.
|
|
19
|
+
*
|
|
20
|
+
* The token rides as the FIRST path segment (`<urlPrefix><token>/<relPath>`) so
|
|
21
|
+
* a player resolving a manifest's RELATIVE child URIs carries it on every
|
|
22
|
+
* sub-request automatically — no per-URI rewriting, no cookies/headers a dumb
|
|
23
|
+
* player can't set. The caller supplies `verifyToken(token, relPath)`; the
|
|
24
|
+
* primitive stays auth-scheme-agnostic.
|
|
25
|
+
*
|
|
26
|
+
* Pure helpers (`parseTokenizedUrl`, `resolveFilePath`, `contentTypeFor`,
|
|
27
|
+
* `parseRangeHeader`) are exported for unit testing; `createAuthenticatedFileServer`
|
|
28
|
+
* is the thin `node:http` glue.
|
|
29
|
+
*/
|
|
30
|
+
function corsHeaders(origin) {
|
|
31
|
+
return {
|
|
32
|
+
"access-control-allow-origin": origin,
|
|
33
|
+
"access-control-allow-methods": "GET, HEAD, OPTIONS",
|
|
34
|
+
"access-control-allow-headers": "range",
|
|
35
|
+
"access-control-expose-headers": "content-length, content-range, accept-ranges",
|
|
36
|
+
"access-control-max-age": "86400"
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Split a request URL (`<urlPrefix><token>/<relPath>`) into its token + rel
|
|
41
|
+
* path. Returns null for anything outside the prefix or missing either part.
|
|
42
|
+
* `relPath` is URL-decoded; a decode failure → null.
|
|
43
|
+
*/
|
|
44
|
+
function parseTokenizedUrl(urlPrefix, urlPath) {
|
|
45
|
+
if (!urlPath.startsWith(urlPrefix)) return null;
|
|
46
|
+
const after = urlPath.slice(urlPrefix.length);
|
|
47
|
+
const slash = after.indexOf("/");
|
|
48
|
+
if (slash <= 0) return null;
|
|
49
|
+
const token = after.slice(0, slash);
|
|
50
|
+
let rel;
|
|
51
|
+
try {
|
|
52
|
+
rel = decodeURIComponent(after.slice(slash + 1));
|
|
53
|
+
} catch {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
if (rel.length === 0) return null;
|
|
57
|
+
return {
|
|
58
|
+
token,
|
|
59
|
+
rel
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Map a rel path to one candidate absolute path PER root, keeping only roots the
|
|
64
|
+
* path stays within (traversal guard). The handler serves the first candidate
|
|
65
|
+
* that exists. Rejects a path that escapes every root.
|
|
66
|
+
*/
|
|
67
|
+
function resolveFilePath(roots, rel) {
|
|
68
|
+
if (rel.length === 0) return { error: "forbidden" };
|
|
69
|
+
const candidates = [];
|
|
70
|
+
for (const rootDir of roots) {
|
|
71
|
+
const root = node_path.default.resolve(rootDir);
|
|
72
|
+
const abs = node_path.default.resolve(root, rel);
|
|
73
|
+
if (abs === root || abs.startsWith(root + node_path.default.sep)) candidates.push(abs);
|
|
74
|
+
}
|
|
75
|
+
if (candidates.length === 0) return { error: "forbidden" };
|
|
76
|
+
return { candidates };
|
|
77
|
+
}
|
|
78
|
+
var DEFAULT_CONTENT_TYPES = {
|
|
79
|
+
".m3u8": "application/vnd.apple.mpegurl",
|
|
80
|
+
".m4s": "video/mp4",
|
|
81
|
+
".mp4": "video/mp4",
|
|
82
|
+
".idx": "application/octet-stream",
|
|
83
|
+
".html": "text/html; charset=utf-8",
|
|
84
|
+
".js": "application/javascript; charset=utf-8",
|
|
85
|
+
".mjs": "application/javascript; charset=utf-8",
|
|
86
|
+
".css": "text/css; charset=utf-8",
|
|
87
|
+
".json": "application/json; charset=utf-8",
|
|
88
|
+
".map": "application/json; charset=utf-8",
|
|
89
|
+
".svg": "image/svg+xml",
|
|
90
|
+
".png": "image/png",
|
|
91
|
+
".jpg": "image/jpeg",
|
|
92
|
+
".jpeg": "image/jpeg",
|
|
93
|
+
".ico": "image/x-icon",
|
|
94
|
+
".woff2": "font/woff2",
|
|
95
|
+
".woff": "font/woff",
|
|
96
|
+
".ttf": "font/ttf"
|
|
97
|
+
};
|
|
98
|
+
function contentTypeFor(filePath, overrides) {
|
|
99
|
+
const ext = node_path.default.extname(filePath).toLowerCase();
|
|
100
|
+
return overrides?.[ext] ?? DEFAULT_CONTENT_TYPES[ext] ?? "application/octet-stream";
|
|
101
|
+
}
|
|
102
|
+
/** Parse an HTTP `Range` header against a known size. Null = serve whole file. */
|
|
103
|
+
function parseRangeHeader(header, size) {
|
|
104
|
+
if (!header) return null;
|
|
105
|
+
const m = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
|
|
106
|
+
if (!m) return null;
|
|
107
|
+
const [, rawStart, rawEnd] = m;
|
|
108
|
+
if (rawStart === "" && rawEnd === "") return null;
|
|
109
|
+
let start;
|
|
110
|
+
let end;
|
|
111
|
+
if (rawStart === "") {
|
|
112
|
+
const n = Number(rawEnd);
|
|
113
|
+
if (n <= 0) return null;
|
|
114
|
+
start = Math.max(0, size - n);
|
|
115
|
+
end = size - 1;
|
|
116
|
+
} else {
|
|
117
|
+
start = Number(rawStart);
|
|
118
|
+
end = rawEnd === "" ? size - 1 : Number(rawEnd);
|
|
119
|
+
}
|
|
120
|
+
if (!Number.isFinite(start) || !Number.isFinite(end) || start > end || start >= size) return null;
|
|
121
|
+
return {
|
|
122
|
+
start,
|
|
123
|
+
end: Math.min(end, size - 1)
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Start an authenticated file-server. Returns the bound port so the caller can
|
|
128
|
+
* build URLs. `getRoots`/`verifyToken` are invoked per request, so they always
|
|
129
|
+
* reflect the live config.
|
|
130
|
+
*/
|
|
131
|
+
async function createAuthenticatedFileServer(opts) {
|
|
132
|
+
const cors = corsHeaders(opts.corsOrigin ?? "*");
|
|
133
|
+
const server = (0, node_http.createServer)((req, res) => {
|
|
134
|
+
handle(opts, cors, req, res);
|
|
135
|
+
});
|
|
136
|
+
await new Promise((resolve, reject) => {
|
|
137
|
+
server.once("error", reject);
|
|
138
|
+
server.listen(opts.port, () => resolve());
|
|
139
|
+
});
|
|
140
|
+
const addr = server.address();
|
|
141
|
+
return {
|
|
142
|
+
port: typeof addr === "object" && addr ? addr.port : opts.port,
|
|
143
|
+
close: () => new Promise((resolve) => server.close(() => resolve()))
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
async function handle(opts, cors, req, res) {
|
|
147
|
+
if (req.method === "OPTIONS") {
|
|
148
|
+
res.writeHead(204, cors).end();
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
if (req.method !== "GET" && req.method !== "HEAD") {
|
|
152
|
+
res.writeHead(405, cors).end();
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
const urlPath = (req.url ?? "").split("?")[0] ?? "";
|
|
156
|
+
const parsed = parseTokenizedUrl(opts.urlPrefix, urlPath);
|
|
157
|
+
if (!parsed) {
|
|
158
|
+
res.writeHead(403, cors).end();
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
if (!opts.verifyToken(parsed.token, parsed.rel)) {
|
|
162
|
+
res.writeHead(401, cors).end();
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
const resolved = resolveFilePath(opts.getRoots(), parsed.rel);
|
|
166
|
+
if ("error" in resolved) {
|
|
167
|
+
res.writeHead(403, cors).end();
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
let absPath = null;
|
|
171
|
+
let size = 0;
|
|
172
|
+
for (const candidate of resolved.candidates) try {
|
|
173
|
+
const st = await node_fs.promises.stat(candidate);
|
|
174
|
+
if (st.isFile()) {
|
|
175
|
+
absPath = candidate;
|
|
176
|
+
size = st.size;
|
|
177
|
+
break;
|
|
178
|
+
}
|
|
179
|
+
} catch {}
|
|
180
|
+
if (absPath === null) {
|
|
181
|
+
res.writeHead(404, cors).end();
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
const baseHeaders = {
|
|
185
|
+
...cors,
|
|
186
|
+
"content-type": contentTypeFor(absPath, opts.contentTypes),
|
|
187
|
+
"accept-ranges": "bytes",
|
|
188
|
+
"cache-control": "no-cache"
|
|
189
|
+
};
|
|
190
|
+
const range = parseRangeHeader(req.headers.range, size);
|
|
191
|
+
if (range) {
|
|
192
|
+
res.writeHead(206, {
|
|
193
|
+
...baseHeaders,
|
|
194
|
+
"content-range": `bytes ${range.start}-${range.end}/${size}`,
|
|
195
|
+
"content-length": String(range.end - range.start + 1)
|
|
196
|
+
});
|
|
197
|
+
if (req.method === "HEAD") {
|
|
198
|
+
res.end();
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
(0, node_fs.createReadStream)(absPath, {
|
|
202
|
+
start: range.start,
|
|
203
|
+
end: range.end
|
|
204
|
+
}).pipe(res);
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
res.writeHead(200, {
|
|
208
|
+
...baseHeaders,
|
|
209
|
+
"content-length": String(size)
|
|
210
|
+
});
|
|
211
|
+
if (req.method === "HEAD") {
|
|
212
|
+
res.end();
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
(0, node_fs.createReadStream)(absPath).pipe(res);
|
|
216
|
+
}
|
|
217
|
+
//#endregion
|
|
218
|
+
//#region src/http/file-data-plane.ts
|
|
219
|
+
/**
|
|
220
|
+
* A ready-made data-plane request handler that serves files from a set of roots
|
|
221
|
+
* with HTTP `Range`, for addons whose data-plane is "stream files off disk"
|
|
222
|
+
* (recording playback, scrub-frame stores, exported clips).
|
|
223
|
+
*
|
|
224
|
+
* This is the addon-side handler the addon hands to `ctx.dataPlane.serve({ handler })`.
|
|
225
|
+
* It receives the REAL Node `req`/`res`, resolves the request path against the
|
|
226
|
+
* addon's roots (traversal-guarded), and streams the file. NO token and NO CORS:
|
|
227
|
+
* the hub already authenticated the caller and the data plane is same-origin
|
|
228
|
+
* through the hub's port. Reuses the shared Range/content-type/traversal helpers
|
|
229
|
+
* so there is one implementation across the standalone file-server and this.
|
|
230
|
+
*/
|
|
231
|
+
/** Build a `(req, res)` handler that serves `getRoots()` files with Range. */
|
|
232
|
+
function createFileDataPlaneHandler(opts) {
|
|
233
|
+
return async (req, res) => {
|
|
234
|
+
if (req.method !== "GET" && req.method !== "HEAD") {
|
|
235
|
+
res.writeHead(405).end();
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
const urlPath = (req.url ?? "/").split("?")[0] ?? "/";
|
|
239
|
+
let rel;
|
|
240
|
+
try {
|
|
241
|
+
rel = decodeURIComponent(urlPath.replace(/^\/+/, ""));
|
|
242
|
+
} catch {
|
|
243
|
+
res.writeHead(400).end();
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
if (rel.length === 0) {
|
|
247
|
+
res.writeHead(404).end();
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
const resolved = resolveFilePath(opts.getRoots(), rel);
|
|
251
|
+
if ("error" in resolved) {
|
|
252
|
+
res.writeHead(403).end();
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
let absPath = null;
|
|
256
|
+
let size = 0;
|
|
257
|
+
for (const candidate of resolved.candidates) try {
|
|
258
|
+
const st = await node_fs.promises.stat(candidate);
|
|
259
|
+
if (st.isFile()) {
|
|
260
|
+
absPath = candidate;
|
|
261
|
+
size = st.size;
|
|
262
|
+
break;
|
|
263
|
+
}
|
|
264
|
+
} catch {}
|
|
265
|
+
if (absPath === null) {
|
|
266
|
+
res.writeHead(404).end();
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
const baseHeaders = {
|
|
270
|
+
"content-type": contentTypeFor(absPath, opts.contentTypes),
|
|
271
|
+
"accept-ranges": "bytes",
|
|
272
|
+
"cache-control": "no-cache"
|
|
273
|
+
};
|
|
274
|
+
const range = parseRangeHeader(req.headers.range, size);
|
|
275
|
+
if (range) {
|
|
276
|
+
res.writeHead(206, {
|
|
277
|
+
...baseHeaders,
|
|
278
|
+
"content-range": `bytes ${range.start}-${range.end}/${size}`,
|
|
279
|
+
"content-length": String(range.end - range.start + 1)
|
|
280
|
+
});
|
|
281
|
+
if (req.method === "HEAD") {
|
|
282
|
+
res.end();
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
(0, node_fs.createReadStream)(absPath, {
|
|
286
|
+
start: range.start,
|
|
287
|
+
end: range.end
|
|
288
|
+
}).pipe(res);
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
res.writeHead(200, {
|
|
292
|
+
...baseHeaders,
|
|
293
|
+
"content-length": String(size)
|
|
294
|
+
});
|
|
295
|
+
if (req.method === "HEAD") {
|
|
296
|
+
res.end();
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
(0, node_fs.createReadStream)(absPath).pipe(res);
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
//#endregion
|
|
303
|
+
//#region src/download/model-downloader.ts
|
|
304
|
+
/** Build fetch headers, including HF auth token for huggingface.co URLs */
|
|
305
|
+
function buildHeaders(url) {
|
|
306
|
+
const headers = { "User-Agent": "CamStack/1.0" };
|
|
307
|
+
const hfToken = process.env["HF_TOKEN"] ?? process.env["HUGGING_FACE_HUB_TOKEN"];
|
|
308
|
+
if (hfToken && url.includes("huggingface.co")) headers["Authorization"] = `Bearer ${hfToken}`;
|
|
309
|
+
return headers;
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Download a single file from a URL to a destination path.
|
|
313
|
+
* Uses native fetch() (Node 22+) which handles redirects natively.
|
|
314
|
+
* Streams to disk with optional progress callback.
|
|
315
|
+
* Returns the destination path. Skips download if file already exists.
|
|
316
|
+
*/
|
|
317
|
+
async function downloadFile(url, destPath, onProgress) {
|
|
318
|
+
if (node_fs.existsSync(destPath)) return destPath;
|
|
319
|
+
node_fs.mkdirSync(node_path.dirname(destPath), { recursive: true });
|
|
320
|
+
const tmpPath = destPath + ".downloading";
|
|
321
|
+
try {
|
|
322
|
+
const response = await fetch(url, {
|
|
323
|
+
redirect: "follow",
|
|
324
|
+
headers: buildHeaders(url)
|
|
325
|
+
});
|
|
326
|
+
if (!response.ok) throw new Error(`HTTP ${response.status} downloading ${url}`);
|
|
327
|
+
if (!response.body) throw new Error(`No response body from ${url}`);
|
|
328
|
+
const total = parseInt(response.headers.get("content-length") ?? "0", 10);
|
|
329
|
+
let downloaded = 0;
|
|
330
|
+
const fileStream = node_fs.createWriteStream(tmpPath);
|
|
331
|
+
const reader = response.body.getReader();
|
|
332
|
+
try {
|
|
333
|
+
for (;;) {
|
|
334
|
+
const { done, value } = await reader.read();
|
|
335
|
+
if (done || !value) break;
|
|
336
|
+
fileStream.write(value);
|
|
337
|
+
downloaded += value.length;
|
|
338
|
+
onProgress?.(downloaded, total);
|
|
339
|
+
}
|
|
340
|
+
} finally {
|
|
341
|
+
fileStream.end();
|
|
342
|
+
await new Promise((resolve, reject) => {
|
|
343
|
+
fileStream.on("finish", resolve);
|
|
344
|
+
fileStream.on("error", reject);
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
node_fs.renameSync(tmpPath, destPath);
|
|
348
|
+
return destPath;
|
|
349
|
+
} catch (err) {
|
|
350
|
+
try {
|
|
351
|
+
node_fs.unlinkSync(tmpPath);
|
|
352
|
+
} catch {}
|
|
353
|
+
throw err;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Fetch JSON from a URL using native fetch().
|
|
358
|
+
*/
|
|
359
|
+
async function fetchJson(url) {
|
|
360
|
+
const response = await fetch(url, {
|
|
361
|
+
redirect: "follow",
|
|
362
|
+
headers: buildHeaders(url)
|
|
363
|
+
});
|
|
364
|
+
if (!response.ok) throw new Error(`HTTP ${response.status} fetching ${url}`);
|
|
365
|
+
return response.json();
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Download a model with fallback URLs and optional SHA256 verification.
|
|
369
|
+
* Legacy API preserved for backward compatibility -- delegates to downloadFile().
|
|
370
|
+
*/
|
|
371
|
+
async function downloadModel(options) {
|
|
372
|
+
const { url, fallbackUrls = [], destDir, filename, expectedSha256, onProgress } = options;
|
|
373
|
+
const fname = filename ?? url.split("/").pop() ?? "model.bin";
|
|
374
|
+
const destPath = node_path.join(destDir, fname);
|
|
375
|
+
if (node_fs.existsSync(destPath)) return {
|
|
376
|
+
filePath: destPath,
|
|
377
|
+
downloadedBytes: 0,
|
|
378
|
+
fromCache: true
|
|
379
|
+
};
|
|
380
|
+
node_fs.mkdirSync(destDir, { recursive: true });
|
|
381
|
+
const urls = [url, ...fallbackUrls];
|
|
382
|
+
let lastError = null;
|
|
383
|
+
for (const tryUrl of urls) try {
|
|
384
|
+
await downloadFile(tryUrl, destPath, onProgress);
|
|
385
|
+
if (expectedSha256) {
|
|
386
|
+
const hash = await computeSha256(destPath);
|
|
387
|
+
if (hash !== expectedSha256) {
|
|
388
|
+
node_fs.unlinkSync(destPath);
|
|
389
|
+
throw new Error(`SHA256 mismatch: expected ${expectedSha256}, got ${hash}`);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
return {
|
|
393
|
+
filePath: destPath,
|
|
394
|
+
downloadedBytes: node_fs.statSync(destPath).size,
|
|
395
|
+
fromCache: false
|
|
396
|
+
};
|
|
397
|
+
} catch (e) {
|
|
398
|
+
lastError = e;
|
|
399
|
+
if (node_fs.existsSync(destPath)) node_fs.unlinkSync(destPath);
|
|
400
|
+
}
|
|
401
|
+
throw lastError ?? /* @__PURE__ */ new Error(`Failed to download model from ${url}`);
|
|
402
|
+
}
|
|
403
|
+
async function computeSha256(filePath) {
|
|
404
|
+
return new Promise((resolve, reject) => {
|
|
405
|
+
const hash = (0, node_crypto.createHash)("sha256");
|
|
406
|
+
const stream = node_fs.createReadStream(filePath);
|
|
407
|
+
stream.on("data", (chunk) => hash.update(chunk));
|
|
408
|
+
stream.on("end", () => resolve(hash.digest("hex")));
|
|
409
|
+
stream.on("error", reject);
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
/**
|
|
413
|
+
* Download every file in a HuggingFace directory bundle (e.g.,
|
|
414
|
+
* `.mlpackage` / OpenVINO IR pair) atomically. `knownFiles` lists the
|
|
415
|
+
* relative paths inside the directory; the function fetches each from
|
|
416
|
+
* `${url}/${file}` and renames the staging directory only on full
|
|
417
|
+
* success. Mirrors `ModelDownloadService.downloadDirectory` but
|
|
418
|
+
* exposed as a standalone for catalog-less callers.
|
|
419
|
+
*/
|
|
420
|
+
async function downloadDirectory(url, destDir, knownFiles, onProgress) {
|
|
421
|
+
const match = url.match(/huggingface\.co\/([^/]+\/[^/]+)\/resolve\/main\/(.+)/);
|
|
422
|
+
if (!match) throw new Error(`Cannot parse HuggingFace URL: ${url}`);
|
|
423
|
+
const [, repo, dirPath] = match;
|
|
424
|
+
const files = (knownFiles ?? []).map((f) => ({
|
|
425
|
+
relativePath: f,
|
|
426
|
+
fileUrl: `https://huggingface.co/${repo}/resolve/main/${dirPath}/${f}`
|
|
427
|
+
}));
|
|
428
|
+
if (files.length === 0) throw new Error(`Directory bundle requires explicit \`files\` list (got none for ${url})`);
|
|
429
|
+
const tmpDir = destDir + ".downloading";
|
|
430
|
+
node_fs.rmSync(tmpDir, {
|
|
431
|
+
recursive: true,
|
|
432
|
+
force: true
|
|
433
|
+
});
|
|
434
|
+
node_fs.mkdirSync(tmpDir, { recursive: true });
|
|
435
|
+
let totalDownloaded = 0;
|
|
436
|
+
try {
|
|
437
|
+
for (const file of files) {
|
|
438
|
+
const destPath = node_path.join(tmpDir, file.relativePath);
|
|
439
|
+
node_fs.mkdirSync(node_path.dirname(destPath), { recursive: true });
|
|
440
|
+
await downloadFile(file.fileUrl, destPath, (downloaded, _total) => {
|
|
441
|
+
onProgress?.(totalDownloaded + downloaded, void 0);
|
|
442
|
+
});
|
|
443
|
+
totalDownloaded += node_fs.statSync(destPath).size;
|
|
444
|
+
}
|
|
445
|
+
node_fs.rmSync(destDir, {
|
|
446
|
+
recursive: true,
|
|
447
|
+
force: true
|
|
448
|
+
});
|
|
449
|
+
node_fs.renameSync(tmpDir, destDir);
|
|
450
|
+
} catch (err) {
|
|
451
|
+
node_fs.rmSync(tmpDir, {
|
|
452
|
+
recursive: true,
|
|
453
|
+
force: true
|
|
454
|
+
});
|
|
455
|
+
throw err;
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
/**
|
|
459
|
+
* Resolve a `ModelCatalogEntry` against `modelsDir`: download model file
|
|
460
|
+
* (or directory bundle) + extra files (labels JSON, charset dict, …),
|
|
461
|
+
* skip if already on disk. Returns the local model path.
|
|
462
|
+
*/
|
|
463
|
+
async function ensureModel(modelsDir, entry, format, onProgress) {
|
|
464
|
+
const formatEntry = entry.formats[format];
|
|
465
|
+
if (!formatEntry) throw new Error(`Model "${entry.id}" has no ${format} format. Available: ${Object.keys(entry.formats).join(", ")}`);
|
|
466
|
+
if (entry.extraFiles) for (const extra of entry.extraFiles) await downloadFile(extra.url, node_path.join(modelsDir, extra.filename));
|
|
467
|
+
const filename = formatEntry.url.split("/").pop() ?? `${entry.id}.${format}`;
|
|
468
|
+
const modelPath = node_path.join(modelsDir, filename);
|
|
469
|
+
if (node_fs.existsSync(modelPath)) if (formatEntry.isDirectory && !node_fs.existsSync(node_path.join(modelPath, "Manifest.json"))) node_fs.rmSync(modelPath, {
|
|
470
|
+
recursive: true,
|
|
471
|
+
force: true
|
|
472
|
+
});
|
|
473
|
+
else return modelPath;
|
|
474
|
+
node_fs.mkdirSync(modelsDir, { recursive: true });
|
|
475
|
+
if (formatEntry.isDirectory) await downloadDirectory(formatEntry.url, modelPath, formatEntry.files, onProgress);
|
|
476
|
+
else await downloadFile(formatEntry.url, modelPath, (downloaded, total) => onProgress?.(downloaded, total === 0 ? void 0 : total));
|
|
477
|
+
return modelPath;
|
|
478
|
+
}
|
|
479
|
+
/** Compute the on-disk path for a given model + format, even when not yet downloaded. */
|
|
480
|
+
function getModelFilePath(modelsDir, entry, format) {
|
|
481
|
+
const formatEntry = entry.formats[format];
|
|
482
|
+
if (!formatEntry) return null;
|
|
483
|
+
const filename = formatEntry.url.split("/").pop() ?? `${entry.id}.${format}`;
|
|
484
|
+
return node_path.join(modelsDir, filename);
|
|
485
|
+
}
|
|
486
|
+
/** True iff the model file (or `Manifest.json` for directory bundles) exists and is non-empty. */
|
|
487
|
+
function isModelDownloaded(modelsDir, entry, format) {
|
|
488
|
+
const formatEntry = entry.formats[format];
|
|
489
|
+
if (!formatEntry) return false;
|
|
490
|
+
const modelPath = getModelFilePath(modelsDir, entry, format);
|
|
491
|
+
if (!modelPath || !node_fs.existsSync(modelPath)) return false;
|
|
492
|
+
if (formatEntry.isDirectory) return node_fs.existsSync(node_path.join(modelPath, "Manifest.json"));
|
|
493
|
+
return node_fs.statSync(modelPath).size > 0;
|
|
494
|
+
}
|
|
495
|
+
/** Remove the on-disk model file/directory. Returns true if something was deleted. */
|
|
496
|
+
function deleteModelFromDisk(modelsDir, entry, format) {
|
|
497
|
+
const modelPath = getModelFilePath(modelsDir, entry, format);
|
|
498
|
+
if (!modelPath || !node_fs.existsSync(modelPath)) return false;
|
|
499
|
+
if (entry.formats[format]?.isDirectory) node_fs.rmSync(modelPath, {
|
|
500
|
+
recursive: true,
|
|
501
|
+
force: true
|
|
502
|
+
});
|
|
503
|
+
else node_fs.unlinkSync(modelPath);
|
|
504
|
+
return true;
|
|
505
|
+
}
|
|
506
|
+
//#endregion
|
|
507
|
+
//#region src/download/model-download-service.ts
|
|
508
|
+
/**
|
|
509
|
+
* Unified model download service.
|
|
510
|
+
*
|
|
511
|
+
* Handles downloading model files and extra files (labels, dicts) from a
|
|
512
|
+
* catalog of ModelCatalogEntry items. Supports single-file models and
|
|
513
|
+
* directory bundles (e.g., .mlpackage for CoreML).
|
|
514
|
+
*
|
|
515
|
+
* Addons use this via `context.models.ensure(modelId, format)`.
|
|
516
|
+
*/
|
|
517
|
+
var ModelDownloadService = class {
|
|
518
|
+
modelsDir;
|
|
519
|
+
onProgress;
|
|
520
|
+
catalog;
|
|
521
|
+
constructor(modelsDir, catalog, onProgress) {
|
|
522
|
+
this.modelsDir = modelsDir;
|
|
523
|
+
this.onProgress = onProgress;
|
|
524
|
+
const map = /* @__PURE__ */ new Map();
|
|
525
|
+
for (const entry of catalog) map.set(entry.id, entry);
|
|
526
|
+
this.catalog = map;
|
|
527
|
+
}
|
|
528
|
+
/**
|
|
529
|
+
* Ensure a model (and its extra files) is downloaded.
|
|
530
|
+
* Returns the local filesystem path to the model file/directory.
|
|
531
|
+
*/
|
|
532
|
+
async ensure(modelId, format) {
|
|
533
|
+
const entry = this.catalog.get(modelId);
|
|
534
|
+
if (!entry) throw new Error(`ModelDownloadService: unknown model "${modelId}"`);
|
|
535
|
+
const selectedFormat = format ?? this.pickDefaultFormat(entry);
|
|
536
|
+
const formatEntry = entry.formats[selectedFormat];
|
|
537
|
+
if (!formatEntry) throw new Error(`ModelDownloadService: model "${modelId}" has no ${selectedFormat} format`);
|
|
538
|
+
await this.ensureExtraFiles(modelId);
|
|
539
|
+
const modelPath = this.modelFilePath(entry, selectedFormat);
|
|
540
|
+
if (node_fs.existsSync(modelPath)) if (formatEntry.isDirectory) if (!node_fs.existsSync(node_path.join(modelPath, "Manifest.json"))) node_fs.rmSync(modelPath, {
|
|
541
|
+
recursive: true,
|
|
542
|
+
force: true
|
|
543
|
+
});
|
|
544
|
+
else return modelPath;
|
|
545
|
+
else return modelPath;
|
|
546
|
+
node_fs.mkdirSync(this.modelsDir, { recursive: true });
|
|
547
|
+
if (formatEntry.isDirectory) await this.downloadDirectory(formatEntry.url, modelPath, formatEntry.files, modelId);
|
|
548
|
+
else await downloadFile(formatEntry.url, modelPath, this.onProgress ? (downloaded, total) => this.onProgress(modelId, downloaded, total) : void 0);
|
|
549
|
+
return modelPath;
|
|
550
|
+
}
|
|
551
|
+
/**
|
|
552
|
+
* Ensure extra files for a model are downloaded.
|
|
553
|
+
* Returns the local paths of all extra files.
|
|
554
|
+
*/
|
|
555
|
+
async ensureExtraFiles(modelId) {
|
|
556
|
+
const entry = this.catalog.get(modelId);
|
|
557
|
+
if (!entry) throw new Error(`ModelDownloadService: unknown model "${modelId}"`);
|
|
558
|
+
const extras = entry.extraFiles;
|
|
559
|
+
if (!extras || extras.length === 0) return [];
|
|
560
|
+
const paths = [];
|
|
561
|
+
for (const extra of extras) {
|
|
562
|
+
const destPath = node_path.join(this.modelsDir, extra.filename);
|
|
563
|
+
await downloadFile(extra.url, destPath);
|
|
564
|
+
paths.push(destPath);
|
|
565
|
+
}
|
|
566
|
+
return paths;
|
|
567
|
+
}
|
|
568
|
+
/** Absolute path to the shared models directory. */
|
|
569
|
+
getModelsDir() {
|
|
570
|
+
return this.modelsDir;
|
|
571
|
+
}
|
|
572
|
+
/** Check if a model file is already present on disk. */
|
|
573
|
+
isDownloaded(modelId, format) {
|
|
574
|
+
const entry = this.catalog.get(modelId);
|
|
575
|
+
if (!entry) return false;
|
|
576
|
+
const selectedFormat = format ?? this.pickDefaultFormat(entry);
|
|
577
|
+
const formatEntry = entry.formats[selectedFormat];
|
|
578
|
+
if (!formatEntry) return false;
|
|
579
|
+
const modelPath = this.modelFilePath(entry, selectedFormat);
|
|
580
|
+
if (!node_fs.existsSync(modelPath)) return false;
|
|
581
|
+
if (formatEntry.isDirectory) return node_fs.existsSync(node_path.join(modelPath, "Manifest.json"));
|
|
582
|
+
return node_fs.statSync(modelPath).size > 0;
|
|
583
|
+
}
|
|
584
|
+
/** Get the catalog entry for a model by ID. */
|
|
585
|
+
getEntry(modelId) {
|
|
586
|
+
return this.catalog.get(modelId);
|
|
587
|
+
}
|
|
588
|
+
pickDefaultFormat(entry) {
|
|
589
|
+
for (const fmt of [
|
|
590
|
+
"onnx",
|
|
591
|
+
"coreml",
|
|
592
|
+
"openvino",
|
|
593
|
+
"tflite",
|
|
594
|
+
"pt"
|
|
595
|
+
]) if (entry.formats[fmt]) return fmt;
|
|
596
|
+
const first = Object.keys(entry.formats)[0];
|
|
597
|
+
if (first) return first;
|
|
598
|
+
throw new Error(`ModelDownloadService: model "${entry.id}" has no formats`);
|
|
599
|
+
}
|
|
600
|
+
modelFilePath(entry, format) {
|
|
601
|
+
const formatEntry = entry.formats[format];
|
|
602
|
+
if (!formatEntry) throw new Error(`Model ${entry.id} has no ${format} format`);
|
|
603
|
+
const urlParts = formatEntry.url.split("/");
|
|
604
|
+
const filename = urlParts[urlParts.length - 1] ?? `${entry.id}.${format}`;
|
|
605
|
+
return node_path.join(this.modelsDir, filename);
|
|
606
|
+
}
|
|
607
|
+
/**
|
|
608
|
+
* Download a directory bundle (e.g., .mlpackage) from HuggingFace.
|
|
609
|
+
* ATOMIC: downloads to temp dir, renames only on complete success.
|
|
610
|
+
*/
|
|
611
|
+
async downloadDirectory(url, destDir, knownFiles, _modelId) {
|
|
612
|
+
const match = url.match(/huggingface\.co\/([^/]+\/[^/]+)\/resolve\/main\/(.+)/);
|
|
613
|
+
if (!match) throw new Error(`Cannot parse HuggingFace URL: ${url}`);
|
|
614
|
+
const [, repo, dirPath] = match;
|
|
615
|
+
let filesToDownload;
|
|
616
|
+
if (knownFiles && knownFiles.length > 0) filesToDownload = knownFiles.map((f) => ({
|
|
617
|
+
relativePath: f,
|
|
618
|
+
fileUrl: `https://huggingface.co/${repo}/resolve/main/${dirPath}/${f}`
|
|
619
|
+
}));
|
|
620
|
+
else {
|
|
621
|
+
const hfFiles = await this.listHfFiles(repo, dirPath);
|
|
622
|
+
if (hfFiles.length === 0) throw new Error(`No files found in HuggingFace directory: ${dirPath}`);
|
|
623
|
+
filesToDownload = hfFiles.map((f) => ({
|
|
624
|
+
relativePath: f.path.substring(dirPath.length + 1),
|
|
625
|
+
fileUrl: `https://huggingface.co/${repo}/resolve/main/${f.path}`
|
|
626
|
+
}));
|
|
627
|
+
}
|
|
628
|
+
const tmpDir = destDir + ".downloading";
|
|
629
|
+
node_fs.rmSync(tmpDir, {
|
|
630
|
+
recursive: true,
|
|
631
|
+
force: true
|
|
632
|
+
});
|
|
633
|
+
node_fs.mkdirSync(tmpDir, { recursive: true });
|
|
634
|
+
try {
|
|
635
|
+
for (const file of filesToDownload) {
|
|
636
|
+
const destPath = node_path.join(tmpDir, file.relativePath);
|
|
637
|
+
node_fs.mkdirSync(node_path.dirname(destPath), { recursive: true });
|
|
638
|
+
await downloadFile(file.fileUrl, destPath);
|
|
639
|
+
}
|
|
640
|
+
node_fs.rmSync(destDir, {
|
|
641
|
+
recursive: true,
|
|
642
|
+
force: true
|
|
643
|
+
});
|
|
644
|
+
node_fs.renameSync(tmpDir, destDir);
|
|
645
|
+
} catch (err) {
|
|
646
|
+
node_fs.rmSync(tmpDir, {
|
|
647
|
+
recursive: true,
|
|
648
|
+
force: true
|
|
649
|
+
});
|
|
650
|
+
throw err;
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
/** Recursively list all files in a HuggingFace directory via API. */
|
|
654
|
+
async listHfFiles(repo, dirPath) {
|
|
655
|
+
const entries = await fetchJson(`https://huggingface.co/api/models/${repo}/tree/main/${dirPath}`);
|
|
656
|
+
const files = [];
|
|
657
|
+
for (const entry of entries) if (entry.type === "file") files.push({
|
|
658
|
+
path: entry.path,
|
|
659
|
+
size: entry.size ?? entry.lfs?.size ?? 0
|
|
660
|
+
});
|
|
661
|
+
else if (entry.type === "directory") {
|
|
662
|
+
const subFiles = await this.listHfFiles(repo, entry.path);
|
|
663
|
+
files.push(...subFiles);
|
|
664
|
+
}
|
|
665
|
+
return files;
|
|
666
|
+
}
|
|
667
|
+
};
|
|
668
|
+
//#endregion
|
|
669
|
+
Object.defineProperty(exports, "ModelDownloadService", {
|
|
670
|
+
enumerable: true,
|
|
671
|
+
get: function() {
|
|
672
|
+
return ModelDownloadService;
|
|
673
|
+
}
|
|
674
|
+
});
|
|
675
|
+
Object.defineProperty(exports, "contentTypeFor", {
|
|
676
|
+
enumerable: true,
|
|
677
|
+
get: function() {
|
|
678
|
+
return contentTypeFor;
|
|
679
|
+
}
|
|
680
|
+
});
|
|
681
|
+
Object.defineProperty(exports, "createAuthenticatedFileServer", {
|
|
682
|
+
enumerable: true,
|
|
683
|
+
get: function() {
|
|
684
|
+
return createAuthenticatedFileServer;
|
|
685
|
+
}
|
|
686
|
+
});
|
|
687
|
+
Object.defineProperty(exports, "createFileDataPlaneHandler", {
|
|
688
|
+
enumerable: true,
|
|
689
|
+
get: function() {
|
|
690
|
+
return createFileDataPlaneHandler;
|
|
691
|
+
}
|
|
692
|
+
});
|
|
693
|
+
Object.defineProperty(exports, "deleteModelFromDisk", {
|
|
694
|
+
enumerable: true,
|
|
695
|
+
get: function() {
|
|
696
|
+
return deleteModelFromDisk;
|
|
697
|
+
}
|
|
698
|
+
});
|
|
699
|
+
Object.defineProperty(exports, "downloadFile", {
|
|
700
|
+
enumerable: true,
|
|
701
|
+
get: function() {
|
|
702
|
+
return downloadFile;
|
|
703
|
+
}
|
|
704
|
+
});
|
|
705
|
+
Object.defineProperty(exports, "downloadModel", {
|
|
706
|
+
enumerable: true,
|
|
707
|
+
get: function() {
|
|
708
|
+
return downloadModel;
|
|
709
|
+
}
|
|
710
|
+
});
|
|
711
|
+
Object.defineProperty(exports, "ensureModel", {
|
|
712
|
+
enumerable: true,
|
|
713
|
+
get: function() {
|
|
714
|
+
return ensureModel;
|
|
715
|
+
}
|
|
716
|
+
});
|
|
717
|
+
Object.defineProperty(exports, "fetchJson", {
|
|
718
|
+
enumerable: true,
|
|
719
|
+
get: function() {
|
|
720
|
+
return fetchJson;
|
|
721
|
+
}
|
|
722
|
+
});
|
|
723
|
+
Object.defineProperty(exports, "getModelFilePath", {
|
|
724
|
+
enumerable: true,
|
|
725
|
+
get: function() {
|
|
726
|
+
return getModelFilePath;
|
|
727
|
+
}
|
|
728
|
+
});
|
|
729
|
+
Object.defineProperty(exports, "isModelDownloaded", {
|
|
730
|
+
enumerable: true,
|
|
731
|
+
get: function() {
|
|
732
|
+
return isModelDownloaded;
|
|
733
|
+
}
|
|
734
|
+
});
|
|
735
|
+
Object.defineProperty(exports, "parseRangeHeader", {
|
|
736
|
+
enumerable: true,
|
|
737
|
+
get: function() {
|
|
738
|
+
return parseRangeHeader;
|
|
739
|
+
}
|
|
740
|
+
});
|
|
741
|
+
Object.defineProperty(exports, "parseTokenizedUrl", {
|
|
742
|
+
enumerable: true,
|
|
743
|
+
get: function() {
|
|
744
|
+
return parseTokenizedUrl;
|
|
745
|
+
}
|
|
746
|
+
});
|
|
747
|
+
Object.defineProperty(exports, "resolveFilePath", {
|
|
748
|
+
enumerable: true,
|
|
749
|
+
get: function() {
|
|
750
|
+
return resolveFilePath;
|
|
751
|
+
}
|
|
752
|
+
});
|