@camstack/addon-post-analysis 1.2.134 → 1.2.136
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{dist-DruBvw_u.mjs → dist-BAN19J7s.mjs} +255 -25
- package/dist/{dist-Dqa0bdV6.js → dist-kn_4KEmG.js} +255 -25
- package/dist/embedding-encoder/index.js +3 -295
- package/dist/embedding-encoder/index.mjs +2 -294
- package/dist/pipeline-analytics/_stub.js +1 -1
- package/dist/pipeline-analytics/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-CWyLksI0.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-CrPuzi1S.mjs} +3 -3
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-D_xOplZC.mjs +26 -0
- package/dist/pipeline-analytics/{hostInit-BTMdnoEZ.mjs → hostInit-DR1JAJld.mjs} +3 -3
- package/dist/pipeline-analytics/index.js +1180 -132
- package/dist/pipeline-analytics/index.mjs +1180 -132
- package/dist/pipeline-analytics/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CZJ4DupR.mjs +0 -26
|
@@ -2,17 +2,13 @@ Object.defineProperties(exports, {
|
|
|
2
2
|
__esModule: { value: true },
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
|
-
const require_dist = require("../dist-
|
|
5
|
+
const require_dist = require("../dist-kn_4KEmG.js");
|
|
6
6
|
let node_fs = require("node:fs");
|
|
7
|
-
let node_fs$1 = require_dist.__toESM(node_fs, 1);
|
|
8
7
|
node_fs = require_dist.__toESM(node_fs);
|
|
9
8
|
let node_path = require("node:path");
|
|
10
|
-
let node_path$1 = require_dist.__toESM(node_path, 1);
|
|
11
9
|
node_path = require_dist.__toESM(node_path);
|
|
12
|
-
require("node:crypto");
|
|
13
10
|
let node_child_process = require("node:child_process");
|
|
14
|
-
let
|
|
15
|
-
let node_zlib = require("node:zlib");
|
|
11
|
+
let _camstack_system_addon_utils = require("@camstack/system/addon-utils");
|
|
16
12
|
//#region src/embedding-encoder/shared/process-memory.ts
|
|
17
13
|
/**
|
|
18
14
|
* Read a child process's memory footprint for the pool memory watchdog.
|
|
@@ -60,294 +56,6 @@ function readViaPs(pid) {
|
|
|
60
56
|
});
|
|
61
57
|
}
|
|
62
58
|
//#endregion
|
|
63
|
-
//#region ../system/dist/file-data-plane-BhKdxJgf.mjs
|
|
64
|
-
function isNonEmptyFile(filePath) {
|
|
65
|
-
return node_fs$1.existsSync(filePath) && node_fs$1.statSync(filePath).size > 0;
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
68
|
-
* Sibling files of a single-file (non-directory) format — extra files the
|
|
69
|
-
* format needs, fetched from the same remote directory as `url` and stored
|
|
70
|
-
* flat alongside the main file in `modelsDir`. Catalog-declared via
|
|
71
|
-
* `formatEntry.files`, e.g. OpenVINO IR lists its `.bin` weights next to the
|
|
72
|
-
* `.xml`. The downloader stays format-agnostic; the convention lives in the
|
|
73
|
-
* catalog data (like `MLPACKAGE_FILES` for the directory case).
|
|
74
|
-
*/
|
|
75
|
-
function siblingFilesFor(formatEntry) {
|
|
76
|
-
return formatEntry.isDirectory ? [] : formatEntry.files ?? [];
|
|
77
|
-
}
|
|
78
|
-
/** Resolve a sibling's remote URL relative to the main file's directory. */
|
|
79
|
-
function siblingUrl(mainUrl, sibling) {
|
|
80
|
-
return mainUrl.replace(/[^/]+$/, sibling);
|
|
81
|
-
}
|
|
82
|
-
/** Build fetch headers, including HF auth token for huggingface.co URLs */
|
|
83
|
-
function buildHeaders(url) {
|
|
84
|
-
const headers = { "User-Agent": "CamStack/1.0" };
|
|
85
|
-
const hfToken = process.env["HF_TOKEN"] ?? process.env["HUGGING_FACE_HUB_TOKEN"];
|
|
86
|
-
if (hfToken && url.includes("huggingface.co")) headers["Authorization"] = `Bearer ${hfToken}`;
|
|
87
|
-
return headers;
|
|
88
|
-
}
|
|
89
|
-
var DEFAULT_MAX_REDIRECTS = 5;
|
|
90
|
-
function normalizeDownloadOptions(third) {
|
|
91
|
-
if (typeof third === "function") return { onProgress: third };
|
|
92
|
-
return third ?? {};
|
|
93
|
-
}
|
|
94
|
-
function isRedirectStatus(status) {
|
|
95
|
-
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
|
96
|
-
}
|
|
97
|
-
function resolveRedirectUrl(current, location) {
|
|
98
|
-
return new URL(location, current);
|
|
99
|
-
}
|
|
100
|
-
async function downloadFile(url, destPath, onProgressOrOptions) {
|
|
101
|
-
if (node_fs$1.existsSync(destPath)) return destPath;
|
|
102
|
-
const opts = normalizeDownloadOptions(onProgressOrOptions);
|
|
103
|
-
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
104
|
-
const maxRedirects = opts.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
|
|
105
|
-
node_fs$1.mkdirSync(node_path$1.dirname(destPath), { recursive: true });
|
|
106
|
-
const tmpPath = destPath + ".downloading";
|
|
107
|
-
try {
|
|
108
|
-
let current = url;
|
|
109
|
-
const seen = /* @__PURE__ */ new Set();
|
|
110
|
-
let response;
|
|
111
|
-
const manual = opts.redirectPolicy !== void 0;
|
|
112
|
-
for (let hop = 0; hop <= maxRedirects; hop++) {
|
|
113
|
-
const parsed = new URL(current);
|
|
114
|
-
if (opts.redirectPolicy) opts.redirectPolicy(parsed, hop);
|
|
115
|
-
if (seen.has(parsed.href)) throw new Error(`Redirect loop detected at ${parsed.href}`);
|
|
116
|
-
seen.add(parsed.href);
|
|
117
|
-
const controller = opts.timeoutMs !== void 0 ? new AbortController() : void 0;
|
|
118
|
-
const timer = controller && opts.timeoutMs !== void 0 ? setTimeout(() => controller.abort(), opts.timeoutMs) : void 0;
|
|
119
|
-
try {
|
|
120
|
-
response = await fetchImpl(current, {
|
|
121
|
-
redirect: manual ? "manual" : "follow",
|
|
122
|
-
headers: buildHeaders(current),
|
|
123
|
-
...controller ? { signal: controller.signal } : {}
|
|
124
|
-
});
|
|
125
|
-
} finally {
|
|
126
|
-
if (timer) clearTimeout(timer);
|
|
127
|
-
}
|
|
128
|
-
if (manual && isRedirectStatus(response.status)) {
|
|
129
|
-
const location = response.headers.get("location");
|
|
130
|
-
if (!location) throw new Error(`Redirect ${response.status} missing Location header`);
|
|
131
|
-
if (hop >= maxRedirects) throw new Error(`Too many redirects (max ${maxRedirects}) downloading ${url}`);
|
|
132
|
-
current = resolveRedirectUrl(current, location).href;
|
|
133
|
-
continue;
|
|
134
|
-
}
|
|
135
|
-
break;
|
|
136
|
-
}
|
|
137
|
-
if (!response) throw new Error(`No response downloading ${url}`);
|
|
138
|
-
if (manual && isRedirectStatus(response.status)) throw new Error(`Too many redirects (max ${maxRedirects}) downloading ${url}`);
|
|
139
|
-
if (!response.ok) throw new Error(`HTTP ${response.status} downloading ${url}`);
|
|
140
|
-
if (!response.body) throw new Error(`No response body from ${url}`);
|
|
141
|
-
const total = parseInt(response.headers.get("content-length") ?? "0", 10);
|
|
142
|
-
let downloaded = 0;
|
|
143
|
-
const fileStream = node_fs$1.createWriteStream(tmpPath);
|
|
144
|
-
const reader = response.body.getReader();
|
|
145
|
-
try {
|
|
146
|
-
for (;;) {
|
|
147
|
-
const { done, value } = await reader.read();
|
|
148
|
-
if (done || !value) break;
|
|
149
|
-
downloaded += value.length;
|
|
150
|
-
if (opts.maxBytes !== void 0 && downloaded > opts.maxBytes) throw new Error(`Downloaded ${downloaded} bytes exceeds the ${opts.maxBytes}-byte limit`);
|
|
151
|
-
fileStream.write(value);
|
|
152
|
-
opts.onProgress?.(downloaded, total);
|
|
153
|
-
}
|
|
154
|
-
} finally {
|
|
155
|
-
fileStream.end();
|
|
156
|
-
await new Promise((resolve, reject) => {
|
|
157
|
-
fileStream.on("finish", resolve);
|
|
158
|
-
fileStream.on("error", reject);
|
|
159
|
-
});
|
|
160
|
-
}
|
|
161
|
-
node_fs$1.renameSync(tmpPath, destPath);
|
|
162
|
-
return destPath;
|
|
163
|
-
} catch (err) {
|
|
164
|
-
try {
|
|
165
|
-
node_fs$1.unlinkSync(tmpPath);
|
|
166
|
-
} catch {}
|
|
167
|
-
throw err;
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
/**
|
|
171
|
-
* Fetch JSON from a URL using native fetch().
|
|
172
|
-
*/
|
|
173
|
-
async function fetchJson(url) {
|
|
174
|
-
const response = await fetch(url, {
|
|
175
|
-
redirect: "follow",
|
|
176
|
-
headers: buildHeaders(url)
|
|
177
|
-
});
|
|
178
|
-
if (!response.ok) throw new Error(`HTTP ${response.status} fetching ${url}`);
|
|
179
|
-
return response.json();
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Unified model download service.
|
|
183
|
-
*
|
|
184
|
-
* Handles downloading model files and extra files (labels, dicts) from a
|
|
185
|
-
* catalog of ModelCatalogEntry items. Supports single-file models and
|
|
186
|
-
* directory bundles (e.g., .mlpackage for CoreML).
|
|
187
|
-
*
|
|
188
|
-
* Addons use this via `context.models.ensure(modelId, format)`.
|
|
189
|
-
*/
|
|
190
|
-
var ModelDownloadService = class {
|
|
191
|
-
modelsDir;
|
|
192
|
-
onProgress;
|
|
193
|
-
catalog;
|
|
194
|
-
constructor(modelsDir, catalog, onProgress) {
|
|
195
|
-
this.modelsDir = modelsDir;
|
|
196
|
-
this.onProgress = onProgress;
|
|
197
|
-
const map = /* @__PURE__ */ new Map();
|
|
198
|
-
for (const entry of catalog) map.set(entry.id, entry);
|
|
199
|
-
this.catalog = map;
|
|
200
|
-
}
|
|
201
|
-
/**
|
|
202
|
-
* Ensure a model (and its extra files) is downloaded.
|
|
203
|
-
* Returns the local filesystem path to the model file/directory.
|
|
204
|
-
*/
|
|
205
|
-
async ensure(modelId, format) {
|
|
206
|
-
const entry = this.catalog.get(modelId);
|
|
207
|
-
if (!entry) throw new Error(`ModelDownloadService: unknown model "${modelId}"`);
|
|
208
|
-
const selectedFormat = format ?? this.pickDefaultFormat(entry);
|
|
209
|
-
const formatEntry = entry.formats[selectedFormat];
|
|
210
|
-
if (!formatEntry) throw new Error(`ModelDownloadService: model "${modelId}" has no ${selectedFormat} format`);
|
|
211
|
-
await this.ensureExtraFiles(modelId);
|
|
212
|
-
const modelPath = this.modelFilePath(entry, selectedFormat);
|
|
213
|
-
const siblings = siblingFilesFor(formatEntry);
|
|
214
|
-
if (node_fs$1.existsSync(modelPath)) {
|
|
215
|
-
if (formatEntry.isDirectory) if (!node_fs$1.existsSync(node_path$1.join(modelPath, "Manifest.json"))) node_fs$1.rmSync(modelPath, {
|
|
216
|
-
recursive: true,
|
|
217
|
-
force: true
|
|
218
|
-
});
|
|
219
|
-
else return modelPath;
|
|
220
|
-
else if (siblings.every((f) => isNonEmptyFile(node_path$1.join(this.modelsDir, f)))) return modelPath;
|
|
221
|
-
}
|
|
222
|
-
node_fs$1.mkdirSync(this.modelsDir, { recursive: true });
|
|
223
|
-
if (formatEntry.isDirectory) await this.downloadDirectory(formatEntry.url, modelPath, formatEntry.files, modelId);
|
|
224
|
-
else {
|
|
225
|
-
await downloadFile(formatEntry.url, modelPath, this.onProgress ? (downloaded, total) => this.onProgress(modelId, downloaded, total) : void 0);
|
|
226
|
-
for (const sibling of siblings) await downloadFile(siblingUrl(formatEntry.url, sibling), node_path$1.join(this.modelsDir, sibling));
|
|
227
|
-
}
|
|
228
|
-
return modelPath;
|
|
229
|
-
}
|
|
230
|
-
/**
|
|
231
|
-
* Ensure extra files for a model are downloaded.
|
|
232
|
-
* Returns the local paths of all extra files.
|
|
233
|
-
*/
|
|
234
|
-
async ensureExtraFiles(modelId) {
|
|
235
|
-
const entry = this.catalog.get(modelId);
|
|
236
|
-
if (!entry) throw new Error(`ModelDownloadService: unknown model "${modelId}"`);
|
|
237
|
-
const extras = entry.extraFiles;
|
|
238
|
-
if (!extras || extras.length === 0) return [];
|
|
239
|
-
const paths = [];
|
|
240
|
-
for (const extra of extras) {
|
|
241
|
-
const destPath = node_path$1.join(this.modelsDir, extra.filename);
|
|
242
|
-
await downloadFile(extra.url, destPath);
|
|
243
|
-
paths.push(destPath);
|
|
244
|
-
}
|
|
245
|
-
return paths;
|
|
246
|
-
}
|
|
247
|
-
/** Absolute path to the shared models directory. */
|
|
248
|
-
getModelsDir() {
|
|
249
|
-
return this.modelsDir;
|
|
250
|
-
}
|
|
251
|
-
/** Check if a model file is already present on disk. */
|
|
252
|
-
isDownloaded(modelId, format) {
|
|
253
|
-
const entry = this.catalog.get(modelId);
|
|
254
|
-
if (!entry) return false;
|
|
255
|
-
const selectedFormat = format ?? this.pickDefaultFormat(entry);
|
|
256
|
-
const formatEntry = entry.formats[selectedFormat];
|
|
257
|
-
if (!formatEntry) return false;
|
|
258
|
-
const modelPath = this.modelFilePath(entry, selectedFormat);
|
|
259
|
-
if (!node_fs$1.existsSync(modelPath)) return false;
|
|
260
|
-
if (formatEntry.isDirectory) return node_fs$1.existsSync(node_path$1.join(modelPath, "Manifest.json"));
|
|
261
|
-
if (node_fs$1.statSync(modelPath).size <= 0) return false;
|
|
262
|
-
return siblingFilesFor(formatEntry).every((f) => isNonEmptyFile(node_path$1.join(this.modelsDir, f)));
|
|
263
|
-
}
|
|
264
|
-
/** Get the catalog entry for a model by ID. */
|
|
265
|
-
getEntry(modelId) {
|
|
266
|
-
return this.catalog.get(modelId);
|
|
267
|
-
}
|
|
268
|
-
pickDefaultFormat(entry) {
|
|
269
|
-
for (const fmt of [
|
|
270
|
-
"onnx",
|
|
271
|
-
"coreml",
|
|
272
|
-
"openvino",
|
|
273
|
-
"tflite",
|
|
274
|
-
"pt"
|
|
275
|
-
]) if (entry.formats[fmt]) return fmt;
|
|
276
|
-
const first = Object.keys(entry.formats)[0];
|
|
277
|
-
if (first) return first;
|
|
278
|
-
throw new Error(`ModelDownloadService: model "${entry.id}" has no formats`);
|
|
279
|
-
}
|
|
280
|
-
modelFilePath(entry, format) {
|
|
281
|
-
const formatEntry = entry.formats[format];
|
|
282
|
-
if (!formatEntry) throw new Error(`Model ${entry.id} has no ${format} format`);
|
|
283
|
-
const urlParts = formatEntry.url.split("/");
|
|
284
|
-
const filename = urlParts[urlParts.length - 1] ?? `${entry.id}.${format}`;
|
|
285
|
-
return node_path$1.join(this.modelsDir, filename);
|
|
286
|
-
}
|
|
287
|
-
/**
|
|
288
|
-
* Download a directory bundle (e.g., .mlpackage) from HuggingFace.
|
|
289
|
-
* ATOMIC: downloads to temp dir, renames only on complete success.
|
|
290
|
-
*/
|
|
291
|
-
async downloadDirectory(url, destDir, knownFiles, _modelId) {
|
|
292
|
-
const match = url.match(/huggingface\.co\/([^/]+\/[^/]+)\/resolve\/main\/(.+)/);
|
|
293
|
-
if (!match) throw new Error(`Cannot parse HuggingFace URL: ${url}`);
|
|
294
|
-
const [, repo, dirPath] = match;
|
|
295
|
-
let filesToDownload;
|
|
296
|
-
if (knownFiles && knownFiles.length > 0) filesToDownload = knownFiles.map((f) => ({
|
|
297
|
-
relativePath: f,
|
|
298
|
-
fileUrl: `https://huggingface.co/${repo}/resolve/main/${dirPath}/${f}`
|
|
299
|
-
}));
|
|
300
|
-
else {
|
|
301
|
-
const hfFiles = await this.listHfFiles(repo, dirPath);
|
|
302
|
-
if (hfFiles.length === 0) throw new Error(`No files found in HuggingFace directory: ${dirPath}`);
|
|
303
|
-
filesToDownload = hfFiles.map((f) => ({
|
|
304
|
-
relativePath: f.path.substring(dirPath.length + 1),
|
|
305
|
-
fileUrl: `https://huggingface.co/${repo}/resolve/main/${f.path}`
|
|
306
|
-
}));
|
|
307
|
-
}
|
|
308
|
-
const tmpDir = destDir + ".downloading";
|
|
309
|
-
node_fs$1.rmSync(tmpDir, {
|
|
310
|
-
recursive: true,
|
|
311
|
-
force: true
|
|
312
|
-
});
|
|
313
|
-
node_fs$1.mkdirSync(tmpDir, { recursive: true });
|
|
314
|
-
try {
|
|
315
|
-
for (const file of filesToDownload) {
|
|
316
|
-
const destPath = node_path$1.join(tmpDir, file.relativePath);
|
|
317
|
-
node_fs$1.mkdirSync(node_path$1.dirname(destPath), { recursive: true });
|
|
318
|
-
await downloadFile(file.fileUrl, destPath);
|
|
319
|
-
}
|
|
320
|
-
node_fs$1.rmSync(destDir, {
|
|
321
|
-
recursive: true,
|
|
322
|
-
force: true
|
|
323
|
-
});
|
|
324
|
-
node_fs$1.renameSync(tmpDir, destDir);
|
|
325
|
-
} catch (err) {
|
|
326
|
-
node_fs$1.rmSync(tmpDir, {
|
|
327
|
-
recursive: true,
|
|
328
|
-
force: true
|
|
329
|
-
});
|
|
330
|
-
throw err;
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
/** Recursively list all files in a HuggingFace directory via API. */
|
|
334
|
-
async listHfFiles(repo, dirPath) {
|
|
335
|
-
const entries = await fetchJson(`https://huggingface.co/api/models/${repo}/tree/main/${dirPath}`);
|
|
336
|
-
const files = [];
|
|
337
|
-
for (const entry of entries) if (entry.type === "file") files.push({
|
|
338
|
-
path: entry.path,
|
|
339
|
-
size: entry.size ?? entry.lfs?.size ?? 0
|
|
340
|
-
});
|
|
341
|
-
else if (entry.type === "directory") {
|
|
342
|
-
const subFiles = await this.listHfFiles(repo, entry.path);
|
|
343
|
-
files.push(...subFiles);
|
|
344
|
-
}
|
|
345
|
-
return files;
|
|
346
|
-
}
|
|
347
|
-
};
|
|
348
|
-
(0, node_util.promisify)(node_zlib.brotliCompress);
|
|
349
|
-
(0, node_util.promisify)(node_zlib.gzip);
|
|
350
|
-
//#endregion
|
|
351
59
|
//#region src/embedding-encoder/catalogs/embedding-models.ts
|
|
352
60
|
var HF_REPO = "camstack/camstack-models";
|
|
353
61
|
var hf = (path) => require_dist.hfModelUrl(HF_REPO, path);
|
|
@@ -843,7 +551,7 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
|
843
551
|
}
|
|
844
552
|
async onInitialize() {
|
|
845
553
|
const modelsDir = await this.resolveModelsDir();
|
|
846
|
-
this.models = new ModelDownloadService(modelsDir, [...CLIP_IMAGE_MODELS, ...CLIP_TEXT_MODELS]);
|
|
554
|
+
this.models = new _camstack_system_addon_utils.ModelDownloadService(modelsDir, [...CLIP_IMAGE_MODELS, ...CLIP_TEXT_MODELS]);
|
|
847
555
|
this.memoryGuard = new require_dist.PoolMemoryWatchdog({
|
|
848
556
|
policy: require_dist.resolvePoolMemoryPolicy(process.env),
|
|
849
557
|
log: this.ctx.logger.child("pool-memory"),
|
|
@@ -1,13 +1,9 @@
|
|
|
1
|
-
import { M as PoolMemoryWatchdog, at as hfModelUrl, gt as resolvePoolMemoryPolicy, nt as embeddingEncoderCapability, ut as parseProcStatus, wt as BaseAddon } from "../dist-
|
|
1
|
+
import { M as PoolMemoryWatchdog, at as hfModelUrl, gt as resolvePoolMemoryPolicy, nt as embeddingEncoderCapability, ut as parseProcStatus, wt as BaseAddon } from "../dist-BAN19J7s.mjs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import * as fs from "node:fs";
|
|
4
|
-
import "node:fs";
|
|
5
4
|
import * as path$1 from "node:path";
|
|
6
|
-
import "node:path";
|
|
7
|
-
import "node:crypto";
|
|
8
5
|
import { execFile, spawn } from "node:child_process";
|
|
9
|
-
import {
|
|
10
|
-
import { brotliCompress, gzip } from "node:zlib";
|
|
6
|
+
import { ModelDownloadService } from "@camstack/system/addon-utils";
|
|
11
7
|
//#region \0rolldown/runtime.js
|
|
12
8
|
var __defProp = Object.defineProperty;
|
|
13
9
|
var __exportAll = (all, no_symbols) => {
|
|
@@ -68,294 +64,6 @@ function readViaPs(pid) {
|
|
|
68
64
|
});
|
|
69
65
|
}
|
|
70
66
|
//#endregion
|
|
71
|
-
//#region ../system/dist/file-data-plane-BhKdxJgf.mjs
|
|
72
|
-
function isNonEmptyFile(filePath) {
|
|
73
|
-
return fs.existsSync(filePath) && fs.statSync(filePath).size > 0;
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* Sibling files of a single-file (non-directory) format — extra files the
|
|
77
|
-
* format needs, fetched from the same remote directory as `url` and stored
|
|
78
|
-
* flat alongside the main file in `modelsDir`. Catalog-declared via
|
|
79
|
-
* `formatEntry.files`, e.g. OpenVINO IR lists its `.bin` weights next to the
|
|
80
|
-
* `.xml`. The downloader stays format-agnostic; the convention lives in the
|
|
81
|
-
* catalog data (like `MLPACKAGE_FILES` for the directory case).
|
|
82
|
-
*/
|
|
83
|
-
function siblingFilesFor(formatEntry) {
|
|
84
|
-
return formatEntry.isDirectory ? [] : formatEntry.files ?? [];
|
|
85
|
-
}
|
|
86
|
-
/** Resolve a sibling's remote URL relative to the main file's directory. */
|
|
87
|
-
function siblingUrl(mainUrl, sibling) {
|
|
88
|
-
return mainUrl.replace(/[^/]+$/, sibling);
|
|
89
|
-
}
|
|
90
|
-
/** Build fetch headers, including HF auth token for huggingface.co URLs */
|
|
91
|
-
function buildHeaders(url) {
|
|
92
|
-
const headers = { "User-Agent": "CamStack/1.0" };
|
|
93
|
-
const hfToken = process.env["HF_TOKEN"] ?? process.env["HUGGING_FACE_HUB_TOKEN"];
|
|
94
|
-
if (hfToken && url.includes("huggingface.co")) headers["Authorization"] = `Bearer ${hfToken}`;
|
|
95
|
-
return headers;
|
|
96
|
-
}
|
|
97
|
-
var DEFAULT_MAX_REDIRECTS = 5;
|
|
98
|
-
function normalizeDownloadOptions(third) {
|
|
99
|
-
if (typeof third === "function") return { onProgress: third };
|
|
100
|
-
return third ?? {};
|
|
101
|
-
}
|
|
102
|
-
function isRedirectStatus(status) {
|
|
103
|
-
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
|
104
|
-
}
|
|
105
|
-
function resolveRedirectUrl(current, location) {
|
|
106
|
-
return new URL(location, current);
|
|
107
|
-
}
|
|
108
|
-
async function downloadFile(url, destPath, onProgressOrOptions) {
|
|
109
|
-
if (fs.existsSync(destPath)) return destPath;
|
|
110
|
-
const opts = normalizeDownloadOptions(onProgressOrOptions);
|
|
111
|
-
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
112
|
-
const maxRedirects = opts.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
|
|
113
|
-
fs.mkdirSync(path$1.dirname(destPath), { recursive: true });
|
|
114
|
-
const tmpPath = destPath + ".downloading";
|
|
115
|
-
try {
|
|
116
|
-
let current = url;
|
|
117
|
-
const seen = /* @__PURE__ */ new Set();
|
|
118
|
-
let response;
|
|
119
|
-
const manual = opts.redirectPolicy !== void 0;
|
|
120
|
-
for (let hop = 0; hop <= maxRedirects; hop++) {
|
|
121
|
-
const parsed = new URL(current);
|
|
122
|
-
if (opts.redirectPolicy) opts.redirectPolicy(parsed, hop);
|
|
123
|
-
if (seen.has(parsed.href)) throw new Error(`Redirect loop detected at ${parsed.href}`);
|
|
124
|
-
seen.add(parsed.href);
|
|
125
|
-
const controller = opts.timeoutMs !== void 0 ? new AbortController() : void 0;
|
|
126
|
-
const timer = controller && opts.timeoutMs !== void 0 ? setTimeout(() => controller.abort(), opts.timeoutMs) : void 0;
|
|
127
|
-
try {
|
|
128
|
-
response = await fetchImpl(current, {
|
|
129
|
-
redirect: manual ? "manual" : "follow",
|
|
130
|
-
headers: buildHeaders(current),
|
|
131
|
-
...controller ? { signal: controller.signal } : {}
|
|
132
|
-
});
|
|
133
|
-
} finally {
|
|
134
|
-
if (timer) clearTimeout(timer);
|
|
135
|
-
}
|
|
136
|
-
if (manual && isRedirectStatus(response.status)) {
|
|
137
|
-
const location = response.headers.get("location");
|
|
138
|
-
if (!location) throw new Error(`Redirect ${response.status} missing Location header`);
|
|
139
|
-
if (hop >= maxRedirects) throw new Error(`Too many redirects (max ${maxRedirects}) downloading ${url}`);
|
|
140
|
-
current = resolveRedirectUrl(current, location).href;
|
|
141
|
-
continue;
|
|
142
|
-
}
|
|
143
|
-
break;
|
|
144
|
-
}
|
|
145
|
-
if (!response) throw new Error(`No response downloading ${url}`);
|
|
146
|
-
if (manual && isRedirectStatus(response.status)) throw new Error(`Too many redirects (max ${maxRedirects}) downloading ${url}`);
|
|
147
|
-
if (!response.ok) throw new Error(`HTTP ${response.status} downloading ${url}`);
|
|
148
|
-
if (!response.body) throw new Error(`No response body from ${url}`);
|
|
149
|
-
const total = parseInt(response.headers.get("content-length") ?? "0", 10);
|
|
150
|
-
let downloaded = 0;
|
|
151
|
-
const fileStream = fs.createWriteStream(tmpPath);
|
|
152
|
-
const reader = response.body.getReader();
|
|
153
|
-
try {
|
|
154
|
-
for (;;) {
|
|
155
|
-
const { done, value } = await reader.read();
|
|
156
|
-
if (done || !value) break;
|
|
157
|
-
downloaded += value.length;
|
|
158
|
-
if (opts.maxBytes !== void 0 && downloaded > opts.maxBytes) throw new Error(`Downloaded ${downloaded} bytes exceeds the ${opts.maxBytes}-byte limit`);
|
|
159
|
-
fileStream.write(value);
|
|
160
|
-
opts.onProgress?.(downloaded, total);
|
|
161
|
-
}
|
|
162
|
-
} finally {
|
|
163
|
-
fileStream.end();
|
|
164
|
-
await new Promise((resolve, reject) => {
|
|
165
|
-
fileStream.on("finish", resolve);
|
|
166
|
-
fileStream.on("error", reject);
|
|
167
|
-
});
|
|
168
|
-
}
|
|
169
|
-
fs.renameSync(tmpPath, destPath);
|
|
170
|
-
return destPath;
|
|
171
|
-
} catch (err) {
|
|
172
|
-
try {
|
|
173
|
-
fs.unlinkSync(tmpPath);
|
|
174
|
-
} catch {}
|
|
175
|
-
throw err;
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
/**
|
|
179
|
-
* Fetch JSON from a URL using native fetch().
|
|
180
|
-
*/
|
|
181
|
-
async function fetchJson(url) {
|
|
182
|
-
const response = await fetch(url, {
|
|
183
|
-
redirect: "follow",
|
|
184
|
-
headers: buildHeaders(url)
|
|
185
|
-
});
|
|
186
|
-
if (!response.ok) throw new Error(`HTTP ${response.status} fetching ${url}`);
|
|
187
|
-
return response.json();
|
|
188
|
-
}
|
|
189
|
-
/**
|
|
190
|
-
* Unified model download service.
|
|
191
|
-
*
|
|
192
|
-
* Handles downloading model files and extra files (labels, dicts) from a
|
|
193
|
-
* catalog of ModelCatalogEntry items. Supports single-file models and
|
|
194
|
-
* directory bundles (e.g., .mlpackage for CoreML).
|
|
195
|
-
*
|
|
196
|
-
* Addons use this via `context.models.ensure(modelId, format)`.
|
|
197
|
-
*/
|
|
198
|
-
var ModelDownloadService = class {
|
|
199
|
-
modelsDir;
|
|
200
|
-
onProgress;
|
|
201
|
-
catalog;
|
|
202
|
-
constructor(modelsDir, catalog, onProgress) {
|
|
203
|
-
this.modelsDir = modelsDir;
|
|
204
|
-
this.onProgress = onProgress;
|
|
205
|
-
const map = /* @__PURE__ */ new Map();
|
|
206
|
-
for (const entry of catalog) map.set(entry.id, entry);
|
|
207
|
-
this.catalog = map;
|
|
208
|
-
}
|
|
209
|
-
/**
|
|
210
|
-
* Ensure a model (and its extra files) is downloaded.
|
|
211
|
-
* Returns the local filesystem path to the model file/directory.
|
|
212
|
-
*/
|
|
213
|
-
async ensure(modelId, format) {
|
|
214
|
-
const entry = this.catalog.get(modelId);
|
|
215
|
-
if (!entry) throw new Error(`ModelDownloadService: unknown model "${modelId}"`);
|
|
216
|
-
const selectedFormat = format ?? this.pickDefaultFormat(entry);
|
|
217
|
-
const formatEntry = entry.formats[selectedFormat];
|
|
218
|
-
if (!formatEntry) throw new Error(`ModelDownloadService: model "${modelId}" has no ${selectedFormat} format`);
|
|
219
|
-
await this.ensureExtraFiles(modelId);
|
|
220
|
-
const modelPath = this.modelFilePath(entry, selectedFormat);
|
|
221
|
-
const siblings = siblingFilesFor(formatEntry);
|
|
222
|
-
if (fs.existsSync(modelPath)) {
|
|
223
|
-
if (formatEntry.isDirectory) if (!fs.existsSync(path$1.join(modelPath, "Manifest.json"))) fs.rmSync(modelPath, {
|
|
224
|
-
recursive: true,
|
|
225
|
-
force: true
|
|
226
|
-
});
|
|
227
|
-
else return modelPath;
|
|
228
|
-
else if (siblings.every((f) => isNonEmptyFile(path$1.join(this.modelsDir, f)))) return modelPath;
|
|
229
|
-
}
|
|
230
|
-
fs.mkdirSync(this.modelsDir, { recursive: true });
|
|
231
|
-
if (formatEntry.isDirectory) await this.downloadDirectory(formatEntry.url, modelPath, formatEntry.files, modelId);
|
|
232
|
-
else {
|
|
233
|
-
await downloadFile(formatEntry.url, modelPath, this.onProgress ? (downloaded, total) => this.onProgress(modelId, downloaded, total) : void 0);
|
|
234
|
-
for (const sibling of siblings) await downloadFile(siblingUrl(formatEntry.url, sibling), path$1.join(this.modelsDir, sibling));
|
|
235
|
-
}
|
|
236
|
-
return modelPath;
|
|
237
|
-
}
|
|
238
|
-
/**
|
|
239
|
-
* Ensure extra files for a model are downloaded.
|
|
240
|
-
* Returns the local paths of all extra files.
|
|
241
|
-
*/
|
|
242
|
-
async ensureExtraFiles(modelId) {
|
|
243
|
-
const entry = this.catalog.get(modelId);
|
|
244
|
-
if (!entry) throw new Error(`ModelDownloadService: unknown model "${modelId}"`);
|
|
245
|
-
const extras = entry.extraFiles;
|
|
246
|
-
if (!extras || extras.length === 0) return [];
|
|
247
|
-
const paths = [];
|
|
248
|
-
for (const extra of extras) {
|
|
249
|
-
const destPath = path$1.join(this.modelsDir, extra.filename);
|
|
250
|
-
await downloadFile(extra.url, destPath);
|
|
251
|
-
paths.push(destPath);
|
|
252
|
-
}
|
|
253
|
-
return paths;
|
|
254
|
-
}
|
|
255
|
-
/** Absolute path to the shared models directory. */
|
|
256
|
-
getModelsDir() {
|
|
257
|
-
return this.modelsDir;
|
|
258
|
-
}
|
|
259
|
-
/** Check if a model file is already present on disk. */
|
|
260
|
-
isDownloaded(modelId, format) {
|
|
261
|
-
const entry = this.catalog.get(modelId);
|
|
262
|
-
if (!entry) return false;
|
|
263
|
-
const selectedFormat = format ?? this.pickDefaultFormat(entry);
|
|
264
|
-
const formatEntry = entry.formats[selectedFormat];
|
|
265
|
-
if (!formatEntry) return false;
|
|
266
|
-
const modelPath = this.modelFilePath(entry, selectedFormat);
|
|
267
|
-
if (!fs.existsSync(modelPath)) return false;
|
|
268
|
-
if (formatEntry.isDirectory) return fs.existsSync(path$1.join(modelPath, "Manifest.json"));
|
|
269
|
-
if (fs.statSync(modelPath).size <= 0) return false;
|
|
270
|
-
return siblingFilesFor(formatEntry).every((f) => isNonEmptyFile(path$1.join(this.modelsDir, f)));
|
|
271
|
-
}
|
|
272
|
-
/** Get the catalog entry for a model by ID. */
|
|
273
|
-
getEntry(modelId) {
|
|
274
|
-
return this.catalog.get(modelId);
|
|
275
|
-
}
|
|
276
|
-
pickDefaultFormat(entry) {
|
|
277
|
-
for (const fmt of [
|
|
278
|
-
"onnx",
|
|
279
|
-
"coreml",
|
|
280
|
-
"openvino",
|
|
281
|
-
"tflite",
|
|
282
|
-
"pt"
|
|
283
|
-
]) if (entry.formats[fmt]) return fmt;
|
|
284
|
-
const first = Object.keys(entry.formats)[0];
|
|
285
|
-
if (first) return first;
|
|
286
|
-
throw new Error(`ModelDownloadService: model "${entry.id}" has no formats`);
|
|
287
|
-
}
|
|
288
|
-
modelFilePath(entry, format) {
|
|
289
|
-
const formatEntry = entry.formats[format];
|
|
290
|
-
if (!formatEntry) throw new Error(`Model ${entry.id} has no ${format} format`);
|
|
291
|
-
const urlParts = formatEntry.url.split("/");
|
|
292
|
-
const filename = urlParts[urlParts.length - 1] ?? `${entry.id}.${format}`;
|
|
293
|
-
return path$1.join(this.modelsDir, filename);
|
|
294
|
-
}
|
|
295
|
-
/**
|
|
296
|
-
* Download a directory bundle (e.g., .mlpackage) from HuggingFace.
|
|
297
|
-
* ATOMIC: downloads to temp dir, renames only on complete success.
|
|
298
|
-
*/
|
|
299
|
-
async downloadDirectory(url, destDir, knownFiles, _modelId) {
|
|
300
|
-
const match = url.match(/huggingface\.co\/([^/]+\/[^/]+)\/resolve\/main\/(.+)/);
|
|
301
|
-
if (!match) throw new Error(`Cannot parse HuggingFace URL: ${url}`);
|
|
302
|
-
const [, repo, dirPath] = match;
|
|
303
|
-
let filesToDownload;
|
|
304
|
-
if (knownFiles && knownFiles.length > 0) filesToDownload = knownFiles.map((f) => ({
|
|
305
|
-
relativePath: f,
|
|
306
|
-
fileUrl: `https://huggingface.co/${repo}/resolve/main/${dirPath}/${f}`
|
|
307
|
-
}));
|
|
308
|
-
else {
|
|
309
|
-
const hfFiles = await this.listHfFiles(repo, dirPath);
|
|
310
|
-
if (hfFiles.length === 0) throw new Error(`No files found in HuggingFace directory: ${dirPath}`);
|
|
311
|
-
filesToDownload = hfFiles.map((f) => ({
|
|
312
|
-
relativePath: f.path.substring(dirPath.length + 1),
|
|
313
|
-
fileUrl: `https://huggingface.co/${repo}/resolve/main/${f.path}`
|
|
314
|
-
}));
|
|
315
|
-
}
|
|
316
|
-
const tmpDir = destDir + ".downloading";
|
|
317
|
-
fs.rmSync(tmpDir, {
|
|
318
|
-
recursive: true,
|
|
319
|
-
force: true
|
|
320
|
-
});
|
|
321
|
-
fs.mkdirSync(tmpDir, { recursive: true });
|
|
322
|
-
try {
|
|
323
|
-
for (const file of filesToDownload) {
|
|
324
|
-
const destPath = path$1.join(tmpDir, file.relativePath);
|
|
325
|
-
fs.mkdirSync(path$1.dirname(destPath), { recursive: true });
|
|
326
|
-
await downloadFile(file.fileUrl, destPath);
|
|
327
|
-
}
|
|
328
|
-
fs.rmSync(destDir, {
|
|
329
|
-
recursive: true,
|
|
330
|
-
force: true
|
|
331
|
-
});
|
|
332
|
-
fs.renameSync(tmpDir, destDir);
|
|
333
|
-
} catch (err) {
|
|
334
|
-
fs.rmSync(tmpDir, {
|
|
335
|
-
recursive: true,
|
|
336
|
-
force: true
|
|
337
|
-
});
|
|
338
|
-
throw err;
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
|
-
/** Recursively list all files in a HuggingFace directory via API. */
|
|
342
|
-
async listHfFiles(repo, dirPath) {
|
|
343
|
-
const entries = await fetchJson(`https://huggingface.co/api/models/${repo}/tree/main/${dirPath}`);
|
|
344
|
-
const files = [];
|
|
345
|
-
for (const entry of entries) if (entry.type === "file") files.push({
|
|
346
|
-
path: entry.path,
|
|
347
|
-
size: entry.size ?? entry.lfs?.size ?? 0
|
|
348
|
-
});
|
|
349
|
-
else if (entry.type === "directory") {
|
|
350
|
-
const subFiles = await this.listHfFiles(repo, entry.path);
|
|
351
|
-
files.push(...subFiles);
|
|
352
|
-
}
|
|
353
|
-
return files;
|
|
354
|
-
}
|
|
355
|
-
};
|
|
356
|
-
promisify(brotliCompress);
|
|
357
|
-
promisify(gzip);
|
|
358
|
-
//#endregion
|
|
359
67
|
//#region src/embedding-encoder/catalogs/embedding-models.ts
|
|
360
68
|
var HF_REPO = "camstack/camstack-models";
|
|
361
69
|
var hf = (path) => hfModelUrl(HF_REPO, path);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { a as e, i as t, n, o as r, r as i, t as a } from "./_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare__react__loadShare__.js-C0AuF9av.mjs";
|
|
2
2
|
import { t as o } from "./_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_tanstack_mf_1_react_mf_2_query__loadShare__.js-B3Wx5J80.mjs";
|
|
3
|
-
import { a as s, i as c, n as l, o as u, r as d, s as f, t as p } from "./_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-
|
|
3
|
+
import { a as s, i as c, n as l, o as u, r as d, s as f, t as p } from "./_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-D_xOplZC.mjs";
|
|
4
4
|
import { n as m, r as h, t as g } from "./_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare__react_mf_1_jsx_mf_2_runtime__loadShare__.js-Bm-iyjmq.mjs";
|
|
5
5
|
//#region ../../node_modules/lucide-react/dist/esm/shared/src/utils.js
|
|
6
6
|
var _ = (e) => e.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase(), v = (e) => e.replace(/^([A-Z])|[\s-_]+(\w)/g, (e, t, n) => n ? n.toUpperCase() : t.toLowerCase()), y = (e) => {
|
|
@@ -3,7 +3,7 @@ import "./dist-CYZr2fwk.mjs";
|
|
|
3
3
|
var e = {
|
|
4
4
|
"@camstack/sdk": {
|
|
5
5
|
name: "@camstack/sdk",
|
|
6
|
-
version: "1.2.
|
|
6
|
+
version: "1.2.40",
|
|
7
7
|
scope: ["default"],
|
|
8
8
|
loaded: !1,
|
|
9
9
|
from: "addon_pipeline_analytics_widgets",
|
|
@@ -18,7 +18,7 @@ var e = {
|
|
|
18
18
|
},
|
|
19
19
|
"@camstack/types": {
|
|
20
20
|
name: "@camstack/types",
|
|
21
|
-
version: "1.2.
|
|
21
|
+
version: "1.2.118",
|
|
22
22
|
scope: ["default"],
|
|
23
23
|
loaded: !1,
|
|
24
24
|
from: "addon_pipeline_analytics_widgets",
|
|
@@ -33,7 +33,7 @@ var e = {
|
|
|
33
33
|
},
|
|
34
34
|
"@camstack/ui-library": {
|
|
35
35
|
name: "@camstack/ui-library",
|
|
36
|
-
version: "1.2.
|
|
36
|
+
version: "1.2.81",
|
|
37
37
|
scope: ["default"],
|
|
38
38
|
loaded: !1,
|
|
39
39
|
from: "addon_pipeline_analytics_widgets",
|