@camstack/addon-post-analysis 1.1.4 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{dist-B2FbCLNx.mjs → dist-BpGP9ago.mjs} +41 -5
- package/dist/{dist-B6aOcq5T.js → dist-n-zJ0Uox.js} +46 -4
- package/dist/embedding-encoder/index.js +272 -93
- package/dist/embedding-encoder/index.mjs +270 -91
- package/dist/enrichment-engine/index.js +2 -2
- package/dist/enrichment-engine/index.mjs +1 -1
- package/dist/pipeline-analytics/_stub.js +1 -1
- package/dist/pipeline-analytics/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-Bcn1PmOA.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-DgAlWohT.mjs} +3 -3
- package/dist/pipeline-analytics/{_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BFIbMkkd.mjs → _virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DBkzwlqD.mjs} +1 -1
- package/dist/pipeline-analytics/{hostInit-gu9R5ABc.mjs → hostInit-D51b7QGW.mjs} +3 -3
- package/dist/pipeline-analytics/index.js +316 -2
- package/dist/pipeline-analytics/index.mjs +315 -1
- package/dist/pipeline-analytics/remoteEntry.js +1 -1
- package/dist/{resolve-frame-B6a1wzQb.js → resolve-frame-DB2NdMu2.js} +1 -1
- package/package.json +2 -1
- package/python/raw_tensor_inference.py +73 -0
- package/python/requirements-embedding.txt +8 -0
- package/python/tensor_frames.py +70 -0
- package/python/test_text_encoder.py +48 -0
- package/python/text_encoder_inference.py +76 -0
|
@@ -4627,7 +4627,7 @@ function _instanceof(cls, params = {}) {
|
|
|
4627
4627
|
return inst;
|
|
4628
4628
|
}
|
|
4629
4629
|
//#endregion
|
|
4630
|
-
//#region ../types/dist/sleep-
|
|
4630
|
+
//#region ../types/dist/sleep-NOH4yRwj.mjs
|
|
4631
4631
|
var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
4632
4632
|
EventCategory["SystemBoot"] = "system.boot";
|
|
4633
4633
|
EventCategory["SystemAddonsReady"] = "system.addons-ready";
|
|
@@ -7134,6 +7134,9 @@ var EncodeProfileSchema = object({
|
|
|
7134
7134
|
*/
|
|
7135
7135
|
outputArgs: array(string()).optional()
|
|
7136
7136
|
});
|
|
7137
|
+
function hfModelUrl(repo, path) {
|
|
7138
|
+
return `https://huggingface.co/${repo}/resolve/main/${path}`;
|
|
7139
|
+
}
|
|
7137
7140
|
/** Cosine similarity between two embedding vectors */
|
|
7138
7141
|
function cosineSimilarity(a, b) {
|
|
7139
7142
|
if (a.length !== b.length) return 0;
|
|
@@ -12770,7 +12773,8 @@ method(object({
|
|
|
12770
12773
|
sessionId: string().optional()
|
|
12771
12774
|
}), ConvertResultSchema, {
|
|
12772
12775
|
kind: "mutation",
|
|
12773
|
-
auth: "admin"
|
|
12776
|
+
auth: "admin",
|
|
12777
|
+
timeoutMs: 6e5
|
|
12774
12778
|
});
|
|
12775
12779
|
var AddonHttpRouteSchema = object({
|
|
12776
12780
|
method: _enum([
|
|
@@ -14326,7 +14330,14 @@ var NotificationRuleConditionsSchema = object({
|
|
|
14326
14330
|
/** Match against `event.data.eventType` token (e.g. `'press_long'`). When non-empty, only events
|
|
14327
14331
|
* carrying a matching `data.eventType` string pass this condition. Rules without this field are
|
|
14328
14332
|
* unaffected (back-compat). Distinct from `rule.eventTypes` which holds EventCategory strings. */
|
|
14329
|
-
eventTypeTokens: array(string()).readonly().optional()
|
|
14333
|
+
eventTypeTokens: array(string()).readonly().optional(),
|
|
14334
|
+
/** Match detections whose CLIP image embedding is semantically similar to this free-text
|
|
14335
|
+
* description. Requires the embedding-encoder cap to have pre-warmed the text vector.
|
|
14336
|
+
* `minSimilarity` is the cosine similarity threshold in [0, 1]. */
|
|
14337
|
+
clipDescription: object({
|
|
14338
|
+
text: string().min(1),
|
|
14339
|
+
minSimilarity: number().min(0).max(1)
|
|
14340
|
+
}).optional()
|
|
14330
14341
|
});
|
|
14331
14342
|
var NotificationRuleTemplateSchema = object({
|
|
14332
14343
|
title: string(),
|
|
@@ -14568,6 +14579,16 @@ var TrackedDetectionSchema = object({
|
|
|
14568
14579
|
zones: array(string()).readonly(),
|
|
14569
14580
|
state: TrackStateSchema
|
|
14570
14581
|
});
|
|
14582
|
+
var ScoredObjectEventSchema = ObjectEventSchema.extend({ score: number() });
|
|
14583
|
+
var SearchObjectEventsInput = object({
|
|
14584
|
+
text: string(),
|
|
14585
|
+
deviceId: number().optional(),
|
|
14586
|
+
since: number().optional(),
|
|
14587
|
+
until: number().optional(),
|
|
14588
|
+
classFilter: string().optional(),
|
|
14589
|
+
limit: number().default(50),
|
|
14590
|
+
minScore: number().min(0).max(1).default(.2)
|
|
14591
|
+
});
|
|
14571
14592
|
var pipelineAnalyticsCapability = {
|
|
14572
14593
|
name: "pipeline-analytics",
|
|
14573
14594
|
scope: "device",
|
|
@@ -14634,7 +14655,16 @@ var pipelineAnalyticsCapability = {
|
|
|
14634
14655
|
eventId: string(),
|
|
14635
14656
|
kind: MediaFileKindEnum.optional()
|
|
14636
14657
|
}), array(MediaFileSchema).readonly()),
|
|
14637
|
-
getTrackMedia: method(object({ trackId: string() }), array(MediaFileSchema).readonly())
|
|
14658
|
+
getTrackMedia: method(object({ trackId: string() }), array(MediaFileSchema).readonly()),
|
|
14659
|
+
/**
|
|
14660
|
+
* Search object events by text query using CLIP cosine similarity.
|
|
14661
|
+
* Encodes `text` via the `embedding-encoder` cap, queries the
|
|
14662
|
+
* `ObjectEmbeddingStore` with optional prefilters, ranks all matching
|
|
14663
|
+
* embeddings by cosine similarity, and joins winners to their
|
|
14664
|
+
* ObjectEvents by trackId. Returns up to `limit` events scored ≥
|
|
14665
|
+
* `minScore`, sorted descending by score.
|
|
14666
|
+
*/
|
|
14667
|
+
searchObjectEvents: method(SearchObjectEventsInput, array(ScoredObjectEventSchema).readonly())
|
|
14638
14668
|
},
|
|
14639
14669
|
events: {
|
|
14640
14670
|
/**
|
|
@@ -19351,6 +19381,12 @@ Object.freeze({
|
|
|
19351
19381
|
addonId: null,
|
|
19352
19382
|
access: "create"
|
|
19353
19383
|
},
|
|
19384
|
+
"pipelineAnalytics.searchObjectEvents": {
|
|
19385
|
+
capName: "pipeline-analytics",
|
|
19386
|
+
capScope: "device",
|
|
19387
|
+
addonId: null,
|
|
19388
|
+
access: "view"
|
|
19389
|
+
},
|
|
19354
19390
|
"pipelineExecutor.cacheFrameInPool": {
|
|
19355
19391
|
capName: "pipeline-executor",
|
|
19356
19392
|
capScope: "system",
|
|
@@ -21235,4 +21271,4 @@ object({
|
|
|
21235
21271
|
schemaVersion: literal(1)
|
|
21236
21272
|
});
|
|
21237
21273
|
//#endregion
|
|
21238
|
-
export {
|
|
21274
|
+
export { string as C, object as S, hydrateSchema as _, faceGalleryCapability as a, boolean as b, plateGalleryCapability as c, errMsg as d, BaseAddon as f, createEvent as g, asJsonObject as h, embeddingEncoderCapability as i, videoclipsCapability as l, EventCategory as m, audioMetricsCapability as n, hfModelUrl as o, DeviceType as p, cosineSimilarity as r, pipelineAnalyticsCapability as s, addonWidgetsSourceCapability as t, zoneAnalyticsCapability as u, _enum as v, tuple as w, number as x, array as y };
|
|
@@ -4649,7 +4649,7 @@ function _instanceof(cls, params = {}) {
|
|
|
4649
4649
|
return inst;
|
|
4650
4650
|
}
|
|
4651
4651
|
//#endregion
|
|
4652
|
-
//#region ../types/dist/sleep-
|
|
4652
|
+
//#region ../types/dist/sleep-NOH4yRwj.mjs
|
|
4653
4653
|
var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
4654
4654
|
EventCategory["SystemBoot"] = "system.boot";
|
|
4655
4655
|
EventCategory["SystemAddonsReady"] = "system.addons-ready";
|
|
@@ -7156,6 +7156,9 @@ var EncodeProfileSchema = object({
|
|
|
7156
7156
|
*/
|
|
7157
7157
|
outputArgs: array(string()).optional()
|
|
7158
7158
|
});
|
|
7159
|
+
function hfModelUrl(repo, path) {
|
|
7160
|
+
return `https://huggingface.co/${repo}/resolve/main/${path}`;
|
|
7161
|
+
}
|
|
7159
7162
|
/** Cosine similarity between two embedding vectors */
|
|
7160
7163
|
function cosineSimilarity(a, b) {
|
|
7161
7164
|
if (a.length !== b.length) return 0;
|
|
@@ -12792,7 +12795,8 @@ method(object({
|
|
|
12792
12795
|
sessionId: string().optional()
|
|
12793
12796
|
}), ConvertResultSchema, {
|
|
12794
12797
|
kind: "mutation",
|
|
12795
|
-
auth: "admin"
|
|
12798
|
+
auth: "admin",
|
|
12799
|
+
timeoutMs: 6e5
|
|
12796
12800
|
});
|
|
12797
12801
|
var AddonHttpRouteSchema = object({
|
|
12798
12802
|
method: _enum([
|
|
@@ -14348,7 +14352,14 @@ var NotificationRuleConditionsSchema = object({
|
|
|
14348
14352
|
/** Match against `event.data.eventType` token (e.g. `'press_long'`). When non-empty, only events
|
|
14349
14353
|
* carrying a matching `data.eventType` string pass this condition. Rules without this field are
|
|
14350
14354
|
* unaffected (back-compat). Distinct from `rule.eventTypes` which holds EventCategory strings. */
|
|
14351
|
-
eventTypeTokens: array(string()).readonly().optional()
|
|
14355
|
+
eventTypeTokens: array(string()).readonly().optional(),
|
|
14356
|
+
/** Match detections whose CLIP image embedding is semantically similar to this free-text
|
|
14357
|
+
* description. Requires the embedding-encoder cap to have pre-warmed the text vector.
|
|
14358
|
+
* `minSimilarity` is the cosine similarity threshold in [0, 1]. */
|
|
14359
|
+
clipDescription: object({
|
|
14360
|
+
text: string().min(1),
|
|
14361
|
+
minSimilarity: number().min(0).max(1)
|
|
14362
|
+
}).optional()
|
|
14352
14363
|
});
|
|
14353
14364
|
var NotificationRuleTemplateSchema = object({
|
|
14354
14365
|
title: string(),
|
|
@@ -14590,6 +14601,16 @@ var TrackedDetectionSchema = object({
|
|
|
14590
14601
|
zones: array(string()).readonly(),
|
|
14591
14602
|
state: TrackStateSchema
|
|
14592
14603
|
});
|
|
14604
|
+
var ScoredObjectEventSchema = ObjectEventSchema.extend({ score: number() });
|
|
14605
|
+
var SearchObjectEventsInput = object({
|
|
14606
|
+
text: string(),
|
|
14607
|
+
deviceId: number().optional(),
|
|
14608
|
+
since: number().optional(),
|
|
14609
|
+
until: number().optional(),
|
|
14610
|
+
classFilter: string().optional(),
|
|
14611
|
+
limit: number().default(50),
|
|
14612
|
+
minScore: number().min(0).max(1).default(.2)
|
|
14613
|
+
});
|
|
14593
14614
|
var pipelineAnalyticsCapability = {
|
|
14594
14615
|
name: "pipeline-analytics",
|
|
14595
14616
|
scope: "device",
|
|
@@ -14656,7 +14677,16 @@ var pipelineAnalyticsCapability = {
|
|
|
14656
14677
|
eventId: string(),
|
|
14657
14678
|
kind: MediaFileKindEnum.optional()
|
|
14658
14679
|
}), array(MediaFileSchema).readonly()),
|
|
14659
|
-
getTrackMedia: method(object({ trackId: string() }), array(MediaFileSchema).readonly())
|
|
14680
|
+
getTrackMedia: method(object({ trackId: string() }), array(MediaFileSchema).readonly()),
|
|
14681
|
+
/**
|
|
14682
|
+
* Search object events by text query using CLIP cosine similarity.
|
|
14683
|
+
* Encodes `text` via the `embedding-encoder` cap, queries the
|
|
14684
|
+
* `ObjectEmbeddingStore` with optional prefilters, ranks all matching
|
|
14685
|
+
* embeddings by cosine similarity, and joins winners to their
|
|
14686
|
+
* ObjectEvents by trackId. Returns up to `limit` events scored ≥
|
|
14687
|
+
* `minScore`, sorted descending by score.
|
|
14688
|
+
*/
|
|
14689
|
+
searchObjectEvents: method(SearchObjectEventsInput, array(ScoredObjectEventSchema).readonly())
|
|
14660
14690
|
},
|
|
14661
14691
|
events: {
|
|
14662
14692
|
/**
|
|
@@ -19373,6 +19403,12 @@ Object.freeze({
|
|
|
19373
19403
|
addonId: null,
|
|
19374
19404
|
access: "create"
|
|
19375
19405
|
},
|
|
19406
|
+
"pipelineAnalytics.searchObjectEvents": {
|
|
19407
|
+
capName: "pipeline-analytics",
|
|
19408
|
+
capScope: "device",
|
|
19409
|
+
addonId: null,
|
|
19410
|
+
access: "view"
|
|
19411
|
+
},
|
|
19376
19412
|
"pipelineExecutor.cacheFrameInPool": {
|
|
19377
19413
|
capName: "pipeline-executor",
|
|
19378
19414
|
capScope: "system",
|
|
@@ -21347,6 +21383,12 @@ Object.defineProperty(exports, "faceGalleryCapability", {
|
|
|
21347
21383
|
return faceGalleryCapability;
|
|
21348
21384
|
}
|
|
21349
21385
|
});
|
|
21386
|
+
Object.defineProperty(exports, "hfModelUrl", {
|
|
21387
|
+
enumerable: true,
|
|
21388
|
+
get: function() {
|
|
21389
|
+
return hfModelUrl;
|
|
21390
|
+
}
|
|
21391
|
+
});
|
|
21350
21392
|
Object.defineProperty(exports, "hydrateSchema", {
|
|
21351
21393
|
enumerable: true,
|
|
21352
21394
|
get: function() {
|
|
@@ -2,13 +2,13 @@ Object.defineProperties(exports, {
|
|
|
2
2
|
__esModule: { value: true },
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
|
-
const require_dist = require("../dist-
|
|
6
|
-
let node_path = require("node:path");
|
|
7
|
-
let node_path$1 = require_dist.__toESM(node_path, 1);
|
|
8
|
-
node_path = require_dist.__toESM(node_path);
|
|
5
|
+
const require_dist = require("../dist-n-zJ0Uox.js");
|
|
9
6
|
let node_fs = require("node:fs");
|
|
10
7
|
let node_fs$1 = require_dist.__toESM(node_fs, 1);
|
|
11
8
|
node_fs = require_dist.__toESM(node_fs);
|
|
9
|
+
let node_path = require("node:path");
|
|
10
|
+
let node_path$1 = require_dist.__toESM(node_path, 1);
|
|
11
|
+
node_path = require_dist.__toESM(node_path);
|
|
12
12
|
require("node:crypto");
|
|
13
13
|
let node_child_process = require("node:child_process");
|
|
14
14
|
//#region ../system/dist/model-download-service-C-IHWnXx.mjs
|
|
@@ -262,98 +262,164 @@ var ModelDownloadService = class {
|
|
|
262
262
|
};
|
|
263
263
|
//#endregion
|
|
264
264
|
//#region src/embedding-encoder/catalogs/embedding-models.ts
|
|
265
|
+
var HF_REPO = "camstack/camstack-models";
|
|
266
|
+
var hf = (path) => require_dist.hfModelUrl(HF_REPO, path);
|
|
267
|
+
/**
|
|
268
|
+
* The CLIP BPE tokenizer (HF `tokenizers` format: vocab 49408 + 48894 merges).
|
|
269
|
+
* Hosted next to every text-encoder onnx (`.../onnx/tokenizer.json`) and fetched
|
|
270
|
+
* as a sibling file so it lands beside the model in the shared models dir.
|
|
271
|
+
*/
|
|
272
|
+
var TOKENIZER_FILE = "tokenizer.json";
|
|
273
|
+
var ovFormat = (url, sizeMB) => {
|
|
274
|
+
const base = url.split("/").pop() ?? "";
|
|
275
|
+
const files = base.endsWith(".xml") ? [base.replace(/\.xml$/, ".bin")] : void 0;
|
|
276
|
+
return {
|
|
277
|
+
url,
|
|
278
|
+
sizeMB,
|
|
279
|
+
runtimes: ["python"],
|
|
280
|
+
...files ? { files } : {}
|
|
281
|
+
};
|
|
282
|
+
};
|
|
283
|
+
/**
|
|
284
|
+
* Files inside an .mlpackage directory bundle.
|
|
285
|
+
* Must be fetched alongside the package root when isDirectory is true.
|
|
286
|
+
*/
|
|
287
|
+
var MLPACKAGE_FILES = [
|
|
288
|
+
"Manifest.json",
|
|
289
|
+
"Data/com.apple.CoreML/model.mlmodel",
|
|
290
|
+
"Data/com.apple.CoreML/weights/weight.bin"
|
|
291
|
+
];
|
|
265
292
|
var CLIP_IMAGE_MODELS = [
|
|
266
293
|
{
|
|
267
|
-
id: "
|
|
268
|
-
name: "
|
|
269
|
-
description: "
|
|
294
|
+
id: "mobileclip-s0",
|
|
295
|
+
name: "MobileCLIP S0",
|
|
296
|
+
description: "Apple MobileCLIP S0 — fast vision encoder, 512-dim, 256×256 (40 MB ONNX)",
|
|
270
297
|
inputSize: {
|
|
271
|
-
width:
|
|
272
|
-
height:
|
|
298
|
+
width: 256,
|
|
299
|
+
height: 256
|
|
273
300
|
},
|
|
274
301
|
labels: [],
|
|
275
|
-
inputLayout: "nchw",
|
|
276
302
|
inputNormalization: "none",
|
|
277
|
-
formats: {
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
303
|
+
formats: {
|
|
304
|
+
onnx: {
|
|
305
|
+
url: hf("clip/mobileclip-s0/onnx/camstack-mobileclip-s0-vision.onnx"),
|
|
306
|
+
sizeMB: 40
|
|
307
|
+
},
|
|
308
|
+
openvino: ovFormat(hf("clip/mobileclip-s0/openvino/camstack-mobileclip-s0-vision.xml"), 25),
|
|
309
|
+
coreml: {
|
|
310
|
+
url: hf("clip/mobileclip-s0/coreml/camstack-mobileclip-s0-vision.mlpackage"),
|
|
311
|
+
sizeMB: 30,
|
|
312
|
+
isDirectory: true,
|
|
313
|
+
files: [...MLPACKAGE_FILES],
|
|
314
|
+
runtimes: ["python"]
|
|
315
|
+
}
|
|
316
|
+
}
|
|
281
317
|
},
|
|
282
318
|
{
|
|
283
|
-
id: "
|
|
284
|
-
name: "
|
|
285
|
-
description: "
|
|
319
|
+
id: "mobileclip-s1",
|
|
320
|
+
name: "MobileCLIP S1",
|
|
321
|
+
description: "Apple MobileCLIP S1 — balanced vision encoder, 512-dim, 256×256 (90 MB ONNX)",
|
|
286
322
|
inputSize: {
|
|
287
|
-
width:
|
|
288
|
-
height:
|
|
323
|
+
width: 256,
|
|
324
|
+
height: 256
|
|
289
325
|
},
|
|
290
326
|
labels: [],
|
|
291
|
-
inputLayout: "nchw",
|
|
292
327
|
inputNormalization: "none",
|
|
293
|
-
formats: {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
328
|
+
formats: {
|
|
329
|
+
onnx: {
|
|
330
|
+
url: hf("clip/mobileclip-s1/onnx/camstack-mobileclip-s1-vision.onnx"),
|
|
331
|
+
sizeMB: 90
|
|
332
|
+
},
|
|
333
|
+
openvino: ovFormat(hf("clip/mobileclip-s1/openvino/camstack-mobileclip-s1-vision.xml"), 55),
|
|
334
|
+
coreml: {
|
|
335
|
+
url: hf("clip/mobileclip-s1/coreml/camstack-mobileclip-s1-vision.mlpackage"),
|
|
336
|
+
sizeMB: 65,
|
|
337
|
+
isDirectory: true,
|
|
338
|
+
files: [...MLPACKAGE_FILES],
|
|
339
|
+
runtimes: ["python"]
|
|
340
|
+
}
|
|
341
|
+
}
|
|
297
342
|
},
|
|
298
343
|
{
|
|
299
|
-
id: "
|
|
300
|
-
name: "
|
|
301
|
-
description: "
|
|
344
|
+
id: "mobileclip-s2",
|
|
345
|
+
name: "MobileCLIP S2",
|
|
346
|
+
description: "Apple MobileCLIP S2 — high-accuracy vision encoder, 512-dim, 256×256 (150 MB ONNX)",
|
|
302
347
|
inputSize: {
|
|
303
348
|
width: 256,
|
|
304
349
|
height: 256
|
|
305
350
|
},
|
|
306
351
|
labels: [],
|
|
307
|
-
inputLayout: "nchw",
|
|
308
352
|
inputNormalization: "none",
|
|
309
|
-
formats: {
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
353
|
+
formats: {
|
|
354
|
+
onnx: {
|
|
355
|
+
url: hf("clip/mobileclip-s2/onnx/camstack-mobileclip-s2-vision.onnx"),
|
|
356
|
+
sizeMB: 150
|
|
357
|
+
},
|
|
358
|
+
openvino: ovFormat(hf("clip/mobileclip-s2/openvino/camstack-mobileclip-s2-vision.xml"), 90),
|
|
359
|
+
coreml: {
|
|
360
|
+
url: hf("clip/mobileclip-s2/coreml/camstack-mobileclip-s2-vision.mlpackage"),
|
|
361
|
+
sizeMB: 110,
|
|
362
|
+
isDirectory: true,
|
|
363
|
+
files: [...MLPACKAGE_FILES],
|
|
364
|
+
runtimes: ["python"]
|
|
365
|
+
}
|
|
366
|
+
}
|
|
313
367
|
}
|
|
314
368
|
];
|
|
315
369
|
var CLIP_TEXT_MODELS = [
|
|
316
370
|
{
|
|
317
|
-
id: "
|
|
318
|
-
name: "
|
|
319
|
-
description: "Text encoder for
|
|
371
|
+
id: "mobileclip-s0-text",
|
|
372
|
+
name: "MobileCLIP S0 Text Encoder",
|
|
373
|
+
description: "Text encoder for MobileCLIP S0, 512-dim, int8 quantized (35 MB)",
|
|
320
374
|
inputSize: {
|
|
321
375
|
width: 0,
|
|
322
376
|
height: 0
|
|
323
377
|
},
|
|
324
378
|
labels: [],
|
|
325
|
-
formats: {
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
379
|
+
formats: {
|
|
380
|
+
onnx: {
|
|
381
|
+
url: hf("clip/mobileclip-s0/onnx/camstack-mobileclip-s0-text.onnx"),
|
|
382
|
+
sizeMB: 35,
|
|
383
|
+
files: [TOKENIZER_FILE]
|
|
384
|
+
},
|
|
385
|
+
openvino: ovFormat(hf("clip/mobileclip-s0/openvino/camstack-mobileclip-s0-text.xml"), 22)
|
|
386
|
+
}
|
|
329
387
|
},
|
|
330
388
|
{
|
|
331
|
-
id: "
|
|
332
|
-
name: "
|
|
333
|
-
description: "Text encoder for
|
|
389
|
+
id: "mobileclip-s1-text",
|
|
390
|
+
name: "MobileCLIP S1 Text Encoder",
|
|
391
|
+
description: "Text encoder for MobileCLIP S1, 512-dim, int8 quantized (35 MB)",
|
|
334
392
|
inputSize: {
|
|
335
393
|
width: 0,
|
|
336
394
|
height: 0
|
|
337
395
|
},
|
|
338
396
|
labels: [],
|
|
339
|
-
formats: {
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
397
|
+
formats: {
|
|
398
|
+
onnx: {
|
|
399
|
+
url: hf("clip/mobileclip-s1/onnx/camstack-mobileclip-s1-text.onnx"),
|
|
400
|
+
sizeMB: 35,
|
|
401
|
+
files: [TOKENIZER_FILE]
|
|
402
|
+
},
|
|
403
|
+
openvino: ovFormat(hf("clip/mobileclip-s1/openvino/camstack-mobileclip-s1-text.xml"), 22)
|
|
404
|
+
}
|
|
343
405
|
},
|
|
344
406
|
{
|
|
345
|
-
id: "
|
|
346
|
-
name: "
|
|
347
|
-
description: "Text encoder for
|
|
407
|
+
id: "mobileclip-s2-text",
|
|
408
|
+
name: "MobileCLIP S2 Text Encoder",
|
|
409
|
+
description: "Text encoder for MobileCLIP S2, 512-dim, int8 quantized (35 MB)",
|
|
348
410
|
inputSize: {
|
|
349
411
|
width: 0,
|
|
350
412
|
height: 0
|
|
351
413
|
},
|
|
352
414
|
labels: [],
|
|
353
|
-
formats: {
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
415
|
+
formats: {
|
|
416
|
+
onnx: {
|
|
417
|
+
url: hf("clip/mobileclip-s2/onnx/camstack-mobileclip-s2-text.onnx"),
|
|
418
|
+
sizeMB: 35,
|
|
419
|
+
files: [TOKENIZER_FILE]
|
|
420
|
+
},
|
|
421
|
+
openvino: ovFormat(hf("clip/mobileclip-s2/openvino/camstack-mobileclip-s2-text.xml"), 22)
|
|
422
|
+
}
|
|
357
423
|
}
|
|
358
424
|
];
|
|
359
425
|
//#endregion
|
|
@@ -485,33 +551,155 @@ var PythonRawTensorEngine = class {
|
|
|
485
551
|
}
|
|
486
552
|
};
|
|
487
553
|
//#endregion
|
|
554
|
+
//#region src/embedding-encoder/shared/python-text-encoder-engine.ts
|
|
555
|
+
/**
|
|
556
|
+
* CLIP text-encoder engine backed by an embedded-Python subprocess
|
|
557
|
+
* (`text_encoder_inference.py`). Tokenization happens IN Python via the HF
|
|
558
|
+
* `tokenizers` Rust BPE (exact by construction) — this replaces the former
|
|
559
|
+
* hand-rolled TypeScript CLIP BPE.
|
|
560
|
+
*
|
|
561
|
+
* The caller sends raw UTF-8 text; Python tokenizes (truncate/pad to 77), runs
|
|
562
|
+
* onnxruntime, and returns the embedding tensor. Wire protocol = length-prefixed
|
|
563
|
+
* binary frames ([4B LE length][payload]):
|
|
564
|
+
* ready (in): [0x01]
|
|
565
|
+
* request (out): UTF-8 text bytes
|
|
566
|
+
* response (in): [1B ndims][dims × 4B LE uint32][float32 LE data]
|
|
567
|
+
*/
|
|
568
|
+
var PythonTextEncoderEngine = class {
|
|
569
|
+
pythonPath;
|
|
570
|
+
scriptPath;
|
|
571
|
+
modelPath;
|
|
572
|
+
tokenizerPath;
|
|
573
|
+
process = null;
|
|
574
|
+
receiveBuffer = Buffer.alloc(0);
|
|
575
|
+
pendingResolve = null;
|
|
576
|
+
pendingReject = null;
|
|
577
|
+
log;
|
|
578
|
+
constructor(pythonPath, scriptPath, modelPath, tokenizerPath, logger) {
|
|
579
|
+
this.pythonPath = pythonPath;
|
|
580
|
+
this.scriptPath = scriptPath;
|
|
581
|
+
this.modelPath = modelPath;
|
|
582
|
+
this.tokenizerPath = tokenizerPath;
|
|
583
|
+
this.log = logger ?? createNoopLogger();
|
|
584
|
+
}
|
|
585
|
+
async initialize() {
|
|
586
|
+
this.process = (0, node_child_process.spawn)(this.pythonPath, [
|
|
587
|
+
this.scriptPath,
|
|
588
|
+
this.modelPath,
|
|
589
|
+
this.tokenizerPath
|
|
590
|
+
], { stdio: [
|
|
591
|
+
"pipe",
|
|
592
|
+
"pipe",
|
|
593
|
+
"pipe"
|
|
594
|
+
] });
|
|
595
|
+
this.process.stderr?.on("data", (chunk) => {
|
|
596
|
+
const text = chunk.toString().trim();
|
|
597
|
+
if (text) this.log.warn(text);
|
|
598
|
+
});
|
|
599
|
+
this.process.on("error", (err) => {
|
|
600
|
+
this.log.error("Python text-encoder process error", { meta: { error: err.message } });
|
|
601
|
+
this.pendingReject?.(err);
|
|
602
|
+
this.pendingReject = null;
|
|
603
|
+
this.pendingResolve = null;
|
|
604
|
+
});
|
|
605
|
+
this.process.on("exit", (code) => {
|
|
606
|
+
if (code !== 0 && code !== null) {
|
|
607
|
+
const err = /* @__PURE__ */ new Error(`PythonTextEncoderEngine: process exited with code ${code}`);
|
|
608
|
+
this.pendingReject?.(err);
|
|
609
|
+
this.pendingReject = null;
|
|
610
|
+
this.pendingResolve = null;
|
|
611
|
+
}
|
|
612
|
+
});
|
|
613
|
+
this.process.stdout.on("data", (chunk) => {
|
|
614
|
+
this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
|
|
615
|
+
this.tryReceive();
|
|
616
|
+
});
|
|
617
|
+
const ready = await this.receiveFrame();
|
|
618
|
+
if (ready.length !== 1 || ready[0] !== 1) throw new Error("PythonTextEncoderEngine: unexpected ready frame");
|
|
619
|
+
this.log.info("CLIP text-encoder engine ready (embedded Python)", { meta: {
|
|
620
|
+
modelPath: this.modelPath,
|
|
621
|
+
tokenizerPath: this.tokenizerPath
|
|
622
|
+
} });
|
|
623
|
+
}
|
|
624
|
+
/** Tokenize + encode `text` into the model embedding (float32). */
|
|
625
|
+
async encode(text) {
|
|
626
|
+
if (!this.process?.stdin) throw new Error("PythonTextEncoderEngine: not initialized — call initialize() first");
|
|
627
|
+
const payload = Buffer.from(text, "utf-8");
|
|
628
|
+
const lenBuf = Buffer.allocUnsafe(4);
|
|
629
|
+
lenBuf.writeUInt32LE(payload.length, 0);
|
|
630
|
+
this.process.stdin.write(Buffer.concat([lenBuf, payload]));
|
|
631
|
+
const resp = await this.receiveFrame();
|
|
632
|
+
const floatStart = 1 + resp.readUInt8(0) * 4;
|
|
633
|
+
const count = (resp.length - floatStart) / 4;
|
|
634
|
+
const out = new Float32Array(count);
|
|
635
|
+
for (let i = 0; i < count; i++) out[i] = resp.readFloatLE(floatStart + i * 4);
|
|
636
|
+
return out;
|
|
637
|
+
}
|
|
638
|
+
async dispose() {
|
|
639
|
+
const proc = this.process;
|
|
640
|
+
if (!proc) return;
|
|
641
|
+
this.process = null;
|
|
642
|
+
proc.stdin?.end();
|
|
643
|
+
proc.kill("SIGTERM");
|
|
644
|
+
await new Promise((resolve) => {
|
|
645
|
+
const timer = setTimeout(() => {
|
|
646
|
+
try {
|
|
647
|
+
proc.kill("SIGKILL");
|
|
648
|
+
} catch {}
|
|
649
|
+
resolve();
|
|
650
|
+
}, 5e3);
|
|
651
|
+
proc.once("exit", () => {
|
|
652
|
+
clearTimeout(timer);
|
|
653
|
+
resolve();
|
|
654
|
+
});
|
|
655
|
+
});
|
|
656
|
+
}
|
|
657
|
+
receiveFrame() {
|
|
658
|
+
return new Promise((resolve, reject) => {
|
|
659
|
+
this.pendingResolve = resolve;
|
|
660
|
+
this.pendingReject = reject;
|
|
661
|
+
});
|
|
662
|
+
}
|
|
663
|
+
tryReceive() {
|
|
664
|
+
if (this.receiveBuffer.length < 4) return;
|
|
665
|
+
const length = this.receiveBuffer.readUInt32LE(0);
|
|
666
|
+
if (this.receiveBuffer.length < 4 + length) return;
|
|
667
|
+
const payload = Buffer.from(this.receiveBuffer.subarray(4, 4 + length));
|
|
668
|
+
this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
|
|
669
|
+
const resolve = this.pendingResolve;
|
|
670
|
+
this.pendingResolve = null;
|
|
671
|
+
this.pendingReject = null;
|
|
672
|
+
resolve?.(payload);
|
|
673
|
+
}
|
|
674
|
+
};
|
|
675
|
+
//#endregion
|
|
488
676
|
//#region src/embedding-encoder/addon/clip-models.ts
|
|
489
677
|
var CLIP_MODEL_META = {
|
|
490
|
-
"
|
|
491
|
-
imageModelId: "
|
|
492
|
-
textModelId: "
|
|
678
|
+
"mobileclip-s0": {
|
|
679
|
+
imageModelId: "mobileclip-s0",
|
|
680
|
+
textModelId: "mobileclip-s0-text",
|
|
493
681
|
embeddingDim: 512,
|
|
494
|
-
inputSize:
|
|
682
|
+
inputSize: 256,
|
|
495
683
|
tokenizerType: "clip"
|
|
496
684
|
},
|
|
497
|
-
"
|
|
498
|
-
imageModelId: "
|
|
499
|
-
textModelId: "
|
|
685
|
+
"mobileclip-s1": {
|
|
686
|
+
imageModelId: "mobileclip-s1",
|
|
687
|
+
textModelId: "mobileclip-s1-text",
|
|
500
688
|
embeddingDim: 512,
|
|
501
|
-
inputSize:
|
|
689
|
+
inputSize: 256,
|
|
502
690
|
tokenizerType: "clip"
|
|
503
691
|
},
|
|
504
|
-
"
|
|
505
|
-
imageModelId: "
|
|
506
|
-
textModelId: "
|
|
507
|
-
embeddingDim:
|
|
692
|
+
"mobileclip-s2": {
|
|
693
|
+
imageModelId: "mobileclip-s2",
|
|
694
|
+
textModelId: "mobileclip-s2-text",
|
|
695
|
+
embeddingDim: 512,
|
|
508
696
|
inputSize: 256,
|
|
509
|
-
tokenizerType: "
|
|
697
|
+
tokenizerType: "clip"
|
|
510
698
|
}
|
|
511
699
|
};
|
|
512
|
-
var DEFAULT_CLIP_MODEL = "
|
|
700
|
+
var DEFAULT_CLIP_MODEL = "mobileclip-s1";
|
|
513
701
|
function getModelMeta(modelId) {
|
|
514
|
-
return CLIP_MODEL_META[modelId] ?? CLIP_MODEL_META["
|
|
702
|
+
return CLIP_MODEL_META[modelId] ?? CLIP_MODEL_META["mobileclip-s1"];
|
|
515
703
|
}
|
|
516
704
|
//#endregion
|
|
517
705
|
//#region src/embedding-encoder/addon/clip-preprocessing.ts
|
|
@@ -559,7 +747,7 @@ function l2Normalize(vec) {
|
|
|
559
747
|
//#region src/embedding-encoder/addon/index.ts
|
|
560
748
|
var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
561
749
|
imageRawEngine = null;
|
|
562
|
-
|
|
750
|
+
textEngine = null;
|
|
563
751
|
models = null;
|
|
564
752
|
constructor() {
|
|
565
753
|
super({
|
|
@@ -600,9 +788,8 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
|
600
788
|
await this.ensureTextEngine();
|
|
601
789
|
const meta = getModelMeta(this.config.modelId);
|
|
602
790
|
const start = Date.now();
|
|
603
|
-
|
|
604
|
-
const
|
|
605
|
-
const output = await this.textRawEngine.run(inputTensor, [1, tokenIds.length]);
|
|
791
|
+
if (!this.textEngine) throw new Error("EmbeddingEncoder: text engine not loaded — ensureTextEngine() must run first");
|
|
792
|
+
const output = await this.textEngine.encode(text);
|
|
606
793
|
const sliced = output.length > meta.embeddingDim ? output.slice(0, meta.embeddingDim) : output;
|
|
607
794
|
const normalized = l2Normalize(new Float32Array(sliced));
|
|
608
795
|
return {
|
|
@@ -626,7 +813,7 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
|
626
813
|
await this.resolveForEntry(imageEntry, "image");
|
|
627
814
|
}
|
|
628
815
|
async ensureTextEngine() {
|
|
629
|
-
if (this.
|
|
816
|
+
if (this.textEngine) return;
|
|
630
817
|
const meta = getModelMeta(this.config.modelId);
|
|
631
818
|
const textEntry = CLIP_TEXT_MODELS.find((m) => m.id === meta.textModelId);
|
|
632
819
|
if (!textEntry) throw new Error(`EmbeddingEncoderAddon: unknown text model "${meta.textModelId}"`);
|
|
@@ -643,14 +830,21 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
|
643
830
|
if (!pythonPath) throw new Error("EmbeddingEncoder: embedded Python is unavailable — cannot run ONNX embeddings. ctx.deps.ensurePython() returned null (portable Python download likely failed).");
|
|
644
831
|
const pythonDir = resolveEmbeddingPythonDir();
|
|
645
832
|
await this.ctx.deps.installPythonRequirements(node_path.join(pythonDir, "requirements-embedding.txt"));
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
833
|
+
if (target === "image") {
|
|
834
|
+
const rawEngine = new PythonRawTensorEngine(pythonPath, node_path.join(pythonDir, "raw_tensor_inference.py"), modelPath, engineLogger);
|
|
835
|
+
await rawEngine.initialize();
|
|
836
|
+
this.imageRawEngine = rawEngine;
|
|
837
|
+
return;
|
|
838
|
+
}
|
|
839
|
+
const tokenizerPath = node_path.join(node_path.dirname(modelPath), TOKENIZER_FILE);
|
|
840
|
+
if (!node_fs.existsSync(tokenizerPath)) throw new Error(`EmbeddingEncoder: CLIP tokenizer not found at "${tokenizerPath}" — the tokenizer.json sibling download likely failed.`);
|
|
841
|
+
const textEngine = new PythonTextEncoderEngine(pythonPath, node_path.join(pythonDir, "text_encoder_inference.py"), modelPath, tokenizerPath, engineLogger);
|
|
842
|
+
await textEngine.initialize();
|
|
843
|
+
this.textEngine = textEngine;
|
|
650
844
|
}
|
|
651
845
|
async onShutdown() {
|
|
652
846
|
await this.imageRawEngine?.dispose();
|
|
653
|
-
await this.
|
|
847
|
+
await this.textEngine?.dispose();
|
|
654
848
|
}
|
|
655
849
|
globalSettingsSchema() {
|
|
656
850
|
return this.schema({ sections: [{
|
|
@@ -706,21 +900,6 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
|
706
900
|
async onConfigChanged() {}
|
|
707
901
|
};
|
|
708
902
|
/**
|
|
709
|
-
* Minimal CLIP tokenizer — encodes ASCII text to token IDs.
|
|
710
|
-
* Production implementations should use a proper BPE tokenizer;
|
|
711
|
-
* this is a simplified placeholder that maps characters to IDs
|
|
712
|
-
* with SOT/EOT tokens for basic functionality.
|
|
713
|
-
*/
|
|
714
|
-
function clipTokenize(text, maxLength = 77) {
|
|
715
|
-
const SOT_TOKEN = 49406;
|
|
716
|
-
const EOT_TOKEN = 49407;
|
|
717
|
-
const tokens = [SOT_TOKEN];
|
|
718
|
-
for (let i = 0; i < text.length && tokens.length < maxLength - 1; i++) tokens.push(text.charCodeAt(i) + 256);
|
|
719
|
-
tokens.push(EOT_TOKEN);
|
|
720
|
-
while (tokens.length < maxLength) tokens.push(0);
|
|
721
|
-
return tokens;
|
|
722
|
-
}
|
|
723
|
-
/**
|
|
724
903
|
* Locate the addon's bundled `python/` dir (holds `raw_tensor_inference.py` +
|
|
725
904
|
* `requirements-embedding.txt`). Published package first, then `__dirname`
|
|
726
905
|
* candidates for the in-tree dev build.
|