@camstack/addon-post-analysis 1.1.4 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (20) hide show
  1. package/dist/{dist-B2FbCLNx.mjs → dist-BpGP9ago.mjs} +41 -5
  2. package/dist/{dist-B6aOcq5T.js → dist-n-zJ0Uox.js} +46 -4
  3. package/dist/embedding-encoder/index.js +272 -93
  4. package/dist/embedding-encoder/index.mjs +270 -91
  5. package/dist/enrichment-engine/index.js +2 -2
  6. package/dist/enrichment-engine/index.mjs +1 -1
  7. package/dist/pipeline-analytics/_stub.js +1 -1
  8. package/dist/pipeline-analytics/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-Bcn1PmOA.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-DgAlWohT.mjs} +3 -3
  9. package/dist/pipeline-analytics/{_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BFIbMkkd.mjs → _virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DBkzwlqD.mjs} +1 -1
  10. package/dist/pipeline-analytics/{hostInit-gu9R5ABc.mjs → hostInit-D51b7QGW.mjs} +3 -3
  11. package/dist/pipeline-analytics/index.js +316 -2
  12. package/dist/pipeline-analytics/index.mjs +315 -1
  13. package/dist/pipeline-analytics/remoteEntry.js +1 -1
  14. package/dist/{resolve-frame-B6a1wzQb.js → resolve-frame-DB2NdMu2.js} +1 -1
  15. package/package.json +2 -1
  16. package/python/raw_tensor_inference.py +73 -0
  17. package/python/requirements-embedding.txt +8 -0
  18. package/python/tensor_frames.py +70 -0
  19. package/python/test_text_encoder.py +48 -0
  20. package/python/text_encoder_inference.py +76 -0
@@ -4627,7 +4627,7 @@ function _instanceof(cls, params = {}) {
4627
4627
  return inst;
4628
4628
  }
4629
4629
  //#endregion
4630
- //#region ../types/dist/sleep-_J5S7OEP.mjs
4630
+ //#region ../types/dist/sleep-NOH4yRwj.mjs
4631
4631
  var EventCategory = /* @__PURE__ */ function(EventCategory) {
4632
4632
  EventCategory["SystemBoot"] = "system.boot";
4633
4633
  EventCategory["SystemAddonsReady"] = "system.addons-ready";
@@ -7134,6 +7134,9 @@ var EncodeProfileSchema = object({
7134
7134
  */
7135
7135
  outputArgs: array(string()).optional()
7136
7136
  });
7137
+ function hfModelUrl(repo, path) {
7138
+ return `https://huggingface.co/${repo}/resolve/main/${path}`;
7139
+ }
7137
7140
  /** Cosine similarity between two embedding vectors */
7138
7141
  function cosineSimilarity(a, b) {
7139
7142
  if (a.length !== b.length) return 0;
@@ -12770,7 +12773,8 @@ method(object({
12770
12773
  sessionId: string().optional()
12771
12774
  }), ConvertResultSchema, {
12772
12775
  kind: "mutation",
12773
- auth: "admin"
12776
+ auth: "admin",
12777
+ timeoutMs: 6e5
12774
12778
  });
12775
12779
  var AddonHttpRouteSchema = object({
12776
12780
  method: _enum([
@@ -14326,7 +14330,14 @@ var NotificationRuleConditionsSchema = object({
14326
14330
  /** Match against `event.data.eventType` token (e.g. `'press_long'`). When non-empty, only events
14327
14331
  * carrying a matching `data.eventType` string pass this condition. Rules without this field are
14328
14332
  * unaffected (back-compat). Distinct from `rule.eventTypes` which holds EventCategory strings. */
14329
- eventTypeTokens: array(string()).readonly().optional()
14333
+ eventTypeTokens: array(string()).readonly().optional(),
14334
+ /** Match detections whose CLIP image embedding is semantically similar to this free-text
14335
+ * description. Requires the embedding-encoder cap to have pre-warmed the text vector.
14336
+ * `minSimilarity` is the cosine similarity threshold in [0, 1]. */
14337
+ clipDescription: object({
14338
+ text: string().min(1),
14339
+ minSimilarity: number().min(0).max(1)
14340
+ }).optional()
14330
14341
  });
14331
14342
  var NotificationRuleTemplateSchema = object({
14332
14343
  title: string(),
@@ -14568,6 +14579,16 @@ var TrackedDetectionSchema = object({
14568
14579
  zones: array(string()).readonly(),
14569
14580
  state: TrackStateSchema
14570
14581
  });
14582
+ var ScoredObjectEventSchema = ObjectEventSchema.extend({ score: number() });
14583
+ var SearchObjectEventsInput = object({
14584
+ text: string(),
14585
+ deviceId: number().optional(),
14586
+ since: number().optional(),
14587
+ until: number().optional(),
14588
+ classFilter: string().optional(),
14589
+ limit: number().default(50),
14590
+ minScore: number().min(0).max(1).default(.2)
14591
+ });
14571
14592
  var pipelineAnalyticsCapability = {
14572
14593
  name: "pipeline-analytics",
14573
14594
  scope: "device",
@@ -14634,7 +14655,16 @@ var pipelineAnalyticsCapability = {
14634
14655
  eventId: string(),
14635
14656
  kind: MediaFileKindEnum.optional()
14636
14657
  }), array(MediaFileSchema).readonly()),
14637
- getTrackMedia: method(object({ trackId: string() }), array(MediaFileSchema).readonly())
14658
+ getTrackMedia: method(object({ trackId: string() }), array(MediaFileSchema).readonly()),
14659
+ /**
14660
+ * Search object events by text query using CLIP cosine similarity.
14661
+ * Encodes `text` via the `embedding-encoder` cap, queries the
14662
+ * `ObjectEmbeddingStore` with optional prefilters, ranks all matching
14663
+ * embeddings by cosine similarity, and joins winners to their
14664
+ * ObjectEvents by trackId. Returns up to `limit` events scored ≥
14665
+ * `minScore`, sorted descending by score.
14666
+ */
14667
+ searchObjectEvents: method(SearchObjectEventsInput, array(ScoredObjectEventSchema).readonly())
14638
14668
  },
14639
14669
  events: {
14640
14670
  /**
@@ -19351,6 +19381,12 @@ Object.freeze({
19351
19381
  addonId: null,
19352
19382
  access: "create"
19353
19383
  },
19384
+ "pipelineAnalytics.searchObjectEvents": {
19385
+ capName: "pipeline-analytics",
19386
+ capScope: "device",
19387
+ addonId: null,
19388
+ access: "view"
19389
+ },
19354
19390
  "pipelineExecutor.cacheFrameInPool": {
19355
19391
  capName: "pipeline-executor",
19356
19392
  capScope: "system",
@@ -21235,4 +21271,4 @@ object({
21235
21271
  schemaVersion: literal(1)
21236
21272
  });
21237
21273
  //#endregion
21238
- export { tuple as C, string as S, _enum as _, faceGalleryCapability as a, number as b, videoclipsCapability as c, BaseAddon as d, DeviceType as f, hydrateSchema as g, createEvent as h, embeddingEncoderCapability as i, zoneAnalyticsCapability as l, asJsonObject as m, audioMetricsCapability as n, pipelineAnalyticsCapability as o, EventCategory as p, cosineSimilarity as r, plateGalleryCapability as s, addonWidgetsSourceCapability as t, errMsg as u, array as v, object as x, boolean as y };
21274
+ export { string as C, object as S, hydrateSchema as _, faceGalleryCapability as a, boolean as b, plateGalleryCapability as c, errMsg as d, BaseAddon as f, createEvent as g, asJsonObject as h, embeddingEncoderCapability as i, videoclipsCapability as l, EventCategory as m, audioMetricsCapability as n, hfModelUrl as o, DeviceType as p, cosineSimilarity as r, pipelineAnalyticsCapability as s, addonWidgetsSourceCapability as t, zoneAnalyticsCapability as u, _enum as v, tuple as w, number as x, array as y };
@@ -4649,7 +4649,7 @@ function _instanceof(cls, params = {}) {
4649
4649
  return inst;
4650
4650
  }
4651
4651
  //#endregion
4652
- //#region ../types/dist/sleep-_J5S7OEP.mjs
4652
+ //#region ../types/dist/sleep-NOH4yRwj.mjs
4653
4653
  var EventCategory = /* @__PURE__ */ function(EventCategory) {
4654
4654
  EventCategory["SystemBoot"] = "system.boot";
4655
4655
  EventCategory["SystemAddonsReady"] = "system.addons-ready";
@@ -7156,6 +7156,9 @@ var EncodeProfileSchema = object({
7156
7156
  */
7157
7157
  outputArgs: array(string()).optional()
7158
7158
  });
7159
+ function hfModelUrl(repo, path) {
7160
+ return `https://huggingface.co/${repo}/resolve/main/${path}`;
7161
+ }
7159
7162
  /** Cosine similarity between two embedding vectors */
7160
7163
  function cosineSimilarity(a, b) {
7161
7164
  if (a.length !== b.length) return 0;
@@ -12792,7 +12795,8 @@ method(object({
12792
12795
  sessionId: string().optional()
12793
12796
  }), ConvertResultSchema, {
12794
12797
  kind: "mutation",
12795
- auth: "admin"
12798
+ auth: "admin",
12799
+ timeoutMs: 6e5
12796
12800
  });
12797
12801
  var AddonHttpRouteSchema = object({
12798
12802
  method: _enum([
@@ -14348,7 +14352,14 @@ var NotificationRuleConditionsSchema = object({
14348
14352
  /** Match against `event.data.eventType` token (e.g. `'press_long'`). When non-empty, only events
14349
14353
  * carrying a matching `data.eventType` string pass this condition. Rules without this field are
14350
14354
  * unaffected (back-compat). Distinct from `rule.eventTypes` which holds EventCategory strings. */
14351
- eventTypeTokens: array(string()).readonly().optional()
14355
+ eventTypeTokens: array(string()).readonly().optional(),
14356
+ /** Match detections whose CLIP image embedding is semantically similar to this free-text
14357
+ * description. Requires the embedding-encoder cap to have pre-warmed the text vector.
14358
+ * `minSimilarity` is the cosine similarity threshold in [0, 1]. */
14359
+ clipDescription: object({
14360
+ text: string().min(1),
14361
+ minSimilarity: number().min(0).max(1)
14362
+ }).optional()
14352
14363
  });
14353
14364
  var NotificationRuleTemplateSchema = object({
14354
14365
  title: string(),
@@ -14590,6 +14601,16 @@ var TrackedDetectionSchema = object({
14590
14601
  zones: array(string()).readonly(),
14591
14602
  state: TrackStateSchema
14592
14603
  });
14604
+ var ScoredObjectEventSchema = ObjectEventSchema.extend({ score: number() });
14605
+ var SearchObjectEventsInput = object({
14606
+ text: string(),
14607
+ deviceId: number().optional(),
14608
+ since: number().optional(),
14609
+ until: number().optional(),
14610
+ classFilter: string().optional(),
14611
+ limit: number().default(50),
14612
+ minScore: number().min(0).max(1).default(.2)
14613
+ });
14593
14614
  var pipelineAnalyticsCapability = {
14594
14615
  name: "pipeline-analytics",
14595
14616
  scope: "device",
@@ -14656,7 +14677,16 @@ var pipelineAnalyticsCapability = {
14656
14677
  eventId: string(),
14657
14678
  kind: MediaFileKindEnum.optional()
14658
14679
  }), array(MediaFileSchema).readonly()),
14659
- getTrackMedia: method(object({ trackId: string() }), array(MediaFileSchema).readonly())
14680
+ getTrackMedia: method(object({ trackId: string() }), array(MediaFileSchema).readonly()),
14681
+ /**
14682
+ * Search object events by text query using CLIP cosine similarity.
14683
+ * Encodes `text` via the `embedding-encoder` cap, queries the
14684
+ * `ObjectEmbeddingStore` with optional prefilters, ranks all matching
14685
+ * embeddings by cosine similarity, and joins winners to their
14686
+ * ObjectEvents by trackId. Returns up to `limit` events scored ≥
14687
+ * `minScore`, sorted descending by score.
14688
+ */
14689
+ searchObjectEvents: method(SearchObjectEventsInput, array(ScoredObjectEventSchema).readonly())
14660
14690
  },
14661
14691
  events: {
14662
14692
  /**
@@ -19373,6 +19403,12 @@ Object.freeze({
19373
19403
  addonId: null,
19374
19404
  access: "create"
19375
19405
  },
19406
+ "pipelineAnalytics.searchObjectEvents": {
19407
+ capName: "pipeline-analytics",
19408
+ capScope: "device",
19409
+ addonId: null,
19410
+ access: "view"
19411
+ },
19376
19412
  "pipelineExecutor.cacheFrameInPool": {
19377
19413
  capName: "pipeline-executor",
19378
19414
  capScope: "system",
@@ -21347,6 +21383,12 @@ Object.defineProperty(exports, "faceGalleryCapability", {
21347
21383
  return faceGalleryCapability;
21348
21384
  }
21349
21385
  });
21386
+ Object.defineProperty(exports, "hfModelUrl", {
21387
+ enumerable: true,
21388
+ get: function() {
21389
+ return hfModelUrl;
21390
+ }
21391
+ });
21350
21392
  Object.defineProperty(exports, "hydrateSchema", {
21351
21393
  enumerable: true,
21352
21394
  get: function() {
@@ -2,13 +2,13 @@ Object.defineProperties(exports, {
2
2
  __esModule: { value: true },
3
3
  [Symbol.toStringTag]: { value: "Module" }
4
4
  });
5
- const require_dist = require("../dist-B6aOcq5T.js");
6
- let node_path = require("node:path");
7
- let node_path$1 = require_dist.__toESM(node_path, 1);
8
- node_path = require_dist.__toESM(node_path);
5
+ const require_dist = require("../dist-n-zJ0Uox.js");
9
6
  let node_fs = require("node:fs");
10
7
  let node_fs$1 = require_dist.__toESM(node_fs, 1);
11
8
  node_fs = require_dist.__toESM(node_fs);
9
+ let node_path = require("node:path");
10
+ let node_path$1 = require_dist.__toESM(node_path, 1);
11
+ node_path = require_dist.__toESM(node_path);
12
12
  require("node:crypto");
13
13
  let node_child_process = require("node:child_process");
14
14
  //#region ../system/dist/model-download-service-C-IHWnXx.mjs
@@ -262,98 +262,164 @@ var ModelDownloadService = class {
262
262
  };
263
263
  //#endregion
264
264
  //#region src/embedding-encoder/catalogs/embedding-models.ts
265
+ var HF_REPO = "camstack/camstack-models";
266
+ var hf = (path) => require_dist.hfModelUrl(HF_REPO, path);
267
+ /**
268
+ * The CLIP BPE tokenizer (HF `tokenizers` format: vocab 49408 + 48894 merges).
269
+ * Hosted next to every text-encoder onnx (`.../onnx/tokenizer.json`) and fetched
270
+ * as a sibling file so it lands beside the model in the shared models dir.
271
+ */
272
+ var TOKENIZER_FILE = "tokenizer.json";
273
+ var ovFormat = (url, sizeMB) => {
274
+ const base = url.split("/").pop() ?? "";
275
+ const files = base.endsWith(".xml") ? [base.replace(/\.xml$/, ".bin")] : void 0;
276
+ return {
277
+ url,
278
+ sizeMB,
279
+ runtimes: ["python"],
280
+ ...files ? { files } : {}
281
+ };
282
+ };
283
+ /**
284
+ * Files inside an .mlpackage directory bundle.
285
+ * Must be fetched alongside the package root when isDirectory is true.
286
+ */
287
+ var MLPACKAGE_FILES = [
288
+ "Manifest.json",
289
+ "Data/com.apple.CoreML/model.mlmodel",
290
+ "Data/com.apple.CoreML/weights/weight.bin"
291
+ ];
265
292
  var CLIP_IMAGE_MODELS = [
266
293
  {
267
- id: "clip-vit-b32",
268
- name: "CLIP ViT-B/32",
269
- description: "OpenAI CLIP ViT-B/32 — fast, 512-dim, int8 quantized (85 MB)",
294
+ id: "mobileclip-s0",
295
+ name: "MobileCLIP S0",
296
+ description: "Apple MobileCLIP S0 — fast vision encoder, 512-dim, 256×256 (40 MB ONNX)",
270
297
  inputSize: {
271
- width: 224,
272
- height: 224
298
+ width: 256,
299
+ height: 256
273
300
  },
274
301
  labels: [],
275
- inputLayout: "nchw",
276
302
  inputNormalization: "none",
277
- formats: { onnx: {
278
- url: "https://huggingface.co/Xenova/clip-vit-base-patch32/resolve/main/onnx/vision_model_quantized.onnx",
279
- sizeMB: 85
280
- } }
303
+ formats: {
304
+ onnx: {
305
+ url: hf("clip/mobileclip-s0/onnx/camstack-mobileclip-s0-vision.onnx"),
306
+ sizeMB: 40
307
+ },
308
+ openvino: ovFormat(hf("clip/mobileclip-s0/openvino/camstack-mobileclip-s0-vision.xml"), 25),
309
+ coreml: {
310
+ url: hf("clip/mobileclip-s0/coreml/camstack-mobileclip-s0-vision.mlpackage"),
311
+ sizeMB: 30,
312
+ isDirectory: true,
313
+ files: [...MLPACKAGE_FILES],
314
+ runtimes: ["python"]
315
+ }
316
+ }
281
317
  },
282
318
  {
283
- id: "clip-vit-b16",
284
- name: "CLIP ViT-B/16",
285
- description: "OpenAI CLIP ViT-B/16higher accuracy, 512-dim, int8 quantized (83 MB)",
319
+ id: "mobileclip-s1",
320
+ name: "MobileCLIP S1",
321
+ description: "Apple MobileCLIP S1balanced vision encoder, 512-dim, 256×256 (90 MB ONNX)",
286
322
  inputSize: {
287
- width: 224,
288
- height: 224
323
+ width: 256,
324
+ height: 256
289
325
  },
290
326
  labels: [],
291
- inputLayout: "nchw",
292
327
  inputNormalization: "none",
293
- formats: { onnx: {
294
- url: "https://huggingface.co/Xenova/clip-vit-base-patch16/resolve/main/onnx/vision_model_quantized.onnx",
295
- sizeMB: 83
296
- } }
328
+ formats: {
329
+ onnx: {
330
+ url: hf("clip/mobileclip-s1/onnx/camstack-mobileclip-s1-vision.onnx"),
331
+ sizeMB: 90
332
+ },
333
+ openvino: ovFormat(hf("clip/mobileclip-s1/openvino/camstack-mobileclip-s1-vision.xml"), 55),
334
+ coreml: {
335
+ url: hf("clip/mobileclip-s1/coreml/camstack-mobileclip-s1-vision.mlpackage"),
336
+ sizeMB: 65,
337
+ isDirectory: true,
338
+ files: [...MLPACKAGE_FILES],
339
+ runtimes: ["python"]
340
+ }
341
+ }
297
342
  },
298
343
  {
299
- id: "siglip2-b16-256",
300
- name: "SigLIP2 Base/16 256",
301
- description: "Google SigLIP2superior scene understanding, 768-dim, int8 quantized (90 MB)",
344
+ id: "mobileclip-s2",
345
+ name: "MobileCLIP S2",
346
+ description: "Apple MobileCLIP S2 high-accuracy vision encoder, 512-dim, 256×256 (150 MB ONNX)",
302
347
  inputSize: {
303
348
  width: 256,
304
349
  height: 256
305
350
  },
306
351
  labels: [],
307
- inputLayout: "nchw",
308
352
  inputNormalization: "none",
309
- formats: { onnx: {
310
- url: "https://huggingface.co/onnx-community/siglip2-base-patch16-256-ONNX/resolve/main/onnx/vision_model_quantized.onnx",
311
- sizeMB: 90
312
- } }
353
+ formats: {
354
+ onnx: {
355
+ url: hf("clip/mobileclip-s2/onnx/camstack-mobileclip-s2-vision.onnx"),
356
+ sizeMB: 150
357
+ },
358
+ openvino: ovFormat(hf("clip/mobileclip-s2/openvino/camstack-mobileclip-s2-vision.xml"), 90),
359
+ coreml: {
360
+ url: hf("clip/mobileclip-s2/coreml/camstack-mobileclip-s2-vision.mlpackage"),
361
+ sizeMB: 110,
362
+ isDirectory: true,
363
+ files: [...MLPACKAGE_FILES],
364
+ runtimes: ["python"]
365
+ }
366
+ }
313
367
  }
314
368
  ];
315
369
  var CLIP_TEXT_MODELS = [
316
370
  {
317
- id: "clip-vit-b32-text",
318
- name: "CLIP ViT-B/32 Text Encoder",
319
- description: "Text encoder for CLIP ViT-B/32, int8 quantized (62 MB)",
371
+ id: "mobileclip-s0-text",
372
+ name: "MobileCLIP S0 Text Encoder",
373
+ description: "Text encoder for MobileCLIP S0, 512-dim, int8 quantized (35 MB)",
320
374
  inputSize: {
321
375
  width: 0,
322
376
  height: 0
323
377
  },
324
378
  labels: [],
325
- formats: { onnx: {
326
- url: "https://huggingface.co/Xenova/clip-vit-base-patch32/resolve/main/onnx/text_model_quantized.onnx",
327
- sizeMB: 62
328
- } }
379
+ formats: {
380
+ onnx: {
381
+ url: hf("clip/mobileclip-s0/onnx/camstack-mobileclip-s0-text.onnx"),
382
+ sizeMB: 35,
383
+ files: [TOKENIZER_FILE]
384
+ },
385
+ openvino: ovFormat(hf("clip/mobileclip-s0/openvino/camstack-mobileclip-s0-text.xml"), 22)
386
+ }
329
387
  },
330
388
  {
331
- id: "clip-vit-b16-text",
332
- name: "CLIP ViT-B/16 Text Encoder",
333
- description: "Text encoder for CLIP ViT-B/16, int8 quantized (62 MB)",
389
+ id: "mobileclip-s1-text",
390
+ name: "MobileCLIP S1 Text Encoder",
391
+ description: "Text encoder for MobileCLIP S1, 512-dim, int8 quantized (35 MB)",
334
392
  inputSize: {
335
393
  width: 0,
336
394
  height: 0
337
395
  },
338
396
  labels: [],
339
- formats: { onnx: {
340
- url: "https://huggingface.co/Xenova/clip-vit-base-patch16/resolve/main/onnx/text_model_quantized.onnx",
341
- sizeMB: 62
342
- } }
397
+ formats: {
398
+ onnx: {
399
+ url: hf("clip/mobileclip-s1/onnx/camstack-mobileclip-s1-text.onnx"),
400
+ sizeMB: 35,
401
+ files: [TOKENIZER_FILE]
402
+ },
403
+ openvino: ovFormat(hf("clip/mobileclip-s1/openvino/camstack-mobileclip-s1-text.xml"), 22)
404
+ }
343
405
  },
344
406
  {
345
- id: "siglip2-b16-256-text",
346
- name: "SigLIP2 Base/16 256 Text Encoder",
347
- description: "Text encoder for SigLIP2, int8 quantized (270 MB)",
407
+ id: "mobileclip-s2-text",
408
+ name: "MobileCLIP S2 Text Encoder",
409
+ description: "Text encoder for MobileCLIP S2, 512-dim, int8 quantized (35 MB)",
348
410
  inputSize: {
349
411
  width: 0,
350
412
  height: 0
351
413
  },
352
414
  labels: [],
353
- formats: { onnx: {
354
- url: "https://huggingface.co/onnx-community/siglip2-base-patch16-256-ONNX/resolve/main/onnx/text_model_quantized.onnx",
355
- sizeMB: 270
356
- } }
415
+ formats: {
416
+ onnx: {
417
+ url: hf("clip/mobileclip-s2/onnx/camstack-mobileclip-s2-text.onnx"),
418
+ sizeMB: 35,
419
+ files: [TOKENIZER_FILE]
420
+ },
421
+ openvino: ovFormat(hf("clip/mobileclip-s2/openvino/camstack-mobileclip-s2-text.xml"), 22)
422
+ }
357
423
  }
358
424
  ];
359
425
  //#endregion
@@ -485,33 +551,155 @@ var PythonRawTensorEngine = class {
485
551
  }
486
552
  };
487
553
  //#endregion
554
+ //#region src/embedding-encoder/shared/python-text-encoder-engine.ts
555
+ /**
556
+ * CLIP text-encoder engine backed by an embedded-Python subprocess
557
+ * (`text_encoder_inference.py`). Tokenization happens IN Python via the HF
558
+ * `tokenizers` Rust BPE (exact by construction) — this replaces the former
559
+ * hand-rolled TypeScript CLIP BPE.
560
+ *
561
+ * The caller sends raw UTF-8 text; Python tokenizes (truncate/pad to 77), runs
562
+ * onnxruntime, and returns the embedding tensor. Wire protocol = length-prefixed
563
+ * binary frames ([4B LE length][payload]):
564
+ * ready (in): [0x01]
565
+ * request (out): UTF-8 text bytes
566
+ * response (in): [1B ndims][dims × 4B LE uint32][float32 LE data]
567
+ */
568
+ var PythonTextEncoderEngine = class {
569
+ pythonPath;
570
+ scriptPath;
571
+ modelPath;
572
+ tokenizerPath;
573
+ process = null;
574
+ receiveBuffer = Buffer.alloc(0);
575
+ pendingResolve = null;
576
+ pendingReject = null;
577
+ log;
578
+ constructor(pythonPath, scriptPath, modelPath, tokenizerPath, logger) {
579
+ this.pythonPath = pythonPath;
580
+ this.scriptPath = scriptPath;
581
+ this.modelPath = modelPath;
582
+ this.tokenizerPath = tokenizerPath;
583
+ this.log = logger ?? createNoopLogger();
584
+ }
585
+ async initialize() {
586
+ this.process = (0, node_child_process.spawn)(this.pythonPath, [
587
+ this.scriptPath,
588
+ this.modelPath,
589
+ this.tokenizerPath
590
+ ], { stdio: [
591
+ "pipe",
592
+ "pipe",
593
+ "pipe"
594
+ ] });
595
+ this.process.stderr?.on("data", (chunk) => {
596
+ const text = chunk.toString().trim();
597
+ if (text) this.log.warn(text);
598
+ });
599
+ this.process.on("error", (err) => {
600
+ this.log.error("Python text-encoder process error", { meta: { error: err.message } });
601
+ this.pendingReject?.(err);
602
+ this.pendingReject = null;
603
+ this.pendingResolve = null;
604
+ });
605
+ this.process.on("exit", (code) => {
606
+ if (code !== 0 && code !== null) {
607
+ const err = /* @__PURE__ */ new Error(`PythonTextEncoderEngine: process exited with code ${code}`);
608
+ this.pendingReject?.(err);
609
+ this.pendingReject = null;
610
+ this.pendingResolve = null;
611
+ }
612
+ });
613
+ this.process.stdout.on("data", (chunk) => {
614
+ this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
615
+ this.tryReceive();
616
+ });
617
+ const ready = await this.receiveFrame();
618
+ if (ready.length !== 1 || ready[0] !== 1) throw new Error("PythonTextEncoderEngine: unexpected ready frame");
619
+ this.log.info("CLIP text-encoder engine ready (embedded Python)", { meta: {
620
+ modelPath: this.modelPath,
621
+ tokenizerPath: this.tokenizerPath
622
+ } });
623
+ }
624
+ /** Tokenize + encode `text` into the model embedding (float32). */
625
+ async encode(text) {
626
+ if (!this.process?.stdin) throw new Error("PythonTextEncoderEngine: not initialized — call initialize() first");
627
+ const payload = Buffer.from(text, "utf-8");
628
+ const lenBuf = Buffer.allocUnsafe(4);
629
+ lenBuf.writeUInt32LE(payload.length, 0);
630
+ this.process.stdin.write(Buffer.concat([lenBuf, payload]));
631
+ const resp = await this.receiveFrame();
632
+ const floatStart = 1 + resp.readUInt8(0) * 4;
633
+ const count = (resp.length - floatStart) / 4;
634
+ const out = new Float32Array(count);
635
+ for (let i = 0; i < count; i++) out[i] = resp.readFloatLE(floatStart + i * 4);
636
+ return out;
637
+ }
638
+ async dispose() {
639
+ const proc = this.process;
640
+ if (!proc) return;
641
+ this.process = null;
642
+ proc.stdin?.end();
643
+ proc.kill("SIGTERM");
644
+ await new Promise((resolve) => {
645
+ const timer = setTimeout(() => {
646
+ try {
647
+ proc.kill("SIGKILL");
648
+ } catch {}
649
+ resolve();
650
+ }, 5e3);
651
+ proc.once("exit", () => {
652
+ clearTimeout(timer);
653
+ resolve();
654
+ });
655
+ });
656
+ }
657
+ receiveFrame() {
658
+ return new Promise((resolve, reject) => {
659
+ this.pendingResolve = resolve;
660
+ this.pendingReject = reject;
661
+ });
662
+ }
663
+ tryReceive() {
664
+ if (this.receiveBuffer.length < 4) return;
665
+ const length = this.receiveBuffer.readUInt32LE(0);
666
+ if (this.receiveBuffer.length < 4 + length) return;
667
+ const payload = Buffer.from(this.receiveBuffer.subarray(4, 4 + length));
668
+ this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
669
+ const resolve = this.pendingResolve;
670
+ this.pendingResolve = null;
671
+ this.pendingReject = null;
672
+ resolve?.(payload);
673
+ }
674
+ };
675
+ //#endregion
488
676
  //#region src/embedding-encoder/addon/clip-models.ts
489
677
  var CLIP_MODEL_META = {
490
- "clip-vit-b32": {
491
- imageModelId: "clip-vit-b32",
492
- textModelId: "clip-vit-b32-text",
678
+ "mobileclip-s0": {
679
+ imageModelId: "mobileclip-s0",
680
+ textModelId: "mobileclip-s0-text",
493
681
  embeddingDim: 512,
494
- inputSize: 224,
682
+ inputSize: 256,
495
683
  tokenizerType: "clip"
496
684
  },
497
- "clip-vit-b16": {
498
- imageModelId: "clip-vit-b16",
499
- textModelId: "clip-vit-b16-text",
685
+ "mobileclip-s1": {
686
+ imageModelId: "mobileclip-s1",
687
+ textModelId: "mobileclip-s1-text",
500
688
  embeddingDim: 512,
501
- inputSize: 224,
689
+ inputSize: 256,
502
690
  tokenizerType: "clip"
503
691
  },
504
- "siglip2-b16-256": {
505
- imageModelId: "siglip2-b16-256",
506
- textModelId: "siglip2-b16-256-text",
507
- embeddingDim: 768,
692
+ "mobileclip-s2": {
693
+ imageModelId: "mobileclip-s2",
694
+ textModelId: "mobileclip-s2-text",
695
+ embeddingDim: 512,
508
696
  inputSize: 256,
509
- tokenizerType: "siglip"
697
+ tokenizerType: "clip"
510
698
  }
511
699
  };
512
- var DEFAULT_CLIP_MODEL = "clip-vit-b32";
700
+ var DEFAULT_CLIP_MODEL = "mobileclip-s1";
513
701
  function getModelMeta(modelId) {
514
- return CLIP_MODEL_META[modelId] ?? CLIP_MODEL_META["clip-vit-b32"];
702
+ return CLIP_MODEL_META[modelId] ?? CLIP_MODEL_META["mobileclip-s1"];
515
703
  }
516
704
  //#endregion
517
705
  //#region src/embedding-encoder/addon/clip-preprocessing.ts
@@ -559,7 +747,7 @@ function l2Normalize(vec) {
559
747
  //#region src/embedding-encoder/addon/index.ts
560
748
  var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
561
749
  imageRawEngine = null;
562
- textRawEngine = null;
750
+ textEngine = null;
563
751
  models = null;
564
752
  constructor() {
565
753
  super({
@@ -600,9 +788,8 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
600
788
  await this.ensureTextEngine();
601
789
  const meta = getModelMeta(this.config.modelId);
602
790
  const start = Date.now();
603
- const tokenIds = clipTokenize(text);
604
- const inputTensor = new Float32Array(tokenIds);
605
- const output = await this.textRawEngine.run(inputTensor, [1, tokenIds.length]);
791
+ if (!this.textEngine) throw new Error("EmbeddingEncoder: text engine not loaded — ensureTextEngine() must run first");
792
+ const output = await this.textEngine.encode(text);
606
793
  const sliced = output.length > meta.embeddingDim ? output.slice(0, meta.embeddingDim) : output;
607
794
  const normalized = l2Normalize(new Float32Array(sliced));
608
795
  return {
@@ -626,7 +813,7 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
626
813
  await this.resolveForEntry(imageEntry, "image");
627
814
  }
628
815
  async ensureTextEngine() {
629
- if (this.textRawEngine) return;
816
+ if (this.textEngine) return;
630
817
  const meta = getModelMeta(this.config.modelId);
631
818
  const textEntry = CLIP_TEXT_MODELS.find((m) => m.id === meta.textModelId);
632
819
  if (!textEntry) throw new Error(`EmbeddingEncoderAddon: unknown text model "${meta.textModelId}"`);
@@ -643,14 +830,21 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
643
830
  if (!pythonPath) throw new Error("EmbeddingEncoder: embedded Python is unavailable — cannot run ONNX embeddings. ctx.deps.ensurePython() returned null (portable Python download likely failed).");
644
831
  const pythonDir = resolveEmbeddingPythonDir();
645
832
  await this.ctx.deps.installPythonRequirements(node_path.join(pythonDir, "requirements-embedding.txt"));
646
- const rawEngine = new PythonRawTensorEngine(pythonPath, node_path.join(pythonDir, "raw_tensor_inference.py"), modelPath, engineLogger);
647
- await rawEngine.initialize();
648
- if (target === "image") this.imageRawEngine = rawEngine;
649
- else this.textRawEngine = rawEngine;
833
+ if (target === "image") {
834
+ const rawEngine = new PythonRawTensorEngine(pythonPath, node_path.join(pythonDir, "raw_tensor_inference.py"), modelPath, engineLogger);
835
+ await rawEngine.initialize();
836
+ this.imageRawEngine = rawEngine;
837
+ return;
838
+ }
839
+ const tokenizerPath = node_path.join(node_path.dirname(modelPath), TOKENIZER_FILE);
840
+ if (!node_fs.existsSync(tokenizerPath)) throw new Error(`EmbeddingEncoder: CLIP tokenizer not found at "${tokenizerPath}" — the tokenizer.json sibling download likely failed.`);
841
+ const textEngine = new PythonTextEncoderEngine(pythonPath, node_path.join(pythonDir, "text_encoder_inference.py"), modelPath, tokenizerPath, engineLogger);
842
+ await textEngine.initialize();
843
+ this.textEngine = textEngine;
650
844
  }
651
845
  async onShutdown() {
652
846
  await this.imageRawEngine?.dispose();
653
- await this.textRawEngine?.dispose();
847
+ await this.textEngine?.dispose();
654
848
  }
655
849
  globalSettingsSchema() {
656
850
  return this.schema({ sections: [{
@@ -706,21 +900,6 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
706
900
  async onConfigChanged() {}
707
901
  };
708
902
  /**
709
- * Minimal CLIP tokenizer — encodes ASCII text to token IDs.
710
- * Production implementations should use a proper BPE tokenizer;
711
- * this is a simplified placeholder that maps characters to IDs
712
- * with SOT/EOT tokens for basic functionality.
713
- */
714
- function clipTokenize(text, maxLength = 77) {
715
- const SOT_TOKEN = 49406;
716
- const EOT_TOKEN = 49407;
717
- const tokens = [SOT_TOKEN];
718
- for (let i = 0; i < text.length && tokens.length < maxLength - 1; i++) tokens.push(text.charCodeAt(i) + 256);
719
- tokens.push(EOT_TOKEN);
720
- while (tokens.length < maxLength) tokens.push(0);
721
- return tokens;
722
- }
723
- /**
724
903
  * Locate the addon's bundled `python/` dir (holds `raw_tensor_inference.py` +
725
904
  * `requirements-embedding.txt`). Published package first, then `__dirname`
726
905
  * candidates for the in-tree dev build.