@camstack/addon-post-analysis 1.1.3 → 1.1.5

package/dist/embedding-encoder/index.js

@@ -2,13 +2,13 @@ Object.defineProperties(exports, {
 	__esModule: { value: true },
 	[Symbol.toStringTag]: { value: "Module" }
 });
-const require_dist = require("../dist-BY1NQ5oi.js");
-let node_path = require("node:path");
-let node_path$1 = require_dist.__toESM(node_path, 1);
-node_path = require_dist.__toESM(node_path);
+const require_dist = require("../dist-n-zJ0Uox.js");
 let node_fs = require("node:fs");
 let node_fs$1 = require_dist.__toESM(node_fs, 1);
 node_fs = require_dist.__toESM(node_fs);
+let node_path = require("node:path");
+let node_path$1 = require_dist.__toESM(node_path, 1);
+node_path = require_dist.__toESM(node_path);
 require("node:crypto");
 let node_child_process = require("node:child_process");
 //#region ../system/dist/model-download-service-C-IHWnXx.mjs
@@ -262,98 +262,164 @@ var ModelDownloadService = class {
 };
 //#endregion
 //#region src/embedding-encoder/catalogs/embedding-models.ts
+var HF_REPO = "camstack/camstack-models";
+var hf = (path) => require_dist.hfModelUrl(HF_REPO, path);
+/**
+* The CLIP BPE tokenizer (HF `tokenizers` format: vocab 49408 + 48894 merges).
+* Hosted next to every text-encoder onnx (`.../onnx/tokenizer.json`) and fetched
+* as a sibling file so it lands beside the model in the shared models dir.
+*/
+var TOKENIZER_FILE = "tokenizer.json";
+var ovFormat = (url, sizeMB) => {
+	const base = url.split("/").pop() ?? "";
+	const files = base.endsWith(".xml") ? [base.replace(/\.xml$/, ".bin")] : void 0;
+	return {
+		url,
+		sizeMB,
+		runtimes: ["python"],
+		...files ? { files } : {}
+	};
+};
+/**
+* Files inside an .mlpackage directory bundle.
+* Must be fetched alongside the package root when isDirectory is true.
+*/
+var MLPACKAGE_FILES = [
+	"Manifest.json",
+	"Data/com.apple.CoreML/model.mlmodel",
+	"Data/com.apple.CoreML/weights/weight.bin"
+];
 var CLIP_IMAGE_MODELS = [
 	{
-		id: "clip-vit-b32",
-		name: "CLIP ViT-B/32",
-		description: "OpenAI CLIP ViT-B/32 — fast, 512-dim, int8 quantized (85 MB)",
+		id: "mobileclip-s0",
+		name: "MobileCLIP S0",
+		description: "Apple MobileCLIP S0 — fast vision encoder, 512-dim, 256×256 (40 MB ONNX)",
 		inputSize: {
-			width: 224,
-			height: 224
+			width: 256,
+			height: 256
 		},
 		labels: [],
-		inputLayout: "nchw",
 		inputNormalization: "none",
-		formats: { onnx: {
-			url: "https://huggingface.co/Xenova/clip-vit-base-patch32/resolve/main/onnx/vision_model_quantized.onnx",
-			sizeMB: 85
-		} }
+		formats: {
+			onnx: {
+				url: hf("clip/mobileclip-s0/onnx/camstack-mobileclip-s0-vision.onnx"),
+				sizeMB: 40
+			},
+			openvino: ovFormat(hf("clip/mobileclip-s0/openvino/camstack-mobileclip-s0-vision.xml"), 25),
+			coreml: {
+				url: hf("clip/mobileclip-s0/coreml/camstack-mobileclip-s0-vision.mlpackage"),
+				sizeMB: 30,
+				isDirectory: true,
+				files: [...MLPACKAGE_FILES],
+				runtimes: ["python"]
+			}
+		}
 	},
 	{
-		id: "clip-vit-b16",
-		name: "CLIP ViT-B/16",
-		description: "OpenAI CLIP ViT-B/16 — higher accuracy, 512-dim, int8 quantized (83 MB)",
+		id: "mobileclip-s1",
+		name: "MobileCLIP S1",
+		description: "Apple MobileCLIP S1 — balanced vision encoder, 512-dim, 256×256 (90 MB ONNX)",
 		inputSize: {
-			width: 224,
-			height: 224
+			width: 256,
+			height: 256
 		},
 		labels: [],
-		inputLayout: "nchw",
 		inputNormalization: "none",
-		formats: { onnx: {
-			url: "https://huggingface.co/Xenova/clip-vit-base-patch16/resolve/main/onnx/vision_model_quantized.onnx",
-			sizeMB: 83
-		} }
+		formats: {
+			onnx: {
+				url: hf("clip/mobileclip-s1/onnx/camstack-mobileclip-s1-vision.onnx"),
+				sizeMB: 90
+			},
+			openvino: ovFormat(hf("clip/mobileclip-s1/openvino/camstack-mobileclip-s1-vision.xml"), 55),
+			coreml: {
+				url: hf("clip/mobileclip-s1/coreml/camstack-mobileclip-s1-vision.mlpackage"),
+				sizeMB: 65,
+				isDirectory: true,
+				files: [...MLPACKAGE_FILES],
+				runtimes: ["python"]
+			}
+		}
 	},
 	{
-		id: "siglip2-b16-256",
-		name: "SigLIP2 Base/16 256",
-		description: "Google SigLIP2 — superior scene understanding, 768-dim, int8 quantized (90 MB)",
+		id: "mobileclip-s2",
+		name: "MobileCLIP S2",
+		description: "Apple MobileCLIP S2 — high-accuracy vision encoder, 512-dim, 256×256 (150 MB ONNX)",
 		inputSize: {
 			width: 256,
 			height: 256
 		},
 		labels: [],
-		inputLayout: "nchw",
 		inputNormalization: "none",
-		formats: { onnx: {
-			url: "https://huggingface.co/onnx-community/siglip2-base-patch16-256-ONNX/resolve/main/onnx/vision_model_quantized.onnx",
-			sizeMB: 90
-		} }
+		formats: {
+			onnx: {
+				url: hf("clip/mobileclip-s2/onnx/camstack-mobileclip-s2-vision.onnx"),
+				sizeMB: 150
+			},
+			openvino: ovFormat(hf("clip/mobileclip-s2/openvino/camstack-mobileclip-s2-vision.xml"), 90),
+			coreml: {
+				url: hf("clip/mobileclip-s2/coreml/camstack-mobileclip-s2-vision.mlpackage"),
+				sizeMB: 110,
+				isDirectory: true,
+				files: [...MLPACKAGE_FILES],
+				runtimes: ["python"]
+			}
+		}
 	}
 ];
 var CLIP_TEXT_MODELS = [
 	{
-		id: "clip-vit-b32-text",
-		name: "CLIP ViT-B/32 Text Encoder",
-		description: "Text encoder for CLIP ViT-B/32, int8 quantized (62 MB)",
+		id: "mobileclip-s0-text",
+		name: "MobileCLIP S0 Text Encoder",
+		description: "Text encoder for MobileCLIP S0, 512-dim, int8 quantized (35 MB)",
 		inputSize: {
 			width: 0,
 			height: 0
 		},
 		labels: [],
-		formats: { onnx: {
-			url: "https://huggingface.co/Xenova/clip-vit-base-patch32/resolve/main/onnx/text_model_quantized.onnx",
-			sizeMB: 62
-		} }
+		formats: {
+			onnx: {
+				url: hf("clip/mobileclip-s0/onnx/camstack-mobileclip-s0-text.onnx"),
+				sizeMB: 35,
+				files: [TOKENIZER_FILE]
+			},
+			openvino: ovFormat(hf("clip/mobileclip-s0/openvino/camstack-mobileclip-s0-text.xml"), 22)
+		}
 	},
 	{
-		id: "clip-vit-b16-text",
-		name: "CLIP ViT-B/16 Text Encoder",
-		description: "Text encoder for CLIP ViT-B/16, int8 quantized (62 MB)",
+		id: "mobileclip-s1-text",
+		name: "MobileCLIP S1 Text Encoder",
+		description: "Text encoder for MobileCLIP S1, 512-dim, int8 quantized (35 MB)",
 		inputSize: {
 			width: 0,
 			height: 0
 		},
 		labels: [],
-		formats: { onnx: {
-			url: "https://huggingface.co/Xenova/clip-vit-base-patch16/resolve/main/onnx/text_model_quantized.onnx",
-			sizeMB: 62
-		} }
+		formats: {
+			onnx: {
+				url: hf("clip/mobileclip-s1/onnx/camstack-mobileclip-s1-text.onnx"),
+				sizeMB: 35,
+				files: [TOKENIZER_FILE]
+			},
+			openvino: ovFormat(hf("clip/mobileclip-s1/openvino/camstack-mobileclip-s1-text.xml"), 22)
+		}
 	},
 	{
-		id: "siglip2-b16-256-text",
-		name: "SigLIP2 Base/16 256 Text Encoder",
-		description: "Text encoder for SigLIP2, int8 quantized (270 MB)",
+		id: "mobileclip-s2-text",
+		name: "MobileCLIP S2 Text Encoder",
+		description: "Text encoder for MobileCLIP S2, 512-dim, int8 quantized (35 MB)",
 		inputSize: {
 			width: 0,
 			height: 0
 		},
 		labels: [],
-		formats: { onnx: {
-			url: "https://huggingface.co/onnx-community/siglip2-base-patch16-256-ONNX/resolve/main/onnx/text_model_quantized.onnx",
-			sizeMB: 270
-		} }
+		formats: {
+			onnx: {
+				url: hf("clip/mobileclip-s2/onnx/camstack-mobileclip-s2-text.onnx"),
+				sizeMB: 35,
+				files: [TOKENIZER_FILE]
+			},
+			openvino: ovFormat(hf("clip/mobileclip-s2/openvino/camstack-mobileclip-s2-text.xml"), 22)
+		}
 	}
 ];
 //#endregion
@@ -485,33 +551,155 @@ var PythonRawTensorEngine = class {
 	}
 };
 //#endregion
+//#region src/embedding-encoder/shared/python-text-encoder-engine.ts
+/**
+* CLIP text-encoder engine backed by an embedded-Python subprocess
+* (`text_encoder_inference.py`). Tokenization happens IN Python via the HF
+* `tokenizers` Rust BPE (exact by construction) — this replaces the former
+* hand-rolled TypeScript CLIP BPE.
+*
+* The caller sends raw UTF-8 text; Python tokenizes (truncate/pad to 77), runs
+* onnxruntime, and returns the embedding tensor. Wire protocol = length-prefixed
+* binary frames ([4B LE length][payload]):
+*   ready    (in):  [0x01]
+*   request  (out): UTF-8 text bytes
+*   response (in):  [1B ndims][dims × 4B LE uint32][float32 LE data]
+*/
+var PythonTextEncoderEngine = class {
+	pythonPath;
+	scriptPath;
+	modelPath;
+	tokenizerPath;
+	process = null;
+	receiveBuffer = Buffer.alloc(0);
+	pendingResolve = null;
+	pendingReject = null;
+	log;
+	constructor(pythonPath, scriptPath, modelPath, tokenizerPath, logger) {
+		this.pythonPath = pythonPath;
+		this.scriptPath = scriptPath;
+		this.modelPath = modelPath;
+		this.tokenizerPath = tokenizerPath;
+		this.log = logger ?? createNoopLogger();
+	}
+	async initialize() {
+		this.process = (0, node_child_process.spawn)(this.pythonPath, [
+			this.scriptPath,
+			this.modelPath,
+			this.tokenizerPath
+		], { stdio: [
+			"pipe",
+			"pipe",
+			"pipe"
+		] });
+		this.process.stderr?.on("data", (chunk) => {
+			const text = chunk.toString().trim();
+			if (text) this.log.warn(text);
+		});
+		this.process.on("error", (err) => {
+			this.log.error("Python text-encoder process error", { meta: { error: err.message } });
+			this.pendingReject?.(err);
+			this.pendingReject = null;
+			this.pendingResolve = null;
+		});
+		this.process.on("exit", (code) => {
+			if (code !== 0 && code !== null) {
+				const err = /* @__PURE__ */ new Error(`PythonTextEncoderEngine: process exited with code ${code}`);
+				this.pendingReject?.(err);
+				this.pendingReject = null;
+				this.pendingResolve = null;
+			}
+		});
+		this.process.stdout.on("data", (chunk) => {
+			this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
+			this.tryReceive();
+		});
+		const ready = await this.receiveFrame();
+		if (ready.length !== 1 || ready[0] !== 1) throw new Error("PythonTextEncoderEngine: unexpected ready frame");
+		this.log.info("CLIP text-encoder engine ready (embedded Python)", { meta: {
+			modelPath: this.modelPath,
+			tokenizerPath: this.tokenizerPath
+		} });
+	}
+	/** Tokenize + encode `text` into the model embedding (float32). */
+	async encode(text) {
+		if (!this.process?.stdin) throw new Error("PythonTextEncoderEngine: not initialized — call initialize() first");
+		const payload = Buffer.from(text, "utf-8");
+		const lenBuf = Buffer.allocUnsafe(4);
+		lenBuf.writeUInt32LE(payload.length, 0);
+		this.process.stdin.write(Buffer.concat([lenBuf, payload]));
+		const resp = await this.receiveFrame();
+		const floatStart = 1 + resp.readUInt8(0) * 4;
+		const count = (resp.length - floatStart) / 4;
+		const out = new Float32Array(count);
+		for (let i = 0; i < count; i++) out[i] = resp.readFloatLE(floatStart + i * 4);
+		return out;
+	}
+	async dispose() {
+		const proc = this.process;
+		if (!proc) return;
+		this.process = null;
+		proc.stdin?.end();
+		proc.kill("SIGTERM");
+		await new Promise((resolve) => {
+			const timer = setTimeout(() => {
+				try {
+					proc.kill("SIGKILL");
+				} catch {}
+				resolve();
+			}, 5e3);
+			proc.once("exit", () => {
+				clearTimeout(timer);
+				resolve();
+			});
+		});
+	}
+	receiveFrame() {
+		return new Promise((resolve, reject) => {
+			this.pendingResolve = resolve;
+			this.pendingReject = reject;
+		});
+	}
+	tryReceive() {
+		if (this.receiveBuffer.length < 4) return;
+		const length = this.receiveBuffer.readUInt32LE(0);
+		if (this.receiveBuffer.length < 4 + length) return;
+		const payload = Buffer.from(this.receiveBuffer.subarray(4, 4 + length));
+		this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
+		const resolve = this.pendingResolve;
+		this.pendingResolve = null;
+		this.pendingReject = null;
+		resolve?.(payload);
+	}
+};
+//#endregion
 //#region src/embedding-encoder/addon/clip-models.ts
 var CLIP_MODEL_META = {
-	"clip-vit-b32": {
-		imageModelId: "clip-vit-b32",
-		textModelId: "clip-vit-b32-text",
+	"mobileclip-s0": {
+		imageModelId: "mobileclip-s0",
+		textModelId: "mobileclip-s0-text",
 		embeddingDim: 512,
-		inputSize: 224,
+		inputSize: 256,
 		tokenizerType: "clip"
 	},
-	"clip-vit-b16": {
-		imageModelId: "clip-vit-b16",
-		textModelId: "clip-vit-b16-text",
+	"mobileclip-s1": {
+		imageModelId: "mobileclip-s1",
+		textModelId: "mobileclip-s1-text",
 		embeddingDim: 512,
-		inputSize: 224,
+		inputSize: 256,
 		tokenizerType: "clip"
 	},
-	"siglip2-b16-256": {
-		imageModelId: "siglip2-b16-256",
-		textModelId: "siglip2-b16-256-text",
-		embeddingDim: 768,
+	"mobileclip-s2": {
+		imageModelId: "mobileclip-s2",
+		textModelId: "mobileclip-s2-text",
+		embeddingDim: 512,
 		inputSize: 256,
-		tokenizerType: "siglip"
+		tokenizerType: "clip"
 	}
 };
-var DEFAULT_CLIP_MODEL = "clip-vit-b32";
+var DEFAULT_CLIP_MODEL = "mobileclip-s1";
 function getModelMeta(modelId) {
-	return CLIP_MODEL_META[modelId] ?? CLIP_MODEL_META["clip-vit-b32"];
+	return CLIP_MODEL_META[modelId] ?? CLIP_MODEL_META["mobileclip-s1"];
 }
 //#endregion
 //#region src/embedding-encoder/addon/clip-preprocessing.ts
@@ -559,7 +747,7 @@ function l2Normalize(vec) {
 //#region src/embedding-encoder/addon/index.ts
 var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
 	imageRawEngine = null;
-	textRawEngine = null;
+	textEngine = null;
 	models = null;
 	constructor() {
 		super({
@@ -569,10 +757,7 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
 		});
 	}
 	async onInitialize() {
-		const modelsDir = await this.ctx.api.storage.resolve.query({
-			location: "models",
-			relativePath: ""
-		}).catch(() => "camstack-data/models");
+		const modelsDir = await this.resolveModelsDir();
 		this.models = new ModelDownloadService(modelsDir, []);
 		return [{
 			capability: require_dist.embeddingEncoderCapability,
@@ -603,9 +788,8 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
 		await this.ensureTextEngine();
 		const meta = getModelMeta(this.config.modelId);
 		const start = Date.now();
-		const tokenIds = clipTokenize(text);
-		const inputTensor = new Float32Array(tokenIds);
-		const output = await this.textRawEngine.run(inputTensor, [1, tokenIds.length]);
+		if (!this.textEngine) throw new Error("EmbeddingEncoder: text engine not loaded — ensureTextEngine() must run first");
+		const output = await this.textEngine.encode(text);
 		const sliced = output.length > meta.embeddingDim ? output.slice(0, meta.embeddingDim) : output;
 		const normalized = l2Normalize(new Float32Array(sliced));
 		return {
@@ -629,7 +813,7 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
 		await this.resolveForEntry(imageEntry, "image");
 	}
 	async ensureTextEngine() {
-		if (this.textRawEngine) return;
+		if (this.textEngine) return;
 		const meta = getModelMeta(this.config.modelId);
 		const textEntry = CLIP_TEXT_MODELS.find((m) => m.id === meta.textModelId);
 		if (!textEntry) throw new Error(`EmbeddingEncoderAddon: unknown text model "${meta.textModelId}"`);
@@ -646,14 +830,21 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
 		if (!pythonPath) throw new Error("EmbeddingEncoder: embedded Python is unavailable — cannot run ONNX embeddings. ctx.deps.ensurePython() returned null (portable Python download likely failed).");
 		const pythonDir = resolveEmbeddingPythonDir();
 		await this.ctx.deps.installPythonRequirements(node_path.join(pythonDir, "requirements-embedding.txt"));
-		const rawEngine = new PythonRawTensorEngine(pythonPath, node_path.join(pythonDir, "raw_tensor_inference.py"), modelPath, engineLogger);
-		await rawEngine.initialize();
-		if (target === "image") this.imageRawEngine = rawEngine;
-		else this.textRawEngine = rawEngine;
+		if (target === "image") {
+			const rawEngine = new PythonRawTensorEngine(pythonPath, node_path.join(pythonDir, "raw_tensor_inference.py"), modelPath, engineLogger);
+			await rawEngine.initialize();
+			this.imageRawEngine = rawEngine;
+			return;
+		}
+		const tokenizerPath = node_path.join(node_path.dirname(modelPath), TOKENIZER_FILE);
+		if (!node_fs.existsSync(tokenizerPath)) throw new Error(`EmbeddingEncoder: CLIP tokenizer not found at "${tokenizerPath}" — the tokenizer.json sibling download likely failed.`);
+		const textEngine = new PythonTextEncoderEngine(pythonPath, node_path.join(pythonDir, "text_encoder_inference.py"), modelPath, tokenizerPath, engineLogger);
+		await textEngine.initialize();
+		this.textEngine = textEngine;
 	}
 	async onShutdown() {
 		await this.imageRawEngine?.dispose();
-		await this.textRawEngine?.dispose();
+		await this.textEngine?.dispose();
 	}
 	globalSettingsSchema() {
 		return this.schema({ sections: [{
@@ -709,21 +900,6 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
 	async onConfigChanged() {}
 };
 /**
-* Minimal CLIP tokenizer — encodes ASCII text to token IDs.
-* Production implementations should use a proper BPE tokenizer;
-* this is a simplified placeholder that maps characters to IDs
-* with SOT/EOT tokens for basic functionality.
-*/
-function clipTokenize(text, maxLength = 77) {
-	const SOT_TOKEN = 49406;
-	const EOT_TOKEN = 49407;
-	const tokens = [SOT_TOKEN];
-	for (let i = 0; i < text.length && tokens.length < maxLength - 1; i++) tokens.push(text.charCodeAt(i) + 256);
-	tokens.push(EOT_TOKEN);
-	while (tokens.length < maxLength) tokens.push(0);
-	return tokens;
-}
-/**
 * Locate the addon's bundled `python/` dir (holds `raw_tensor_inference.py` +
 * `requirements-embedding.txt`). Published package first, then `__dirname`
 * candidates for the in-tree dev build.