omnius 1.0.633 → 1.0.634

package/dist/scripts/audio-yamnet-tensorrt-worker.py

@@ -154,7 +154,12 @@ def read_wav(path: str, np):
         raise RuntimeError(
             f"Expected at most 6 seconds of caller-conditioned audio, received {waveform.size / sample_rate:.3f} seconds"
         )
-    return waveform, sample_rate, (time.perf_counter() - started) * 1000
+    return (
+        waveform,
+        sample_rate,
+        (time.perf_counter() - started) * 1000,
+        "sha256:" + hashlib.sha256(raw).hexdigest(),
+    )
 
 
 class YAMNetTensorRT:
@@ -193,6 +198,8 @@ class YAMNetTensorRT:
                 self.output_names.append(name)
         if self.input_name is None or not self.output_names:
             raise RuntimeError("YAMNet TensorRT engine did not expose input and output tensors")
+        self.score_output_name = self._unique_output_name(521, "AudioSet score")
+        self.embedding_output_name = self._unique_output_name(1024, "YAMNet embedding")
         with open(class_map_path, newline="", encoding="utf8") as handle:
             self.classes = [row["display_name"] for row in csv.DictReader(handle)]
         if len(self.classes) != 521:
@@ -205,13 +212,28 @@ class YAMNetTensorRT:
         # namespace to exactly the approved Jetson GPU before process launch.
         return self.cuda.device_metadata()
 
+    def _unique_output_name(self, width: int, label: str):
+        matches = []
+        for name in self.output_names:
+            shape = tuple(self.engine.get_tensor_shape(name))
+            if shape and int(shape[-1]) == width:
+                matches.append(name)
+        if len(matches) != 1:
+            raise RuntimeError(
+                f"YAMNet TensorRT engine must expose exactly one {label} output with width {width}; found {matches}"
+            )
+        return matches[0]
+
     def warm(self):
         # YAMNet needs >=0.975 seconds.  This is a model-load probe, not a
         # captured-audio classification, and it verifies the TensorRT path is
         # actually executable before readiness is reported.
-        self.infer(self.np.zeros(15600, dtype=self.np.float32), 1)
+        # Exercise and validate both exported model heads before reporting the
+        # persistent worker as ready.  A stale score-only plan must fail here,
+        # rather than after an embedding caller has acquired the queue slot.
+        self.infer(self.np.zeros(15600, dtype=self.np.float32), 1, include_embedding=True)
 
-    def infer(self, waveform, top_k: int):
+    def infer(self, waveform, top_k: int | None = None, include_embedding: bool = False):
         started = time.perf_counter()
         np = self.np
         if waveform.ndim != 1 or waveform.size < 15600:
@@ -246,16 +268,34 @@ class YAMNetTensorRT:
             for name, (host, device) in host_outputs.items():
                 self.cuda.d2h(host, device, self.stream, name)
             self.cuda.synchronize(self.stream)
-            # The converted Google model exposes output_0 with AudioSet scores.
-            scores = next((value[0] for name, value in host_outputs.items() if value[0].shape[-1] == 521), None)
-            if scores is None:
-                raise RuntimeError("TensorRT YAMNet engine did not produce a 521-class score tensor")
-            mean_scores = np.mean(scores, axis=0)
-            indices = np.argsort(mean_scores)[-top_k:][::-1]
-            return [
-                {"label": self.classes[int(index)], "confidence": float(mean_scores[int(index)])}
-                for index in indices
-            ], (time.perf_counter() - started) * 1000
+            result = {}
+            if top_k is not None:
+                scores = host_outputs[self.score_output_name][0]
+                if scores.ndim != 2 or scores.shape[-1] != 521:
+                    raise RuntimeError(f"YAMNet score output shape is invalid: {scores.shape}")
+                mean_scores = np.mean(scores, axis=0)
+                indices = np.argsort(mean_scores)[-top_k:][::-1]
+                result["classifications"] = [
+                    {"label": self.classes[int(index)], "confidence": float(mean_scores[int(index)])}
+                    for index in indices
+                ]
+            if include_embedding:
+                frame_embeddings = host_outputs[self.embedding_output_name][0]
+                if frame_embeddings.ndim != 2 or frame_embeddings.shape[-1] != 1024 or frame_embeddings.shape[0] < 1:
+                    raise RuntimeError(f"YAMNet embedding output shape is invalid: {frame_embeddings.shape}")
+                vector = np.mean(frame_embeddings.astype(np.float32, copy=False), axis=0)
+                norm = float(np.linalg.norm(vector))
+                if not np.isfinite(norm) or norm <= 0.0:
+                    raise RuntimeError("YAMNet embedding has an invalid L2 norm")
+                result["embedding"] = {
+                    "dimension": 1024,
+                    "frame_count": int(frame_embeddings.shape[0]),
+                    "pooling": "mean_l2",
+                    "dtype": "float32",
+                    "normalization": "l2",
+                    "values": (vector / norm).astype(np.float32).tolist(),
+                }
+            return result, (time.perf_counter() - started) * 1000
         finally:
             for allocation in allocations:
                 try:
@@ -307,40 +347,48 @@ def main() -> int:
             if request.get("type") == "shutdown":
                 emit({"type": "stopped"})
                 return 0
-            if request.get("type") != "classify":
-                raise RuntimeError("unknown request type")
+            action = str(request.get("action") or request.get("type") or "").strip().lower()
+            if action not in ("classify", "embed"):
+                raise RuntimeError("unknown audio analysis action")
             request_id = str(request.get("id") or "")
             file_path = str(request.get("file") or "")
             if not request_id or not file_path:
-                raise RuntimeError("classify requires id and file")
-            waveform, sample_rate, decode_ms = read_wav(file_path, worker.np)
+                raise RuntimeError("audio analysis requires id and file")
+            waveform, sample_rate, decode_ms, input_digest = read_wav(file_path, worker.np)
             rms = float(worker.np.sqrt(worker.np.mean(worker.np.square(waveform)))) if waveform.size else 0.0
             peak = float(worker.np.max(worker.np.abs(waveform))) if waveform.size else 0.0
             # Do not force YAMNet to invent an AudioSet label for digital or
             # near-digital silence. This remains a valid, grounded result.
             low_information = rms < 0.0005 and peak < 0.002
             if low_information:
-                classifications, inference_ms = [], 0.0
+                result, inference_ms = (
+                    {"classifications": []} if action == "classify" else {"embedding": None},
+                    0.0,
+                )
             else:
-                classifications, inference_ms = worker.infer(
+                result, inference_ms = worker.infer(
                     waveform,
-                    max(1, min(int(request.get("top_k", 5)), 25)),
+                    top_k=max(1, min(int(request.get("top_k", 5)), 25)) if action == "classify" else None,
+                    include_embedding=action == "embed",
                 )
             emit({
                 "type": "result",
                 "id": request_id,
+                "action": action,
                 "success": True,
                 "sample_rate_hz": sample_rate,
                 "duration_seconds": round(float(waveform.size) / sample_rate, 6),
-                "classifications": classifications,
+                "input_digest": input_digest,
                 "low_information": low_information,
                 "acoustic": {"rms": round(rms, 8), "peak": round(peak, 8)},
                 "timings_ms": {"decode": round(decode_ms, 3), "inference": round(inference_ms, 3)},
+                **result,
             })
         except Exception as exc:  # keep the persistent worker alive per request
             emit({
                 "type": "result",
                 "id": str(locals().get("request", {}).get("id", "")),
+                "action": str(locals().get("action", "")),
                 "success": False,
                 "error": str(exc),
             })

package/dist/scripts/embed-image.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+import sys, json, base64, io
+try:
+    import torch
+    import open_clip
+    from PIL import Image
+except Exception as e:
+    print(json.dumps({"error":"deps_missing","message":str(e)}))
+    sys.exit(0)
+
+def load_image_from_input(data):
+    if data.get('path'):
+        return Image.open(data['path']).convert('RGB')
+    if data.get('bytesBase64'):
+        b = base64.b64decode(data['bytesBase64'])
+        return Image.open(io.BytesIO(b)).convert('RGB')
+    raise RuntimeError('no input provided')
+
+def main():
+    raw = sys.stdin.read()
+    data = json.loads(raw) if raw.strip() else {}
+    image = load_image_from_input(data)
+    model, _, preprocess = open_clip.create_model_and_transforms('ViT-B-32', pretrained='laion2b_s34b_b79k')
+    model.eval()
+    with torch.no_grad():
+        img_t = preprocess(image).unsqueeze(0)
+        emb = model.encode_image(img_t)
+        emb = emb / emb.norm(dim=-1, keepdim=True)
+        arr = emb.squeeze(0).cpu().numpy().astype('float32').tolist()
+    print(json.dumps({"embedding": arr}))
+
+if __name__ == '__main__':
+    main()
+

package/dist/scripts/embed-text.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+import sys, json
+try:
+    import torch
+    import open_clip
+except Exception as e:
+    print(json.dumps({"error":"deps_missing","message":str(e)}))
+    sys.exit(0)
+
+def main():
+    raw = sys.stdin.read()
+    data = json.loads(raw) if raw.strip() else {}
+    text = data.get('text') or ''
+    model, _, preprocess = open_clip.create_model_and_transforms('ViT-B-32', pretrained='laion2b_s34b_b79k')
+    tokenizer = open_clip.get_tokenizer('ViT-B-32')
+    model.eval()
+    with torch.no_grad():
+        tokens = tokenizer([text])
+        emb = model.encode_text(tokens)
+        emb = emb / emb.norm(dim=-1, keepdim=True)
+        arr = emb.squeeze(0).cpu().numpy().astype('float32').tolist()
+    print(json.dumps({"embedding": arr}))
+
+if __name__ == '__main__':
+    main()
+

package/dist/update-worker.js

@@ -245085,6 +245085,24 @@ var MODEL_URL = `https://huggingface.co/zeropointnine/yamnet-onnx/resolve/${MODE
 var CLASS_MAP_URL = `https://huggingface.co/zeropointnine/yamnet-onnx/resolve/${MODEL_REVISION}/yamnet_class_map.csv`;
 var SETUP_TIMEOUT_MS = 20 * 6e4;
 
+// packages/execution/dist/speaker-diarization-runtime.js
+init_model_store();
+
+// packages/execution/dist/audio-semantic-embedding-runtime.js
+init_process_async();
+init_model_store();
+init_venv_paths();
+var SETUP_TIMEOUT_MS2 = 30 * 6e4;
+
+// packages/execution/dist/speaker-embedding-runtime.js
+init_jetson_monitor();
+init_process_async();
+init_model_store();
+init_venv_paths();
+var MODEL_REVISION2 = "acf623ad8ca746e50baa432255cf8fc57c669c45";
+var MODEL_URL2 = `https://huggingface.co/Wespeaker/wespeaker-voxceleb-campplus/resolve/${MODEL_REVISION2}/voxceleb_CAM%2B%2B.onnx`;
+var SETUP_TIMEOUT_MS3 = 10 * 6e4;
+
 // packages/execution/dist/ocr-advanced-runtime.js
 init_process_async();
 init_jetson_monitor();
@@ -290433,6 +290451,21 @@ except Exception as exc:
     sys.exit(1)
 `;
 
+// packages/memory/dist/audioNoveltyPrototype.js
+var AUDIO_NOVELTY_WEIGHTS = Object.freeze({
+  distance: 0.6,
+  uncertainty: 0.1,
+  recurrence: 0.15,
+  direction: 0.075,
+  scene: 0.075
+});
+
+// packages/memory/dist/speakerIdentityStore.js
+var DEFAULT_THRESHOLDS = Object.freeze({
+  provisional: 0.78,
+  ambiguityMargin: 0.04
+});
+
 // packages/memory/dist/crl/index.js
 var CRL_SYMBOLS = {
   // Logical operators (Core)