omnius 1.0.633 → 1.0.636
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -4
- package/dist/index.js +19384 -14859
- package/dist/memory-maintenance-worker.js +116 -0
- package/dist/scripts/audio-clap-semantic-worker.py +171 -0
- package/dist/scripts/audio-speaker-embedding-worker.py +259 -0
- package/dist/scripts/audio-yamnet-tensorrt-worker.py +69 -21
- package/dist/scripts/embed-image.py +34 -0
- package/dist/scripts/embed-text.py +26 -0
- package/dist/scripts/speaker-diarization-worker.py +190 -0
- package/dist/scripts/vision-open-clip-worker.py +170 -0
- package/dist/update-worker.js +35 -0
- package/docs/DISCOVERY.json +1563 -36
- package/docs/DISCOVERY.md +18 -4
- package/docs/reference/rest-api.md +15 -1
- package/docs/rest/INDEX.md +1 -1
- package/docs/rest/REST-DOCS-MANIFEST.json +2 -2
- package/docs/rest/endpoints/config.md +8 -1
- package/docs/rest/endpoints/memory.md +54 -0
- package/docs/rest/endpoints/voice-vision.md +177 -1
- package/npm-shrinkwrap.json +8 -8
- package/package.json +2 -2
|
@@ -154,7 +154,12 @@ def read_wav(path: str, np):
|
|
|
154
154
|
raise RuntimeError(
|
|
155
155
|
f"Expected at most 6 seconds of caller-conditioned audio, received {waveform.size / sample_rate:.3f} seconds"
|
|
156
156
|
)
|
|
157
|
-
return
|
|
157
|
+
return (
|
|
158
|
+
waveform,
|
|
159
|
+
sample_rate,
|
|
160
|
+
(time.perf_counter() - started) * 1000,
|
|
161
|
+
"sha256:" + hashlib.sha256(raw).hexdigest(),
|
|
162
|
+
)
|
|
158
163
|
|
|
159
164
|
|
|
160
165
|
class YAMNetTensorRT:
|
|
@@ -193,6 +198,8 @@ class YAMNetTensorRT:
|
|
|
193
198
|
self.output_names.append(name)
|
|
194
199
|
if self.input_name is None or not self.output_names:
|
|
195
200
|
raise RuntimeError("YAMNet TensorRT engine did not expose input and output tensors")
|
|
201
|
+
self.score_output_name = self._unique_output_name(521, "AudioSet score")
|
|
202
|
+
self.embedding_output_name = self._unique_output_name(1024, "YAMNet embedding")
|
|
196
203
|
with open(class_map_path, newline="", encoding="utf8") as handle:
|
|
197
204
|
self.classes = [row["display_name"] for row in csv.DictReader(handle)]
|
|
198
205
|
if len(self.classes) != 521:
|
|
@@ -205,13 +212,28 @@ class YAMNetTensorRT:
|
|
|
205
212
|
# namespace to exactly the approved Jetson GPU before process launch.
|
|
206
213
|
return self.cuda.device_metadata()
|
|
207
214
|
|
|
215
|
+
def _unique_output_name(self, width: int, label: str):
|
|
216
|
+
matches = []
|
|
217
|
+
for name in self.output_names:
|
|
218
|
+
shape = tuple(self.engine.get_tensor_shape(name))
|
|
219
|
+
if shape and int(shape[-1]) == width:
|
|
220
|
+
matches.append(name)
|
|
221
|
+
if len(matches) != 1:
|
|
222
|
+
raise RuntimeError(
|
|
223
|
+
f"YAMNet TensorRT engine must expose exactly one {label} output with width {width}; found {matches}"
|
|
224
|
+
)
|
|
225
|
+
return matches[0]
|
|
226
|
+
|
|
208
227
|
def warm(self):
|
|
209
228
|
# YAMNet needs >=0.975 seconds. This is a model-load probe, not a
|
|
210
229
|
# captured-audio classification, and it verifies the TensorRT path is
|
|
211
230
|
# actually executable before readiness is reported.
|
|
212
|
-
|
|
231
|
+
# Exercise and validate both exported model heads before reporting the
|
|
232
|
+
# persistent worker as ready. A stale score-only plan must fail here,
|
|
233
|
+
# rather than after an embedding caller has acquired the queue slot.
|
|
234
|
+
self.infer(self.np.zeros(15600, dtype=self.np.float32), 1, include_embedding=True)
|
|
213
235
|
|
|
214
|
-
def infer(self, waveform, top_k: int):
|
|
236
|
+
def infer(self, waveform, top_k: int | None = None, include_embedding: bool = False):
|
|
215
237
|
started = time.perf_counter()
|
|
216
238
|
np = self.np
|
|
217
239
|
if waveform.ndim != 1 or waveform.size < 15600:
|
|
@@ -246,16 +268,34 @@ class YAMNetTensorRT:
|
|
|
246
268
|
for name, (host, device) in host_outputs.items():
|
|
247
269
|
self.cuda.d2h(host, device, self.stream, name)
|
|
248
270
|
self.cuda.synchronize(self.stream)
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
271
|
+
result = {}
|
|
272
|
+
if top_k is not None:
|
|
273
|
+
scores = host_outputs[self.score_output_name][0]
|
|
274
|
+
if scores.ndim != 2 or scores.shape[-1] != 521:
|
|
275
|
+
raise RuntimeError(f"YAMNet score output shape is invalid: {scores.shape}")
|
|
276
|
+
mean_scores = np.mean(scores, axis=0)
|
|
277
|
+
indices = np.argsort(mean_scores)[-top_k:][::-1]
|
|
278
|
+
result["classifications"] = [
|
|
279
|
+
{"label": self.classes[int(index)], "confidence": float(mean_scores[int(index)])}
|
|
280
|
+
for index in indices
|
|
281
|
+
]
|
|
282
|
+
if include_embedding:
|
|
283
|
+
frame_embeddings = host_outputs[self.embedding_output_name][0]
|
|
284
|
+
if frame_embeddings.ndim != 2 or frame_embeddings.shape[-1] != 1024 or frame_embeddings.shape[0] < 1:
|
|
285
|
+
raise RuntimeError(f"YAMNet embedding output shape is invalid: {frame_embeddings.shape}")
|
|
286
|
+
vector = np.mean(frame_embeddings.astype(np.float32, copy=False), axis=0)
|
|
287
|
+
norm = float(np.linalg.norm(vector))
|
|
288
|
+
if not np.isfinite(norm) or norm <= 0.0:
|
|
289
|
+
raise RuntimeError("YAMNet embedding has an invalid L2 norm")
|
|
290
|
+
result["embedding"] = {
|
|
291
|
+
"dimension": 1024,
|
|
292
|
+
"frame_count": int(frame_embeddings.shape[0]),
|
|
293
|
+
"pooling": "mean_l2",
|
|
294
|
+
"dtype": "float32",
|
|
295
|
+
"normalization": "l2",
|
|
296
|
+
"values": (vector / norm).astype(np.float32).tolist(),
|
|
297
|
+
}
|
|
298
|
+
return result, (time.perf_counter() - started) * 1000
|
|
259
299
|
finally:
|
|
260
300
|
for allocation in allocations:
|
|
261
301
|
try:
|
|
@@ -307,40 +347,48 @@ def main() -> int:
|
|
|
307
347
|
if request.get("type") == "shutdown":
|
|
308
348
|
emit({"type": "stopped"})
|
|
309
349
|
return 0
|
|
310
|
-
|
|
311
|
-
|
|
350
|
+
action = str(request.get("action") or request.get("type") or "").strip().lower()
|
|
351
|
+
if action not in ("classify", "embed"):
|
|
352
|
+
raise RuntimeError("unknown audio analysis action")
|
|
312
353
|
request_id = str(request.get("id") or "")
|
|
313
354
|
file_path = str(request.get("file") or "")
|
|
314
355
|
if not request_id or not file_path:
|
|
315
|
-
raise RuntimeError("
|
|
316
|
-
waveform, sample_rate, decode_ms = read_wav(file_path, worker.np)
|
|
356
|
+
raise RuntimeError("audio analysis requires id and file")
|
|
357
|
+
waveform, sample_rate, decode_ms, input_digest = read_wav(file_path, worker.np)
|
|
317
358
|
rms = float(worker.np.sqrt(worker.np.mean(worker.np.square(waveform)))) if waveform.size else 0.0
|
|
318
359
|
peak = float(worker.np.max(worker.np.abs(waveform))) if waveform.size else 0.0
|
|
319
360
|
# Do not force YAMNet to invent an AudioSet label for digital or
|
|
320
361
|
# near-digital silence. This remains a valid, grounded result.
|
|
321
362
|
low_information = rms < 0.0005 and peak < 0.002
|
|
322
363
|
if low_information:
|
|
323
|
-
|
|
364
|
+
result, inference_ms = (
|
|
365
|
+
{"classifications": []} if action == "classify" else {"embedding": None},
|
|
366
|
+
0.0,
|
|
367
|
+
)
|
|
324
368
|
else:
|
|
325
|
-
|
|
369
|
+
result, inference_ms = worker.infer(
|
|
326
370
|
waveform,
|
|
327
|
-
max(1, min(int(request.get("top_k", 5)), 25)),
|
|
371
|
+
top_k=max(1, min(int(request.get("top_k", 5)), 25)) if action == "classify" else None,
|
|
372
|
+
include_embedding=action == "embed",
|
|
328
373
|
)
|
|
329
374
|
emit({
|
|
330
375
|
"type": "result",
|
|
331
376
|
"id": request_id,
|
|
377
|
+
"action": action,
|
|
332
378
|
"success": True,
|
|
333
379
|
"sample_rate_hz": sample_rate,
|
|
334
380
|
"duration_seconds": round(float(waveform.size) / sample_rate, 6),
|
|
335
|
-
"
|
|
381
|
+
"input_digest": input_digest,
|
|
336
382
|
"low_information": low_information,
|
|
337
383
|
"acoustic": {"rms": round(rms, 8), "peak": round(peak, 8)},
|
|
338
384
|
"timings_ms": {"decode": round(decode_ms, 3), "inference": round(inference_ms, 3)},
|
|
385
|
+
**result,
|
|
339
386
|
})
|
|
340
387
|
except Exception as exc: # keep the persistent worker alive per request
|
|
341
388
|
emit({
|
|
342
389
|
"type": "result",
|
|
343
390
|
"id": str(locals().get("request", {}).get("id", "")),
|
|
391
|
+
"action": str(locals().get("action", "")),
|
|
344
392
|
"success": False,
|
|
345
393
|
"error": str(exc),
|
|
346
394
|
})
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
import sys, json, base64, io
|
|
3
|
+
try:
|
|
4
|
+
import torch
|
|
5
|
+
import open_clip
|
|
6
|
+
from PIL import Image
|
|
7
|
+
except Exception as e:
|
|
8
|
+
print(json.dumps({"error":"deps_missing","message":str(e)}))
|
|
9
|
+
sys.exit(0)
|
|
10
|
+
|
|
11
|
+
def load_image_from_input(data):
|
|
12
|
+
if data.get('path'):
|
|
13
|
+
return Image.open(data['path']).convert('RGB')
|
|
14
|
+
if data.get('bytesBase64'):
|
|
15
|
+
b = base64.b64decode(data['bytesBase64'])
|
|
16
|
+
return Image.open(io.BytesIO(b)).convert('RGB')
|
|
17
|
+
raise RuntimeError('no input provided')
|
|
18
|
+
|
|
19
|
+
def main():
|
|
20
|
+
raw = sys.stdin.read()
|
|
21
|
+
data = json.loads(raw) if raw.strip() else {}
|
|
22
|
+
image = load_image_from_input(data)
|
|
23
|
+
model, _, preprocess = open_clip.create_model_and_transforms('ViT-B-32', pretrained='laion2b_s34b_b79k')
|
|
24
|
+
model.eval()
|
|
25
|
+
with torch.no_grad():
|
|
26
|
+
img_t = preprocess(image).unsqueeze(0)
|
|
27
|
+
emb = model.encode_image(img_t)
|
|
28
|
+
emb = emb / emb.norm(dim=-1, keepdim=True)
|
|
29
|
+
arr = emb.squeeze(0).cpu().numpy().astype('float32').tolist()
|
|
30
|
+
print(json.dumps({"embedding": arr}))
|
|
31
|
+
|
|
32
|
+
if __name__ == '__main__':
|
|
33
|
+
main()
|
|
34
|
+
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
import sys, json
|
|
3
|
+
try:
|
|
4
|
+
import torch
|
|
5
|
+
import open_clip
|
|
6
|
+
except Exception as e:
|
|
7
|
+
print(json.dumps({"error":"deps_missing","message":str(e)}))
|
|
8
|
+
sys.exit(0)
|
|
9
|
+
|
|
10
|
+
def main():
|
|
11
|
+
raw = sys.stdin.read()
|
|
12
|
+
data = json.loads(raw) if raw.strip() else {}
|
|
13
|
+
text = data.get('text') or ''
|
|
14
|
+
model, _, preprocess = open_clip.create_model_and_transforms('ViT-B-32', pretrained='laion2b_s34b_b79k')
|
|
15
|
+
tokenizer = open_clip.get_tokenizer('ViT-B-32')
|
|
16
|
+
model.eval()
|
|
17
|
+
with torch.no_grad():
|
|
18
|
+
tokens = tokenizer([text])
|
|
19
|
+
emb = model.encode_text(tokens)
|
|
20
|
+
emb = emb / emb.norm(dim=-1, keepdim=True)
|
|
21
|
+
arr = emb.squeeze(0).cpu().numpy().astype('float32').tolist()
|
|
22
|
+
print(json.dumps({"embedding": arr}))
|
|
23
|
+
|
|
24
|
+
if __name__ == '__main__':
|
|
25
|
+
main()
|
|
26
|
+
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Offline, line-delimited workers for Omnius managed speaker diarization.
|
|
3
|
+
|
|
4
|
+
The parent process supplies an already-verified local artifact and forces
|
|
5
|
+
offline Hugging Face mode. This worker deliberately has no setup/download
|
|
6
|
+
branch and never reads a token: a request can only use a model proven during
|
|
7
|
+
the admin setup path.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import sys
|
|
16
|
+
import time
|
|
17
|
+
import wave
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def emit(payload: dict[str, Any]) -> None:
|
|
23
|
+
print(json.dumps(payload, separators=(",", ":")), flush=True)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def validate_live_wav(path: str) -> None:
|
|
27
|
+
"""Live Sortformer is a bounded retained 16k mono PCM16 window."""
|
|
28
|
+
try:
|
|
29
|
+
with wave.open(path, "rb") as wav:
|
|
30
|
+
channels = wav.getnchannels()
|
|
31
|
+
width = wav.getsampwidth()
|
|
32
|
+
rate = wav.getframerate()
|
|
33
|
+
duration = wav.getnframes() / float(rate) if rate else 0.0
|
|
34
|
+
except Exception as exc: # wave errors are intentionally surfaced to caller
|
|
35
|
+
raise ValueError(f"live diarization requires readable WAV: {exc}") from exc
|
|
36
|
+
if channels != 1 or width != 2 or rate != 16000:
|
|
37
|
+
raise ValueError("live diarization requires retained mono PCM16/16 kHz WAV")
|
|
38
|
+
if duration <= 0 or duration > 6.0:
|
|
39
|
+
raise ValueError("live diarization accepts retained audio windows greater than 0 and at most 6 seconds")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def number(value: Any) -> float | None:
|
|
43
|
+
try:
|
|
44
|
+
result = float(value)
|
|
45
|
+
return result if result == result and result not in (float("inf"), float("-inf")) else None
|
|
46
|
+
except Exception:
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def append_segment(result: list[dict[str, Any]], start: Any, end: Any, speaker: Any, confidence: Any = None) -> None:
|
|
51
|
+
start_s = number(start)
|
|
52
|
+
end_s = number(end)
|
|
53
|
+
label = str(speaker or "").strip()
|
|
54
|
+
if start_s is None or end_s is None or end_s <= start_s or not label:
|
|
55
|
+
return
|
|
56
|
+
confidence_value = number(confidence)
|
|
57
|
+
result.append({
|
|
58
|
+
"start_ms": round(start_s * 1000),
|
|
59
|
+
"end_ms": round(end_s * 1000),
|
|
60
|
+
"speaker": label,
|
|
61
|
+
"confidence": confidence_value,
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def normalize_sortformer(value: Any) -> list[dict[str, Any]]:
|
|
66
|
+
"""Accept NeMo's current tuple/list result variants without inventing data."""
|
|
67
|
+
while isinstance(value, list) and len(value) == 1 and isinstance(value[0], (list, tuple)):
|
|
68
|
+
value = value[0]
|
|
69
|
+
result: list[dict[str, Any]] = []
|
|
70
|
+
if not isinstance(value, (list, tuple)):
|
|
71
|
+
raise RuntimeError("Sortformer returned an unsupported diarization result shape")
|
|
72
|
+
for item in value:
|
|
73
|
+
if isinstance(item, dict):
|
|
74
|
+
append_segment(result, item.get("start", item.get("start_time")), item.get("end", item.get("end_time")), item.get("speaker", item.get("label")), item.get("confidence"))
|
|
75
|
+
elif isinstance(item, (tuple, list)) and len(item) >= 3:
|
|
76
|
+
append_segment(result, item[0], item[1], item[2], item[3] if len(item) > 3 else None)
|
|
77
|
+
elif isinstance(item, str):
|
|
78
|
+
# Some NeMo surfaces return RTTM lines. Parse only the standard
|
|
79
|
+
# SPEAKER fields; malformed lines are ignored rather than guessed.
|
|
80
|
+
fields = item.split()
|
|
81
|
+
if len(fields) >= 8 and fields[0].upper() == "SPEAKER":
|
|
82
|
+
start = number(fields[3])
|
|
83
|
+
duration = number(fields[4])
|
|
84
|
+
if start is not None and duration is not None:
|
|
85
|
+
append_segment(result, start, start + duration, fields[7])
|
|
86
|
+
if not result and value:
|
|
87
|
+
raise RuntimeError("Sortformer result contained no parseable speaker turns")
|
|
88
|
+
return result
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def load_live(args: argparse.Namespace):
|
|
92
|
+
import torch
|
|
93
|
+
from nemo.collections.asr.models import SortformerEncLabelModel
|
|
94
|
+
|
|
95
|
+
if not torch.cuda.is_available() or tuple(torch.cuda.get_device_capability(0)) != (8, 7):
|
|
96
|
+
raise RuntimeError("live Sortformer requires Jetson Orin CUDA:0 (compute capability 8.7)")
|
|
97
|
+
model = SortformerEncLabelModel.restore_from(
|
|
98
|
+
args.artifact_path, map_location=torch.device("cuda:0"), strict=False
|
|
99
|
+
)
|
|
100
|
+
model.eval()
|
|
101
|
+
|
|
102
|
+
def diarize(path: str) -> list[dict[str, Any]]:
|
|
103
|
+
validate_live_wav(path)
|
|
104
|
+
with torch.inference_mode():
|
|
105
|
+
return normalize_sortformer(model.diarize(audio=[path], batch_size=1))
|
|
106
|
+
|
|
107
|
+
return model, diarize, "nemo-sortformer-python-cuda"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def load_reconcile(args: argparse.Namespace):
|
|
111
|
+
# Community-1's supported offline workflow is Pipeline.from_pretrained on
|
|
112
|
+
# a local clone. HF token/terms are setup concerns and intentionally never
|
|
113
|
+
# appear here. CPU protects the live CUDA sidecar on Jetson.
|
|
114
|
+
import torch
|
|
115
|
+
from pyannote.audio import Pipeline
|
|
116
|
+
|
|
117
|
+
pipeline = Pipeline.from_pretrained(args.snapshot_path)
|
|
118
|
+
if pipeline is None:
|
|
119
|
+
raise RuntimeError("pyannote failed to load the local Community-1 snapshot")
|
|
120
|
+
pipeline.to(torch.device("cpu"))
|
|
121
|
+
|
|
122
|
+
def diarize(path: str) -> list[dict[str, Any]]:
|
|
123
|
+
if not Path(path).is_file():
|
|
124
|
+
raise ValueError("offline reconciliation audio file does not exist")
|
|
125
|
+
output = pipeline(path)
|
|
126
|
+
annotation = getattr(output, "speaker_diarization", output)
|
|
127
|
+
turns = getattr(annotation, "itertracks", None)
|
|
128
|
+
if not callable(turns):
|
|
129
|
+
raise RuntimeError("pyannote Community-1 result has no speaker_diarization tracks")
|
|
130
|
+
result: list[dict[str, Any]] = []
|
|
131
|
+
for segment, _track, speaker in annotation.itertracks(yield_label=True):
|
|
132
|
+
append_segment(result, getattr(segment, "start", None), getattr(segment, "end", None), speaker)
|
|
133
|
+
return result
|
|
134
|
+
|
|
135
|
+
return pipeline, diarize, "pyannote-community-python-cpu"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def main() -> int:
|
|
139
|
+
parser = argparse.ArgumentParser()
|
|
140
|
+
parser.add_argument("--mode", choices=["live", "reconcile"], required=True)
|
|
141
|
+
parser.add_argument("--snapshot-path", required=True)
|
|
142
|
+
parser.add_argument("--artifact-path", required=True)
|
|
143
|
+
parser.add_argument("--revision", required=True)
|
|
144
|
+
parser.add_argument("--model-digest", required=True)
|
|
145
|
+
args = parser.parse_args()
|
|
146
|
+
|
|
147
|
+
# Do not allow an accidentally inherited setup token to become a request
|
|
148
|
+
# time credential or network fallback.
|
|
149
|
+
os.environ["HF_HUB_OFFLINE"] = "1"
|
|
150
|
+
os.environ["TRANSFORMERS_OFFLINE"] = "1"
|
|
151
|
+
os.environ.pop("HF_TOKEN", None)
|
|
152
|
+
os.environ.pop("HUGGING_FACE_HUB_TOKEN", None)
|
|
153
|
+
try:
|
|
154
|
+
_model, diarize, backend = load_live(args) if args.mode == "live" else load_reconcile(args)
|
|
155
|
+
model = "nvidia/diar_streaming_sortformer_4spk-v2" if args.mode == "live" else "pyannote/speaker-diarization-community-1"
|
|
156
|
+
emit({
|
|
157
|
+
"type": "ready",
|
|
158
|
+
"pid": os.getpid(),
|
|
159
|
+
"mode": args.mode,
|
|
160
|
+
"model": model,
|
|
161
|
+
"revision": args.revision,
|
|
162
|
+
"model_digest": args.model_digest,
|
|
163
|
+
"backend": backend,
|
|
164
|
+
"warmed": True,
|
|
165
|
+
})
|
|
166
|
+
except Exception as exc:
|
|
167
|
+
emit({"type": "fatal", "error": str(exc)})
|
|
168
|
+
return 2
|
|
169
|
+
|
|
170
|
+
for line in sys.stdin:
|
|
171
|
+
try:
|
|
172
|
+
message = json.loads(line)
|
|
173
|
+
if message.get("type") != "diarize" or not isinstance(message.get("id"), str):
|
|
174
|
+
continue
|
|
175
|
+
started = time.perf_counter()
|
|
176
|
+
segments = diarize(str(message.get("file", "")))
|
|
177
|
+
emit({
|
|
178
|
+
"type": "result",
|
|
179
|
+
"id": message["id"],
|
|
180
|
+
"success": True,
|
|
181
|
+
"segments": segments,
|
|
182
|
+
"timings_ms": {"inference": round((time.perf_counter() - started) * 1000)},
|
|
183
|
+
})
|
|
184
|
+
except Exception as exc:
|
|
185
|
+
emit({"type": "result", "id": message.get("id", "unknown") if "message" in locals() else "unknown", "success": False, "error": str(exc)})
|
|
186
|
+
return 0
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
if __name__ == "__main__":
|
|
190
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Explicit-setup OpenCLIP vision embedding worker.
|
|
3
|
+
|
|
4
|
+
`--setup` is the only mode allowed to obtain model artifacts. `--embed` runs
|
|
5
|
+
with outbound sockets denied and checks the setup manifest digest before it
|
|
6
|
+
loads the model, so a REST embedding request can never silently provision.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import base64
|
|
11
|
+
import hashlib
|
|
12
|
+
import io
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import socket
|
|
16
|
+
import sys
|
|
17
|
+
|
|
18
|
+
MODEL = "ViT-B-32"
|
|
19
|
+
PRETRAINED = "laion2b_s34b_b79k"
|
|
20
|
+
EMBEDDING_DIMENSION = 512
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def emit(value):
|
|
24
|
+
print(json.dumps(value, separators=(",", ":")), flush=True)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def deny_network():
|
|
28
|
+
def denied(*_args, **_kwargs):
|
|
29
|
+
raise RuntimeError("OpenCLIP inference is offline; run explicit vision setup to obtain artifacts")
|
|
30
|
+
socket.create_connection = denied
|
|
31
|
+
socket.socket.connect = denied
|
|
32
|
+
socket.socket.connect_ex = denied
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def artifact_entries(cache_dir):
|
|
36
|
+
entries = []
|
|
37
|
+
for root, _dirs, names in os.walk(cache_dir):
|
|
38
|
+
for name in sorted(names):
|
|
39
|
+
path = os.path.join(root, name)
|
|
40
|
+
if not os.path.isfile(path) or os.path.getsize(path) < 1024:
|
|
41
|
+
continue
|
|
42
|
+
relative = os.path.relpath(path, cache_dir).replace(os.sep, "/")
|
|
43
|
+
digest = hashlib.sha256()
|
|
44
|
+
with open(path, "rb") as source:
|
|
45
|
+
while True:
|
|
46
|
+
block = source.read(1024 * 1024)
|
|
47
|
+
if not block:
|
|
48
|
+
break
|
|
49
|
+
digest.update(block)
|
|
50
|
+
entries.append({"path": relative, "bytes": os.path.getsize(path), "sha256": digest.hexdigest()})
|
|
51
|
+
return sorted(entries, key=lambda item: item["path"])
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def artifact_digest(entries):
|
|
55
|
+
digest = hashlib.sha256()
|
|
56
|
+
for entry in entries:
|
|
57
|
+
digest.update(entry["path"].encode("utf-8"))
|
|
58
|
+
digest.update(b"\0")
|
|
59
|
+
digest.update(entry["sha256"].encode("ascii"))
|
|
60
|
+
digest.update(b"\n")
|
|
61
|
+
return "sha256:" + digest.hexdigest()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def load_model(cache_dir, offline):
|
|
65
|
+
if offline:
|
|
66
|
+
deny_network()
|
|
67
|
+
os.environ["HF_HUB_OFFLINE"] = "1"
|
|
68
|
+
os.environ["TRANSFORMERS_OFFLINE"] = "1"
|
|
69
|
+
import torch
|
|
70
|
+
import open_clip
|
|
71
|
+
if not torch.cuda.is_available():
|
|
72
|
+
raise RuntimeError("JetPack CUDA Torch is unavailable for OpenCLIP")
|
|
73
|
+
device = torch.device("cuda:0")
|
|
74
|
+
model, _, preprocess = open_clip.create_model_and_transforms(
|
|
75
|
+
MODEL,
|
|
76
|
+
pretrained=PRETRAINED,
|
|
77
|
+
cache_dir=cache_dir,
|
|
78
|
+
device=device,
|
|
79
|
+
)
|
|
80
|
+
model.eval()
|
|
81
|
+
return torch, model, preprocess, device
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def setup(cache_dir):
|
|
85
|
+
torch, model, _preprocess, device = load_model(cache_dir, False)
|
|
86
|
+
entries = artifact_entries(cache_dir)
|
|
87
|
+
if not entries:
|
|
88
|
+
raise RuntimeError("OpenCLIP setup did not create verifiable model artifacts")
|
|
89
|
+
with torch.inference_mode():
|
|
90
|
+
# Model loading is setup verification, not a user inference request.
|
|
91
|
+
torch.cuda.synchronize(device)
|
|
92
|
+
major, minor = torch.cuda.get_device_capability(0)
|
|
93
|
+
emit({
|
|
94
|
+
"type": "setup",
|
|
95
|
+
"success": True,
|
|
96
|
+
"model": MODEL,
|
|
97
|
+
"pretrained": PRETRAINED,
|
|
98
|
+
"backend": "open-clip",
|
|
99
|
+
"embedding_dimension": EMBEDDING_DIMENSION,
|
|
100
|
+
"normalization": "l2",
|
|
101
|
+
"device": torch.cuda.get_device_name(0),
|
|
102
|
+
"compute_capability": "%d.%d" % (major, minor),
|
|
103
|
+
"torch_cuda_version": str(torch.version.cuda),
|
|
104
|
+
"artifacts": entries,
|
|
105
|
+
"model_digest": artifact_digest(entries),
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def image_from_input(input_):
|
|
110
|
+
from PIL import Image
|
|
111
|
+
path = input_.get("path")
|
|
112
|
+
encoded = input_.get("bytesBase64")
|
|
113
|
+
if isinstance(path, str) and path:
|
|
114
|
+
return Image.open(path).convert("RGB")
|
|
115
|
+
if isinstance(encoded, str) and encoded:
|
|
116
|
+
return Image.open(io.BytesIO(base64.b64decode(encoded, validate=True))).convert("RGB")
|
|
117
|
+
raise ValueError("Provide path or bytesBase64")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def embed(cache_dir, expected_digest):
|
|
121
|
+
entries = artifact_entries(cache_dir)
|
|
122
|
+
actual = artifact_digest(entries)
|
|
123
|
+
if actual != expected_digest:
|
|
124
|
+
raise RuntimeError("OpenCLIP model artifacts are missing or changed; run explicit vision setup")
|
|
125
|
+
input_ = json.loads(sys.stdin.read() or "{}")
|
|
126
|
+
torch, model, preprocess, device = load_model(cache_dir, True)
|
|
127
|
+
image = image_from_input(input_)
|
|
128
|
+
with torch.inference_mode():
|
|
129
|
+
tensor = preprocess(image).unsqueeze(0).to(device)
|
|
130
|
+
vector = model.encode_image(tensor).float()
|
|
131
|
+
vector = vector / vector.norm(dim=-1, keepdim=True).clamp_min(1e-12)
|
|
132
|
+
values = vector[0].detach().cpu().numpy().astype("float32").tolist()
|
|
133
|
+
if len(values) != EMBEDDING_DIMENSION or not all(isinstance(value, float) for value in values):
|
|
134
|
+
raise RuntimeError("OpenCLIP returned an invalid image embedding")
|
|
135
|
+
emit({
|
|
136
|
+
"type": "embedding",
|
|
137
|
+
"success": True,
|
|
138
|
+
"model": MODEL,
|
|
139
|
+
"pretrained": PRETRAINED,
|
|
140
|
+
"backend": "open-clip",
|
|
141
|
+
"model_digest": actual,
|
|
142
|
+
"embedding": values,
|
|
143
|
+
"dimension": EMBEDDING_DIMENSION,
|
|
144
|
+
"normalization": "l2",
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def main():
|
|
149
|
+
parser = argparse.ArgumentParser()
|
|
150
|
+
parser.add_argument("--setup", action="store_true")
|
|
151
|
+
parser.add_argument("--embed", action="store_true")
|
|
152
|
+
parser.add_argument("--cache-dir", required=True)
|
|
153
|
+
parser.add_argument("--model-digest")
|
|
154
|
+
args = parser.parse_args()
|
|
155
|
+
if args.setup == args.embed:
|
|
156
|
+
raise ValueError("Choose exactly one of --setup or --embed")
|
|
157
|
+
if args.setup:
|
|
158
|
+
setup(args.cache_dir)
|
|
159
|
+
else:
|
|
160
|
+
if not args.model_digest:
|
|
161
|
+
raise ValueError("--model-digest is required for offline embedding")
|
|
162
|
+
embed(args.cache_dir, args.model_digest)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
if __name__ == "__main__":
|
|
166
|
+
try:
|
|
167
|
+
main()
|
|
168
|
+
except Exception as error:
|
|
169
|
+
emit({"type": "error", "success": False, "error": str(error)})
|
|
170
|
+
raise SystemExit(1)
|
package/dist/update-worker.js
CHANGED
|
@@ -245085,6 +245085,26 @@ var MODEL_URL = `https://huggingface.co/zeropointnine/yamnet-onnx/resolve/${MODE
|
|
|
245085
245085
|
var CLASS_MAP_URL = `https://huggingface.co/zeropointnine/yamnet-onnx/resolve/${MODEL_REVISION}/yamnet_class_map.csv`;
|
|
245086
245086
|
var SETUP_TIMEOUT_MS = 20 * 6e4;
|
|
245087
245087
|
|
|
245088
|
+
// packages/execution/dist/speaker-diarization-runtime.js
|
|
245089
|
+
init_model_store();
|
|
245090
|
+
init_process_async();
|
|
245091
|
+
|
|
245092
|
+
// packages/execution/dist/audio-semantic-embedding-runtime.js
|
|
245093
|
+
init_process_async();
|
|
245094
|
+
init_jetson_monitor();
|
|
245095
|
+
init_model_store();
|
|
245096
|
+
init_venv_paths();
|
|
245097
|
+
var SETUP_TIMEOUT_MS2 = 30 * 6e4;
|
|
245098
|
+
|
|
245099
|
+
// packages/execution/dist/speaker-embedding-runtime.js
|
|
245100
|
+
init_jetson_monitor();
|
|
245101
|
+
init_process_async();
|
|
245102
|
+
init_model_store();
|
|
245103
|
+
init_venv_paths();
|
|
245104
|
+
var MODEL_REVISION2 = "acf623ad8ca746e50baa432255cf8fc57c669c45";
|
|
245105
|
+
var MODEL_URL2 = `https://huggingface.co/Wespeaker/wespeaker-voxceleb-campplus/resolve/${MODEL_REVISION2}/voxceleb_CAM%2B%2B.onnx`;
|
|
245106
|
+
var SETUP_TIMEOUT_MS3 = 10 * 6e4;
|
|
245107
|
+
|
|
245088
245108
|
// packages/execution/dist/ocr-advanced-runtime.js
|
|
245089
245109
|
init_process_async();
|
|
245090
245110
|
init_jetson_monitor();
|
|
@@ -290433,6 +290453,21 @@ except Exception as exc:
|
|
|
290433
290453
|
sys.exit(1)
|
|
290434
290454
|
`;
|
|
290435
290455
|
|
|
290456
|
+
// packages/memory/dist/audioNoveltyPrototype.js
|
|
290457
|
+
var AUDIO_NOVELTY_WEIGHTS = Object.freeze({
|
|
290458
|
+
distance: 0.6,
|
|
290459
|
+
uncertainty: 0.1,
|
|
290460
|
+
recurrence: 0.15,
|
|
290461
|
+
direction: 0.075,
|
|
290462
|
+
scene: 0.075
|
|
290463
|
+
});
|
|
290464
|
+
|
|
290465
|
+
// packages/memory/dist/speakerIdentityStore.js
|
|
290466
|
+
var DEFAULT_THRESHOLDS = Object.freeze({
|
|
290467
|
+
provisional: 0.78,
|
|
290468
|
+
ambiguityMargin: 0.04
|
|
290469
|
+
});
|
|
290470
|
+
|
|
290436
290471
|
// packages/memory/dist/crl/index.js
|
|
290437
290472
|
var CRL_SYMBOLS = {
|
|
290438
290473
|
// Logical operators (Core)
|