omnius 1.0.633 → 1.0.634
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -4
- package/dist/index.js +17581 -14391
- package/dist/memory-maintenance-worker.js +116 -0
- package/dist/scripts/audio-clap-semantic-worker.py +171 -0
- package/dist/scripts/audio-speaker-embedding-worker.py +259 -0
- package/dist/scripts/audio-yamnet-tensorrt-worker.py +69 -21
- package/dist/scripts/embed-image.py +34 -0
- package/dist/scripts/embed-text.py +26 -0
- package/dist/update-worker.js +33 -0
- package/docs/DISCOVERY.json +1285 -7
- package/docs/DISCOVERY.md +13 -1
- package/docs/reference/rest-api.md +13 -1
- package/docs/rest/INDEX.md +1 -1
- package/docs/rest/REST-DOCS-MANIFEST.json +1 -1
- package/docs/rest/endpoints/memory.md +54 -0
- package/docs/rest/endpoints/voice-vision.md +122 -1
- package/npm-shrinkwrap.json +8 -8
- package/package.json +2 -2
|
@@ -340,6 +340,107 @@ function runMigrations(db) {
|
|
|
340
340
|
CREATE INDEX IF NOT EXISTS idx_memory_embedding_v2_set
|
|
341
341
|
ON memory_embedding_v2 (embedding_set_id);
|
|
342
342
|
|
|
343
|
+
-- Role-isolated audio observations and embeddings. These intentionally
|
|
344
|
+
-- do not reuse episodes.embedding or memory_embedding_v2: speaker
|
|
345
|
+
-- identity, acoustic features, and sound semantics are incomparable
|
|
346
|
+
-- vector spaces even when their dimensions happen to match.
|
|
347
|
+
CREATE TABLE IF NOT EXISTS audio_observation (
|
|
348
|
+
id TEXT PRIMARY KEY,
|
|
349
|
+
episode_id TEXT NOT NULL DEFAULT '',
|
|
350
|
+
asset_digest TEXT NOT NULL,
|
|
351
|
+
source_uri TEXT NOT NULL DEFAULT '',
|
|
352
|
+
start_ms INTEGER NOT NULL DEFAULT 0,
|
|
353
|
+
end_ms INTEGER NOT NULL DEFAULT 0,
|
|
354
|
+
channel INTEGER NOT NULL DEFAULT 0,
|
|
355
|
+
sample_rate_hz INTEGER NOT NULL DEFAULT 0,
|
|
356
|
+
diarization_cluster_id TEXT NOT NULL DEFAULT '',
|
|
357
|
+
observed_at TEXT,
|
|
358
|
+
quality_json TEXT NOT NULL DEFAULT '{}',
|
|
359
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
360
|
+
provenance_json TEXT NOT NULL DEFAULT '{}',
|
|
361
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
|
362
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
|
363
|
+
CHECK (start_ms >= 0),
|
|
364
|
+
CHECK (end_ms >= start_ms),
|
|
365
|
+
CHECK (channel >= 0),
|
|
366
|
+
CHECK (sample_rate_hz >= 0),
|
|
367
|
+
UNIQUE(episode_id, asset_digest, start_ms, end_ms, channel, sample_rate_hz, diarization_cluster_id)
|
|
368
|
+
);
|
|
369
|
+
|
|
370
|
+
CREATE INDEX IF NOT EXISTS idx_audio_observation_episode
|
|
371
|
+
ON audio_observation (episode_id);
|
|
372
|
+
CREATE INDEX IF NOT EXISTS idx_audio_observation_asset
|
|
373
|
+
ON audio_observation (asset_digest);
|
|
374
|
+
CREATE INDEX IF NOT EXISTS idx_audio_observation_diarization
|
|
375
|
+
ON audio_observation (diarization_cluster_id);
|
|
376
|
+
|
|
377
|
+
CREATE TABLE IF NOT EXISTS audio_observation_embedding (
|
|
378
|
+
id TEXT PRIMARY KEY,
|
|
379
|
+
observation_id TEXT NOT NULL REFERENCES audio_observation(id) ON DELETE CASCADE,
|
|
380
|
+
role TEXT NOT NULL,
|
|
381
|
+
embedding_set_slug TEXT NOT NULL,
|
|
382
|
+
model TEXT NOT NULL,
|
|
383
|
+
model_revision TEXT NOT NULL,
|
|
384
|
+
model_digest TEXT NOT NULL,
|
|
385
|
+
vector BLOB NOT NULL,
|
|
386
|
+
dimensions INTEGER NOT NULL,
|
|
387
|
+
normalization TEXT NOT NULL DEFAULT '',
|
|
388
|
+
input_digest TEXT NOT NULL,
|
|
389
|
+
quality_json TEXT NOT NULL DEFAULT '{}',
|
|
390
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
391
|
+
provenance_json TEXT NOT NULL DEFAULT '{}',
|
|
392
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
|
393
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
|
394
|
+
CHECK (role IN ('speaker_identity_embedding', 'sound_acoustic_embedding', 'sound_semantic_embedding')),
|
|
395
|
+
CHECK (dimensions > 0),
|
|
396
|
+
UNIQUE(observation_id, role, embedding_set_slug, model, model_revision, model_digest, input_digest)
|
|
397
|
+
);
|
|
398
|
+
|
|
399
|
+
CREATE INDEX IF NOT EXISTS idx_audio_observation_embedding_role_set
|
|
400
|
+
ON audio_observation_embedding (role, embedding_set_slug, dimensions);
|
|
401
|
+
CREATE INDEX IF NOT EXISTS idx_audio_observation_embedding_observation
|
|
402
|
+
ON audio_observation_embedding (observation_id);
|
|
403
|
+
|
|
404
|
+
-- Durable speaker identities are consent-gated profiles over only
|
|
405
|
+
-- speaker_identity_embedding exemplars. These tables intentionally do not
|
|
406
|
+
-- create temporal-graph edges: sender IDs and diarization labels are never
|
|
407
|
+
-- biometric proof and must not silently become voice_sample_of evidence.
|
|
408
|
+
CREATE TABLE IF NOT EXISTS speaker_identity_profile (
|
|
409
|
+
id TEXT PRIMARY KEY,
|
|
410
|
+
scope_id TEXT NOT NULL,
|
|
411
|
+
identity TEXT NOT NULL,
|
|
412
|
+
identity_key TEXT NOT NULL,
|
|
413
|
+
embedding_set_slug TEXT NOT NULL,
|
|
414
|
+
model TEXT NOT NULL,
|
|
415
|
+
model_revision TEXT NOT NULL,
|
|
416
|
+
model_digest TEXT NOT NULL,
|
|
417
|
+
dimensions INTEGER NOT NULL,
|
|
418
|
+
normalization TEXT NOT NULL,
|
|
419
|
+
centroid BLOB NOT NULL,
|
|
420
|
+
consent_json TEXT NOT NULL,
|
|
421
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
|
422
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
|
423
|
+
CHECK (dimensions > 0),
|
|
424
|
+
CHECK (normalization = 'l2'),
|
|
425
|
+
UNIQUE(scope_id, identity_key, embedding_set_slug, model, model_revision, model_digest, dimensions, normalization)
|
|
426
|
+
);
|
|
427
|
+
|
|
428
|
+
CREATE INDEX IF NOT EXISTS idx_speaker_identity_profile_exact_space
|
|
429
|
+
ON speaker_identity_profile (scope_id, embedding_set_slug, model, model_revision, model_digest, dimensions, normalization);
|
|
430
|
+
|
|
431
|
+
CREATE TABLE IF NOT EXISTS speaker_identity_exemplar (
|
|
432
|
+
profile_id TEXT NOT NULL REFERENCES speaker_identity_profile(id) ON DELETE CASCADE,
|
|
433
|
+
audio_embedding_id TEXT NOT NULL REFERENCES audio_observation_embedding(id) ON DELETE RESTRICT,
|
|
434
|
+
observation_id TEXT NOT NULL REFERENCES audio_observation(id) ON DELETE RESTRICT,
|
|
435
|
+
consent_json TEXT NOT NULL,
|
|
436
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
|
437
|
+
PRIMARY KEY(profile_id, audio_embedding_id),
|
|
438
|
+
UNIQUE(audio_embedding_id)
|
|
439
|
+
);
|
|
440
|
+
|
|
441
|
+
CREATE INDEX IF NOT EXISTS idx_speaker_identity_exemplar_profile
|
|
442
|
+
ON speaker_identity_exemplar (profile_id, created_at);
|
|
443
|
+
|
|
343
444
|
CREATE TABLE IF NOT EXISTS memory_link_v2 (
|
|
344
445
|
id TEXT PRIMARY KEY,
|
|
345
446
|
source_item_id TEXT NOT NULL REFERENCES memory_item(id) ON DELETE CASCADE,
|
|
@@ -1605,6 +1706,21 @@ var EpisodeStore = class {
|
|
|
1605
1706
|
}
|
|
1606
1707
|
};
|
|
1607
1708
|
|
|
1709
|
+
// packages/memory/dist/audioNoveltyPrototype.js
|
|
1710
|
+
var AUDIO_NOVELTY_WEIGHTS = Object.freeze({
|
|
1711
|
+
distance: 0.6,
|
|
1712
|
+
uncertainty: 0.1,
|
|
1713
|
+
recurrence: 0.15,
|
|
1714
|
+
direction: 0.075,
|
|
1715
|
+
scene: 0.075
|
|
1716
|
+
});
|
|
1717
|
+
|
|
1718
|
+
// packages/memory/dist/speakerIdentityStore.js
|
|
1719
|
+
var DEFAULT_THRESHOLDS = Object.freeze({
|
|
1720
|
+
provisional: 0.78,
|
|
1721
|
+
ambiguityMargin: 0.04
|
|
1722
|
+
});
|
|
1723
|
+
|
|
1608
1724
|
// packages/memory/dist/crl/index.js
|
|
1609
1725
|
var CRL_SYMBOLS = {
|
|
1610
1726
|
// Logical operators (Core)
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Persistent, offline CLAP semantic-audio embedding worker.
|
|
3
|
+
|
|
4
|
+
This process deliberately has no setup behavior. The Node runtime has already
|
|
5
|
+
created the system-site venv, installed its pinned non-Torch requirements, and
|
|
6
|
+
verified the immutable model artifact before this process is started.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
import wave
|
|
16
|
+
|
|
17
|
+
if os.environ.get("HF_HUB_OFFLINE") != "1" or os.environ.get("TRANSFORMERS_OFFLINE") != "1":
|
|
18
|
+
raise RuntimeError("CLAP worker must be started with offline model access enabled")
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
import torch
|
|
22
|
+
import torch.nn.functional as F
|
|
23
|
+
from transformers import ClapModel, ClapProcessor
|
|
24
|
+
|
|
25
|
+
MODEL_ID = "laion/clap-htsat-unfused"
|
|
26
|
+
# The worker receives exactly Egg's caller-conditioned audio window. It does
|
|
27
|
+
# not decode arbitrary media, downmix, trim, or otherwise silently redefine
|
|
28
|
+
# the observation that will later be persisted.
|
|
29
|
+
SOURCE_SAMPLE_RATE_HZ = 16000
|
|
30
|
+
MAX_SOURCE_SAMPLES = 96000
|
|
31
|
+
SAMPLE_RATE_HZ = 48000
|
|
32
|
+
MAX_SAMPLES = 288000
|
|
33
|
+
EMBEDDING_DIMENSION = 512
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def json_line(value):
|
|
37
|
+
print(json.dumps(value, separators=(",", ":")), flush=True)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def sha256_file(path):
|
|
41
|
+
digest = hashlib.sha256()
|
|
42
|
+
with open(path, "rb") as source:
|
|
43
|
+
while True:
|
|
44
|
+
block = source.read(1024 * 1024)
|
|
45
|
+
if not block:
|
|
46
|
+
break
|
|
47
|
+
digest.update(block)
|
|
48
|
+
return "sha256:" + digest.hexdigest()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def read_pcm_wav(path):
|
|
52
|
+
with wave.open(path, "rb") as handle:
|
|
53
|
+
if handle.getcomptype() != "NONE":
|
|
54
|
+
raise ValueError("Expected caller-conditioned uncompressed mono PCM16/16 kHz WAV")
|
|
55
|
+
channels = handle.getnchannels()
|
|
56
|
+
width = handle.getsampwidth()
|
|
57
|
+
sample_rate = handle.getframerate()
|
|
58
|
+
frames = handle.readframes(handle.getnframes())
|
|
59
|
+
if channels != 1 or width != 2 or sample_rate != SOURCE_SAMPLE_RATE_HZ:
|
|
60
|
+
raise ValueError("Expected caller-conditioned mono PCM16/16 kHz WAV")
|
|
61
|
+
if not frames or len(frames) % 2:
|
|
62
|
+
raise ValueError("WAV contains no complete PCM16 samples")
|
|
63
|
+
if len(frames) // 2 > MAX_SOURCE_SAMPLES:
|
|
64
|
+
raise ValueError("Expected caller-conditioned WAV of at most 6 seconds")
|
|
65
|
+
# The digest is the exact PCM window actually consumed, not mutable RIFF
|
|
66
|
+
# metadata such as timestamp, comment, or chunk ordering.
|
|
67
|
+
input_digest = "sha256:" + hashlib.sha256(frames).hexdigest()
|
|
68
|
+
return np.frombuffer(frames, dtype="<i2").astype(np.float32) / 32768.0, input_digest
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def resample_conditioned_16k_to_48k(samples):
|
|
72
|
+
# SOURCE_SAMPLE_RATE_HZ divides CLAP's rate exactly. Linear interpolation
|
|
73
|
+
# keeps preprocessing explicit and deterministic without adding another
|
|
74
|
+
# resampling dependency to JetPack's pinned venv.
|
|
75
|
+
target_count = samples.size * 3
|
|
76
|
+
old = np.arange(samples.size, dtype=np.float64)
|
|
77
|
+
new = np.linspace(0, max(0, samples.size - 1), target_count, dtype=np.float64)
|
|
78
|
+
return np.interp(new, old, samples).astype(np.float32)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class ClapSemanticWorker:
|
|
82
|
+
def __init__(self, model_dir, model_digest, model_revision):
|
|
83
|
+
actual = sha256_file(os.path.join(model_dir, "pytorch_model.bin"))
|
|
84
|
+
if actual != model_digest:
|
|
85
|
+
raise RuntimeError("CLAP weight digest mismatch: expected %s, got %s" % (model_digest, actual))
|
|
86
|
+
self.model_digest = model_digest
|
|
87
|
+
self.model_revision = model_revision
|
|
88
|
+
self.device = torch.device("cuda:0")
|
|
89
|
+
if not torch.cuda.is_available():
|
|
90
|
+
raise RuntimeError("JetPack CUDA Torch is unavailable")
|
|
91
|
+
if torch.cuda.get_device_capability(0) != (8, 7) or not str(torch.version.cuda or "").startswith("12.2"):
|
|
92
|
+
raise RuntimeError("CLAP requires the preflighted JetPack Orin CUDA:0 (CC 8.7 / CUDA 12.2)")
|
|
93
|
+
self.processor = ClapProcessor.from_pretrained(model_dir, local_files_only=True)
|
|
94
|
+
self.model = ClapModel.from_pretrained(model_dir, local_files_only=True).to(self.device).eval()
|
|
95
|
+
if getattr(self.model.config, "projection_dim", None) != EMBEDDING_DIMENSION:
|
|
96
|
+
raise RuntimeError("Pinned CLAP model did not expose a 512-dimensional projection")
|
|
97
|
+
self._warm()
|
|
98
|
+
|
|
99
|
+
def _features(self, samples):
|
|
100
|
+
encoded = self.processor(audios=samples, sampling_rate=SAMPLE_RATE_HZ, return_tensors="pt")
|
|
101
|
+
encoded = {key: value.to(self.device) for key, value in encoded.items() if isinstance(value, torch.Tensor)}
|
|
102
|
+
with torch.inference_mode():
|
|
103
|
+
output = self.model.get_audio_features(**encoded)
|
|
104
|
+
output = F.normalize(output.float(), p=2, dim=-1)
|
|
105
|
+
vector = output[0].detach().cpu().numpy().astype(np.float32)
|
|
106
|
+
if vector.shape != (EMBEDDING_DIMENSION,) or not np.isfinite(vector).all():
|
|
107
|
+
raise RuntimeError("CLAP did not return one finite 512-dimensional vector")
|
|
108
|
+
return vector
|
|
109
|
+
|
|
110
|
+
def _warm(self):
|
|
111
|
+
self._features(np.zeros(MAX_SAMPLES, dtype=np.float32))
|
|
112
|
+
torch.cuda.synchronize(self.device)
|
|
113
|
+
|
|
114
|
+
def embed(self, path):
|
|
115
|
+
started = time.perf_counter()
|
|
116
|
+
samples, input_digest = read_pcm_wav(path)
|
|
117
|
+
decode_ms = (time.perf_counter() - started) * 1000
|
|
118
|
+
source_duration_seconds = samples.size / SOURCE_SAMPLE_RATE_HZ
|
|
119
|
+
samples = resample_conditioned_16k_to_48k(samples)
|
|
120
|
+
rms = float(np.sqrt(np.mean(np.square(samples))))
|
|
121
|
+
peak = float(np.max(np.abs(samples)))
|
|
122
|
+
duration_seconds = source_duration_seconds
|
|
123
|
+
common = {
|
|
124
|
+
"sample_rate_hz": SOURCE_SAMPLE_RATE_HZ,
|
|
125
|
+
"semantic_sample_rate_hz": SAMPLE_RATE_HZ,
|
|
126
|
+
"duration_seconds": duration_seconds,
|
|
127
|
+
"input_digest": input_digest,
|
|
128
|
+
"acoustic": {"rms": rms, "peak": peak},
|
|
129
|
+
"timings_ms": {"decode": decode_ms, "inference": 0.0},
|
|
130
|
+
}
|
|
131
|
+
if rms < 0.0005 and peak < 0.002:
|
|
132
|
+
return dict(common, low_information=True, embedding=None, segment_count=0)
|
|
133
|
+
inference_started = time.perf_counter()
|
|
134
|
+
vectors = []
|
|
135
|
+
# The contract bounds inputs to six seconds, so one explicit 48 kHz
|
|
136
|
+
# window is embedded; semantic segment pooling is kept in the schema
|
|
137
|
+
# for model-set compatibility, not used to hide arbitrary long input.
|
|
138
|
+
vectors.append(self._features(samples))
|
|
139
|
+
vector = np.mean(np.stack(vectors), axis=0)
|
|
140
|
+
vector /= max(float(np.linalg.norm(vector)), 1e-12)
|
|
141
|
+
inference_ms = (time.perf_counter() - inference_started) * 1000
|
|
142
|
+
return dict(common,
|
|
143
|
+
low_information=False,
|
|
144
|
+
segment_count=len(vectors),
|
|
145
|
+
embedding={"dimension": EMBEDDING_DIMENSION, "pooling": "segment_mean_l2", "dtype": "float32", "normalization": "l2", "values": vector.tolist()},
|
|
146
|
+
timings_ms={"decode": decode_ms, "inference": inference_ms},
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def main():
|
|
151
|
+
parser = argparse.ArgumentParser()
|
|
152
|
+
parser.add_argument("--model-dir", required=True)
|
|
153
|
+
parser.add_argument("--model-digest", required=True)
|
|
154
|
+
parser.add_argument("--model-revision", required=True)
|
|
155
|
+
args = parser.parse_args()
|
|
156
|
+
worker = ClapSemanticWorker(args.model_dir, args.model_digest, args.model_revision)
|
|
157
|
+
capability = torch.cuda.get_device_capability(0)
|
|
158
|
+
json_line({"type": "ready", "pid": os.getpid(), "backend": "transformers-clap", "device": torch.cuda.get_device_name(0), "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES", ""), "compute_capability": "%d.%d" % capability, "torch_cuda_version": str(torch.version.cuda), "model": MODEL_ID, "model_revision": args.model_revision, "model_digest": args.model_digest, "embedding_dimension": EMBEDDING_DIMENSION, "sample_rate_hz": SAMPLE_RATE_HZ, "warmed": True})
|
|
159
|
+
for line in sys.stdin:
|
|
160
|
+
try:
|
|
161
|
+
request = json.loads(line)
|
|
162
|
+
if request.get("type") != "embed" or not isinstance(request.get("id"), str) or not isinstance(request.get("file"), str):
|
|
163
|
+
raise ValueError("Expected an embed request with id and file")
|
|
164
|
+
result = worker.embed(request["file"])
|
|
165
|
+
json_line(dict(result, type="result", id=request["id"], action="embed", success=True))
|
|
166
|
+
except Exception as error:
|
|
167
|
+
json_line({"type": "result", "id": request.get("id") if "request" in locals() and isinstance(request, dict) else "", "action": "embed", "success": False, "error": str(error)})
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
if __name__ == "__main__":
|
|
171
|
+
main()
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Persistent CPU-only WeSpeaker CAM++ speaker-identity embedding worker.
|
|
3
|
+
|
|
4
|
+
This process accepts only caller-owned, already-conditioned mono PCM16/16 kHz
|
|
5
|
+
WAV paths. It never records audio, installs dependencies, downloads models,
|
|
6
|
+
or enables CUDA. The parent runtime checksums the pinned ONNX artifact before
|
|
7
|
+
this worker can start.
|
|
8
|
+
|
|
9
|
+
The fbank path is deliberately the official WeSpeaker ONNX preprocessing:
|
|
10
|
+
torchaudio.compliance.kaldi.fbank with 80 bins, 25 ms / 10 ms Hamming frames,
|
|
11
|
+
dither=0 and utterance CMN (without CVN).
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import hashlib
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
import time
|
|
21
|
+
import traceback
|
|
22
|
+
import wave
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# Set before importing Torch or ONNX Runtime so a globally configured Jetson
|
|
26
|
+
# CUDA device cannot become an accidental provider for speaker embeddings.
|
|
27
|
+
os.environ["CUDA_VISIBLE_DEVICES"] = ""
|
|
28
|
+
|
|
29
|
+
MODEL_NAME = "wespeaker-voxceleb-campplus"
|
|
30
|
+
SAMPLE_RATE = 16000
|
|
31
|
+
EMBEDDING_DIMENSION = 512
|
|
32
|
+
MIN_DURATION_SECONDS = 1.5
|
|
33
|
+
MAX_DURATION_SECONDS = 30.0
|
|
34
|
+
MIN_RMS = 0.003
|
|
35
|
+
MIN_PEAK = 0.01
|
|
36
|
+
MAX_CLIPPED_FRACTION = 0.01
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def emit(payload: dict) -> None:
|
|
40
|
+
sys.stdout.write(json.dumps(payload, separators=(",", ":")) + "\n")
|
|
41
|
+
sys.stdout.flush()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def fail(message: str) -> None:
|
|
45
|
+
emit({"type": "fatal", "error": message})
|
|
46
|
+
raise RuntimeError(message)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def read_pcm16_16khz_wav(path: str, np):
|
|
50
|
+
started = time.perf_counter()
|
|
51
|
+
with wave.open(path, "rb") as reader:
|
|
52
|
+
channels = reader.getnchannels()
|
|
53
|
+
sample_width = reader.getsampwidth()
|
|
54
|
+
sample_rate = reader.getframerate()
|
|
55
|
+
frames = reader.getnframes()
|
|
56
|
+
compression = reader.getcomptype()
|
|
57
|
+
raw = reader.readframes(frames)
|
|
58
|
+
if compression != "NONE":
|
|
59
|
+
raise RuntimeError("Only uncompressed RIFF/WAV input is supported")
|
|
60
|
+
if channels != 1:
|
|
61
|
+
raise RuntimeError(f"Expected mono WAV from the caller, received {channels} channels")
|
|
62
|
+
if sample_width != 2:
|
|
63
|
+
raise RuntimeError(f"Expected PCM16 WAV from the caller, received {sample_width * 8}-bit samples")
|
|
64
|
+
if sample_rate != SAMPLE_RATE:
|
|
65
|
+
raise RuntimeError(f"Expected 16 kHz WAV from the caller, received {sample_rate} Hz")
|
|
66
|
+
samples = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
|
|
67
|
+
if samples.size > int(MAX_DURATION_SECONDS * SAMPLE_RATE):
|
|
68
|
+
raise RuntimeError(
|
|
69
|
+
f"Expected at most {MAX_DURATION_SECONDS:.0f} seconds of caller-conditioned speech, received {samples.size / sample_rate:.3f} seconds"
|
|
70
|
+
)
|
|
71
|
+
return (
|
|
72
|
+
samples,
|
|
73
|
+
sample_rate,
|
|
74
|
+
"sha256:" + hashlib.sha256(raw).hexdigest(),
|
|
75
|
+
(time.perf_counter() - started) * 1000,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def acoustic_metrics(samples, np):
|
|
80
|
+
if samples.size == 0:
|
|
81
|
+
return {"rms": 0.0, "peak": 0.0, "clipped_fraction": 0.0}
|
|
82
|
+
absolute = np.abs(samples)
|
|
83
|
+
return {
|
|
84
|
+
"rms": float(np.sqrt(np.mean(np.square(samples)))),
|
|
85
|
+
"peak": float(np.max(absolute)),
|
|
86
|
+
"clipped_fraction": float(np.mean(absolute >= 0.999)),
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def quality_gate(samples, metrics):
|
|
91
|
+
duration = samples.size / SAMPLE_RATE
|
|
92
|
+
if duration < MIN_DURATION_SECONDS:
|
|
93
|
+
return "insufficient_duration"
|
|
94
|
+
if metrics["rms"] < MIN_RMS or metrics["peak"] < MIN_PEAK:
|
|
95
|
+
return "low_energy"
|
|
96
|
+
if metrics["clipped_fraction"] > MAX_CLIPPED_FRACTION:
|
|
97
|
+
return "clipped"
|
|
98
|
+
return "usable"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class WeSpeakerCamPlus:
|
|
102
|
+
def __init__(self, model_path: str):
|
|
103
|
+
started = time.perf_counter()
|
|
104
|
+
import numpy as np
|
|
105
|
+
import onnxruntime as ort
|
|
106
|
+
import torch
|
|
107
|
+
from torchaudio.compliance import kaldi
|
|
108
|
+
|
|
109
|
+
self.np = np
|
|
110
|
+
self.torch = torch
|
|
111
|
+
self.kaldi = kaldi
|
|
112
|
+
torch.set_num_threads(1)
|
|
113
|
+
options = ort.SessionOptions()
|
|
114
|
+
options.inter_op_num_threads = 1
|
|
115
|
+
options.intra_op_num_threads = 1
|
|
116
|
+
self.session = ort.InferenceSession(
|
|
117
|
+
model_path,
|
|
118
|
+
sess_options=options,
|
|
119
|
+
providers=["CPUExecutionProvider"],
|
|
120
|
+
)
|
|
121
|
+
providers = self.session.get_providers()
|
|
122
|
+
if providers != ["CPUExecutionProvider"]:
|
|
123
|
+
raise RuntimeError(f"WeSpeaker must run only on CPUExecutionProvider, got {providers}")
|
|
124
|
+
inputs = self.session.get_inputs()
|
|
125
|
+
outputs = self.session.get_outputs()
|
|
126
|
+
if len(inputs) != 1 or inputs[0].name != "feats" or inputs[0].type != "tensor(float)":
|
|
127
|
+
raise RuntimeError("Pinned CAM++ ONNX model must expose float input 'feats'")
|
|
128
|
+
if len(inputs[0].shape) != 3 or inputs[0].shape[-1] != 80:
|
|
129
|
+
raise RuntimeError(f"Pinned CAM++ input must be [batch, frames, 80], got {inputs[0].shape}")
|
|
130
|
+
if len(outputs) != 1 or outputs[0].name != "embs" or outputs[0].type != "tensor(float)":
|
|
131
|
+
raise RuntimeError("Pinned CAM++ ONNX model must expose float output 'embs'")
|
|
132
|
+
if len(outputs[0].shape) != 2 or outputs[0].shape[-1] != EMBEDDING_DIMENSION:
|
|
133
|
+
raise RuntimeError(f"Pinned CAM++ output must be [batch, {EMBEDDING_DIMENSION}], got {outputs[0].shape}")
|
|
134
|
+
self.model_load_ms = (time.perf_counter() - started) * 1000
|
|
135
|
+
|
|
136
|
+
def features(self, samples):
|
|
137
|
+
# Mirrors wespeaker/bin/infer_onnx.py exactly. Its torchaudio.load
|
|
138
|
+
# values are normalized PCM16 floats and multiplied by 2**15 before
|
|
139
|
+
# Kaldi fbank; samples are decoded PCM16 values in that same range.
|
|
140
|
+
waveform = self.torch.from_numpy(
|
|
141
|
+
self.np.ascontiguousarray(samples * (1 << 15), dtype=self.np.float32)
|
|
142
|
+
).unsqueeze(0)
|
|
143
|
+
mat = self.kaldi.fbank(
|
|
144
|
+
waveform,
|
|
145
|
+
num_mel_bins=80,
|
|
146
|
+
frame_length=25,
|
|
147
|
+
frame_shift=10,
|
|
148
|
+
dither=0.0,
|
|
149
|
+
sample_frequency=SAMPLE_RATE,
|
|
150
|
+
window_type="hamming",
|
|
151
|
+
use_energy=False,
|
|
152
|
+
)
|
|
153
|
+
if mat.ndim != 2 or mat.shape[0] < 1 or mat.shape[1] != 80:
|
|
154
|
+
raise RuntimeError(f"Official WeSpeaker fbank produced an invalid shape: {tuple(mat.shape)}")
|
|
155
|
+
# CMN without CVN, over the full input utterance, per official script.
|
|
156
|
+
mat = mat - self.torch.mean(mat, dim=0)
|
|
157
|
+
return mat.unsqueeze(0).detach().cpu().numpy().astype(self.np.float32, copy=False)
|
|
158
|
+
|
|
159
|
+
def embed(self, samples):
|
|
160
|
+
started = time.perf_counter()
|
|
161
|
+
features = self.features(samples)
|
|
162
|
+
output = self.session.run(["embs"], {"feats": features})[0]
|
|
163
|
+
vector = self.np.asarray(output, dtype=self.np.float32)
|
|
164
|
+
if vector.shape != (1, EMBEDDING_DIMENSION):
|
|
165
|
+
raise RuntimeError(f"CAM++ produced invalid embedding shape: {vector.shape}")
|
|
166
|
+
vector = vector[0]
|
|
167
|
+
norm = float(self.np.linalg.norm(vector))
|
|
168
|
+
if not self.np.isfinite(norm) or norm <= 0.0:
|
|
169
|
+
raise RuntimeError("CAM++ embedding has an invalid L2 norm")
|
|
170
|
+
return {
|
|
171
|
+
"dimension": EMBEDDING_DIMENSION,
|
|
172
|
+
"dtype": "float32",
|
|
173
|
+
"normalization": "l2",
|
|
174
|
+
"values": (vector / norm).astype(self.np.float32).tolist(),
|
|
175
|
+
}, (time.perf_counter() - started) * 1000
|
|
176
|
+
|
|
177
|
+
def warm(self):
|
|
178
|
+
# A deterministic voiced-like probe validates the official fbank and
|
|
179
|
+
# the CPU session before readiness is published. It is not user audio.
|
|
180
|
+
t = self.np.arange(SAMPLE_RATE * 2, dtype=self.np.float32) / SAMPLE_RATE
|
|
181
|
+
samples = (0.1 * self.np.sin(2.0 * self.np.pi * 220.0 * t)).astype(self.np.float32)
|
|
182
|
+
embedding, _ = self.embed(samples)
|
|
183
|
+
if len(embedding["values"]) != EMBEDDING_DIMENSION:
|
|
184
|
+
raise RuntimeError("CAM++ warmup did not produce a 512-dimensional embedding")
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def main() -> int:
|
|
188
|
+
parser = argparse.ArgumentParser()
|
|
189
|
+
parser.add_argument("--model", required=True)
|
|
190
|
+
parser.add_argument("--model-digest", required=True)
|
|
191
|
+
args = parser.parse_args()
|
|
192
|
+
worker = WeSpeakerCamPlus(args.model)
|
|
193
|
+
worker.warm()
|
|
194
|
+
emit(
|
|
195
|
+
{
|
|
196
|
+
"type": "ready",
|
|
197
|
+
"pid": os.getpid(),
|
|
198
|
+
"backend": "onnxruntime-cpu",
|
|
199
|
+
"provider": "CPUExecutionProvider",
|
|
200
|
+
"model": MODEL_NAME,
|
|
201
|
+
"model_digest": args.model_digest,
|
|
202
|
+
"dimension": EMBEDDING_DIMENSION,
|
|
203
|
+
"model_load_ms": round(worker.model_load_ms, 3),
|
|
204
|
+
"warmed": True,
|
|
205
|
+
}
|
|
206
|
+
)
|
|
207
|
+
for line in sys.stdin:
|
|
208
|
+
request = None
|
|
209
|
+
action = ""
|
|
210
|
+
try:
|
|
211
|
+
request = json.loads(line)
|
|
212
|
+
if request.get("type") == "shutdown":
|
|
213
|
+
emit({"type": "stopped"})
|
|
214
|
+
return 0
|
|
215
|
+
action = str(request.get("type") or "").strip().lower()
|
|
216
|
+
if action != "embed":
|
|
217
|
+
raise RuntimeError("unknown speaker embedding action")
|
|
218
|
+
request_id = str(request.get("id") or "")
|
|
219
|
+
file_path = str(request.get("file") or "")
|
|
220
|
+
if not request_id or not file_path:
|
|
221
|
+
raise RuntimeError("speaker embedding requires id and file")
|
|
222
|
+
samples, sample_rate, input_digest, decode_ms = read_pcm16_16khz_wav(file_path, worker.np)
|
|
223
|
+
metrics = acoustic_metrics(samples, worker.np)
|
|
224
|
+
quality = quality_gate(samples, metrics)
|
|
225
|
+
if quality == "usable":
|
|
226
|
+
embedding, inference_ms = worker.embed(samples)
|
|
227
|
+
else:
|
|
228
|
+
embedding, inference_ms = None, 0.0
|
|
229
|
+
emit(
|
|
230
|
+
{
|
|
231
|
+
"type": "result",
|
|
232
|
+
"id": request_id,
|
|
233
|
+
"action": "embed",
|
|
234
|
+
"success": True,
|
|
235
|
+
"sample_rate_hz": sample_rate,
|
|
236
|
+
"input_digest": input_digest,
|
|
237
|
+
"duration_seconds": round(float(samples.size) / sample_rate, 6),
|
|
238
|
+
"quality": quality,
|
|
239
|
+
"embedding": embedding,
|
|
240
|
+
"acoustic": {key: round(value, 8) for key, value in metrics.items()},
|
|
241
|
+
"timings_ms": {"decode": round(decode_ms, 3), "inference": round(inference_ms, 3)},
|
|
242
|
+
}
|
|
243
|
+
)
|
|
244
|
+
except Exception as exc: # Keep the persistent worker alive per request.
|
|
245
|
+
emit(
|
|
246
|
+
{
|
|
247
|
+
"type": "result",
|
|
248
|
+
"id": str(request.get("id", "")) if isinstance(request, dict) else "",
|
|
249
|
+
"action": action,
|
|
250
|
+
"success": False,
|
|
251
|
+
"error": str(exc),
|
|
252
|
+
}
|
|
253
|
+
)
|
|
254
|
+
print(traceback.format_exc(), file=sys.stderr, flush=True)
|
|
255
|
+
return 0
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
if __name__ == "__main__":
|
|
259
|
+
raise SystemExit(main())
|