dsh-podwave 0.11.28 → 0.11.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +17 -10
- package/dist/client.js +99 -5
- package/package.json +1 -1
- package/scripts/speaker_server.py +114 -0
- package/src/asr-api.js +3 -0
- package/src/config.js +12 -1
- package/src/host.js +19 -4
- package/src/live-speaker.js +242 -0
- package/src/transcribe.js +128 -18
package/package.json
CHANGED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""常驻说话人声纹服务:给每行字幕算一个声纹向量,供在线聚类用。
|
|
3
|
+
|
|
4
|
+
模型加载一次要一两秒,而实时跟播时每 12 秒就会来好几行,
|
|
5
|
+
所以这里做成常驻进程:父进程按行写 JSON 请求,这里按行回 JSON。
|
|
6
|
+
|
|
7
|
+
请求:{"id": 1, "audio": "/path/part.wav", "start": 1.23, "end": 4.56}
|
|
8
|
+
响应:{"id": 1, "embedding": [...]} 或 {"id": 1, "error": "..."}
|
|
9
|
+
|
|
10
|
+
start/end 是「相对该音频文件」的秒数;音频必须是 16kHz 单声道 16bit PCM WAV
|
|
11
|
+
(transcribe.js 切出来的块就是这个格式,所以不用重新解码,直接用 numpy 切片)。
|
|
12
|
+
"""
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import sys
|
|
16
|
+
import wave
|
|
17
|
+
|
|
18
|
+
import numpy as np
|
|
19
|
+
|
|
20
|
+
SAMPLE_RATE = 16000
|
|
21
|
+
MIN_SECONDS = 0.35 # 太短的片段算不出稳定声纹,直接让调用方回退
|
|
22
|
+
MIN_SAMPLES = int(SAMPLE_RATE * MIN_SECONDS)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def read_samples(path):
|
|
26
|
+
with wave.open(path, 'rb') as handle:
|
|
27
|
+
rate = handle.getframerate()
|
|
28
|
+
channels = handle.getnchannels()
|
|
29
|
+
width = handle.getsampwidth()
|
|
30
|
+
frames = handle.readframes(handle.getnframes())
|
|
31
|
+
if width != 2:
|
|
32
|
+
raise RuntimeError(f'只支持 16-bit PCM,收到 {width * 8}-bit')
|
|
33
|
+
if rate != SAMPLE_RATE:
|
|
34
|
+
raise RuntimeError(f'只支持 {SAMPLE_RATE}Hz,收到 {rate}Hz')
|
|
35
|
+
data = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
|
|
36
|
+
if channels > 1:
|
|
37
|
+
data = data.reshape(-1, channels).mean(axis=1)
|
|
38
|
+
return data
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Server:
|
|
42
|
+
def __init__(self, model):
|
|
43
|
+
import sherpa_onnx
|
|
44
|
+
|
|
45
|
+
config = sherpa_onnx.SpeakerEmbeddingExtractorConfig(model=model, num_threads=1)
|
|
46
|
+
self.extractor = sherpa_onnx.SpeakerEmbeddingExtractor(config)
|
|
47
|
+
self.cached_path = None
|
|
48
|
+
self.cached = None
|
|
49
|
+
|
|
50
|
+
def samples(self, path):
|
|
51
|
+
if self.cached_path != path or self.cached is None:
|
|
52
|
+
self.cached = read_samples(path)
|
|
53
|
+
self.cached_path = path
|
|
54
|
+
return self.cached
|
|
55
|
+
|
|
56
|
+
def embed(self, path, start, end):
|
|
57
|
+
data = self.samples(path)
|
|
58
|
+
begin = max(0, int(round(float(start) * SAMPLE_RATE)))
|
|
59
|
+
finish = min(len(data), int(round(float(end) * SAMPLE_RATE)))
|
|
60
|
+
if finish - begin < MIN_SAMPLES:
|
|
61
|
+
return None
|
|
62
|
+
chunk = data[begin:finish]
|
|
63
|
+
stream = self.extractor.create_stream()
|
|
64
|
+
stream.accept_waveform(SAMPLE_RATE, chunk)
|
|
65
|
+
stream.input_finished()
|
|
66
|
+
if not self.extractor.is_ready(stream):
|
|
67
|
+
return None
|
|
68
|
+
return [float(value) for value in self.extractor.compute(stream)]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def main():
|
|
72
|
+
model = None
|
|
73
|
+
args = sys.argv[1:]
|
|
74
|
+
for index, item in enumerate(args):
|
|
75
|
+
if item == '--model' and index + 1 < len(args):
|
|
76
|
+
model = args[index + 1]
|
|
77
|
+
if not model or not os.path.exists(model):
|
|
78
|
+
print(json.dumps({'error': f'找不到声纹模型:{model}'}), flush=True)
|
|
79
|
+
return 2
|
|
80
|
+
|
|
81
|
+
try:
|
|
82
|
+
server = Server(model)
|
|
83
|
+
except Exception as error: # noqa: BLE001 - 起不来就明确告诉父进程
|
|
84
|
+
print(json.dumps({'error': f'声纹模型加载失败:{error}'}), flush=True)
|
|
85
|
+
return 2
|
|
86
|
+
|
|
87
|
+
# 模型加载完先打个招呼,父进程据此认为「服务就绪」。
|
|
88
|
+
print(json.dumps({'ready': True, 'dim': int(server.extractor.dim)}), flush=True)
|
|
89
|
+
|
|
90
|
+
for line in sys.stdin:
|
|
91
|
+
line = line.strip()
|
|
92
|
+
if not line:
|
|
93
|
+
continue
|
|
94
|
+
try:
|
|
95
|
+
request = json.loads(line)
|
|
96
|
+
except Exception: # noqa: BLE001 - 坏行直接跳过,不要拖垮常驻进程
|
|
97
|
+
continue
|
|
98
|
+
request_id = request.get('id')
|
|
99
|
+
try:
|
|
100
|
+
embedding = server.embed(
|
|
101
|
+
request.get('audio', ''),
|
|
102
|
+
request.get('start', 0),
|
|
103
|
+
request.get('end', 0),
|
|
104
|
+
)
|
|
105
|
+
if embedding is None:
|
|
106
|
+
print(json.dumps({'id': request_id, 'embedding': None}), flush=True)
|
|
107
|
+
else:
|
|
108
|
+
print(json.dumps({'id': request_id, 'embedding': embedding}), flush=True)
|
|
109
|
+
except Exception as error: # noqa: BLE001 - 单次失败不影响后续请求
|
|
110
|
+
print(json.dumps({'id': request_id, 'error': str(error)}), flush=True)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if __name__ == '__main__':
|
|
114
|
+
sys.exit(main())
|
package/src/asr-api.js
CHANGED
|
@@ -14,6 +14,9 @@ export const API_ASR_PATH = '/v1/tools/asr'
|
|
|
14
14
|
// 有些音频的重采样/补帧会多出零点几秒,卡在 120 上迟早翻车。
|
|
15
15
|
// 块越大,句子被块边界切开的概率越低(切点还会吸附到静音处,见 transcribe.js 的 planCuts)。
|
|
16
16
|
export const API_CHUNK_SECONDS = 100
|
|
17
|
+
// 实时跟播模式(边播边出字幕)用小段滚动:块越小,字幕越跟手,代价是块边界的
|
|
18
|
+
// 重复开销(识别成本大约上浮 15%~30%)。12 秒在「延迟 3~5 秒」和成本之间折中。
|
|
19
|
+
export const REALTIME_CHUNK_SECONDS = 12
|
|
17
20
|
export const API_MAX_SECONDS = 100
|
|
18
21
|
export const API_MAX_BYTES = 8 * 1024 * 1024
|
|
19
22
|
const REQUEST_TIMEOUT_MS = 180000
|
package/src/config.js
CHANGED
|
@@ -30,7 +30,16 @@ function cachedDefaultModels() {
|
|
|
30
30
|
|
|
31
31
|
export const CONFIG_DEFAULTS = {
|
|
32
32
|
schemaVersion: 1,
|
|
33
|
-
ui: {
|
|
33
|
+
ui: {
|
|
34
|
+
showSidebarPanel: true,
|
|
35
|
+
sidebarCollapsed: false,
|
|
36
|
+
// 字幕字号(px):字幕面板和悬浮字幕共用这一个设置,用拖动条调。
|
|
37
|
+
subtitleSize: 15,
|
|
38
|
+
// 悬浮字幕:是否开启、停留在屏幕上的位置(null = 用默认的右下角)。
|
|
39
|
+
captionOpen: false,
|
|
40
|
+
captionX: null,
|
|
41
|
+
captionY: null,
|
|
42
|
+
},
|
|
34
43
|
// DeepSeek API 的地址 / 密钥 / 对话模型:转写和翻译共用这一份,用户只填一次。
|
|
35
44
|
// 老版本把这三项存在 translate 段里,读到时由 migrateApiSection() 挪过来。
|
|
36
45
|
api: {
|
|
@@ -55,6 +64,8 @@ export const CONFIG_DEFAULTS = {
|
|
|
55
64
|
hotwords: [],
|
|
56
65
|
chunkMinutes: 1,
|
|
57
66
|
diarize: true,
|
|
67
|
+
// 边播边出字幕:播放某集时自动用小段滚动转写,字幕/译文/说话人跟着播放进度出现。
|
|
68
|
+
realtime: true,
|
|
58
69
|
},
|
|
59
70
|
import: { mode: 'copy' },
|
|
60
71
|
}
|
package/src/host.js
CHANGED
|
@@ -74,6 +74,7 @@ import {
|
|
|
74
74
|
splitTranscriptLine,
|
|
75
75
|
validateTranscript,
|
|
76
76
|
} from './transcribe.js'
|
|
77
|
+
import { closeLiveSpeakers } from './live-speaker.js'
|
|
77
78
|
import { isLinux, isMac, isWindows } from './platform.js'
|
|
78
79
|
import { listJobs, removeJob, upsertJob, getJob } from './jobs.js'
|
|
79
80
|
import { scan as scanSetup, plan as planSetup, install as installSetup, cancelSetup, setupJob, llamaBinary, verify as verifySetup } from './setup.js'
|
|
@@ -513,7 +514,7 @@ async function route(ctx, req, res) {
|
|
|
513
514
|
return json(res, 200, {
|
|
514
515
|
ok: true,
|
|
515
516
|
generatedAt: new Date().toISOString(),
|
|
516
|
-
version: '0.11.
|
|
517
|
+
version: '0.11.29',
|
|
517
518
|
platform,
|
|
518
519
|
engine: recommendEngine(platform),
|
|
519
520
|
paths: {
|
|
@@ -543,7 +544,7 @@ async function route(ctx, req, res) {
|
|
|
543
544
|
}
|
|
544
545
|
if ((method === 'GET' || method === 'HEAD') && routePath === 'api/health') {
|
|
545
546
|
await mediaReady
|
|
546
|
-
return json(res, 200, { ok: true, app: 'podwave', version: '0.11.
|
|
547
|
+
return json(res, 200, { ok: true, app: 'podwave', version: '0.11.29', mediaBase, transcribe: runtimeInfo(), diarize: diarizeInfo(), translate: publicConfig() })
|
|
547
548
|
}
|
|
548
549
|
if (method === 'GET' && routePath === 'api/state') {
|
|
549
550
|
const state = getState()
|
|
@@ -926,7 +927,7 @@ async function route(ctx, req, res) {
|
|
|
926
927
|
}
|
|
927
928
|
if (!episode) return json(res, 404, { error: '单集不存在' })
|
|
928
929
|
const body = await readJson(req)
|
|
929
|
-
const result = await startTranscription(episode, { force: Boolean(body?.force), resume: Boolean(body?.resume) })
|
|
930
|
+
const result = await startTranscription(episode, { force: Boolean(body?.force), resume: Boolean(body?.resume), realtime: Boolean(body?.realtime) })
|
|
930
931
|
return json(res, 200, result)
|
|
931
932
|
}
|
|
932
933
|
|
|
@@ -976,6 +977,7 @@ async function route(ctx, req, res) {
|
|
|
976
977
|
if (Array.isArray(body?.hotwords)) patch.hotwords = body.hotwords.map((word) => String(word || '').trim()).filter(Boolean).slice(0, 500)
|
|
977
978
|
if (Number.isFinite(Number(body?.chunkMinutes))) patch.chunkMinutes = Math.max(1, Math.min(60, Number(body.chunkMinutes)))
|
|
978
979
|
if (typeof body?.diarize === 'boolean') patch.diarize = body.diarize
|
|
980
|
+
if (typeof body?.realtime === 'boolean') patch.realtime = body.realtime
|
|
979
981
|
if (!Object.keys(patch).length) return json(res, 400, { error: '没有可保存的设置' })
|
|
980
982
|
const saved = updateSection('transcribe', patch)
|
|
981
983
|
return json(res, 200, { ...saved, runtime: runtimeInfo() })
|
|
@@ -1204,6 +1206,16 @@ async function route(ctx, req, res) {
|
|
|
1204
1206
|
const patch = {}
|
|
1205
1207
|
if (typeof body?.showSidebarPanel === 'boolean') patch.showSidebarPanel = body.showSidebarPanel
|
|
1206
1208
|
if (typeof body?.sidebarCollapsed === 'boolean') patch.sidebarCollapsed = body.sidebarCollapsed
|
|
1209
|
+
if (Number.isFinite(Number(body?.subtitleSize))) patch.subtitleSize = Math.max(12, Math.min(42, Math.round(Number(body.subtitleSize))))
|
|
1210
|
+
if (typeof body?.captionOpen === 'boolean') patch.captionOpen = body.captionOpen
|
|
1211
|
+
// 悬浮字幕的位置:传 null / 空串表示回到默认位置(注意 Number(null) 是 0,不能直接判有限)。
|
|
1212
|
+
const coordinate = (value) => {
|
|
1213
|
+
if (value === null || value === undefined || value === '') return null
|
|
1214
|
+
const number = Number(value)
|
|
1215
|
+
return Number.isFinite(number) ? Math.round(number) : null
|
|
1216
|
+
}
|
|
1217
|
+
if (body && 'captionX' in body) patch.captionX = coordinate(body.captionX)
|
|
1218
|
+
if (body && 'captionY' in body) patch.captionY = coordinate(body.captionY)
|
|
1207
1219
|
if (!Object.keys(patch).length) return json(res, 400, { error: '没有可保存的设置' })
|
|
1208
1220
|
return json(res, 200, saveUiSettings(patch))
|
|
1209
1221
|
}
|
|
@@ -1329,6 +1341,9 @@ export default {
|
|
|
1329
1341
|
}),
|
|
1330
1342
|
'dsh-podwave: /podwave routes',
|
|
1331
1343
|
)
|
|
1332
|
-
ctx.effect(() => () =>
|
|
1344
|
+
ctx.effect(() => () => {
|
|
1345
|
+
stopAll()
|
|
1346
|
+
closeLiveSpeakers()
|
|
1347
|
+
}, 'dsh-podwave: stop transcription workers')
|
|
1333
1348
|
},
|
|
1334
1349
|
}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
// 实时说话人:常驻声纹服务 + 在线聚类。
|
|
2
|
+
//
|
|
3
|
+
// 整段说话人分离(diarize.js)要等全部音频转完才跑,实时跟播用不了。
|
|
4
|
+
// 这里的做法是「边播边猜」:每行字幕出来就取那一小段音频算一个声纹向量,
|
|
5
|
+
// 和已知说话人的质心比余弦相似度,够像就归入,不像就新建一个。
|
|
6
|
+
// 播完以后 transcribe.js 还会用整段 diarize 再校正一遍。
|
|
7
|
+
//
|
|
8
|
+
// 声纹模型和 diarize 用的是同一份 3D-Speaker embedding 模型(本地模型目录里已有)。
|
|
9
|
+
import fs from 'node:fs'
|
|
10
|
+
import path from 'node:path'
|
|
11
|
+
import { spawn } from 'node:child_process'
|
|
12
|
+
import { fileURLToPath } from 'node:url'
|
|
13
|
+
import { venvPython } from './platform.js'
|
|
14
|
+
import { diarizationPaths } from './diarize.js'
|
|
15
|
+
|
|
16
|
+
const SCRIPT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'scripts', 'speaker_server.py')
|
|
17
|
+
const READY_TIMEOUT_MS = 20000
|
|
18
|
+
const REQUEST_TIMEOUT_MS = 12000
|
|
19
|
+
// 同一个人在相邻片段上的余弦相似度一般 0.75 以上,不同人通常低于 0.5。
|
|
20
|
+
// 取 0.72 宁可多认出一个人,也别把两个人并成一个(后面还有整段校正兜底)。
|
|
21
|
+
const MATCH_THRESHOLD = 0.72
|
|
22
|
+
const MAX_SPEAKERS = 12
|
|
23
|
+
|
|
24
|
+
export function liveSpeakerInfo() {
|
|
25
|
+
const python = venvPython()
|
|
26
|
+
const { embModel } = diarizationPaths()
|
|
27
|
+
return {
|
|
28
|
+
available: fs.existsSync(python) && fs.existsSync(SCRIPT) && fs.existsSync(embModel),
|
|
29
|
+
python,
|
|
30
|
+
script: SCRIPT,
|
|
31
|
+
embModel,
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function normalize(vector) {
|
|
36
|
+
let sum = 0
|
|
37
|
+
for (const value of vector) sum += value * value
|
|
38
|
+
const length = Math.sqrt(sum) || 1
|
|
39
|
+
return vector.map((value) => value / length)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function cosine(left, right) {
|
|
43
|
+
let sum = 0
|
|
44
|
+
for (let index = 0; index < left.length; index += 1) sum += left[index] * right[index]
|
|
45
|
+
return sum
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// 常驻进程:模型加载一次要一两秒,每行都起进程肯定来不及。
|
|
49
|
+
class SpeakerServer {
|
|
50
|
+
constructor(child) {
|
|
51
|
+
this.child = child
|
|
52
|
+
this.pending = new Map()
|
|
53
|
+
this.nextId = 0
|
|
54
|
+
this.buffer = ''
|
|
55
|
+
this.closed = false
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
static async start(info) {
|
|
59
|
+
const child = spawn(info.python, [SCRIPT, '--model', info.embModel], {
|
|
60
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
61
|
+
env: { ...process.env, PYTHONUNBUFFERED: '1' },
|
|
62
|
+
})
|
|
63
|
+
const server = new SpeakerServer(child)
|
|
64
|
+
const ready = new Promise((resolve, reject) => {
|
|
65
|
+
const timer = setTimeout(() => reject(new Error('声纹服务启动超时')), READY_TIMEOUT_MS)
|
|
66
|
+
const onExit = () => { clearTimeout(timer); reject(new Error('声纹服务没能启动')) }
|
|
67
|
+
child.once('exit', onExit)
|
|
68
|
+
server.readyResolve = (value) => { clearTimeout(timer); child.off('exit', onExit); resolve(value) }
|
|
69
|
+
server.readyReject = (error) => { clearTimeout(timer); child.off('exit', onExit); reject(error) }
|
|
70
|
+
})
|
|
71
|
+
server.wire()
|
|
72
|
+
await ready
|
|
73
|
+
return server
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
wire() {
|
|
77
|
+
this.child.stdout.on('data', (chunk) => {
|
|
78
|
+
this.buffer += chunk.toString()
|
|
79
|
+
let index = this.buffer.indexOf('\n')
|
|
80
|
+
while (index >= 0) {
|
|
81
|
+
const line = this.buffer.slice(0, index).trim()
|
|
82
|
+
this.buffer = this.buffer.slice(index + 1)
|
|
83
|
+
if (line) this.handle(line)
|
|
84
|
+
index = this.buffer.indexOf('\n')
|
|
85
|
+
}
|
|
86
|
+
})
|
|
87
|
+
this.child.stderr.on('data', () => { /* 服务端日志忽略,出错会从 stdout 回来 */ })
|
|
88
|
+
this.child.on('exit', () => {
|
|
89
|
+
this.closed = true
|
|
90
|
+
for (const [, entry] of this.pending) entry.reject(new Error('声纹服务已退出'))
|
|
91
|
+
this.pending.clear()
|
|
92
|
+
if (this.readyReject) this.readyReject(new Error('声纹服务已退出'))
|
|
93
|
+
})
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
handle(line) {
|
|
97
|
+
let payload
|
|
98
|
+
try {
|
|
99
|
+
payload = JSON.parse(line)
|
|
100
|
+
} catch {
|
|
101
|
+
return
|
|
102
|
+
}
|
|
103
|
+
if (payload.ready) {
|
|
104
|
+
this.readyResolve?.(true)
|
|
105
|
+
return
|
|
106
|
+
}
|
|
107
|
+
if (payload.error && payload.id === undefined) {
|
|
108
|
+
this.readyReject?.(new Error(payload.error))
|
|
109
|
+
return
|
|
110
|
+
}
|
|
111
|
+
const entry = this.pending.get(payload.id)
|
|
112
|
+
if (!entry) return
|
|
113
|
+
this.pending.delete(payload.id)
|
|
114
|
+
clearTimeout(entry.timer)
|
|
115
|
+
if (payload.error) entry.reject(new Error(payload.error))
|
|
116
|
+
else entry.resolve(payload.embedding || null)
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
request(audio, start, end) {
|
|
120
|
+
if (this.closed) return Promise.reject(new Error('声纹服务已退出'))
|
|
121
|
+
const id = (this.nextId += 1)
|
|
122
|
+
return new Promise((resolve, reject) => {
|
|
123
|
+
const timer = setTimeout(() => {
|
|
124
|
+
this.pending.delete(id)
|
|
125
|
+
reject(new Error('声纹计算超时'))
|
|
126
|
+
}, REQUEST_TIMEOUT_MS)
|
|
127
|
+
this.pending.set(id, { resolve, reject, timer })
|
|
128
|
+
this.child.stdin.write(`${JSON.stringify({ id, audio, start, end })}\n`)
|
|
129
|
+
})
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
close() {
|
|
133
|
+
this.closed = true
|
|
134
|
+
try { this.child.stdin.end() } catch { /* 已经关了 */ }
|
|
135
|
+
try { this.child.kill() } catch { /* 已经退了 */ }
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// 多个转写任务共用一个常驻进程(同一个 profile 只有一个用户)。
|
|
140
|
+
let sharedServer = null
|
|
141
|
+
|
|
142
|
+
async function ensureServer() {
|
|
143
|
+
const info = liveSpeakerInfo()
|
|
144
|
+
if (!info.available) return null
|
|
145
|
+
if (!sharedServer || sharedServer.closed) sharedServer = await SpeakerServer.start(info)
|
|
146
|
+
return sharedServer
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function sharedEmbed(audio, start, end) {
|
|
150
|
+
const server = await ensureServer()
|
|
151
|
+
if (!server) return null
|
|
152
|
+
return server.request(audio, start, end)
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function closeLiveSpeakers() {
|
|
156
|
+
if (sharedServer) {
|
|
157
|
+
sharedServer.close()
|
|
158
|
+
sharedServer = null
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function partForTime(parts, time) {
|
|
163
|
+
for (const part of parts) {
|
|
164
|
+
if (time >= part.start - 0.001 && time < part.end) return part
|
|
165
|
+
}
|
|
166
|
+
return parts[parts.length - 1] || null
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export class OnlineSpeakers {
|
|
170
|
+
constructor(embed) {
|
|
171
|
+
this.embed = embed
|
|
172
|
+
this.centroids = []
|
|
173
|
+
this.counts = []
|
|
174
|
+
this.cache = new Map()
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
assign(vector) {
|
|
178
|
+
const unit = normalize(vector)
|
|
179
|
+
let best = -1
|
|
180
|
+
let bestScore = -Infinity
|
|
181
|
+
for (let index = 0; index < this.centroids.length; index += 1) {
|
|
182
|
+
const score = cosine(unit, this.centroids[index])
|
|
183
|
+
if (score > bestScore) {
|
|
184
|
+
bestScore = score
|
|
185
|
+
best = index
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if (best >= 0 && bestScore >= MATCH_THRESHOLD) {
|
|
189
|
+
const count = this.counts[best] + 1
|
|
190
|
+
const merged = this.centroids[best].map((value, index) => (value * this.counts[best] + unit[index]) / count)
|
|
191
|
+
this.centroids[best] = normalize(merged)
|
|
192
|
+
this.counts[best] = count
|
|
193
|
+
return best
|
|
194
|
+
}
|
|
195
|
+
if (this.centroids.length >= MAX_SPEAKERS) return best >= 0 ? best : null
|
|
196
|
+
this.centroids.push(unit)
|
|
197
|
+
this.counts.push(1)
|
|
198
|
+
return this.centroids.length - 1
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
speakers() {
|
|
202
|
+
return this.centroids.map((_, index) => ({ id: index, label: `说话人 ${index + 1}` }))
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// 给还没有 speaker 的行打实时标签。已有的不覆盖(可能来自整段校正)。
|
|
206
|
+
async labelLines(lines, parts) {
|
|
207
|
+
for (const line of lines) {
|
|
208
|
+
if (line.speaker !== null && line.speaker !== undefined) continue
|
|
209
|
+
const key = `${Math.round(line.start * 100)}|${line.text}`
|
|
210
|
+
let index = this.cache.get(key)
|
|
211
|
+
if (index === undefined) {
|
|
212
|
+
const part = partForTime(parts, line.start)
|
|
213
|
+
if (!part) continue
|
|
214
|
+
const start = Math.max(0, line.start - part.start)
|
|
215
|
+
const end = Math.min(part.duration, line.end - part.start)
|
|
216
|
+
if (end - start < 0.4) continue
|
|
217
|
+
let vector = null
|
|
218
|
+
try {
|
|
219
|
+
vector = await this.embed(part.file, start, end)
|
|
220
|
+
} catch {
|
|
221
|
+
vector = null
|
|
222
|
+
}
|
|
223
|
+
if (!vector) continue
|
|
224
|
+
index = this.assign(vector)
|
|
225
|
+
if (index === null) continue
|
|
226
|
+
this.cache.set(key, index)
|
|
227
|
+
}
|
|
228
|
+
line.speaker = index
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// 拿不到本地声纹服务(没装 python 运行时/模型)就返回 null,实时只做转写和翻译。
|
|
234
|
+
export async function createLiveSpeakers() {
|
|
235
|
+
try {
|
|
236
|
+
const server = await ensureServer()
|
|
237
|
+
if (!server) return null
|
|
238
|
+
return new OnlineSpeakers(sharedEmbed)
|
|
239
|
+
} catch {
|
|
240
|
+
return null
|
|
241
|
+
}
|
|
242
|
+
}
|