dsh-live-voice 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/asr/asr.worker.js +1 -1
- package/lib/asr/manifest.json +1 -1
- package/lib/client.js +295 -51
- package/lib/server.js +2 -2
- package/lib/vad/manifest.json +4 -4
- package/lib/vad/vad.capture.js +12 -0
- package/lib/vad/vad.worker.js +1 -1
- package/package.json +2 -2
- package/src/app/ARCHITECTURE.md +2 -0
- package/src/app/client/apply.tsx +39 -11
- package/src/modules/conversation/models/AttendeeClient.ts +3 -1
- package/src/modules/conversation/models/meeting.ts +15 -8
- package/src/modules/core/ARCHITECTURE.md +2 -0
- package/src/modules/core/coordinator.ts +199 -24
- package/src/modules/recognition/engines/whisper/WhisperRecognitionEngine.ts +12 -0
- package/src/modules/recognition/vad/SileroVadRuntime.ts +43 -3
- package/src/modules/recognition/vad/vad.capture.ts +12 -0
- package/src/modules/recording/RecordingRecognition.ts +59 -4
|
@@ -4,7 +4,6 @@ import { TranscriptDraft } from './transcript.ts';
|
|
|
4
4
|
import { normalizeSettings } from './settings.ts';
|
|
5
5
|
import { SPEECH_QUEUE_PREFETCH_LIMIT } from '../speak/services/speechQueue.js';
|
|
6
6
|
import {
|
|
7
|
-
filterSpeechOutput,
|
|
8
7
|
hasMinimumWords,
|
|
9
8
|
hasUnclosedCodeFence,
|
|
10
9
|
hasIncompleteSpeechMarkdown,
|
|
@@ -62,6 +61,9 @@ export class VoiceCoordinator {
|
|
|
62
61
|
this.unfinished = new Set();
|
|
63
62
|
this.suppressed = new Set();
|
|
64
63
|
this.inputTail = Promise.resolve();
|
|
64
|
+
this.navigationPaused = false;
|
|
65
|
+
this.navigationEpoch = 0;
|
|
66
|
+
this.navigationResults = [];
|
|
65
67
|
this.speechBarrier = Promise.resolve();
|
|
66
68
|
this.disposed = false;
|
|
67
69
|
this.inputReleaseError = null;
|
|
@@ -325,6 +327,12 @@ export class VoiceCoordinator {
|
|
|
325
327
|
});
|
|
326
328
|
}
|
|
327
329
|
_recoverInput(error) {
|
|
330
|
+
if (this.navigationPaused) {
|
|
331
|
+
// Keep the recording and remaining STT queue; only an attached chat may
|
|
332
|
+
// decide to retry or replace its capture resources.
|
|
333
|
+
this.patch({ inputError: message(error), error: message(error) });
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
328
336
|
const attempt = this.inputRecoveryAttempt || 0;
|
|
329
337
|
const retry =
|
|
330
338
|
!this.inputReleaseError && isRetryableVoiceError(error) && attempt < MAX_RECOVERY_ATTEMPTS;
|
|
@@ -393,6 +401,27 @@ export class VoiceCoordinator {
|
|
|
393
401
|
this.inputTail = result.catch(() => {});
|
|
394
402
|
return result;
|
|
395
403
|
}
|
|
404
|
+
async _suspendInputCapture() {
|
|
405
|
+
return this._input(async () => {
|
|
406
|
+
if (!this.snapshot.listening || typeof this.recognition.suspendCapture !== 'function')
|
|
407
|
+
return false;
|
|
408
|
+
return (await this.recognition.suspendCapture()) !== false;
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
async _resumeInputCapture(valid = () => true) {
|
|
412
|
+
return this._input(async () => {
|
|
413
|
+
if (
|
|
414
|
+
!valid() ||
|
|
415
|
+
!this.snapshot.listening ||
|
|
416
|
+
typeof this.recognition.resumeCapture !== 'function'
|
|
417
|
+
)
|
|
418
|
+
return false;
|
|
419
|
+
return (await this.recognition.resumeCapture()) !== false;
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
_hasSuspendedInputCapture() {
|
|
423
|
+
return this.snapshot.listening && this.recognition.captureSuspended === true;
|
|
424
|
+
}
|
|
396
425
|
async _releaseInput() {
|
|
397
426
|
// Drain recorder-owned worklet buffers while its context/source still exist.
|
|
398
427
|
const recognitionResult = await Promise.resolve()
|
|
@@ -769,8 +798,19 @@ export class VoiceCoordinator {
|
|
|
769
798
|
}, this.snapshot.settings.assistantSpeechDelaySeconds * 1000);
|
|
770
799
|
}
|
|
771
800
|
|
|
772
|
-
onResult(
|
|
801
|
+
onResult(result) {
|
|
802
|
+
const {
|
|
803
|
+
final = '',
|
|
804
|
+
interim = '',
|
|
805
|
+
startedAt,
|
|
806
|
+
paragraphBefore = false,
|
|
807
|
+
onCommandRecognized,
|
|
808
|
+
} = result;
|
|
773
809
|
if (this.disposed || (!this.snapshot.listening && !this.snapshot.starting)) return;
|
|
810
|
+
if (this.navigationPaused) {
|
|
811
|
+
if (final) this.navigationResults.push(result);
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
774
814
|
const recognizing = this.recognition.activityManaged ? this.snapshot.recognizing : !!interim;
|
|
775
815
|
if (
|
|
776
816
|
this.snapshot.speaking &&
|
|
@@ -885,7 +925,7 @@ export class VoiceCoordinator {
|
|
|
885
925
|
return this.snapshot.pendingDeliveryTranscriptions ?? this.snapshot.pendingTranscriptions;
|
|
886
926
|
}
|
|
887
927
|
maybeScheduleAutoSend(draft = this.autoSendIntent) {
|
|
888
|
-
if (draft == null || draft !== this.composer.getDraft()) return;
|
|
928
|
+
if (this.navigationPaused || draft == null || draft !== this.composer.getDraft()) return;
|
|
889
929
|
if (this.autoSendTimer !== null) return;
|
|
890
930
|
if (
|
|
891
931
|
this.snapshot.settings.sendingMode === 'manual' ||
|
|
@@ -968,6 +1008,9 @@ export class VoiceCoordinator {
|
|
|
968
1008
|
if (this.autoSendIntent != null && draft !== this.autoSendIntent) this.cancelAutoSend();
|
|
969
1009
|
}
|
|
970
1010
|
stopListening() {
|
|
1011
|
+
++this.navigationEpoch;
|
|
1012
|
+
this.navigationPaused = false;
|
|
1013
|
+
this.navigationResults = [];
|
|
971
1014
|
this._cancelRecovery('input');
|
|
972
1015
|
this.holdToTalkRelease = false;
|
|
973
1016
|
this.cancelAutoSend();
|
|
@@ -1135,11 +1178,69 @@ export class VoiceCoordinator {
|
|
|
1135
1178
|
this.patch({ paused: checkpoint.paused && this.queue.length > 0 });
|
|
1136
1179
|
this._syncSpeechSegments();
|
|
1137
1180
|
}
|
|
1138
|
-
async
|
|
1181
|
+
async pauseInputForNavigation() {
|
|
1182
|
+
if (!this.snapshot.listening || typeof this.recognition.pauseForNavigation !== 'function')
|
|
1183
|
+
return false;
|
|
1184
|
+
const generation = ++this.navigationEpoch;
|
|
1185
|
+
this.navigationPaused = true;
|
|
1186
|
+
this.cancelAutoSend();
|
|
1187
|
+
try {
|
|
1188
|
+
const paused = await this._input(() => this.recognition.pauseForNavigation());
|
|
1189
|
+
if (!paused) throw new Error('Microphone capture could not be paused safely.');
|
|
1190
|
+
return true;
|
|
1191
|
+
} catch (error) {
|
|
1192
|
+
if (generation !== this.navigationEpoch || this.disposed) return false;
|
|
1193
|
+
// A failed pause must not leave a failed input ownership barrier forever.
|
|
1194
|
+
// Release hardware before another chat can acquire it; preserve already
|
|
1195
|
+
// completed results for the original composer.
|
|
1196
|
+
try {
|
|
1197
|
+
await this._input(() => this._releaseInput());
|
|
1198
|
+
this.inputReleaseError = null;
|
|
1199
|
+
} catch (cleanup) {
|
|
1200
|
+
this.inputReleaseError = cleanup;
|
|
1201
|
+
this.patch({ inputError: message(cleanup), error: message(cleanup) });
|
|
1202
|
+
throw cleanup;
|
|
1203
|
+
}
|
|
1204
|
+
if (generation !== this.navigationEpoch || this.disposed) return false;
|
|
1205
|
+
this.patch({
|
|
1206
|
+
listening: false,
|
|
1207
|
+
starting: false,
|
|
1208
|
+
recognizing: false,
|
|
1209
|
+
inputError: message(error),
|
|
1210
|
+
error: message(error),
|
|
1211
|
+
});
|
|
1212
|
+
return false;
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
async resumeInputFromNavigation() {
|
|
1216
|
+
if (!this.navigationPaused || this.disposed) return false;
|
|
1217
|
+
const generation = this.navigationEpoch;
|
|
1218
|
+
if (!this.snapshot.listening) {
|
|
1219
|
+
// The prior capture could not be paused and was released safely. A fresh
|
|
1220
|
+
// capture is permitted only after the user returns to this chat.
|
|
1221
|
+
const queued = this.navigationResults.splice(0);
|
|
1222
|
+
this.navigationPaused = false;
|
|
1223
|
+
await this._startInput(this.snapshot.conversation);
|
|
1224
|
+
if (this.disposed || generation !== this.navigationEpoch) return false;
|
|
1225
|
+
if (!this.snapshot.listening) {
|
|
1226
|
+
this.navigationPaused = true;
|
|
1227
|
+
this.navigationResults.unshift(...queued);
|
|
1228
|
+
return false;
|
|
1229
|
+
}
|
|
1230
|
+
for (const result of queued) this.onResult(result);
|
|
1231
|
+
return true;
|
|
1232
|
+
}
|
|
1233
|
+
const resumed = await this._input(() => this.recognition.resumeCapture());
|
|
1234
|
+
if (!resumed || this.disposed || generation !== this.navigationEpoch) return false;
|
|
1235
|
+
this.navigationPaused = false;
|
|
1236
|
+
for (const result of this.navigationResults.splice(0)) this.onResult(result);
|
|
1237
|
+
return true;
|
|
1238
|
+
}
|
|
1239
|
+
async suspendSpeech(preserveInput = false) {
|
|
1139
1240
|
this.speechSuspended = true;
|
|
1140
1241
|
await Promise.all([
|
|
1141
1242
|
VoiceCoordinator.prototype.stopSpeech.call(this, false),
|
|
1142
|
-
this.stopListening(),
|
|
1243
|
+
preserveInput ? this.pauseInputForNavigation() : this.stopListening(),
|
|
1143
1244
|
]);
|
|
1144
1245
|
}
|
|
1145
1246
|
discardSpeechMessage(id) {
|
|
@@ -1206,6 +1307,20 @@ export class VoiceCoordinator {
|
|
|
1206
1307
|
if (epoch === this.speechEpoch) this.patch({ error: message(error) });
|
|
1207
1308
|
return;
|
|
1208
1309
|
}
|
|
1310
|
+
if (
|
|
1311
|
+
epoch === this.speechEpoch &&
|
|
1312
|
+
!this.disposed &&
|
|
1313
|
+
resumeListening &&
|
|
1314
|
+
!this.navigationPaused &&
|
|
1315
|
+
this.snapshot.settings.mode === 'speaker' &&
|
|
1316
|
+
this._hasSuspendedInputCapture()
|
|
1317
|
+
) {
|
|
1318
|
+
try {
|
|
1319
|
+
await this._resumeInputCapture();
|
|
1320
|
+
} catch (error) {
|
|
1321
|
+
this.patch({ inputError: message(error), error: message(error) });
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1209
1324
|
if (
|
|
1210
1325
|
epoch === this.speechEpoch &&
|
|
1211
1326
|
!this.disposed &&
|
|
@@ -1259,23 +1374,67 @@ export class VoiceCoordinator {
|
|
|
1259
1374
|
getSpeechProgress() {
|
|
1260
1375
|
return this.engines[this.snapshot.speechEngine]?.getPlaybackProgress?.() ?? null;
|
|
1261
1376
|
}
|
|
1377
|
+
async _restorePausedSpeechInput(epoch, control) {
|
|
1378
|
+
const valid = () =>
|
|
1379
|
+
!this.disposed &&
|
|
1380
|
+
!this.navigationPaused &&
|
|
1381
|
+
epoch === this.speechEpoch &&
|
|
1382
|
+
control === this.controlEpoch &&
|
|
1383
|
+
this.snapshot.paused &&
|
|
1384
|
+
this.snapshot.settings.mode === 'speaker';
|
|
1385
|
+
if (!valid()) return;
|
|
1386
|
+
try {
|
|
1387
|
+
if (this._hasSuspendedInputCapture()) {
|
|
1388
|
+
await this._resumeInputCapture(valid);
|
|
1389
|
+
} else if (
|
|
1390
|
+
this.snapshot.conversation &&
|
|
1391
|
+
!this.snapshot.listening &&
|
|
1392
|
+
!this.snapshot.starting &&
|
|
1393
|
+
!this.snapshot.inputRecovery
|
|
1394
|
+
) {
|
|
1395
|
+
await this._startInput(true);
|
|
1396
|
+
}
|
|
1397
|
+
} catch (error) {
|
|
1398
|
+
if (valid()) this.patch({ inputError: message(error), error: message(error) });
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
async _gateResumedSpeechInput() {
|
|
1402
|
+
if (this.snapshot.settings.mode !== 'speaker') return;
|
|
1403
|
+
// Preserve recording-backed capture; native engines retain safe stop/restart.
|
|
1404
|
+
const suspended = await this._suspendInputCapture();
|
|
1405
|
+
if (!suspended) {
|
|
1406
|
+
await this.stopListening();
|
|
1407
|
+
if (this.inputReleaseError) throw this.inputReleaseError;
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1262
1410
|
async pauseSpeech(automatic = false) {
|
|
1263
1411
|
if (this.snapshot.speechRecovery?.retrying) this._cancelRecovery('speech', true);
|
|
1264
1412
|
if (!automatic) {
|
|
1265
1413
|
this._cancelInterruptionResume();
|
|
1266
1414
|
this.interruptionPausedSpeech = false;
|
|
1267
1415
|
}
|
|
1268
|
-
if (this.disposed
|
|
1416
|
+
if (this.disposed) return;
|
|
1417
|
+
if (this.snapshot.paused) {
|
|
1418
|
+
// Reaffirming pause invalidates a resume whose capture gate is still pending.
|
|
1419
|
+
const epoch = this.speechEpoch;
|
|
1420
|
+
const control = ++this.controlEpoch;
|
|
1421
|
+
await this.inputTail;
|
|
1422
|
+
await this._restorePausedSpeechInput(epoch, control);
|
|
1423
|
+
return;
|
|
1424
|
+
}
|
|
1269
1425
|
if (!this.snapshot.speaking && this.snapshot.speechRunActive) {
|
|
1270
1426
|
this._cancelAssistantSpeechTimer();
|
|
1427
|
+
const control = ++this.controlEpoch;
|
|
1271
1428
|
this.patch({ paused: true });
|
|
1429
|
+
await this._restorePausedSpeechInput(this.speechEpoch, control);
|
|
1272
1430
|
return;
|
|
1273
1431
|
}
|
|
1274
1432
|
if (!this.snapshot.speaking) return;
|
|
1275
1433
|
if (this.speechLaunchPending) {
|
|
1276
1434
|
// Preparation has no player to pause yet; retain an explicit launch gate.
|
|
1277
|
-
++this.controlEpoch;
|
|
1435
|
+
const control = ++this.controlEpoch;
|
|
1278
1436
|
this.patch({ paused: true });
|
|
1437
|
+
await this._restorePausedSpeechInput(this.speechEpoch, control);
|
|
1279
1438
|
return;
|
|
1280
1439
|
}
|
|
1281
1440
|
const epoch = this.speechEpoch;
|
|
@@ -1292,6 +1451,7 @@ export class VoiceCoordinator {
|
|
|
1292
1451
|
) {
|
|
1293
1452
|
this.patch({ paused: true });
|
|
1294
1453
|
this.activeRecordingTimeline?.('assistant-pause');
|
|
1454
|
+
await this._restorePausedSpeechInput(epoch, control);
|
|
1295
1455
|
}
|
|
1296
1456
|
} catch (error) {
|
|
1297
1457
|
if (epoch === this.speechEpoch) this.patch({ error: message(error) });
|
|
@@ -1311,12 +1471,11 @@ export class VoiceCoordinator {
|
|
|
1311
1471
|
if (this.speechLaunchPending) {
|
|
1312
1472
|
const epoch = this.speechEpoch;
|
|
1313
1473
|
const control = ++this.controlEpoch;
|
|
1314
|
-
|
|
1315
|
-
await this.
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
}
|
|
1474
|
+
try {
|
|
1475
|
+
await this._gateResumedSpeechInput();
|
|
1476
|
+
} catch (error) {
|
|
1477
|
+
if (epoch === this.speechEpoch) this.patch({ error: message(error) });
|
|
1478
|
+
return;
|
|
1320
1479
|
}
|
|
1321
1480
|
if (epoch === this.speechEpoch && control === this.controlEpoch)
|
|
1322
1481
|
this.patch({ paused: false });
|
|
@@ -1325,10 +1484,7 @@ export class VoiceCoordinator {
|
|
|
1325
1484
|
const epoch = this.speechEpoch;
|
|
1326
1485
|
const control = ++this.controlEpoch;
|
|
1327
1486
|
try {
|
|
1328
|
-
|
|
1329
|
-
await this.stopListening();
|
|
1330
|
-
if (this.inputReleaseError) throw this.inputReleaseError;
|
|
1331
|
-
}
|
|
1487
|
+
await this._gateResumedSpeechInput();
|
|
1332
1488
|
if (epoch !== this.speechEpoch || control !== this.controlEpoch) return;
|
|
1333
1489
|
const result =
|
|
1334
1490
|
await this.engines[this.snapshot.speechEngine || this.snapshot.settings.engine].resume();
|
|
@@ -1336,9 +1492,12 @@ export class VoiceCoordinator {
|
|
|
1336
1492
|
this.patch({ paused: false });
|
|
1337
1493
|
this.activeRecordingTimeline?.('assistant-resume');
|
|
1338
1494
|
this._prefetchSpeech();
|
|
1495
|
+
} else {
|
|
1496
|
+
await this._restorePausedSpeechInput(epoch, control);
|
|
1339
1497
|
}
|
|
1340
1498
|
} catch (error) {
|
|
1341
1499
|
if (epoch === this.speechEpoch) this.patch({ error: message(error) });
|
|
1500
|
+
await this._restorePausedSpeechInput(epoch, control);
|
|
1342
1501
|
}
|
|
1343
1502
|
}
|
|
1344
1503
|
async speak(text, messageId = null) {
|
|
@@ -1422,15 +1581,23 @@ export class VoiceCoordinator {
|
|
|
1422
1581
|
if (epoch !== this.speechEpoch || this.disposed) return;
|
|
1423
1582
|
if (this.snapshot.settings.mode === 'speaker') {
|
|
1424
1583
|
const gating = this.snapshot.listening;
|
|
1584
|
+
const persistentCapture = typeof this.recognition.suspendCapture === 'function';
|
|
1425
1585
|
if (gating) timeline?.('capture-gated');
|
|
1426
|
-
|
|
1427
|
-
|
|
1586
|
+
try {
|
|
1587
|
+
const suspended = persistentCapture ? await this._suspendInputCapture() : false;
|
|
1588
|
+
if (gating && !suspended) {
|
|
1589
|
+
await this.stopListening();
|
|
1590
|
+
if (this.inputReleaseError) throw this.inputReleaseError;
|
|
1591
|
+
}
|
|
1592
|
+
} catch (error) {
|
|
1428
1593
|
if (gating) timeline?.('capture-gate-failed');
|
|
1429
|
-
throw
|
|
1594
|
+
throw error;
|
|
1430
1595
|
}
|
|
1431
1596
|
if (gating) timeline?.('input-stop-complete');
|
|
1432
1597
|
}
|
|
1433
1598
|
if (epoch !== this.speechEpoch || this.disposed) return;
|
|
1599
|
+
// A pause can arrive while the initial capture gate is still settling.
|
|
1600
|
+
if (this.snapshot.paused) await this._restorePausedSpeechInput(epoch, this.controlEpoch);
|
|
1434
1601
|
const queued = item;
|
|
1435
1602
|
if (queued?.preparing) await queued.preparing;
|
|
1436
1603
|
if (epoch !== this.speechEpoch || this.disposed) return;
|
|
@@ -1572,6 +1739,12 @@ export class VoiceCoordinator {
|
|
|
1572
1739
|
this.assistantSpeechDelayReason = 'segment-gap';
|
|
1573
1740
|
this.assistantSpeechNotBefore = Date.now() + this.snapshot.settings.segmentGapMs;
|
|
1574
1741
|
this._drain();
|
|
1742
|
+
} else if (this.snapshot.settings.mode === 'speaker' && this.snapshot.listening) {
|
|
1743
|
+
try {
|
|
1744
|
+
await this._resumeInputCapture();
|
|
1745
|
+
} catch (error) {
|
|
1746
|
+
this.patch({ inputError: message(error), error: message(error) });
|
|
1747
|
+
}
|
|
1575
1748
|
} else if (
|
|
1576
1749
|
this.snapshot.conversation &&
|
|
1577
1750
|
!this.snapshot.inputRecovery &&
|
|
@@ -1712,16 +1885,18 @@ export class VoiceCoordinator {
|
|
|
1712
1885
|
if (boundary <= 0) return;
|
|
1713
1886
|
if (!complete && hasIncompleteSpeechMarkdown(remaining.slice(0, boundary))) return;
|
|
1714
1887
|
if (hasUnclosedCodeFence(remaining.slice(0, boundary)) && !complete) return;
|
|
1715
|
-
const
|
|
1888
|
+
const segments = splitSpeechOutput(remaining.slice(0, boundary).trim(), {
|
|
1716
1889
|
filterCodeBlocks: this.snapshot.settings.outputCodeFilterEnabled,
|
|
1717
1890
|
codeBlockMaxLines: this.snapshot.settings.outputCodeMaxLines,
|
|
1718
1891
|
codeBlockNotice: this.snapshot.settings.outputCodeNotice,
|
|
1719
1892
|
lang: this.snapshot.settings.lang,
|
|
1720
|
-
})
|
|
1893
|
+
});
|
|
1721
1894
|
this.consumed.set(id, offset + boundary);
|
|
1722
|
-
if (
|
|
1895
|
+
if (segments.length) {
|
|
1723
1896
|
const initial = !this.snapshot.speaking && !this.queue.some((item) => item.prepared);
|
|
1724
|
-
const parts = initial
|
|
1897
|
+
const parts = initial
|
|
1898
|
+
? [...splitInitialSpeechItem(segments[0]), ...segments.slice(1)]
|
|
1899
|
+
: segments;
|
|
1725
1900
|
this.queue.push(...parts.map((text) => ({ text, id })));
|
|
1726
1901
|
this._prefetchSpeech();
|
|
1727
1902
|
this._syncSpeechSegments();
|
|
@@ -174,6 +174,18 @@ export class WhisperHttpRecognitionEngine {
|
|
|
174
174
|
finish() {
|
|
175
175
|
return this.session?.finishRecording() ?? Promise.resolve();
|
|
176
176
|
}
|
|
177
|
+
get captureSuspended() {
|
|
178
|
+
return this.session?.captureSuspended === true;
|
|
179
|
+
}
|
|
180
|
+
suspendCapture() {
|
|
181
|
+
return this.session?.suspendCapture() ?? Promise.resolve(false);
|
|
182
|
+
}
|
|
183
|
+
resumeCapture() {
|
|
184
|
+
return this.session?.resumeCapture() ?? Promise.resolve(false);
|
|
185
|
+
}
|
|
186
|
+
pauseForNavigation() {
|
|
187
|
+
return this.session?.pauseForNavigation() ?? Promise.resolve(false);
|
|
188
|
+
}
|
|
177
189
|
async stop() {
|
|
178
190
|
this.startGeneration = (this.startGeneration || 0) + 1;
|
|
179
191
|
const session = this.session;
|
|
@@ -364,13 +364,18 @@ export async function attachSileroVadCapture(
|
|
|
364
364
|
source: AudioNode,
|
|
365
365
|
stream: SileroVadStream,
|
|
366
366
|
onPcm?: (pcm: Float32Array, sampleRate: number) => void,
|
|
367
|
-
): Promise<{
|
|
367
|
+
): Promise<{
|
|
368
|
+
finish(): Promise<void>;
|
|
369
|
+
flush(pause?: boolean): Promise<void>;
|
|
370
|
+
resume(): void;
|
|
371
|
+
release(): void;
|
|
372
|
+
}> {
|
|
368
373
|
if (!context.audioWorklet || typeof AudioWorkletNode !== 'function')
|
|
369
374
|
throw new Error('AudioWorklet PCM capture is unavailable.');
|
|
370
375
|
let loading = captureLoads.get(context);
|
|
371
376
|
if (!loading) {
|
|
372
377
|
loading = context.audioWorklet.addModule(
|
|
373
|
-
sileroVadAssetBase() + 'vad.capture.js?revision=
|
|
378
|
+
sileroVadAssetBase() + 'vad.capture.js?revision=pause-v2',
|
|
374
379
|
);
|
|
375
380
|
captureLoads.set(context, loading);
|
|
376
381
|
void loading.catch(() => {
|
|
@@ -389,6 +394,10 @@ export async function attachSileroVadCapture(
|
|
|
389
394
|
let released = false;
|
|
390
395
|
let acknowledged = false;
|
|
391
396
|
let finishing: Promise<void> | undefined;
|
|
397
|
+
let flushing: Promise<void> | undefined;
|
|
398
|
+
let resolveFlush: (() => void) | undefined;
|
|
399
|
+
let rejectFlush: ((error: Error) => void) | undefined;
|
|
400
|
+
let flushTimer: ReturnType<typeof setTimeout> | undefined;
|
|
392
401
|
let resolveFinish: (() => void) | undefined;
|
|
393
402
|
let rejectFinish: ((error: Error) => void) | undefined;
|
|
394
403
|
let failure: Error | undefined;
|
|
@@ -398,6 +407,8 @@ export async function attachSileroVadCapture(
|
|
|
398
407
|
if (released) return;
|
|
399
408
|
released = true;
|
|
400
409
|
clearTimeout(timer);
|
|
410
|
+
clearTimeout(flushTimer);
|
|
411
|
+
rejectFlush?.(cancelled());
|
|
401
412
|
unsubscribe();
|
|
402
413
|
node.port.onmessage = null;
|
|
403
414
|
node.onprocessorerror = null;
|
|
@@ -415,6 +426,8 @@ export async function attachSileroVadCapture(
|
|
|
415
426
|
if (released) return;
|
|
416
427
|
failure = error;
|
|
417
428
|
rejectFinish?.(error);
|
|
429
|
+
rejectFlush?.(error);
|
|
430
|
+
rejectFlush = undefined;
|
|
418
431
|
cleanup();
|
|
419
432
|
};
|
|
420
433
|
unsubscribe = stream.onInvalidated((error) => {
|
|
@@ -427,7 +440,15 @@ export async function attachSileroVadCapture(
|
|
|
427
440
|
}
|
|
428
441
|
node.port.onmessage = (event) => {
|
|
429
442
|
if (released || stream.released) return;
|
|
430
|
-
if (event.data?.type === '
|
|
443
|
+
if (event.data?.type === 'flushed') {
|
|
444
|
+
clearTimeout(flushTimer);
|
|
445
|
+
void stream.flush().then(() => {
|
|
446
|
+
const done = resolveFlush;
|
|
447
|
+
resolveFlush = rejectFlush = undefined;
|
|
448
|
+
flushing = undefined;
|
|
449
|
+
done?.();
|
|
450
|
+
}, fail);
|
|
451
|
+
} else if (event.data?.type === 'stopped') {
|
|
431
452
|
if (!finishing || acknowledged) return;
|
|
432
453
|
acknowledged = true;
|
|
433
454
|
clearTimeout(timer);
|
|
@@ -469,6 +490,25 @@ export async function attachSileroVadCapture(
|
|
|
469
490
|
throw error;
|
|
470
491
|
}
|
|
471
492
|
return {
|
|
493
|
+
resume() {
|
|
494
|
+
if (released || finishing) throw failure || cancelled();
|
|
495
|
+
node.port.postMessage({ type: 'resume' });
|
|
496
|
+
},
|
|
497
|
+
flush(pause = false) {
|
|
498
|
+
if (flushing) return flushing;
|
|
499
|
+
if (released || stream.released || finishing) return Promise.reject(failure || cancelled());
|
|
500
|
+
flushing = new Promise<void>((resolve, reject) => {
|
|
501
|
+
resolveFlush = resolve;
|
|
502
|
+
rejectFlush = reject;
|
|
503
|
+
});
|
|
504
|
+
flushTimer = setTimeout(() => fail(new Error('VAD capture flush timed out.')), 3000);
|
|
505
|
+
try {
|
|
506
|
+
node.port.postMessage({ type: 'flush', pause });
|
|
507
|
+
} catch (error) {
|
|
508
|
+
fail(error instanceof Error ? error : new Error('VAD capture flush failed.'));
|
|
509
|
+
}
|
|
510
|
+
return flushing;
|
|
511
|
+
},
|
|
472
512
|
finish() {
|
|
473
513
|
if (finishing) return finishing;
|
|
474
514
|
if (released || stream.released) return (finishing = Promise.reject(failure || cancelled()));
|
|
@@ -6,11 +6,22 @@ declare class AudioWorkletProcessor {
|
|
|
6
6
|
declare function registerProcessor(name: string, processor: typeof AudioWorkletProcessor): void;
|
|
7
7
|
class VadCaptureProcessor extends AudioWorkletProcessor {
|
|
8
8
|
private stopped = false;
|
|
9
|
+
private paused = false;
|
|
9
10
|
private pcm = new Float32Array(2048);
|
|
10
11
|
private at = 0;
|
|
11
12
|
constructor() {
|
|
12
13
|
super();
|
|
13
14
|
this.port.onmessage = (event) => {
|
|
15
|
+
if (event.data?.type === 'resume') this.paused = false;
|
|
16
|
+
if (event.data?.type === 'flush') {
|
|
17
|
+
if (event.data.pause) this.paused = true;
|
|
18
|
+
if (this.at) {
|
|
19
|
+
const pcm = this.pcm.slice(0, this.at);
|
|
20
|
+
this.port.postMessage({ pcm, sampleRate }, [pcm.buffer]);
|
|
21
|
+
this.at = 0;
|
|
22
|
+
}
|
|
23
|
+
this.port.postMessage({ type: 'flushed' });
|
|
24
|
+
}
|
|
14
25
|
if (event.data?.type === 'stop') {
|
|
15
26
|
this.stopped = true;
|
|
16
27
|
if (event.data.drain && this.at) {
|
|
@@ -25,6 +36,7 @@ class VadCaptureProcessor extends AudioWorkletProcessor {
|
|
|
25
36
|
process(inputs: Float32Array[][], outputs: Float32Array[][]): boolean {
|
|
26
37
|
for (const channels of outputs) for (const channel of channels) channel.fill(0);
|
|
27
38
|
if (this.stopped) return false;
|
|
39
|
+
if (this.paused) return true;
|
|
28
40
|
const input = inputs[0];
|
|
29
41
|
if (!input?.length) return true;
|
|
30
42
|
for (let i = 0; i < input[0].length; i++) {
|
|
@@ -10,7 +10,12 @@ import type {
|
|
|
10
10
|
RecordingVoiceCommand,
|
|
11
11
|
} from './types.js';
|
|
12
12
|
import type { SileroVadOptions, SileroVadStream } from '../recognition/vad/SileroVadRuntime.js';
|
|
13
|
-
type CaptureHandle = {
|
|
13
|
+
type CaptureHandle = {
|
|
14
|
+
finish(): Promise<void>;
|
|
15
|
+
flush?(pause?: boolean): Promise<void>;
|
|
16
|
+
resume?(): void;
|
|
17
|
+
release(): void;
|
|
18
|
+
};
|
|
14
19
|
type Processing = {
|
|
15
20
|
queued: number;
|
|
16
21
|
active: boolean;
|
|
@@ -75,6 +80,8 @@ interface RecordingEngine {
|
|
|
75
80
|
stream: SileroVadStream,
|
|
76
81
|
onPcm: (pcm: Float32Array, rate: number) => void,
|
|
77
82
|
): Promise<CaptureHandle>;
|
|
83
|
+
suspendCapture?(): Promise<boolean>;
|
|
84
|
+
resumeCapture?(): Promise<boolean>;
|
|
78
85
|
stop(): Promise<void>;
|
|
79
86
|
}
|
|
80
87
|
interface RecognitionSession {
|
|
@@ -97,6 +104,10 @@ interface RecognitionSession {
|
|
|
97
104
|
lang: string;
|
|
98
105
|
operation: string;
|
|
99
106
|
capturedSamples: number;
|
|
107
|
+
captureSuspended: boolean;
|
|
108
|
+
suspendCapture(): Promise<boolean>;
|
|
109
|
+
resumeCapture(): Promise<boolean>;
|
|
110
|
+
pauseForNavigation(): Promise<boolean>;
|
|
100
111
|
pinDetector?: PinDetector;
|
|
101
112
|
finishRecording(): Promise<void>;
|
|
102
113
|
stopRecording(): Promise<void>;
|
|
@@ -157,6 +168,7 @@ export async function startRecordingRecognition(
|
|
|
157
168
|
await writer.finish('interrupted');
|
|
158
169
|
throw Object.assign(new Error('Cancelled'), { name: 'AbortError' });
|
|
159
170
|
}
|
|
171
|
+
let suspension: Promise<boolean> | null = null;
|
|
160
172
|
const session: RecognitionSession = {
|
|
161
173
|
writer,
|
|
162
174
|
detector: capture.silero ? 'silero' : 'energy',
|
|
@@ -177,6 +189,31 @@ export async function startRecordingRecognition(
|
|
|
177
189
|
lang,
|
|
178
190
|
operation: uid(),
|
|
179
191
|
capturedSamples: 0,
|
|
192
|
+
captureSuspended: false,
|
|
193
|
+
suspendCapture: () => {
|
|
194
|
+
if (session.captureSuspended) return Promise.resolve(true);
|
|
195
|
+
if (suspension) return suspension;
|
|
196
|
+
if (session.finished || !valid()) return Promise.resolve(false);
|
|
197
|
+
suspension = (async () => {
|
|
198
|
+
await engine.meter.context.suspend();
|
|
199
|
+
if (!valid()) return false;
|
|
200
|
+
session.captureSuspended = true;
|
|
201
|
+
return true;
|
|
202
|
+
})().finally(() => {
|
|
203
|
+
suspension = null;
|
|
204
|
+
});
|
|
205
|
+
return suspension;
|
|
206
|
+
},
|
|
207
|
+
resumeCapture: async () => {
|
|
208
|
+
if (!session.captureSuspended) return true;
|
|
209
|
+
if (session.finished || !valid()) return false;
|
|
210
|
+
session.capture?.resume?.();
|
|
211
|
+
await engine.meter.context.resume();
|
|
212
|
+
if (!valid()) return false;
|
|
213
|
+
session.captureSuspended = false;
|
|
214
|
+
return true;
|
|
215
|
+
},
|
|
216
|
+
pauseForNavigation: async () => false,
|
|
180
217
|
finishRecording: async () => {},
|
|
181
218
|
stopRecording: async () => {},
|
|
182
219
|
abort: () => {},
|
|
@@ -285,10 +322,13 @@ export async function startRecordingRecognition(
|
|
|
285
322
|
queuedThrough = Math.max(queuedThrough, end);
|
|
286
323
|
await releaseCapturedRanges();
|
|
287
324
|
};
|
|
288
|
-
async function releaseCapturedRanges() {
|
|
325
|
+
async function releaseCapturedRanges(clampToCaptured = false) {
|
|
289
326
|
if (!valid()) return;
|
|
290
327
|
const captured = writer.session.capturedSamples;
|
|
291
|
-
while (
|
|
328
|
+
while (
|
|
329
|
+
awaitingTail.length &&
|
|
330
|
+
(session.finished || clampToCaptured || awaitingTail[0].endSample <= captured)
|
|
331
|
+
) {
|
|
292
332
|
const pending = awaitingTail[0];
|
|
293
333
|
const attempt = { ...pending, endSample: Math.min(pending.endSample, captured) };
|
|
294
334
|
await writer.attempt(attempt);
|
|
@@ -346,6 +386,21 @@ export async function startRecordingRecognition(
|
|
|
346
386
|
},
|
|
347
387
|
});
|
|
348
388
|
session.pinDetector = detector;
|
|
389
|
+
session.pauseForNavigation = async () => {
|
|
390
|
+
// Pause at the worklet boundary and drain its partial buffer before freezing
|
|
391
|
+
// the context. No port acknowledgement is awaited on a suspended context.
|
|
392
|
+
// Speaker playback may already be suspending this context. Wait for that
|
|
393
|
+
// transition before deciding whether its worklet can acknowledge a flush.
|
|
394
|
+
if (suspension) await suspension;
|
|
395
|
+
if (!session.captureSuspended) await session.capture?.flush?.(true);
|
|
396
|
+
if (!(await session.suspendCapture()) || !valid()) return false;
|
|
397
|
+
// Close an open utterance at the real capture boundary, without sealing
|
|
398
|
+
// the writer. Requests already running retain their original callbacks.
|
|
399
|
+
detector.finish();
|
|
400
|
+
await session.tail;
|
|
401
|
+
await serialize(() => releaseCapturedRanges(true), false, true);
|
|
402
|
+
return valid();
|
|
403
|
+
};
|
|
349
404
|
async function drain() {
|
|
350
405
|
if (session.draining || !valid()) return;
|
|
351
406
|
session.draining = true;
|
|
@@ -633,7 +688,7 @@ export async function startRecordingRecognition(
|
|
|
633
688
|
awaitingTail.length = 0;
|
|
634
689
|
await recovering;
|
|
635
690
|
session.activeRequest?.abort();
|
|
636
|
-
if (session.finishing) session.capture?.release();
|
|
691
|
+
if (session.finishing || session.captureSuspended) session.capture?.release();
|
|
637
692
|
else {
|
|
638
693
|
try {
|
|
639
694
|
await session.capture?.finish();
|