@agfpd/voicetalk-runtime 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/docs/README.md +30 -2
- package/docs/ru/README.md +30 -2
- package/package.json +1 -1
- package/src/audio.ts +46 -1
- package/src/voiceConnect.ts +6 -5
- package/src/voiceSession.ts +397 -66
- package/src/wake.ts +80 -22
package/README.md
CHANGED
|
@@ -61,11 +61,17 @@ runs the same router text-only, with no microphone at all. Configuration details
|
|
|
61
61
|
- **Modality framing for a naive agent.** An agent has no senses — only envelope text. voicetalk marks the inbound envelope so a peer with no knowledge of the system answers in the right modality ("spoken, live — reply by voice, briefly").
|
|
62
62
|
- **Voice stays out of the core.** Speech-to-text and text-to-speech live in the shared `voice-connect` service (multi-provider, one place to configure), not inside this runtime and not inside the iapeer foundation.
|
|
63
63
|
- **Conversational turn-taking.** A wake-word only opens a conversation; after a reply, an open-mic window lets the next utterance continue without re-triggering. Endpointing and earcons are owned here, inside the audio session — the agent only sees the next message.
|
|
64
|
+
- **Wake evidence fails closed.** Configured names are never injected into Whisper's
|
|
65
|
+
decoder prompt: only an exact leading name heard in the unprimed transcript can open
|
|
66
|
+
a dialogue, and repeated-name decoder loops are discarded before IAP delivery.
|
|
64
67
|
|
|
65
68
|
## What it does not do (yet)
|
|
66
69
|
|
|
67
70
|
- No barge-in: the owner cannot interrupt a spoken answer by talking over it (that needs acoustic echo cancellation plus a neural VAD). While the runtime speaks, it does not listen.
|
|
68
|
-
- Wake detection runs on
|
|
71
|
+
- Wake detection runs on an unprimed transcript, not on an acoustic wake-word model —
|
|
72
|
+
so a wake word costs one line of config and no training, at the price of transcribing
|
|
73
|
+
every voiced segment. A bounded private ring retains wake-candidate segments for
|
|
74
|
+
replayable diagnostics; an acoustic pre-gate remains the stronger future upgrade.
|
|
69
75
|
- Single host, single owner, local microphone and speaker. Multi-device and multi-owner are out of scope for the MVP.
|
|
70
76
|
|
|
71
77
|
## Roadmap
|
package/docs/README.md
CHANGED
|
@@ -41,6 +41,11 @@ self-config hook. `voicetalk-runtime devices` lists the host's inputs;
|
|
|
41
41
|
`voicetalk-runtime doctor` reports the whole contour (binding, wake map,
|
|
42
42
|
voice-connect, microphone).
|
|
43
43
|
|
|
44
|
+
Wake evidence is fail-closed. The configured vocabulary is never passed as a Whisper
|
|
45
|
+
decoder prompt, because prompt text can be generated back on unrelated audio. A closed
|
|
46
|
+
dialogue opens only on an exact leading address in the unprimed transcript; repeated
|
|
47
|
+
token loops and name-only echoes are rejected before an IAP envelope exists.
|
|
48
|
+
|
|
44
49
|
## How a turn works
|
|
45
50
|
|
|
46
51
|
1. The microphone runs always-on; energy endpointing cuts an utterance when the
|
|
@@ -48,9 +53,19 @@ voice-connect, microphone).
|
|
|
48
53
|
2. voice-connect transcribes the utterance (locally, on this host).
|
|
49
54
|
3. If it opens by addressing a peer, the body is delivered to that peer as an IAP
|
|
50
55
|
envelope from the owner's identity, framed for the spoken modality.
|
|
56
|
+
3.5. Fragments are COMPOSED into one turn. Endpointing cuts audio, not thoughts: a
|
|
57
|
+
pause inside a sentence would otherwise become a second envelope and earn a second
|
|
58
|
+
spoken answer. The turn is sent once the owner has been silent for ~3 s since their
|
|
59
|
+
last word — and never while they are still speaking or a fragment is still being
|
|
60
|
+
recognized. A fragment left hanging on a comma or a conjunction waits longer.
|
|
51
61
|
4. The agent's answer arrives on stdin, is spoken through voice-connect on the host
|
|
52
62
|
speaker, and the microphone stays open for ~9 s so the owner can continue without
|
|
53
|
-
naming the peer again.
|
|
63
|
+
naming the peer again. When that window closes, a soft low tone says so.
|
|
64
|
+
|
|
65
|
+
Inside an open conversation a wake word for ANOTHER peer must look deliberate — the
|
|
66
|
+
name heard letter for letter, followed by a real request — otherwise the fragment goes
|
|
67
|
+
to the peer already listening. Mis-hearing a name should cost a stray line, never the
|
|
68
|
+
whole conversation.
|
|
54
69
|
|
|
55
70
|
Earcons mark the states: rising = the microphone is yours, short descending = heard
|
|
56
71
|
you, double descending = nothing was routed. Processing is silent.
|
|
@@ -58,7 +73,20 @@ you, double descending = nothing was routed. Processing is silent.
|
|
|
58
73
|
Everything is tunable by env: `VOICETALK_AUDIO=0` (text-only, no microphone),
|
|
59
74
|
`VOICETALK_FOLLOWUP_MS`, `VOICETALK_HANGOVER_MS`, `VOICETALK_VAD_MIN_RMS`,
|
|
60
75
|
`VOICETALK_VAD_RATIO`, `VOICETALK_STT_LANG`, `VOICETALK_STT_ENGINE`,
|
|
61
|
-
`VOICETALK_TTS_MAX_CHARS`, `VOICETALK_EARCONS=0
|
|
76
|
+
`VOICETALK_TTS_MAX_CHARS`, `VOICETALK_EARCONS=0`, `VOICETALK_COMPOSE_MS`,
|
|
77
|
+
`VOICETALK_COMPOSE_UNFINISHED_MS`, `VOICETALK_TURN_MAX_MS`,
|
|
78
|
+
`VOICETALK_WAKE_CAPTURE=0` (disable the default diagnostic ring), and
|
|
79
|
+
`VOICETALK_WAKE_CAPTURE_MAX` (default 16).
|
|
80
|
+
|
|
81
|
+
Every transcript that resembles a configured wake is retained as a mode-0600 WAV plus
|
|
82
|
+
JSON verdict under `<IAPEER_ROOT>/runtimes/voicetalk/tmp/wake-captures`. The ring is
|
|
83
|
+
bounded and contains candidate utterances only, not continuous room audio. Replay a
|
|
84
|
+
disputed segment through the same paced microphone path described below.
|
|
85
|
+
|
|
86
|
+
Two seams exist for testing turn-taking without touching a live channel:
|
|
87
|
+
`VOICETALK_MIC_FILE=<pcm>` replays a recording through the microphone path (paced in
|
|
88
|
+
real time) and `VOICETALK_PLAYBACK=0` keeps the speaker silent while synthesis is
|
|
89
|
+
still performed and timed. `scripts/replay-dialogue.sh` wires both into a sandbox.
|
|
62
90
|
|
|
63
91
|
## Requirements
|
|
64
92
|
|
package/docs/ru/README.md
CHANGED
|
@@ -41,6 +41,11 @@ self-config-хука (`VOICETALK_WAKE_WORDS`, `VOICETALK_DEFAULT_TARGET`,
|
|
|
41
41
|
`voicetalk-runtime doctor` — весь контур целиком (привязка, карта, voice-connect,
|
|
42
42
|
микрофон).
|
|
43
43
|
|
|
44
|
+
Wake-сигнал работает fail-closed. Настроенные имена не передаются Whisper как decoder
|
|
45
|
+
prompt: такой контекст способен сам породить имя на постороннем аудио. Закрытый диалог
|
|
46
|
+
открывает только точное имя в начале непримированного транскрипта; циклы повторяющихся
|
|
47
|
+
токенов и тело из одних настроенных имён отбрасываются до появления IAP-конверта.
|
|
48
|
+
|
|
44
49
|
## Как устроен ход разговора
|
|
45
50
|
|
|
46
51
|
1. Микрофон слушает постоянно; энергетический эндпойнтер закрывает реплику по паузе
|
|
@@ -48,8 +53,18 @@ self-config-хука (`VOICETALK_WAKE_WORDS`, `VOICETALK_DEFAULT_TARGET`,
|
|
|
48
53
|
2. voice-connect распознаёт реплику (локально, на этом хосте).
|
|
49
54
|
3. Если реплика начинается с обращения к пиру, тело уходит ему IAP-конвертом от
|
|
50
55
|
личности владельца, с рамкой голосовой модальности.
|
|
56
|
+
3.5. Фрагменты СОБИРАЮТСЯ в один ход. Эндпойнтер режет звук, а не мысль: пауза внутри
|
|
57
|
+
фразы иначе стала бы вторым конвертом и заработала бы второй голосовой ответ. Ход
|
|
58
|
+
уходит, когда владелец молчит ~3 с после последнего слова, и никогда — пока он
|
|
59
|
+
ещё говорит или пока фрагмент ещё распознаётся. Фрагмент, оборванный на запятой
|
|
60
|
+
или союзе, ждёт дольше.
|
|
51
61
|
4. Ответ агента приходит на stdin, озвучивается через voice-connect в динамик, и
|
|
52
|
-
микрофон остаётся открытым ~9 с — продолжить можно без повторного обращения.
|
|
62
|
+
микрофон остаётся открытым ~9 с — продолжить можно без повторного обращения. Когда
|
|
63
|
+
окно закрывается, звучит тихий низкий тон.
|
|
64
|
+
|
|
65
|
+
Внутри открытого диалога wake-слово ДРУГОГО пира обязано выглядеть намеренным — имя
|
|
66
|
+
услышано буква в букву и за ним идёт настоящая просьба, — иначе фрагмент уходит тому,
|
|
67
|
+
кто уже слушает. Ослышка должна стоить лишней строки, а не всего разговора.
|
|
53
68
|
|
|
54
69
|
Earcon'ы размечают состояния: восходящий — микрофон ваш, короткий нисходящий —
|
|
55
70
|
принял, двойной нисходящий — не отправлено. Обработка идёт молча.
|
|
@@ -57,7 +72,20 @@ Earcon'ы размечают состояния: восходящий — мик
|
|
|
57
72
|
Всё настраивается через env: `VOICETALK_AUDIO=0` (текстовый режим, без микрофона),
|
|
58
73
|
`VOICETALK_FOLLOWUP_MS`, `VOICETALK_HANGOVER_MS`, `VOICETALK_VAD_MIN_RMS`,
|
|
59
74
|
`VOICETALK_VAD_RATIO`, `VOICETALK_STT_LANG`, `VOICETALK_STT_ENGINE`,
|
|
60
|
-
`VOICETALK_TTS_MAX_CHARS`, `VOICETALK_EARCONS=0
|
|
75
|
+
`VOICETALK_TTS_MAX_CHARS`, `VOICETALK_EARCONS=0`, `VOICETALK_COMPOSE_MS`,
|
|
76
|
+
`VOICETALK_COMPOSE_UNFINISHED_MS`, `VOICETALK_TURN_MAX_MS`,
|
|
77
|
+
`VOICETALK_WAKE_CAPTURE=0` (отключить диагностическое кольцо) и
|
|
78
|
+
`VOICETALK_WAKE_CAPTURE_MAX` (по умолчанию 16).
|
|
79
|
+
|
|
80
|
+
Каждый транскрипт, похожий на настроенный wake, сохраняется как WAV с правами 0600 и
|
|
81
|
+
JSON-вердикт в `<IAPEER_ROOT>/runtimes/voicetalk/tmp/wake-captures`. Кольцо ограничено
|
|
82
|
+
и хранит только сегменты-кандидаты, не непрерывную запись комнаты. Спорный сегмент
|
|
83
|
+
можно прогнать через тот же микрофонный путь способом ниже.
|
|
84
|
+
|
|
85
|
+
Для проверки turn-taking, не трогая живой канал, есть два шва: `VOICETALK_MIC_FILE=<pcm>`
|
|
86
|
+
проигрывает запись через тот же путь, что и микрофон (в реальном темпе), а
|
|
87
|
+
`VOICETALK_PLAYBACK=0` держит динамик молча, оставляя синтез и его замеры. Оба сведены
|
|
88
|
+
в `scripts/replay-dialogue.sh`.
|
|
61
89
|
|
|
62
90
|
## Требования
|
|
63
91
|
|
package/package.json
CHANGED
package/src/audio.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import { spawn, type ChildProcessByStdio } from 'child_process'
|
|
11
11
|
import type { Readable } from 'stream'
|
|
12
|
-
import { writeFileSync } from 'fs'
|
|
12
|
+
import { readFileSync, writeFileSync } from 'fs'
|
|
13
13
|
|
|
14
14
|
export const SAMPLE_RATE = 16_000
|
|
15
15
|
export const FRAME_MS = 20
|
|
@@ -125,6 +125,13 @@ export class Segmenter {
|
|
|
125
125
|
this.warmupFrames = Math.round((opts.warmupMs ?? 600) / FRAME_MS)
|
|
126
126
|
}
|
|
127
127
|
|
|
128
|
+
/** Is the endpointer inside an utterance right now? The turn layer asks before it
|
|
129
|
+
* decides a thought is finished: the owner resuming mid-pause is the difference
|
|
130
|
+
* between one envelope and two. */
|
|
131
|
+
get speaking(): boolean {
|
|
132
|
+
return this.inSpeech
|
|
133
|
+
}
|
|
134
|
+
|
|
128
135
|
get noiseFloor(): number {
|
|
129
136
|
return this.floor < 0 ? 0 : this.floor
|
|
130
137
|
}
|
|
@@ -293,6 +300,11 @@ export async function resolveInputDevice(
|
|
|
293
300
|
|
|
294
301
|
export type MicOptions = {
|
|
295
302
|
deviceIndex: number
|
|
303
|
+
/** Replay a recording INSTEAD of opening the microphone (test seam). Frames are paced
|
|
304
|
+
* in real time, so endpointing and the compose window behave exactly as they do on a
|
|
305
|
+
* live mic — which is the whole point: turn-taking bugs only reproduce against a
|
|
306
|
+
* clock. Accepts a WAV or headerless mono 16-bit 16 kHz PCM file. */
|
|
307
|
+
filePath?: string
|
|
296
308
|
ffmpegBin?: string
|
|
297
309
|
onData: (chunk: Buffer) => void
|
|
298
310
|
onError?: (detail: string) => void
|
|
@@ -333,6 +345,10 @@ export class Microphone {
|
|
|
333
345
|
private spawnChild(): void {
|
|
334
346
|
if (this.stopped) return
|
|
335
347
|
this.attempt += 1
|
|
348
|
+
if (this.opts.filePath) {
|
|
349
|
+
this.replayFile(this.opts.filePath)
|
|
350
|
+
return
|
|
351
|
+
}
|
|
336
352
|
const bin = this.opts.ffmpegBin ?? 'ffmpeg'
|
|
337
353
|
const args = [
|
|
338
354
|
'-hide_banner',
|
|
@@ -368,6 +384,35 @@ export class Microphone {
|
|
|
368
384
|
})
|
|
369
385
|
}
|
|
370
386
|
|
|
387
|
+
/** Feed a recording through the same path the microphone uses, one 20 ms frame per
|
|
388
|
+
* 20 ms, then fall silent (a finished recording is a quiet room, not a dead mic). */
|
|
389
|
+
private replayFile(path: string): void {
|
|
390
|
+
let pcm: Buffer
|
|
391
|
+
try {
|
|
392
|
+
const raw = readFileSync(path)
|
|
393
|
+
// Skip a RIFF header if present; anything else is treated as raw PCM.
|
|
394
|
+
pcm = raw.subarray(0, 4).toString() === 'RIFF' ? raw.subarray(44) : raw
|
|
395
|
+
} catch (err) {
|
|
396
|
+
this.opts.onError?.(`cannot replay ${path}: ${err instanceof Error ? err.message : String(err)}`)
|
|
397
|
+
return
|
|
398
|
+
}
|
|
399
|
+
this.opts.onStart?.(this.attempt)
|
|
400
|
+
let offset = 0
|
|
401
|
+
const tick = () => {
|
|
402
|
+
if (this.stopped) return
|
|
403
|
+
if (offset >= pcm.length) {
|
|
404
|
+
// Keep the stream alive with silence so compose/window timers still fire.
|
|
405
|
+
this.opts.onData(Buffer.alloc(FRAME_BYTES))
|
|
406
|
+
this.timer = setTimeout(tick, FRAME_MS)
|
|
407
|
+
return
|
|
408
|
+
}
|
|
409
|
+
this.opts.onData(Buffer.from(pcm.subarray(offset, offset + FRAME_BYTES)))
|
|
410
|
+
offset += FRAME_BYTES
|
|
411
|
+
this.timer = setTimeout(tick, FRAME_MS)
|
|
412
|
+
}
|
|
413
|
+
tick()
|
|
414
|
+
}
|
|
415
|
+
|
|
371
416
|
private scheduleRestart(): void {
|
|
372
417
|
if (this.stopped || this.timer) return
|
|
373
418
|
const delay = Math.min(30_000, 1_000 * Math.min(8, this.attempt))
|
package/src/voiceConnect.ts
CHANGED
|
@@ -114,13 +114,15 @@ export type TranscribeOptions = {
|
|
|
114
114
|
lang?: string
|
|
115
115
|
/** Force an STT engine (`mlx-whisper` keeps it fully local); default = service cascade. */
|
|
116
116
|
engine?: string
|
|
117
|
-
/** Decoder-priming prompt — biases spelling of fleet names so the wake matcher sees
|
|
118
|
-
* «Борис», not «Барис». */
|
|
119
|
-
prompt?: string
|
|
120
117
|
timeoutMs?: number
|
|
121
118
|
}
|
|
122
119
|
|
|
123
|
-
/** Transcribe a local audio file. Returns the raw transcript (may be empty).
|
|
120
|
+
/** Transcribe a local audio file. Returns the raw transcript (may be empty).
|
|
121
|
+
*
|
|
122
|
+
* Deliberately no `prompt` option: this client feeds a presence detector, so configured
|
|
123
|
+
* wake words must never be injected into the decoder and then mistaken for acoustic
|
|
124
|
+
* evidence. voice-connect supports prompts for other consumers; voicetalk's door does not.
|
|
125
|
+
*/
|
|
124
126
|
export async function transcribe(audioPath: string, opts: TranscribeOptions = {}): Promise<string> {
|
|
125
127
|
const ep = resolveVoiceEndpoint()
|
|
126
128
|
const form = new FormData()
|
|
@@ -128,7 +130,6 @@ export async function transcribe(audioPath: string, opts: TranscribeOptions = {}
|
|
|
128
130
|
form.append('file', new Blob([new Uint8Array(bytes)], { type: 'audio/wav' }), audioPath.split('/').pop() ?? 'audio.wav')
|
|
129
131
|
if (opts.lang) form.append('language', opts.lang)
|
|
130
132
|
if (opts.engine) form.append('engine', opts.engine)
|
|
131
|
-
if (opts.prompt) form.append('prompt', opts.prompt)
|
|
132
133
|
form.append('response_format', 'json')
|
|
133
134
|
const res = await fetch(ep.stt, {
|
|
134
135
|
method: 'POST',
|
package/src/voiceSession.ts
CHANGED
|
@@ -1,9 +1,17 @@
|
|
|
1
1
|
// voiceSession.ts — the live voice session (Фаза 3): the runtime's ears, its turn-taking
|
|
2
2
|
// and its mouth, wired between audio.ts (host mechanics) and voiceConnect.ts (speech).
|
|
3
3
|
//
|
|
4
|
-
// INBOUND mic → endpointing → STT → address match → deliver to the fleet
|
|
4
|
+
// INBOUND mic → endpointing → STT → address match → COMPOSE → deliver to the fleet
|
|
5
5
|
// OUTBOUND agent envelope → TTS → speaker, then an open-mic follow-up window
|
|
6
6
|
//
|
|
7
|
+
// A pause inside a sentence is not the end of a thought. Endpointing (fast, 700 ms) only
|
|
8
|
+
// cuts AUDIO into segments so recognition can start early; what the fleet receives is a
|
|
9
|
+
// TURN — every segment of one thought, joined, delivered ONCE when the owner has actually
|
|
10
|
+
// finished (the compose window). Without that, one spoken thought became two envelopes
|
|
11
|
+
// and earned two spoken answers, the second one replying to a fragment (live complaint,
|
|
12
|
+
// 22.07). Recognition still runs per segment and in the background, so composing costs
|
|
13
|
+
// only the silence the owner was taking anyway.
|
|
14
|
+
//
|
|
7
15
|
// Turn-taking follows the established voice-assistant pattern recorded in Описание
|
|
8
16
|
// voicetalk: the address (wake word) only INITIATES a conversation; after an agent has
|
|
9
17
|
// answered, a short open-mic window lets the owner continue without naming the peer
|
|
@@ -22,9 +30,9 @@
|
|
|
22
30
|
// which exists here yet. Mic gating during playback is the honest substitute — the
|
|
23
31
|
// runtime simply does not listen while it talks.
|
|
24
32
|
|
|
25
|
-
import { mkdirSync, rmSync } from 'fs'
|
|
33
|
+
import { copyFileSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'fs'
|
|
26
34
|
import { join } from 'path'
|
|
27
|
-
import { randomUUID } from 'crypto'
|
|
35
|
+
import { createHash, randomUUID } from 'crypto'
|
|
28
36
|
import {
|
|
29
37
|
Microphone,
|
|
30
38
|
Segmenter,
|
|
@@ -37,15 +45,16 @@ import {
|
|
|
37
45
|
import { synthesize, transcribe } from './voiceConnect.ts'
|
|
38
46
|
import {
|
|
39
47
|
buildAddressBook,
|
|
40
|
-
|
|
48
|
+
hallucinationReason,
|
|
41
49
|
matchWake,
|
|
50
|
+
objectionToActivation,
|
|
51
|
+
objectionToSwitch,
|
|
42
52
|
readFleetPeers,
|
|
43
|
-
sttPrompt,
|
|
44
53
|
type Address,
|
|
45
54
|
type FleetPeer,
|
|
46
55
|
type VoicetalkBinding,
|
|
56
|
+
type WakeMatch,
|
|
47
57
|
} from './wake.ts'
|
|
48
|
-
import { writeFileSync } from 'fs'
|
|
49
58
|
|
|
50
59
|
const num = (v: string | undefined, d: number) => (v && Number(v) > 0 ? Number(v) : d)
|
|
51
60
|
|
|
@@ -57,6 +66,24 @@ const FOLLOWUP_MS = num(process.env.VOICETALK_FOLLOWUP_MS, 9_000)
|
|
|
57
66
|
* 500–800 ms for free dialogue; the upper half of the band suits a heavier recognizer —
|
|
58
67
|
* it fragments one thought into two transcriptions far less often. */
|
|
59
68
|
const HANGOVER_MS = num(process.env.VOICETALK_HANGOVER_MS, 700)
|
|
69
|
+
/** Silence (counted from the owner's last WORD) after which a turn is considered
|
|
70
|
+
* finished and sent. The two errors are not symmetric: waiting one second too long costs
|
|
71
|
+
* one second on a chain where synthesis alone is ~5 s, while not waiting costs a split
|
|
72
|
+
* thought, two envelopes and a spoken answer to a fragment — the owner's actual
|
|
73
|
+
* complaint. Measured thinking pauses in real Russian speech run to ~2.7 s, so the
|
|
74
|
+
* budget sits above that with room. Tunable live: VOICETALK_COMPOSE_MS. */
|
|
75
|
+
const COMPOSE_MS = num(process.env.VOICETALK_COMPOSE_MS, 3_000)
|
|
76
|
+
/** A fragment that ENDS mid-thought (no terminal punctuation, or a trailing comma or
|
|
77
|
+
* conjunction) buys extra patience — the speaker audibly has not finished. */
|
|
78
|
+
const COMPOSE_UNFINISHED_MS = num(process.env.VOICETALK_COMPOSE_UNFINISHED_MS, 4_500)
|
|
79
|
+
/** Segments waiting for recognition. Deep enough for a long sentence spoken over a
|
|
80
|
+
* slow recognizer, shallow enough that a wedged STT cannot grow it without bound. */
|
|
81
|
+
const SEGMENT_QUEUE_MAX = 8
|
|
82
|
+
/** A turn never grows past this — a monologue still reaches its addressee. */
|
|
83
|
+
const TURN_MAX_MS = num(process.env.VOICETALK_TURN_MAX_MS, 45_000)
|
|
84
|
+
/** How often to re-check a turn that is ready to send but whose speaker has not gone
|
|
85
|
+
* quiet yet. */
|
|
86
|
+
const HOLD_RECHECK_MS = num(process.env.VOICETALK_HOLD_RECHECK_MS, 300)
|
|
60
87
|
/** Gate the mic for a moment after the speaker stops — room echo tail. */
|
|
61
88
|
const ECHO_TAIL_MS = num(process.env.VOICETALK_ECHO_TAIL_MS, 400)
|
|
62
89
|
/** Long answers are truncated for speech: a spoken monologue is worse than a summary. */
|
|
@@ -65,6 +92,18 @@ const TTS_MAX_CHARS = num(process.env.VOICETALK_TTS_MAX_CHARS, 1_200)
|
|
|
65
92
|
const STT_LANG = process.env.VOICETALK_STT_LANG ?? 'ru'
|
|
66
93
|
/** Force a local recognizer (mlx-whisper) to keep always-on listening on-device. */
|
|
67
94
|
const STT_ENGINE = process.env.VOICETALK_STT_ENGINE?.trim() || undefined
|
|
95
|
+
/** Test seams. VOICETALK_MIC_FILE replays a recording instead of opening the microphone;
|
|
96
|
+
* VOICETALK_PLAYBACK=0 keeps the speaker silent (synthesis still runs and is still
|
|
97
|
+
* timed). Together they let a turn-taking change be verified end to end on a host whose
|
|
98
|
+
* real voice channel is in use by its owner. */
|
|
99
|
+
const MIC_FILE = process.env.VOICETALK_MIC_FILE?.trim() || undefined
|
|
100
|
+
const PLAYBACK_ENABLED = process.env.VOICETALK_PLAYBACK !== '0'
|
|
101
|
+
/** Keep a small, private ring of audio segments whose transcript looked like a wake.
|
|
102
|
+
* A false wake cannot be diagnosed from text alone: the audio is the evidence of what
|
|
103
|
+
* the owner actually said. Only candidate segments are retained, never room audio; files
|
|
104
|
+
* are mode 0600 under the runtime scratch dir and the oldest pair is evicted. */
|
|
105
|
+
const WAKE_CAPTURE_ENABLED = process.env.VOICETALK_WAKE_CAPTURE !== '0'
|
|
106
|
+
const WAKE_CAPTURE_MAX = Math.min(100, num(process.env.VOICETALK_WAKE_CAPTURE_MAX, 16))
|
|
68
107
|
|
|
69
108
|
export type DeliverFn = (target: string, text: string) => Promise<{ ok: boolean; detail?: string }>
|
|
70
109
|
export type LogFn = (evt: string, fields: Record<string, unknown>) => void
|
|
@@ -79,10 +118,17 @@ export type VoiceSessionOptions = {
|
|
|
79
118
|
peers?: FleetPeer[]
|
|
80
119
|
}
|
|
81
120
|
|
|
121
|
+
type WakeRoutingDecision = {
|
|
122
|
+
candidate: WakeMatch
|
|
123
|
+
accepted: boolean
|
|
124
|
+
reason?: string
|
|
125
|
+
}
|
|
126
|
+
|
|
82
127
|
/** State sounds, per the earcon convention in the research: `open` = the microphone is
|
|
83
|
-
* yours (rising, ≤300 ms), `end` =
|
|
84
|
-
* ≤200 ms), `
|
|
85
|
-
|
|
128
|
+
* yours (rising, ≤300 ms), `end` = turn captured, processing (short descending,
|
|
129
|
+
* ≤200 ms), `close` = the open-mic window expired, a name is needed again (low, soft),
|
|
130
|
+
* `error` = nothing routed (double descending). Processing itself is silent. */
|
|
131
|
+
type Earcons = { open: string; end: string; error: string; close: string }
|
|
86
132
|
|
|
87
133
|
export class VoiceSession {
|
|
88
134
|
private mic: Microphone | null = null
|
|
@@ -92,12 +138,20 @@ export class VoiceSession {
|
|
|
92
138
|
minRms: num(process.env.VOICETALK_VAD_MIN_RMS, 700),
|
|
93
139
|
})
|
|
94
140
|
private book: Address[] = []
|
|
95
|
-
private prompt = ''
|
|
96
141
|
private earcons: Earcons | null = null
|
|
97
142
|
private busy = false
|
|
143
|
+
private queue: Array<Segment & { speechEndedAt: number }> = []
|
|
98
144
|
private muteUntil = 0
|
|
145
|
+
/** When the microphone last stopped hearing speech — the clock the compose window runs
|
|
146
|
+
* on, so the delivery delay the owner feels is exactly the silence they took. */
|
|
147
|
+
private lastSpeechEndedAt = 0
|
|
99
148
|
private followTarget: string | null = null
|
|
100
149
|
private followUntil = 0
|
|
150
|
+
private followTimer: ReturnType<typeof setTimeout> | null = null
|
|
151
|
+
/** The thought being assembled right now — segments of one turn, not yet sent. */
|
|
152
|
+
private pending: { target: string; parts: string[]; openedAt: number; timer: ReturnType<typeof setTimeout> | null } | null = null
|
|
153
|
+
/** When the current turn was handed to the fleet — the clock for «ответ за N с». */
|
|
154
|
+
private deliveredAt = new Map<string, number>()
|
|
101
155
|
private stopped = false
|
|
102
156
|
|
|
103
157
|
constructor(private readonly opts: VoiceSessionOptions) {}
|
|
@@ -109,9 +163,12 @@ export class VoiceSession {
|
|
|
109
163
|
mkdirSync(this.opts.tmpDir, { recursive: true, mode: 0o700 })
|
|
110
164
|
this.refreshAddressBook()
|
|
111
165
|
this.earcons = this.writeEarcons()
|
|
112
|
-
const device =
|
|
166
|
+
const device = MIC_FILE
|
|
167
|
+
? { index: -1, name: `replay:${MIC_FILE}` }
|
|
168
|
+
: await resolveInputDevice(this.opts.binding.device)
|
|
113
169
|
this.mic = new Microphone({
|
|
114
170
|
deviceIndex: device.index,
|
|
171
|
+
filePath: MIC_FILE,
|
|
115
172
|
onData: chunk => this.onAudio(chunk),
|
|
116
173
|
onStart: attempt => this.opts.log('mic.start', { device: device.index, name: device.name, attempt }),
|
|
117
174
|
onError: detail => this.opts.log('mic.error', { detail }),
|
|
@@ -122,6 +179,9 @@ export class VoiceSession {
|
|
|
122
179
|
|
|
123
180
|
stop(): void {
|
|
124
181
|
this.stopped = true
|
|
182
|
+
this.clearWindow()
|
|
183
|
+
if (this.pending?.timer) clearTimeout(this.pending.timer)
|
|
184
|
+
this.pending = null
|
|
125
185
|
this.mic?.stop()
|
|
126
186
|
this.mic = null
|
|
127
187
|
}
|
|
@@ -132,7 +192,6 @@ export class VoiceSession {
|
|
|
132
192
|
binding: this.opts.binding,
|
|
133
193
|
peers: this.opts.peers ?? readFleetPeers(),
|
|
134
194
|
})
|
|
135
|
-
this.prompt = sttPrompt(this.book)
|
|
136
195
|
}
|
|
137
196
|
|
|
138
197
|
private writeEarcons(): Earcons {
|
|
@@ -140,6 +199,7 @@ export class VoiceSession {
|
|
|
140
199
|
open: join(this.opts.tmpDir, 'earcon-open.wav'),
|
|
141
200
|
end: join(this.opts.tmpDir, 'earcon-end.wav'),
|
|
142
201
|
error: join(this.opts.tmpDir, 'earcon-error.wav'),
|
|
202
|
+
close: join(this.opts.tmpDir, 'earcon-close.wav'),
|
|
143
203
|
}
|
|
144
204
|
// A5→C6→E6 rising, 240 ms — the microphone is open.
|
|
145
205
|
writeFileSync(files.open, earconWav([{ hz: 880, ms: 80 }, { hz: 1047, ms: 80 }, { hz: 1319, ms: 80 }]), { mode: 0o600 })
|
|
@@ -147,11 +207,15 @@ export class VoiceSession {
|
|
|
147
207
|
writeFileSync(files.end, earconWav([{ hz: 880, ms: 80 }, { hz: 660, ms: 80 }]), { mode: 0o600 })
|
|
148
208
|
// Double descending, 260 ms — tonally "closed": nothing was routed.
|
|
149
209
|
writeFileSync(files.error, earconWav([{ hz: 520, ms: 120 }, { hz: 330, ms: 140 }]), { mode: 0o600 })
|
|
210
|
+
// One low soft tone, 180 ms — the open-mic window just expired; say a name to speak
|
|
211
|
+
// again. Deliberately quieter and lower than `end` so it reads as "door closing",
|
|
212
|
+
// not as another acknowledgement.
|
|
213
|
+
writeFileSync(files.close, earconWav([{ hz: 400, ms: 180 }], 0.14), { mode: 0o600 })
|
|
150
214
|
return files
|
|
151
215
|
}
|
|
152
216
|
|
|
153
217
|
private async earcon(kind: keyof Earcons): Promise<void> {
|
|
154
|
-
if (!this.earcons || process.env.VOICETALK_EARCONS === '0') return
|
|
218
|
+
if (!this.earcons || process.env.VOICETALK_EARCONS === '0' || !PLAYBACK_ENABLED) return
|
|
155
219
|
try {
|
|
156
220
|
this.muteUntil = Math.max(this.muteUntil, Date.now() + 600)
|
|
157
221
|
await playFile(this.earcons[kind], { timeoutMs: 5_000 })
|
|
@@ -173,16 +237,35 @@ export class VoiceSession {
|
|
|
173
237
|
return
|
|
174
238
|
}
|
|
175
239
|
for (const segment of this.segmenter.push(chunk)) {
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
240
|
+
// Recognition is serialized (one STT at a time — a pile-up only adds latency), but
|
|
241
|
+
// segments QUEUE rather than being dropped: a dropped segment is a hole in the
|
|
242
|
+
// middle of the owner's sentence, which is exactly what this phase is fixing.
|
|
243
|
+
if (this.queue.length >= SEGMENT_QUEUE_MAX) {
|
|
244
|
+
this.opts.log('segment.drop', { reason: 'queue full', ms: segment.ms })
|
|
180
245
|
continue
|
|
181
246
|
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
247
|
+
// The endpointer hands over a segment only after the hangover silence, so speech
|
|
248
|
+
// itself stopped that much earlier. The compose budget is counted from THAT
|
|
249
|
+
// instant: the owner's pause should not be charged twice, once as endpointing and
|
|
250
|
+
// again as composing.
|
|
251
|
+
this.queue.push({ ...segment, speechEndedAt: Date.now() - HANGOVER_MS })
|
|
252
|
+
void this.drainQueue()
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
private async drainQueue(): Promise<void> {
|
|
257
|
+
if (this.busy) return
|
|
258
|
+
this.busy = true
|
|
259
|
+
try {
|
|
260
|
+
while (this.queue.length) {
|
|
261
|
+
const segment = this.queue.shift()!
|
|
262
|
+
this.lastSpeechEndedAt = segment.speechEndedAt
|
|
263
|
+
await this.handleSegment(segment)
|
|
264
|
+
}
|
|
265
|
+
} finally {
|
|
266
|
+
this.busy = false
|
|
267
|
+
// A turn may have been waiting for recognition to finish before it could be sent.
|
|
268
|
+
if (this.pending) this.rearm()
|
|
186
269
|
}
|
|
187
270
|
}
|
|
188
271
|
|
|
@@ -196,21 +279,37 @@ export class VoiceSession {
|
|
|
196
279
|
transcript = await transcribe(wav, {
|
|
197
280
|
lang: STT_LANG || undefined,
|
|
198
281
|
engine: STT_ENGINE,
|
|
199
|
-
prompt: this.prompt || undefined,
|
|
200
282
|
})
|
|
201
283
|
} catch (err) {
|
|
202
284
|
this.opts.log('stt.error', { ms: segment.ms, detail: err instanceof Error ? err.message : String(err) })
|
|
203
285
|
return
|
|
204
286
|
}
|
|
205
287
|
const sttMs = Date.now() - started
|
|
206
|
-
|
|
288
|
+
const candidate = matchWake(transcript, this.book)
|
|
289
|
+
const hallucination = hallucinationReason(transcript)
|
|
290
|
+
if (hallucination) {
|
|
207
291
|
this.opts.log('stt.ignored', {
|
|
208
292
|
ms: segment.ms,
|
|
209
293
|
sttMs,
|
|
210
294
|
peak: Math.round(segment.peakRms),
|
|
211
295
|
floor: Math.round(segment.noiseFloor),
|
|
296
|
+
reason: hallucination,
|
|
212
297
|
text: transcript,
|
|
213
298
|
})
|
|
299
|
+
if (candidate) {
|
|
300
|
+
this.opts.log('wake.rejected', {
|
|
301
|
+
target: candidate.target,
|
|
302
|
+
alias: candidate.alias,
|
|
303
|
+
exact: candidate.exact,
|
|
304
|
+
reason: hallucination,
|
|
305
|
+
text: transcript,
|
|
306
|
+
})
|
|
307
|
+
this.captureWakeCandidate(wav, segment, transcript, {
|
|
308
|
+
candidate,
|
|
309
|
+
accepted: false,
|
|
310
|
+
reason: hallucination,
|
|
311
|
+
})
|
|
312
|
+
}
|
|
214
313
|
return
|
|
215
314
|
}
|
|
216
315
|
this.opts.log('stt.text', {
|
|
@@ -222,52 +321,254 @@ export class VoiceSession {
|
|
|
222
321
|
text: transcript,
|
|
223
322
|
})
|
|
224
323
|
|
|
225
|
-
const
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
324
|
+
const decision = this.absorb(transcript, candidate)
|
|
325
|
+
if (decision) this.captureWakeCandidate(wav, segment, transcript, decision)
|
|
326
|
+
} finally {
|
|
327
|
+
rmSync(wav, { force: true })
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/** Persist the evidence for an apparent wake in a bounded private ring. Text-only logs
|
|
332
|
+
* cannot distinguish a spoken name from a decoder invention; this seam makes the next
|
|
333
|
+
* disputed activation replayable through VOICETALK_MIC_FILE without recording the room
|
|
334
|
+
* continuously. */
|
|
335
|
+
private captureWakeCandidate(
|
|
336
|
+
wav: string,
|
|
337
|
+
segment: Segment,
|
|
338
|
+
transcript: string,
|
|
339
|
+
decision: WakeRoutingDecision,
|
|
340
|
+
): void {
|
|
341
|
+
if (!WAKE_CAPTURE_ENABLED) return
|
|
342
|
+
const dir = join(this.opts.tmpDir, 'wake-captures')
|
|
343
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-')
|
|
344
|
+
const stem = `wake-${stamp}-${randomUUID()}`
|
|
345
|
+
const audioPath = join(dir, `${stem}.wav`)
|
|
346
|
+
const metaPath = join(dir, `${stem}.json`)
|
|
347
|
+
try {
|
|
348
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 })
|
|
349
|
+
copyFileSync(wav, audioPath)
|
|
350
|
+
writeFileSync(
|
|
351
|
+
metaPath,
|
|
352
|
+
JSON.stringify(
|
|
353
|
+
{
|
|
354
|
+
capturedAt: new Date().toISOString(),
|
|
355
|
+
audio: `${stem}.wav`,
|
|
356
|
+
pcmSha256: createHash('sha256').update(segment.pcm).digest('hex'),
|
|
357
|
+
segment: {
|
|
358
|
+
ms: segment.ms,
|
|
359
|
+
peakRms: Math.round(segment.peakRms),
|
|
360
|
+
noiseFloor: Math.round(segment.noiseFloor),
|
|
361
|
+
threshold: segment.threshold,
|
|
362
|
+
truncated: segment.truncated,
|
|
363
|
+
},
|
|
364
|
+
transcript,
|
|
365
|
+
wake: {
|
|
366
|
+
target: decision.candidate.target,
|
|
367
|
+
alias: decision.candidate.alias,
|
|
368
|
+
exact: decision.candidate.exact,
|
|
369
|
+
accepted: decision.accepted,
|
|
370
|
+
...(decision.reason ? { reason: decision.reason } : {}),
|
|
371
|
+
},
|
|
372
|
+
},
|
|
373
|
+
null,
|
|
374
|
+
2,
|
|
375
|
+
) + '\n',
|
|
376
|
+
{ mode: 0o600 },
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
const records = readdirSync(dir).filter(name => /^wake-.*\.json$/.test(name)).sort()
|
|
380
|
+
for (const oldMeta of records.slice(0, Math.max(0, records.length - WAKE_CAPTURE_MAX))) {
|
|
381
|
+
const oldStem = oldMeta.slice(0, -'.json'.length)
|
|
382
|
+
rmSync(join(dir, oldMeta), { force: true })
|
|
383
|
+
rmSync(join(dir, `${oldStem}.wav`), { force: true })
|
|
384
|
+
}
|
|
385
|
+
this.opts.log('wake.capture', {
|
|
386
|
+
target: decision.candidate.target,
|
|
387
|
+
accepted: decision.accepted,
|
|
388
|
+
path: audioPath,
|
|
389
|
+
})
|
|
390
|
+
} catch (err) {
|
|
391
|
+
rmSync(audioPath, { force: true })
|
|
392
|
+
rmSync(metaPath, { force: true })
|
|
393
|
+
this.opts.log('wake.capture-error', { detail: err instanceof Error ? err.message : String(err) })
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/** Route one recognized fragment into the turn being composed.
|
|
398
|
+
*
|
|
399
|
+
* The decision is deliberately conservative in one direction: an unaddressed fragment
|
|
400
|
+
* arriving right after an addressed one CONTINUES that thought rather than being
|
|
401
|
+
* dropped, and a wake word for SOMEONE ELSE has to clear a higher bar than it does
|
|
402
|
+
* when no conversation is open. Mid-dialogue, the cheap error (one more line to the
|
|
403
|
+
* peer already listening) is far cheaper than the expensive one (the conversation
|
|
404
|
+
* silently handed to another agent). */
|
|
405
|
+
private absorb(transcript: string, candidate = matchWake(transcript, this.book)): WakeRoutingDecision | null {
|
|
406
|
+
const active = this.pending?.target ?? (this.followTarget && Date.now() < this.followUntil ? this.followTarget : null)
|
|
407
|
+
if (candidate) {
|
|
408
|
+
const decline = objectionToActivation(candidate, this.book)
|
|
409
|
+
if (decline) {
|
|
410
|
+
this.opts.log('wake.rejected', {
|
|
411
|
+
target: candidate.target,
|
|
412
|
+
alias: candidate.alias,
|
|
413
|
+
exact: candidate.exact,
|
|
414
|
+
reason: decline,
|
|
415
|
+
text: transcript,
|
|
416
|
+
})
|
|
417
|
+
// An open dialogue intentionally needs no wake word. If its next fragment merely
|
|
418
|
+
// resembles a name, preserve it as ordinary continuation rather than dropping or
|
|
419
|
+
// stripping a word from the owner's thought.
|
|
420
|
+
if (active && decline === 'inexact name match') {
|
|
421
|
+
this.opts.log('turn.continue', { target: active, reason: this.pending ? 'compose' : 'follow-up' })
|
|
422
|
+
this.append(active, transcript.trim())
|
|
240
423
|
}
|
|
241
|
-
|
|
242
|
-
target = this.followTarget
|
|
243
|
-
body = transcript.trim()
|
|
244
|
-
this.opts.log('followup.route', { target })
|
|
245
|
-
} else {
|
|
246
|
-
this.opts.log('wake.no-match', { text: transcript })
|
|
247
|
-
return
|
|
424
|
+
return { candidate, accepted: false, reason: decline }
|
|
248
425
|
}
|
|
426
|
+
}
|
|
249
427
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
if (verdict.ok) {
|
|
260
|
-
this.opts.log('voice.delivered', { target, chars: body.length, totalMs: Date.now() - started })
|
|
261
|
-
} else {
|
|
262
|
-
this.opts.log('voice.undelivered', { target, detail: verdict.detail })
|
|
263
|
-
await this.earcon('error')
|
|
264
|
-
await this.say(`Не доставлено: ${verdict.detail ?? 'неизвестная ошибка'}`)
|
|
428
|
+
const match = candidate
|
|
429
|
+
const switching = !!match && !!active && match.target !== active
|
|
430
|
+
|
|
431
|
+
if (switching) {
|
|
432
|
+
const decline = objectionToSwitch(match!)
|
|
433
|
+
if (decline) {
|
|
434
|
+
this.opts.log('wake.switch-declined', { from: active, to: match!.target, reason: decline, text: transcript })
|
|
435
|
+
this.append(active!, transcript.trim())
|
|
436
|
+
return { candidate: match!, accepted: false, reason: decline }
|
|
265
437
|
}
|
|
266
|
-
|
|
267
|
-
|
|
438
|
+
// A deliberate switch: the thought in flight belongs to the previous peer, so it
|
|
439
|
+
// goes out before the new one starts.
|
|
440
|
+
this.opts.log('wake.switch', { from: active, to: match!.target })
|
|
441
|
+
this.flush('switch')
|
|
442
|
+
this.append(match!.target, match!.body)
|
|
443
|
+
return { candidate: match!, accepted: true }
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
if (match) {
|
|
447
|
+
if (!match.body) {
|
|
448
|
+
// Addressed with nothing to say yet («Борис?») — acknowledge and hold the mic
|
|
449
|
+
// open for the actual request, exactly as an assistant beeps and waits.
|
|
450
|
+
this.openWindow(match.target)
|
|
451
|
+
this.opts.log('wake.await-body', { target: match.target, alias: match.alias })
|
|
452
|
+
void this.earcon('open')
|
|
453
|
+
return { candidate: match, accepted: true }
|
|
454
|
+
}
|
|
455
|
+
this.append(match.target, match.body)
|
|
456
|
+
return { candidate: match, accepted: true }
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
if (active) {
|
|
460
|
+
this.opts.log('turn.continue', { target: active, reason: this.pending ? 'compose' : 'follow-up' })
|
|
461
|
+
this.append(active, transcript.trim())
|
|
462
|
+
return null
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
this.opts.log('wake.no-match', { text: transcript })
|
|
466
|
+
return null
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/** Add a fragment to the turn in flight and (re)arm the compose timer. */
|
|
470
|
+
private append(target: string, text: string): void {
|
|
471
|
+
const body = text.trim()
|
|
472
|
+
if (!body) return
|
|
473
|
+
if (this.pending && this.pending.target !== target) this.flush('retarget')
|
|
474
|
+
if (!this.pending) {
|
|
475
|
+
this.pending = { target, parts: [], openedAt: Date.now(), timer: null }
|
|
476
|
+
this.opts.log('turn.open', { target })
|
|
477
|
+
}
|
|
478
|
+
this.pending.parts.push(body)
|
|
479
|
+
this.clearWindow()
|
|
480
|
+
this.rearm()
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/** (Re)arm the compose timer for the turn in flight. */
|
|
484
|
+
private rearm(delayMs?: number): void {
|
|
485
|
+
const turn = this.pending
|
|
486
|
+
if (!turn) return
|
|
487
|
+
if (turn.timer) clearTimeout(turn.timer)
|
|
488
|
+
if (Date.now() - turn.openedAt >= TURN_MAX_MS) {
|
|
489
|
+
this.flush('max-length')
|
|
490
|
+
return
|
|
491
|
+
}
|
|
492
|
+
const budget = looksUnfinished(turn.parts[turn.parts.length - 1] ?? '') ? COMPOSE_UNFINISHED_MS : COMPOSE_MS
|
|
493
|
+
const remaining = this.lastSpeechEndedAt + budget - Date.now()
|
|
494
|
+
const delay = Math.max(0, delayMs ?? remaining)
|
|
495
|
+
turn.timer = setTimeout(() => this.flush('compose'), delay)
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/** Send the composed thought as ONE envelope. */
|
|
499
|
+
private flush(reason: string): void {
|
|
500
|
+
const turn = this.pending
|
|
501
|
+
if (!turn) return
|
|
502
|
+
// A thought is NOT finished while the owner is still speaking it. The compose timer
|
|
503
|
+
// counts silence since the last RECOGNIZED fragment, and recognition lags the voice
|
|
504
|
+
// by an endpointing pause plus the recognizer's own time — so by the time it fires,
|
|
505
|
+
// the owner may already be several words into the rest of the sentence. Waiting for
|
|
506
|
+
// the microphone and the recognizer to go quiet is what turns those words into the
|
|
507
|
+
// same envelope instead of a second one. (Live defect, 22.07: two envelopes, two
|
|
508
|
+
// spoken answers, the second one replying to a fragment.)
|
|
509
|
+
if (reason === 'compose' && (this.segmenter.speaking || this.busy || this.queue.length)) {
|
|
510
|
+
this.opts.log('turn.hold', {
|
|
511
|
+
target: turn.target,
|
|
512
|
+
reason: this.segmenter.speaking ? 'still speaking' : 'recognition in flight',
|
|
513
|
+
})
|
|
514
|
+
this.rearm(HOLD_RECHECK_MS)
|
|
515
|
+
return
|
|
516
|
+
}
|
|
517
|
+
this.pending = null
|
|
518
|
+
if (turn.timer) clearTimeout(turn.timer)
|
|
519
|
+
const text = joinTurn(turn.parts)
|
|
520
|
+
if (!text) return
|
|
521
|
+
const startedAt = Date.now()
|
|
522
|
+
this.opts.log('turn.flush', {
|
|
523
|
+
target: turn.target,
|
|
524
|
+
reason,
|
|
525
|
+
segments: turn.parts.length,
|
|
526
|
+
chars: text.length,
|
|
527
|
+
// The headline latency number of this phase: from the owner's last word to the
|
|
528
|
+
// moment the fleet is handed the thought.
|
|
529
|
+
sinceSpeechEndMs: this.lastSpeechEndedAt ? startedAt - this.lastSpeechEndedAt : null,
|
|
530
|
+
})
|
|
531
|
+
void this.deliverTurn(turn.target, text, startedAt)
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
private async deliverTurn(target: string, text: string, startedAt: number): Promise<void> {
|
|
535
|
+
// The turn is committed → the "heard you, working on it" sound. ONE per thought, not
|
|
536
|
+
// one per segment. Delivery itself stays silent (processing earns no earcon) — the
|
|
537
|
+
// agent's own voice is the next thing the owner hears.
|
|
538
|
+
await this.earcon('end')
|
|
539
|
+
this.openWindow(target)
|
|
540
|
+
const verdict = await this.opts.deliver(target, text)
|
|
541
|
+
if (verdict.ok) {
|
|
542
|
+
this.deliveredAt.set(target, Date.now())
|
|
543
|
+
this.opts.log('voice.delivered', { target, chars: text.length, deliverMs: Date.now() - startedAt })
|
|
544
|
+
} else {
|
|
545
|
+
this.opts.log('voice.undelivered', { target, detail: verdict.detail })
|
|
546
|
+
await this.earcon('error')
|
|
547
|
+
await this.say(`Не доставлено: ${verdict.detail ?? 'неизвестная ошибка'}`)
|
|
268
548
|
}
|
|
269
549
|
}
|
|
270
550
|
|
|
551
|
+
/** Open (or extend) the follow-up window, and arm the sound that tells the owner when
|
|
552
|
+
* it has closed. An invisible window is worse than a short one: the owner kept talking
|
|
553
|
+
* into a closed mic with no idea a name was needed again (live, 22.07). */
|
|
554
|
+
private openWindow(target: string): void {
|
|
555
|
+
this.followTarget = target
|
|
556
|
+
this.followUntil = Date.now() + FOLLOWUP_MS
|
|
557
|
+
if (this.followTimer) clearTimeout(this.followTimer)
|
|
558
|
+
this.followTimer = setTimeout(() => {
|
|
559
|
+
this.followTimer = null
|
|
560
|
+
if (this.pending) return // still composing — the window is not idle
|
|
561
|
+
this.opts.log('followup.closed', { target })
|
|
562
|
+
this.followTarget = null
|
|
563
|
+
void this.earcon('close')
|
|
564
|
+
}, FOLLOWUP_MS)
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
private clearWindow(): void {
|
|
568
|
+
if (this.followTimer) clearTimeout(this.followTimer)
|
|
569
|
+
this.followTimer = null
|
|
570
|
+
}
|
|
571
|
+
|
|
271
572
|
/** Speak an agent's answer on the owner's speaker, then hold the follow-up window
|
|
272
573
|
* open — the conversational turn passes back to the human. */
|
|
273
574
|
async speak(from: string, message: string): Promise<void> {
|
|
@@ -276,16 +577,21 @@ export class VoiceSession {
|
|
|
276
577
|
this.opts.log('tts.skip', { from, reason: 'nothing speakable' })
|
|
277
578
|
return
|
|
278
579
|
}
|
|
580
|
+
// How long the owner waited between finishing their thought and hearing an answer —
|
|
581
|
+
// the number Фаза 4 is judged by. It splits into the agent's thinking time and the
|
|
582
|
+
// synthesis time (logged separately by `tts.spoken`).
|
|
583
|
+
const askedAt = this.deliveredAt.get(from)
|
|
584
|
+
if (askedAt) this.opts.log('turn.answering', { from, sinceDeliveryMs: Date.now() - askedAt })
|
|
279
585
|
// Voice by IDENTITY, not by voice name: voice-connect resolves the sender's voice
|
|
280
586
|
// after picking the engine, so the owner hears WHO is talking even when the primary
|
|
281
587
|
// engine is down and a fallback rung serves. Sending a voice name instead would only
|
|
282
588
|
// bind to the primary engine — the very defect seen live on 22.07 (boris answered in
|
|
283
589
|
// a stranger's voice once gemini was unavailable).
|
|
284
590
|
await this.say(text, { from })
|
|
285
|
-
this.followTarget = from
|
|
286
|
-
this.followUntil = Date.now() + FOLLOWUP_MS
|
|
287
591
|
// The turn passes back to the human: the mic is open for a follow-up without a
|
|
288
|
-
// wake word,
|
|
592
|
+
// wake word, the rising earcon says so, and the window's own timer will say when it
|
|
593
|
+
// closes again.
|
|
594
|
+
this.openWindow(from)
|
|
289
595
|
await this.earcon('open')
|
|
290
596
|
}
|
|
291
597
|
|
|
@@ -299,7 +605,8 @@ export class VoiceSession {
|
|
|
299
605
|
this.muteUntil = Date.now() + 3_600_000
|
|
300
606
|
this.segmenter.reset()
|
|
301
607
|
try {
|
|
302
|
-
await playFile(result.path)
|
|
608
|
+
if (PLAYBACK_ENABLED) await playFile(result.path)
|
|
609
|
+
else this.opts.log('tts.silent', { from: opts.from, chars: text.length, path: result.path })
|
|
303
610
|
} finally {
|
|
304
611
|
this.muteUntil = Date.now() + ECHO_TAIL_MS
|
|
305
612
|
this.segmenter.reset()
|
|
@@ -324,6 +631,30 @@ export class VoiceSession {
|
|
|
324
631
|
}
|
|
325
632
|
}
|
|
326
633
|
|
|
634
|
+
/** Does this fragment sound like the middle of a sentence? Recognition punctuates what
|
|
635
|
+
* it hears, so a fragment left hanging on a comma or a conjunction — or with no terminal
|
|
636
|
+
* mark at all — is a thought still in progress, and the turn waits longer for the rest. */
|
|
637
|
+
export function looksUnfinished(text: string): boolean {
|
|
638
|
+
const t = text.trim()
|
|
639
|
+
if (!t) return false
|
|
640
|
+
if (/[,;:—–-]$/.test(t)) return true
|
|
641
|
+
if (!/[.!?…]$/.test(t)) return true
|
|
642
|
+
const last = t.replace(/[.!?…]+$/, '').split(/\s+/).pop()?.toLowerCase() ?? ''
|
|
643
|
+
return ['и', 'а', 'но', 'что', 'чтобы', 'или', 'если', 'когда', 'потому', 'то'].includes(last)
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
/** Join the fragments of one thought back into a sentence. Recognition returns each
|
|
647
|
+
* segment already punctuated and capitalized, so a plain space is the honest join —
|
|
648
|
+
* the parts were one utterance and are re-read as one. */
|
|
649
|
+
export function joinTurn(parts: readonly string[]): string {
|
|
650
|
+
return parts
|
|
651
|
+
.map(p => p.trim())
|
|
652
|
+
.filter(Boolean)
|
|
653
|
+
.join(' ')
|
|
654
|
+
.replace(/\s+/g, ' ')
|
|
655
|
+
.trim()
|
|
656
|
+
}
|
|
657
|
+
|
|
327
658
|
/** Prepare an agent's text for the speaker: strip what only makes sense on a screen and
|
|
328
659
|
* cap the length — a spoken wall of text is unusable. The modality frame already asks
|
|
329
660
|
* agents for short plain answers; this is the floor for when one forgets. */
|
package/src/wake.ts
CHANGED
|
@@ -10,15 +10,13 @@
|
|
|
10
10
|
// enrich them with their own human name («наталья» → «наталья вячеславовна»).
|
|
11
11
|
//
|
|
12
12
|
// Why the address is matched on the TRANSCRIPT rather than by an acoustic wake-word model
|
|
13
|
-
// (openWakeWord et al.): an acoustic detector needs a trained model per phrase,
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
// on the curated map — cheap in parallel, one shared feature backbone) is the natural
|
|
21
|
-
// Фаза-4 latency upgrade behind this same interface.
|
|
13
|
+
// (openWakeWord et al.): an acoustic detector needs a trained model per phrase, while
|
|
14
|
+
// transcript matching needs no per-name asset. That transcript MUST be decoded without a
|
|
15
|
+
// name prompt. Whisper's `initial_prompt` is generative context, not evidence: priming it
|
|
16
|
+
// with «дарвин, борис, линус» made unrelated audio decode as those names and woke agents
|
|
17
|
+
// in a quiet conversation (live defect, 23–24.07). The address layer therefore accepts
|
|
18
|
+
// only an exact, unprimed name at the start and rejects decoder loops before routing.
|
|
19
|
+
// A dedicated acoustic gate remains the stronger future upgrade behind this interface.
|
|
22
20
|
|
|
23
21
|
import { readFileSync } from 'fs'
|
|
24
22
|
import { homedir } from 'os'
|
|
@@ -205,6 +203,9 @@ export type WakeMatch = {
|
|
|
205
203
|
/** The utterance with the address (and any leading filler) stripped. */
|
|
206
204
|
body: string
|
|
207
205
|
alias: string
|
|
206
|
+
/** True when every word of the address matched letter for letter. Inexact candidates
|
|
207
|
+
* remain observable for diagnostics, but may never activate or retarget a dialogue. */
|
|
208
|
+
exact: boolean
|
|
208
209
|
}
|
|
209
210
|
|
|
210
211
|
/** Does this transcript open by addressing a fleet peer? Longest alias wins, so
|
|
@@ -229,6 +230,7 @@ export function matchWake(transcript: string, book: Address[]): WakeMatch | null
|
|
|
229
230
|
if (tokens.length < cand.words.length) continue
|
|
230
231
|
const ok = cand.words.every((w, i) => tokenMatches(tokens[i], w))
|
|
231
232
|
if (!ok) continue
|
|
233
|
+
const exact = cand.words.every((w, i) => tokens[i] === w)
|
|
232
234
|
// Rebuild the body from the ORIGINAL transcript so punctuation and casing of the
|
|
233
235
|
// message survive — only the address is stripped.
|
|
234
236
|
const body = rawTokens
|
|
@@ -236,14 +238,53 @@ export function matchWake(transcript: string, book: Address[]): WakeMatch | null
|
|
|
236
238
|
.join(' ')
|
|
237
239
|
.replace(/^[\s,.!:;—–-]+/, '')
|
|
238
240
|
.trim()
|
|
239
|
-
return { target: cand.target, alias: cand.alias, body }
|
|
241
|
+
return { target: cand.target, alias: cand.alias, body, exact }
|
|
240
242
|
}
|
|
241
243
|
return null
|
|
242
244
|
}
|
|
243
245
|
|
|
244
|
-
/**
|
|
245
|
-
*
|
|
246
|
-
*
|
|
246
|
+
/** May this match TAKE an open conversation away from the peer currently in it?
|
|
247
|
+
*
|
|
248
|
+
* Mid-dialogue the two errors are wildly asymmetric: one more line delivered to the peer
|
|
249
|
+
* already listening is nearly free, while a mis-heard name silently hands the whole
|
|
250
|
+
* conversation to another agent — the owner keeps talking to someone who is no longer
|
|
251
|
+
* there. So a switch must look deliberate: the name heard letter for letter, followed by
|
|
252
|
+
* a real request. Returns the objection, or null when the switch is allowed.
|
|
253
|
+
*
|
|
254
|
+
* Every rule here is a live artefact, not a hypothesis: «да, различаю» came back from
|
|
255
|
+
* recognition as «Дарвин, дарвин» and moved the conversation off boris (22.07). */
|
|
256
|
+
export function objectionToSwitch(match: WakeMatch): string | null {
|
|
257
|
+
if (!match.exact) return 'inexact name match'
|
|
258
|
+
const body = normalizeSpoken(match.body)
|
|
259
|
+
if (!body) return 'name with no request'
|
|
260
|
+
const words = body.split(' ')
|
|
261
|
+
if (words.every(w => w === match.alias)) return 'body echoes the name'
|
|
262
|
+
if (words.length < 2) return 'request too short to be an address'
|
|
263
|
+
return null
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** May this candidate START a dialogue?
|
|
267
|
+
*
|
|
268
|
+
* The recognizer is allowed to suggest an inexact name so the diagnostic seam can retain
|
|
269
|
+
* the audio, but a suggestion is not presence evidence. The initial address must be exact.
|
|
270
|
+
* Likewise, a transcript made only from configured names is decoder echo, not a request:
|
|
271
|
+
* the live failures were «Дарвин» followed by dozens of «борис» tokens. A bare exact name
|
|
272
|
+
* remains valid — it opens the short awaiting-body window without sending an envelope. */
|
|
273
|
+
export function objectionToActivation(match: WakeMatch, book: Address[]): string | null {
|
|
274
|
+
if (!match.exact) return 'inexact name match'
|
|
275
|
+
const body = normalizeSpoken(match.body)
|
|
276
|
+
if (!body) return null
|
|
277
|
+
const addressTokens = new Set(
|
|
278
|
+
book.flatMap(address => address.aliases.flatMap(alias => normalizeSpoken(alias).split(' '))).filter(Boolean),
|
|
279
|
+
)
|
|
280
|
+
const words = body.split(' ')
|
|
281
|
+
if (words.every(word => addressTokens.has(word))) return 'body only echoes configured names'
|
|
282
|
+
return null
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** Whisper-family recognizers emit both stock captions on near-silence and low-entropy
|
|
286
|
+
* token loops. They are decoder output, not speech; identify them before wake matching so
|
|
287
|
+
* a recognizer cannot fabricate presence from its own repeated hypothesis. */
|
|
247
288
|
const HALLUCINATIONS = [
|
|
248
289
|
'продолжение следует',
|
|
249
290
|
'субтитры',
|
|
@@ -257,16 +298,33 @@ const HALLUCINATIONS = [
|
|
|
257
298
|
'you',
|
|
258
299
|
]
|
|
259
300
|
|
|
260
|
-
export function
|
|
301
|
+
export function hallucinationReason(transcript: string): string | null {
|
|
261
302
|
const n = normalizeSpoken(transcript)
|
|
262
|
-
if (!n) return
|
|
263
|
-
if (n.length <= 2) return
|
|
264
|
-
|
|
303
|
+
if (!n) return 'empty transcript'
|
|
304
|
+
if (n.length <= 2) return 'transcript too short'
|
|
305
|
+
if (HALLUCINATIONS.some(h => n === h || n.startsWith(h))) return 'stock whisper caption'
|
|
306
|
+
|
|
307
|
+
const words = n.split(' ')
|
|
308
|
+
if (words.length >= 3) {
|
|
309
|
+
let longestRun = 1
|
|
310
|
+
let run = 1
|
|
311
|
+
for (let i = 1; i < words.length; i++) {
|
|
312
|
+
run = words[i] === words[i - 1] ? run + 1 : 1
|
|
313
|
+
longestRun = Math.max(longestRun, run)
|
|
314
|
+
}
|
|
315
|
+
// Three repeats can be a human correcting or emphasizing themselves («нет, нет,
|
|
316
|
+
// нет»). Five identical consecutive tokens are outside ordinary turn-taking and
|
|
317
|
+
// still far below the 10/70-token loops observed live.
|
|
318
|
+
if (longestRun >= 5) return 'repeated-token decoder loop'
|
|
319
|
+
|
|
320
|
+
const counts = new Map<string, number>()
|
|
321
|
+
for (const word of words) counts.set(word, (counts.get(word) ?? 0) + 1)
|
|
322
|
+
const dominant = Math.max(...counts.values())
|
|
323
|
+
if (words.length >= 6 && dominant / words.length >= 0.75) return 'low-entropy decoder loop'
|
|
324
|
+
}
|
|
325
|
+
return null
|
|
265
326
|
}
|
|
266
327
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
export function sttPrompt(book: Address[]): string {
|
|
270
|
-
const names = book.flatMap(a => a.aliases).slice(0, 40)
|
|
271
|
-
return names.length ? `Обращение к ассистенту по имени: ${names.join(', ')}.` : ''
|
|
328
|
+
export function isLikelyHallucination(transcript: string): boolean {
|
|
329
|
+
return hallucinationReason(transcript) !== null
|
|
272
330
|
}
|