@agfpd/voicetalk-runtime 0.1.1 → 0.2.0

package/src/voiceConnect.ts

@@ -0,0 +1,191 @@
+// voice-connect client — the ONLY place this package talks to speech.
+//
+// Speech ↔ text is not ours (decision: «Голосовой слой voice-connect — единый сервис,
+// core плюс два фасада»). voice-connect runs as an always-on local HTTP service and
+// publishes a discovery slot at <IAPEER_ROOT>/voice-provider.json; we are a pure
+// CONSUMER of its OpenAI-compatible routes:
+//
+//   POST /v1/audio/transcriptions   multipart(file, language?, engine?)  → { text }
+//   POST /v1/audio/speech           json(input, voice?, lang?, engine?)  → ogg/opus bytes
+//   GET  /health                                                          → { status }
+//
+// Engine selection, provider cascade and fallbacks live INSIDE that service — never
+// re-implement any of it here.
+
+import { readFileSync, writeFileSync } from 'fs'
+import { homedir } from 'os'
+import { join } from 'path'
+
+export type VoiceEndpoint = {
+  base: string
+  tts: string
+  stt: string
+  health: string
+  /** Where the endpoint came from — surfaced by `doctor` so a misrouted host is obvious. */
+  source: 'env' | 'slot' | 'default'
+  version?: string
+}
+
+const DEFAULT_BASE = 'http://127.0.0.1:8127'
+
+function iapeerRoot(): string {
+  const override = process.env.IAPEER_ROOT?.trim()
+  return override || join(homedir(), '.iapeer')
+}
+
+function providerSlotPath(): string {
+  return join(iapeerRoot(), 'voice-provider.json')
+}
+
+function trimSlash(url: string): string {
+  return url.replace(/\/+$/, '')
+}
+
+/** Resolve the voice service endpoint: explicit env override → the discovery slot
+ *  voice-connect writes on install → the documented local default. Never throws;
+ *  a missing/corrupt slot degrades to the default (and `doctor`/health surfaces it). */
+export function resolveVoiceEndpoint(env: NodeJS.ProcessEnv = process.env): VoiceEndpoint {
+  const routes = { tts: '/v1/audio/speech', stt: '/v1/audio/transcriptions', health: '/health' }
+  const override = env.VOICETALK_VOICE_ENDPOINT?.trim()
+  if (override) {
+    const base = trimSlash(override)
+    return { base, tts: base + routes.tts, stt: base + routes.stt, health: base + routes.health, source: 'env' }
+  }
+  try {
+    const slot = JSON.parse(readFileSync(providerSlotPath(), 'utf8')) as {
+      endpoint?: string
+      host?: string
+      port?: number
+      version?: string
+      routes?: Partial<typeof routes>
+    }
+    const base = trimSlash(
+      slot.endpoint?.trim() || (slot.host && slot.port ? `http://${slot.host}:${slot.port}` : ''),
+    )
+    if (base) {
+      return {
+        base,
+        tts: base + (slot.routes?.tts ?? routes.tts),
+        stt: base + (slot.routes?.stt ?? routes.stt),
+        health: base + (slot.routes?.health ?? routes.health),
+        source: 'slot',
+        version: slot.version,
+      }
+    }
+  } catch {
+    // no slot / unreadable → fall through to the default
+  }
+  return {
+    base: DEFAULT_BASE,
+    tts: DEFAULT_BASE + routes.tts,
+    stt: DEFAULT_BASE + routes.stt,
+    health: DEFAULT_BASE + routes.health,
+    source: 'default',
+  }
+}
+
+export type VoiceHealth = { ok: boolean; endpoint: string; source: string; version?: string; detail?: string }
+
+export async function voiceHealth(timeoutMs = 4_000): Promise<VoiceHealth> {
+  const ep = resolveVoiceEndpoint()
+  try {
+    const res = await fetch(ep.health, { signal: AbortSignal.timeout(timeoutMs) })
+    if (!res.ok) return { ok: false, endpoint: ep.base, source: ep.source, detail: `HTTP ${res.status}` }
+    const body = (await res.json()) as { status?: string; version?: string }
+    return {
+      ok: body?.status === 'ok',
+      endpoint: ep.base,
+      source: ep.source,
+      version: body?.version ?? ep.version,
+      detail: body?.status === 'ok' ? undefined : `status=${body?.status ?? 'unknown'}`,
+    }
+  } catch (err) {
+    return {
+      ok: false,
+      endpoint: ep.base,
+      source: ep.source,
+      detail: err instanceof Error ? err.message : String(err),
+    }
+  }
+}
+
+export type TranscribeOptions = {
+  /** ISO-639-1 hint. Omitted → the service auto-detects. */
+  lang?: string
+  /** Force an STT engine (`mlx-whisper` keeps it fully local); default = service cascade. */
+  engine?: string
+  /** Decoder-priming prompt — biases spelling of fleet names so the wake matcher sees
+   *  «Борис», not «Барис». */
+  prompt?: string
+  timeoutMs?: number
+}
+
+/** Transcribe a local audio file. Returns the raw transcript (may be empty). */
+export async function transcribe(audioPath: string, opts: TranscribeOptions = {}): Promise<string> {
+  const ep = resolveVoiceEndpoint()
+  const form = new FormData()
+  const bytes = readFileSync(audioPath)
+  form.append('file', new Blob([new Uint8Array(bytes)], { type: 'audio/wav' }), audioPath.split('/').pop() ?? 'audio.wav')
+  if (opts.lang) form.append('language', opts.lang)
+  if (opts.engine) form.append('engine', opts.engine)
+  if (opts.prompt) form.append('prompt', opts.prompt)
+  form.append('response_format', 'json')
+  const res = await fetch(ep.stt, {
+    method: 'POST',
+    body: form,
+    signal: AbortSignal.timeout(opts.timeoutMs ?? 90_000),
+  })
+  if (!res.ok) {
+    throw new Error(`voice-connect stt ${res.status}: ${(await res.text()).slice(0, 300)}`)
+  }
+  const body = (await res.json()) as { text?: string }
+  return (body?.text ?? '').trim()
+}
+
+export type SynthesizeOptions = {
+  /** Per-peer voice (the sender's `interfaces.voice` entry) — so agents sound distinct. */
+  voice?: string
+  lang?: string
+  style?: string
+  engine?: string
+  timeoutMs?: number
+}
+
+export type SynthesizeResult = {
+  path: string
+  engine?: string
+  voice?: string
+  fallbackFrom?: string
+  bytes: number
+}
+
+/** Synthesize text to a local .ogg file. The HTTP facade is always synchronous — it
+ *  returns the audio bytes, delivery is our job (the speaker). */
+export async function synthesize(text: string, outPath: string, opts: SynthesizeOptions = {}): Promise<SynthesizeResult> {
+  const ep = resolveVoiceEndpoint()
+  const res = await fetch(ep.tts, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({
+      input: text,
+      ...(opts.voice ? { voice: opts.voice } : {}),
+      ...(opts.lang ? { lang: opts.lang } : {}),
+      ...(opts.style ? { style: opts.style } : {}),
+      ...(opts.engine ? { engine: opts.engine } : {}),
+    }),
+    signal: AbortSignal.timeout(opts.timeoutMs ?? 120_000),
+  })
+  if (!res.ok) {
+    throw new Error(`voice-connect tts ${res.status}: ${(await res.text()).slice(0, 300)}`)
+  }
+  const buf = Buffer.from(await res.arrayBuffer())
+  if (buf.length === 0) throw new Error('voice-connect tts returned 0 bytes')
+  writeFileSync(outPath, buf, { mode: 0o600 })
+  return {
+    path: outPath,
+    engine: res.headers.get('x-voice-engine') ?? undefined,
+    voice: res.headers.get('x-voice-voice') ?? undefined,
+    fallbackFrom: res.headers.get('x-voice-fallback-from') ?? undefined,
+    bytes: buf.length,
+  }
+}

package/src/voiceSession.ts

@@ -0,0 +1,353 @@
+// voiceSession.ts — the live voice session (Фаза 3): the runtime's ears, its turn-taking
+// and its mouth, wired between audio.ts (host mechanics) and voiceConnect.ts (speech).
+//
+// INBOUND   mic → endpointing → STT → address match → deliver to the fleet
+// OUTBOUND  agent envelope → TTS → speaker, then an open-mic follow-up window
+//
+// Turn-taking follows the established voice-assistant pattern recorded in Описание
+// voicetalk: the address (wake word) only INITIATES a conversation; after an agent has
+// answered, a short open-mic window lets the owner continue without naming the peer
+// again, and the window closes on silence. The microphone is gated while the speaker is
+// talking, so the runtime never transcribes its own voice.
+//
+// The numeric defaults below are NOT invented here — they come from the turn-taking
+// research prepared for this phase (~/Peers/perplex/research-tree/voice-ux-turntaking/
+// 2026-06-21_turn-taking-defaults.md), which converges Alexa / Google / Siri / Алиса and
+// the open-source stacks: open-mic window 8–10 s, endpointing pause 500 ms (500–800 ms
+// for free dialogue), rising earcon on mic-open / short descending on utterance-end /
+// double descending on error, and NO earcon while processing.
+//
+// Barge-in (interrupting the spoken answer) is deliberately OUT of Фаза 3: the same
+// research shows it requires acoustic echo cancellation plus a neural VAD, neither of
+// which exists here yet. Mic gating during playback is the honest substitute — the
+// runtime simply does not listen while it talks.
+
+import { mkdirSync, rmSync } from 'fs'
+import { join } from 'path'
+import { randomUUID } from 'crypto'
+import {
+  Microphone,
+  Segmenter,
+  earconWav,
+  playFile,
+  resolveInputDevice,
+  writeWav,
+  type Segment,
+} from './audio.ts'
+import { synthesize, transcribe } from './voiceConnect.ts'
+import {
+  buildAddressBook,
+  isLikelyHallucination,
+  matchWake,
+  readFleetPeers,
+  sttPrompt,
+  type Address,
+  type FleetPeer,
+  type VoicetalkBinding,
+} from './wake.ts'
+import { writeFileSync } from 'fs'
+
+const num = (v: string | undefined, d: number) => (v && Number(v) > 0 ? Number(v) : d)
+
+/** How long after a turn the mic stays open for a follow-up without re-addressing.
+ *  Research default: 8–10 s (Google Continued Conversation ≈10 s; Alexa's 5 s reads as
+ *  too short). */
+const FOLLOWUP_MS = num(process.env.VOICETALK_FOLLOWUP_MS, 9_000)
+/** Silence that ends an utterance (endpointing pause). Research default 500 ms, band
+ *  500–800 ms for free dialogue; the upper half of the band suits a heavier recognizer —
+ *  it fragments one thought into two transcriptions far less often. */
+const HANGOVER_MS = num(process.env.VOICETALK_HANGOVER_MS, 700)
+/** Gate the mic for a moment after the speaker stops — room echo tail. */
+const ECHO_TAIL_MS = num(process.env.VOICETALK_ECHO_TAIL_MS, 400)
+/** Long answers are truncated for speech: a spoken monologue is worse than a summary. */
+const TTS_MAX_CHARS = num(process.env.VOICETALK_TTS_MAX_CHARS, 1_200)
+/** Language hint for STT; the owner's language. Empty = let the service auto-detect. */
+const STT_LANG = process.env.VOICETALK_STT_LANG ?? 'ru'
+/** Force a local recognizer (mlx-whisper) to keep always-on listening on-device. */
+const STT_ENGINE = process.env.VOICETALK_STT_ENGINE?.trim() || undefined
+
+export type DeliverFn = (target: string, text: string) => Promise<{ ok: boolean; detail?: string }>
+export type LogFn = (evt: string, fields: Record<string, unknown>) => void
+
+export type VoiceSessionOptions = {
+  owner: string
+  binding: VoicetalkBinding
+  deliver: DeliverFn
+  log: LogFn
+  /** Scratch dir for captured WAVs and synthesized OGGs. */
+  tmpDir: string
+  peers?: FleetPeer[]
+}
+
+/** State sounds, per the earcon convention in the research: `open` = the microphone is
+ *  yours (rising, ≤300 ms), `end` = utterance captured, processing (short descending,
+ *  ≤200 ms), `error` = nothing routed (double descending). Processing itself is silent. */
+type Earcons = { open: string; end: string; error: string }
+
+export class VoiceSession {
+  private mic: Microphone | null = null
+  private readonly segmenter = new Segmenter({
+    hangoverMs: HANGOVER_MS,
+    ratio: num(process.env.VOICETALK_VAD_RATIO, 3.5),
+    minRms: num(process.env.VOICETALK_VAD_MIN_RMS, 700),
+  })
+  private book: Address[] = []
+  private prompt = ''
+  private earcons: Earcons | null = null
+  private busy = false
+  private muteUntil = 0
+  private followTarget: string | null = null
+  private followUntil = 0
+  private stopped = false
+
+  constructor(private readonly opts: VoiceSessionOptions) {}
+
+  /** Resolve the input device, prime the address book, open the mic. Throws if the mic
+   *  cannot be resolved — the caller decides whether that is fatal (it is not: the
+   *  router keeps running text-only). */
+  async start(): Promise<{ deviceIndex: number; deviceName?: string; addresses: number }> {
+    mkdirSync(this.opts.tmpDir, { recursive: true, mode: 0o700 })
+    this.refreshAddressBook()
+    this.earcons = this.writeEarcons()
+    const device = await resolveInputDevice(this.opts.binding.device)
+    this.mic = new Microphone({
+      deviceIndex: device.index,
+      onData: chunk => this.onAudio(chunk),
+      onStart: attempt => this.opts.log('mic.start', { device: device.index, name: device.name, attempt }),
+      onError: detail => this.opts.log('mic.error', { detail }),
+    })
+    this.mic.start()
+    return { deviceIndex: device.index, deviceName: device.name, addresses: this.book.length }
+  }
+
+  stop(): void {
+    this.stopped = true
+    this.mic?.stop()
+    this.mic = null
+  }
+
+  private refreshAddressBook(): void {
+    this.book = buildAddressBook({
+      owner: this.opts.owner,
+      binding: this.opts.binding,
+      peers: this.opts.peers ?? readFleetPeers(),
+    })
+    this.prompt = sttPrompt(this.book)
+  }
+
+  private writeEarcons(): Earcons {
+    const files: Earcons = {
+      open: join(this.opts.tmpDir, 'earcon-open.wav'),
+      end: join(this.opts.tmpDir, 'earcon-end.wav'),
+      error: join(this.opts.tmpDir, 'earcon-error.wav'),
+    }
+    // A5→C6→E6 rising, 240 ms — the microphone is open.
+    writeFileSync(files.open, earconWav([{ hz: 880, ms: 80 }, { hz: 1047, ms: 80 }, { hz: 1319, ms: 80 }]), { mode: 0o600 })
+    // Short descending, 160 ms — heard you, working on it.
+    writeFileSync(files.end, earconWav([{ hz: 880, ms: 80 }, { hz: 660, ms: 80 }]), { mode: 0o600 })
+    // Double descending, 260 ms — tonally "closed": nothing was routed.
+    writeFileSync(files.error, earconWav([{ hz: 520, ms: 120 }, { hz: 330, ms: 140 }]), { mode: 0o600 })
+    return files
+  }
+
+  private async earcon(kind: keyof Earcons): Promise<void> {
+    if (!this.earcons || process.env.VOICETALK_EARCONS === '0') return
+    try {
+      this.muteUntil = Math.max(this.muteUntil, Date.now() + 600)
+      await playFile(this.earcons[kind], { timeoutMs: 5_000 })
+    } catch (err) {
+      this.opts.log('earcon.error', { kind, detail: err instanceof Error ? err.message : String(err) })
+    }
+  }
+
+  private muted(): boolean {
+    return Date.now() < this.muteUntil
+  }
+
+  private onAudio(chunk: Buffer): void {
+    if (this.stopped) return
+    if (this.muted()) {
+      // Our own speech (or an earcon) is in the room — throw the audio away rather than
+      // letting the endpointer build an utterance out of it.
+      this.segmenter.reset()
+      return
+    }
+    for (const segment of this.segmenter.push(chunk)) {
+      if (this.busy) {
+        // One utterance in flight at a time: STT is the expensive step and a pile-up
+        // would only add latency to the turn that matters.
+        this.opts.log('segment.drop', { reason: 'busy', ms: segment.ms })
+        continue
+      }
+      this.busy = true
+      void this.handleSegment(segment).finally(() => {
+        this.busy = false
+      })
+    }
+  }
+
+  private async handleSegment(segment: Segment): Promise<void> {
+    const wav = join(this.opts.tmpDir, `utt-${randomUUID()}.wav`)
+    const started = Date.now()
+    try {
+      writeWav(segment.pcm, wav)
+      let transcript = ''
+      try {
+        transcript = await transcribe(wav, {
+          lang: STT_LANG || undefined,
+          engine: STT_ENGINE,
+          prompt: this.prompt || undefined,
+        })
+      } catch (err) {
+        this.opts.log('stt.error', { ms: segment.ms, detail: err instanceof Error ? err.message : String(err) })
+        return
+      }
+      const sttMs = Date.now() - started
+      if (isLikelyHallucination(transcript)) {
+        this.opts.log('stt.ignored', {
+          ms: segment.ms,
+          sttMs,
+          peak: Math.round(segment.peakRms),
+          floor: Math.round(segment.noiseFloor),
+          text: transcript,
+        })
+        return
+      }
+      this.opts.log('stt.text', {
+        ms: segment.ms,
+        sttMs,
+        peak: Math.round(segment.peakRms),
+        floor: Math.round(segment.noiseFloor),
+        threshold: segment.threshold,
+        text: transcript,
+      })
+
+      const match = matchWake(transcript, this.book)
+      const followOpen = this.followTarget && Date.now() < this.followUntil
+      let target: string | null = null
+      let body = ''
+      if (match) {
+        target = match.target
+        body = match.body
+        if (!body) {
+          // Addressed with nothing to say yet («Борис?») — acknowledge and hold the mic
+          // open for the actual request, exactly as an assistant beeps and waits.
+          this.followTarget = target
+          this.followUntil = Date.now() + FOLLOWUP_MS
+          this.opts.log('wake.await-body', { target, alias: match.alias })
+          await this.earcon('open')
+          return
+        }
+      } else if (followOpen) {
+        target = this.followTarget
+        body = transcript.trim()
+        this.opts.log('followup.route', { target })
+      } else {
+        this.opts.log('wake.no-match', { text: transcript })
+        return
+      }
+
+      // The window is extended for the whole turn: the agent's answer will extend it
+      // again when it is spoken.
+      this.followTarget = target
+      this.followUntil = Date.now() + FOLLOWUP_MS
+      // Utterance accepted → the "end of listening, now processing" sound. Delivery
+      // itself stays SILENT (processing earns no earcon) — the agent's own voice is the
+      // next thing the owner hears.
+      await this.earcon('end')
+      const verdict = await this.opts.deliver(target!, body)
+      if (verdict.ok) {
+        this.opts.log('voice.delivered', { target, chars: body.length, totalMs: Date.now() - started })
+      } else {
+        this.opts.log('voice.undelivered', { target, detail: verdict.detail })
+        await this.earcon('error')
+        await this.say(`Не доставлено: ${verdict.detail ?? 'неизвестная ошибка'}`)
+      }
+    } finally {
+      rmSync(wav, { force: true })
+    }
+  }
+
+  /** Voice of the SENDER — each agent keeps the voice configured in its own passport
+   *  (`interfaces.voice`), so the owner hears who is talking. */
+  private voiceFor(personality: string): string | undefined {
+    const peers = this.opts.peers ?? readFleetPeers()
+    const voice = peers.find(p => p.personality === personality)?.voice
+    if (!voice) return undefined
+    return voice['gemini-3.1-flash-tts-preview'] ?? Object.values(voice)[0]
+  }
+
+  /** Speak an agent's answer on the owner's speaker, then hold the follow-up window
+   *  open — the conversational turn passes back to the human. */
+  async speak(from: string, message: string): Promise<void> {
+    const text = speakableText(message)
+    if (!text) {
+      this.opts.log('tts.skip', { from, reason: 'nothing speakable' })
+      return
+    }
+    await this.say(text, { from, voice: this.voiceFor(from) })
+    this.followTarget = from
+    this.followUntil = Date.now() + FOLLOWUP_MS
+    // The turn passes back to the human: the mic is open for a follow-up without a
+    // wake word, and the rising earcon says so.
+    await this.earcon('open')
+  }
+
+  private async say(text: string, opts: { from?: string; voice?: string } = {}): Promise<void> {
+    const out = join(this.opts.tmpDir, `say-${randomUUID()}.ogg`)
+    const started = Date.now()
+    try {
+      const result = await synthesize(text, out, { voice: opts.voice })
+      const ttsMs = Date.now() - started
+      // Gate the mic for the whole playback: the runtime must not hear itself.
+      this.muteUntil = Date.now() + 3_600_000
+      this.segmenter.reset()
+      try {
+        await playFile(result.path)
+      } finally {
+        this.muteUntil = Date.now() + ECHO_TAIL_MS
+        this.segmenter.reset()
+      }
+      this.opts.log('tts.spoken', {
+        from: opts.from,
+        chars: text.length,
+        engine: result.engine,
+        voice: result.voice,
+        fallbackFrom: result.fallbackFrom,
+        ttsMs,
+        totalMs: Date.now() - started,
+      })
+    } catch (err) {
+      this.muteUntil = 0
+      this.opts.log('tts.error', { from: opts.from, detail: err instanceof Error ? err.message : String(err) })
+      throw err
+    } finally {
+      rmSync(out, { force: true })
+    }
+  }
+}
+
+/** Prepare an agent's text for the speaker: strip what only makes sense on a screen and
+ *  cap the length — a spoken wall of text is unusable. The modality frame already asks
+ *  agents for short plain answers; this is the floor for when one forgets. */
+export function speakableText(message: string, maxChars = TTS_MAX_CHARS): string {
+  let text = message
+    .replace(/```[\s\S]*?```/g, ' (фрагмент кода) ')
+    .replace(/`([^`]*)`/g, '$1')
+    .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ')
+    .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
+    .replace(/https?:\/\/\S+/g, ' ссылка ')
+    .replace(/^\s{0,3}#{1,6}\s+/gm, '')
+    .replace(/\*\*([^*]+)\*\*/g, '$1')
+    .replace(/(^|\s)\*([^*]+)\*/g, '$1$2')
+    .replace(/^\s*[-*•]\s+/gm, '')
+    .replace(/[ \t]+/g, ' ')
+    .replace(/\n{2,}/g, '\n')
+    .trim()
+  if (text.length > maxChars) {
+    const cut = text.slice(0, maxChars)
+    const lastStop = Math.max(cut.lastIndexOf('. '), cut.lastIndexOf('! '), cut.lastIndexOf('? '))
+    text = `${(lastStop > maxChars * 0.5 ? cut.slice(0, lastStop + 1) : cut).trim()} … Дальше — в тексте.`
+  }
+  return text
+}