@stage-labs/metro 0.1.0-beta.26 → 0.1.0-beta.261

package/runtime/node_modules/@metro-labs/daemon/src/voice/call.ts

@@ -0,0 +1,200 @@
+import { randomUUID } from 'node:crypto';
+import { errMsg, log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import { forwardTrainCall } from '../stations/train-call.js';
+import { CallApprovals } from './approvals.js';
+import { Peer } from './peer.js';
+import { Talk } from './talk.js';
+import type { VoiceConfig } from './store.js';
+
+const OFFER_WAIT_MS = 45_000;
+const LONGEST_CALL_MS = 60 * 60_000;
+const CONTEXT_MESSAGES = 12;
+const CONTEXT_CHARS = 400;
+const LEAVE_RETRY_MS = 3_000;
+
+interface CallStart {
+  line: string;
+  lineName: string;
+  direct: boolean;
+  from: string;
+  callerName: string;
+  callId: string;
+  callerPeer: string;
+}
+
+const str = (value: unknown): string => (typeof value === 'string' ? value : '');
+
+export async function trainCall(action: string, args: Record<string, unknown>): Promise<unknown> {
+  const answer = await forwardTrainCall('xmtp', action, args);
+  if (answer.error !== undefined) throw new Error(answer.error);
+  return answer.result;
+}
+
+function speaker(row: Record<string, unknown>, start: CallStart): string {
+  if (row.self === true) return 'you';
+  return row.from === start.from ? start.callerName : `another member (${str(row.from)})`;
+}
+
+function contextLine(row: unknown, start: CallStart): string | null {
+  if (!isRecord(row) || typeof row.text !== 'string' || row.text.startsWith('[call')) return null;
+  const who = speaker(row, start);
+  return `${str(row.ts).slice(11, 16)} ${who}: ${row.text.replace(/\s+/g, ' ').slice(0, CONTEXT_CHARS)}`;
+}
+
+async function recentChat(start: CallStart): Promise<string> {
+  try {
+    const result = await trainCall('read', { line: start.line, limit: CONTEXT_MESSAGES });
+    const rows = isRecord(result) && Array.isArray(result.messages) ? result.messages : [];
+    return rows.map((row) => contextLine(row, start)).filter((l): l is string => l !== null).join('\n');
+  } catch (err) {
+    log.warn({ err: errMsg(err) }, 'voice: could not read the chat before the call');
+    return '';
+  }
+}
+
+function where(start: CallStart): string {
+  if (start.lineName !== '') return `the group chat "${start.lineName}"`;
+  return start.direct ? `your direct chat with ${start.callerName}` : 'a group chat with no name';
+}
+
+export function opening(start: CallStart, chat: string): string {
+  const recent = chat === '' ? 'No earlier messages were found there.' : `Recent messages there before the call, oldest first (times in UTC):\n${chat}`;
+  return [
+    `[A voice call with ${start.callerName} is starting on Stage, in ${where(start)}.`,
+    `Line of this chat: ${start.line} (pass it verbatim to the metro tools, e.g. read, to look at or act on this chat).`,
+    `Caller: ${start.callerName} (${start.from}).`,
+    recent,
+    `Greet ${start.callerName} in a few words.]`,
+  ].join('\n');
+}
+
+export class Call {
+  private readonly selfPeer = randomUUID();
+  private readonly startedAt = Date.now();
+  private readonly talk: Talk;
+  private peer: Peer | null = null;
+  private timer: ReturnType<typeof setTimeout> | null = null;
+  private longest: ReturnType<typeof setTimeout>;
+  private over = false;
+  private left: Promise<void> = Promise.resolve();
+
+  constructor(
+    readonly start: CallStart,
+    cfg: VoiceConfig,
+    private readonly onOver: () => void,
+  ) {
+    const approvals = new CallApprovals(start.line, async (text) => {
+      await trainCall('send', { line: start.line, text });
+    });
+    this.talk = new Talk(
+      cfg,
+      (opus, timestamp, marker) => {
+        this.peer?.sendOpus(opus, timestamp, marker);
+      },
+      (reason) => {
+        this.hangUp(reason);
+      },
+      approvals,
+    );
+    this.longest = setTimeout(() => {
+      this.hangUp('the call reached one hour');
+    }, LONGEST_CALL_MS);
+  }
+
+  async begin(): Promise<void> {
+    log.info({ line: this.start.line, callId: this.start.callId }, 'voice: answering a call');
+    recentChat(this.start)
+      .then((chat) => {
+        if (!this.over) this.talk.prime(opening(this.start, chat));
+      })
+      .catch((err: unknown) => {
+        log.warn({ err: errMsg(err) }, 'voice: could not start the agent on the call');
+      });
+    await this.signal({ kind: 'join', callId: this.start.callId, from: this.selfPeer });
+    this.timer = setTimeout(() => {
+      this.hangUp('no offer arrived');
+    }, OFFER_WAIT_MS);
+  }
+
+  signalled(raw: Record<string, unknown>): void {
+    if (raw.callId !== this.start.callId || this.over) return;
+    const fromCaller = raw.from === this.start.callerPeer;
+    if (raw.kind === 'offer' && fromCaller && raw.to === this.selfPeer && this.peer === null && typeof raw.sdp === 'string')
+      this.answer(this.start.callerPeer, raw.sdp).catch((err: unknown) => {
+        this.hangUp(`answering failed: ${errMsg(err)}`);
+      });
+    else if (raw.kind === 'leave' && fromCaller) this.hangUp('the caller hung up');
+  }
+
+  hangUp(reason: string): void {
+    this.end(reason);
+  }
+
+  leave(reason: string): Promise<void> {
+    this.end(reason);
+    return this.left;
+  }
+
+  private async answer(to: string, offer: string): Promise<void> {
+    if (this.timer !== null) clearTimeout(this.timer);
+    this.peer = new Peer({
+      audio: (opus) => {
+        this.talk.hear(opus);
+      },
+      state: (state) => {
+        this.stateChanged(state);
+      },
+    });
+    const sdp = await this.peer.answer(offer);
+    if (this.over) return;
+    await this.signal({ kind: 'answer', callId: this.start.callId, from: this.selfPeer, to, sdp });
+  }
+
+  private stateChanged(state: string): void {
+    log.info({ state, afterMs: Date.now() - this.startedAt }, 'voice: call connection');
+    if (state === 'connected') this.talk.connect();
+    else if (state === 'failed' || state === 'closed') this.hangUp(`the connection ${state}`);
+  }
+
+  private signal(signal: Record<string, unknown>): Promise<unknown> {
+    return trainCall('callSignal', { line: this.start.line, signal });
+  }
+
+  private end(reason: string): void {
+    if (this.over) return;
+    this.over = true;
+    if (this.timer !== null) clearTimeout(this.timer);
+    clearTimeout(this.longest);
+    log.info({ reason, seconds: Math.round((Date.now() - this.startedAt) / 1000) }, 'voice: call over');
+    this.left = this.sendLeave();
+    this.left
+      .then(() => this.wrapUp(reason))
+      .catch((err: unknown) => {
+        log.warn({ err: errMsg(err) }, 'voice: wrapping up the call failed');
+      })
+      .finally(() => {
+        this.onOver();
+      });
+  }
+
+  private async sendLeave(): Promise<void> {
+    const leave = { kind: 'leave', callId: this.start.callId, from: this.selfPeer };
+    try {
+      await this.signal(leave);
+    } catch (err) {
+      log.warn({ err: errMsg(err) }, 'voice: could not send the hang-up, trying once more');
+      await new Promise((r) => setTimeout(r, LEAVE_RETRY_MS));
+      await this.signal(leave).catch((again: unknown) => {
+        log.warn({ err: errMsg(again) }, 'voice: could not send the hang-up; the next start sends it');
+      });
+    }
+  }
+
+  private async wrapUp(reason: string): Promise<void> {
+    this.peer?.close();
+    this.talk.finish();
+    if (reason.startsWith('the caller') || reason === 'no offer arrived') return;
+    await trainCall('send', { line: this.start.line, text: `The voice call ended: ${reason}.` });
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/calls.ts

@@ -0,0 +1,130 @@
+import { errMsg, log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import type { TrainEvent } from '@metro-labs/core/trains/protocol';
+import { lineReceives, mayApprove } from '../agents/map.js';
+import { Call, trainCall } from './call.js';
+import { readVoice, voiceReady } from './store.js';
+
+const RING_MS = 40_000;
+const CALL_TYPES = new Set(['callInvite', 'callSignal']);
+const SHUTDOWN_LEAVE_MS = 2_000;
+const SWEEP_FIRST_MS = 5_000;
+const SWEEP_EVERY_MS = 10_000;
+const SWEEP_TRIES = 12;
+
+let current: Call | null = null;
+
+const record = (value: unknown): Record<string, unknown> => (isRecord(value) ? value : {});
+const str = (value: unknown): string => (typeof value === 'string' ? value : '');
+
+const contentTypeOf = (env: TrainEvent): string => str(record(env.payload).contentType);
+
+export const isCallEvent = (env: TrainEvent): boolean => CALL_TYPES.has(contentTypeOf(env));
+
+const sameCaller = (call: Call, line: string, from: string): boolean => call.start.line === line && call.start.from === from;
+
+function refusal(line: string, from: string): string | null {
+  if (current !== null && !sameCaller(current, line, from)) return 'already in a call';
+  if (!voiceReady(readVoice())) return 'voice is off or has no API key on the Voice page';
+  if (!lineReceives(line) || !mayApprove(line, from)) return 'the caller is not someone who can approve for this channel';
+  return null;
+}
+
+function decline(line: string, callId: string): void {
+  trainCall('callSignal', { line, signal: { kind: 'decline', callId } }).catch((err: unknown) => {
+    log.warn({ line, err: errMsg(err) }, 'voice: could not decline a call');
+  });
+}
+
+function ringing(env: TrainEvent): boolean {
+  const sent = Date.parse(str(env.ts));
+  return !Number.isNaN(sent) && Date.now() - sent < RING_MS;
+}
+
+function invite(env: TrainEvent, body: Record<string, unknown>): void {
+  const line = str(env.line);
+  const from = str(env.from);
+  const callId = str(body.callId);
+  const callerPeer = str(body.from);
+  if (callId === '' || callerPeer === '' || current?.start.callId === callId) return;
+  if (!ringing(env)) {
+    log.info({ line }, 'voice: an invite came in after the caller stopped ringing, not answered');
+    return;
+  }
+  const refused = refusal(line, from);
+  if (refused !== null) {
+    log.info({ line, reason: refused }, 'voice: a call came in and is declined');
+    decline(line, callId);
+    return;
+  }
+  current?.hangUp('the caller started a new call');
+  const callerName = str(env.from_display_name) || str(env.from_name) || 'the caller';
+  const start = { line, lineName: str(env.line_name), direct: env.is_private === true, from, callerName, callId, callerPeer };
+  const call = new Call(start, readVoice(), () => {
+    if (current === call) current = null;
+  });
+  current = call;
+  call.begin().catch((err: unknown) => {
+    call.hangUp(`starting failed: ${errMsg(err)}`);
+  });
+}
+
+export function onCallEvent(env: TrainEvent): void {
+  const body = record(record(env.payload).call);
+  if (contentTypeOf(env) === 'callInvite') invite(env, body);
+  else if (current !== null && env.line === current.start.line) current.signalled(body);
+}
+
+export function leaveCallsForShutdown(): Promise<void> {
+  if (current === null) return Promise.resolve();
+  const leaving = current.leave('Metro stopped or restarted');
+  const capped = new Promise<void>((resolve) => {
+    setTimeout(resolve, SHUTDOWN_LEAVE_MS).unref();
+  });
+  return Promise.race([leaving, capped]);
+}
+
+interface Leftover {
+  line: string;
+  callId: string;
+  peer: string;
+}
+
+function leftoversOf(result: unknown): Leftover[] {
+  const rows = isRecord(result) && Array.isArray(result.calls) ? result.calls : [];
+  return rows
+    .map((row) => record(row))
+    .map((row) => ({ line: str(row.line), callId: str(row.callId), peer: str(row.peer) }))
+    .filter((row) => row.line !== '' && row.callId !== '' && row.peer !== '');
+}
+
+export async function leaveLeftoverCalls(): Promise<number> {
+  const leftovers = leftoversOf(await trainCall('callLeftovers', {}));
+  let left = 0;
+  for (const { line, callId, peer } of leftovers) {
+    if (current?.start.callId === callId) continue;
+    try {
+      await trainCall('callSignal', { line, signal: { kind: 'leave', callId, from: peer } });
+      log.info({ line, callId }, 'voice: left a call this box was still in');
+      left += 1;
+    } catch (err) {
+      log.warn({ line, err: errMsg(err) }, 'voice: could not leave a call this box was still in');
+    }
+  }
+  return left;
+}
+
+function sweep(tries: number, delayMs: number): void {
+  setTimeout(() => {
+    leaveLeftoverCalls().catch((err: unknown) => {
+      const reason = errMsg(err);
+      if (reason.startsWith('no train named')) return;
+      if (tries > 1) sweep(tries - 1, SWEEP_EVERY_MS);
+      else log.warn({ err: reason }, 'voice: could not check for calls left open');
+    });
+  }, delayMs).unref();
+}
+
+export function sweepLeftoverCalls(): void {
+  sweep(SWEEP_TRIES, SWEEP_FIRST_MS);
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/peer.ts

@@ -0,0 +1,84 @@
+import { RTCPeerConnection, RtpHeader, RtpPacket, type RTCDataChannel, type RTCRtpSender } from 'werift';
+import { errMsg, log } from '@metro-labs/core/log';
+
+const ICE_SERVERS = [{ urls: 'stun:stun.cloudflare.com:3478' }, { urls: 'stun:stun.l.google.com:19302' }];
+const UDP_PORTS: [number, number] = [40_000, 40_100];
+const GATHER_MS = 2_500;
+const OPUS_PT = 111;
+const MEDIA = JSON.stringify({ audio: true, video: false, screen: false });
+
+interface PeerEvents {
+  audio(opus: Buffer): void;
+  state(state: string): void;
+}
+
+function gathered(pc: RTCPeerConnection): Promise<void> {
+  if (pc.iceGatheringState === 'complete') return Promise.resolve();
+  return new Promise((resolve) => {
+    const timer = setTimeout(resolve, GATHER_MS);
+    pc.iceGatheringStateChange.subscribe((state) => {
+      if (state !== 'complete') return;
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+}
+
+export class Peer {
+  private readonly pc = new RTCPeerConnection({
+    iceServers: ICE_SERVERS,
+    bundlePolicy: 'max-bundle',
+    icePortRange: UDP_PORTS,
+    iceUseIpv6: false,
+  });
+  private readonly channel: RTCDataChannel;
+  private sender: RTCRtpSender | null = null;
+  private sequence = Math.floor(Math.random() * 0xffff);
+
+  constructor(private readonly events: PeerEvents) {
+    this.channel = this.pc.createDataChannel('media', { negotiated: true, id: 0 });
+    this.channel.stateChanged.subscribe((state) => {
+      if (state === 'open') this.channel.send(MEDIA);
+    });
+    this.pc.onTrack.subscribe((track) => {
+      if (track.kind === 'audio')
+        track.onReceiveRtp.subscribe((rtp) => {
+          this.events.audio(rtp.payload);
+        });
+    });
+    this.pc.connectionStateChange.subscribe((state) => {
+      this.events.state(state);
+    });
+  }
+
+  async answer(offer: string): Promise<string> {
+    await this.pc.setRemoteDescription({ type: 'offer', sdp: offer });
+    for (const transceiver of this.pc.getTransceivers()) {
+      if (transceiver.kind === 'audio') {
+        transceiver.setDirection('sendrecv');
+        this.sender = transceiver.sender;
+      } else transceiver.setDirection('inactive');
+    }
+    await this.pc.setLocalDescription(await this.pc.createAnswer());
+    await gathered(this.pc);
+    const sdp = this.pc.localDescription?.sdp;
+    if (sdp === undefined) throw new Error('no local description after the answer');
+    return sdp;
+  }
+
+  sendOpus(payload: Buffer, timestamp: number, marker: boolean): void {
+    const sender = this.sender;
+    if (sender === null) return;
+    this.sequence = (this.sequence + 1) & 0xffff;
+    const header = new RtpHeader({ payloadType: OPUS_PT, sequenceNumber: this.sequence, timestamp, marker });
+    sender.sendRtp(new RtpPacket(header, payload)).catch((err: unknown) => {
+      log.debug({ err: errMsg(err) }, 'voice: an audio packet was not sent');
+    });
+  }
+
+  close(): void {
+    this.pc.close().catch((err: unknown) => {
+      log.debug({ err: errMsg(err) }, 'voice: closing the call connection failed');
+    });
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/scribe.ts

@@ -0,0 +1,103 @@
+import WebSocket from 'ws';
+import { errMsg, log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import { bytesOf } from './audio.js';
+
+export const elevenLabs = (): string => process.env.METRO_ELEVENLABS_URL ?? 'wss://api.elevenlabs.io';
+const STT_RATE = 16_000;
+const CHUNK_SAMPLES = STT_RATE / 10;
+const HELD_SAMPLES = STT_RATE * 2;
+export const VAD_SILENCE_SECS = 0.5;
+
+interface Heard {
+  partial(text: string): void;
+  committed(text: string): void;
+  failed(reason: string): void;
+}
+
+const FATAL = new Set([
+  'auth_error',
+  'quota_exceeded',
+  'unaccepted_terms',
+  'rate_limited',
+  'resource_exhausted',
+  'session_time_limit_exceeded',
+  'error',
+]);
+
+export function wsText(data: WebSocket.RawData): string {
+  if (Array.isArray(data)) return Buffer.concat(data).toString('utf8');
+  return (Buffer.isBuffer(data) ? data : Buffer.from(data)).toString('utf8');
+}
+
+export function parsed(raw: string): Record<string, unknown> {
+  try {
+    const msg: unknown = JSON.parse(raw);
+    return isRecord(msg) ? msg : {};
+  } catch {
+    return {};
+  }
+}
+
+export function sttUrl(language: string | null): string {
+  const query = new URLSearchParams({
+    model_id: 'scribe_v2_realtime',
+    audio_format: 'pcm_16000',
+    commit_strategy: 'vad',
+    vad_silence_threshold_secs: String(VAD_SILENCE_SECS),
+  });
+  if (language !== null) query.set('language_code', language);
+  return `${elevenLabs()}/v1/speech-to-text/realtime?${query.toString()}`;
+}
+
+export class Scribe {
+  private readonly ws: WebSocket;
+  private buffer: Int16Array[] = [];
+  private buffered = 0;
+
+  constructor(apiKey: string, language: string | null, private readonly heard: Heard) {
+    this.ws = new WebSocket(sttUrl(language), { headers: { 'xi-api-key': apiKey } });
+    this.ws.on('message', (data) => {
+      this.onMessage(wsText(data));
+    });
+    this.ws.on('error', (err) => {
+      log.warn({ err: errMsg(err) }, 'voice: speech to text connection error');
+    });
+    this.ws.on('close', (code) => {
+      if (code !== 1000) this.heard.failed(`speech to text closed (${String(code)})`);
+    });
+  }
+
+  send(pcm16k: Int16Array): void {
+    this.buffer.push(pcm16k);
+    this.buffered += pcm16k.length;
+    while (this.buffered > HELD_SAMPLES) this.buffered -= this.buffer.shift()?.length ?? this.buffered;
+    if (this.buffered < CHUNK_SAMPLES || this.ws.readyState !== WebSocket.OPEN) return;
+    const chunk = new Int16Array(this.buffered);
+    let at = 0;
+    for (const part of this.buffer) {
+      chunk.set(part, at);
+      at += part.length;
+    }
+    this.buffer = [];
+    this.buffered = 0;
+    this.ws.send(
+      JSON.stringify({ message_type: 'input_audio_chunk', audio_base_64: bytesOf(chunk).toString('base64'), commit: false, sample_rate: STT_RATE }),
+    );
+  }
+
+  close(): void {
+    this.ws.close(1000);
+  }
+
+  private onMessage(raw: string): void {
+    const msg = parsed(raw);
+    const type = typeof msg.message_type === 'string' ? msg.message_type : '';
+    const text = typeof msg.text === 'string' ? msg.text.trim() : '';
+    if (type === 'partial_transcript') this.heard.partial(text);
+    else if (type === 'committed_transcript') {
+      if (text !== '') this.heard.committed(text);
+    } else if (FATAL.has(type)) this.heard.failed(`speech to text: ${typeof msg.error === 'string' ? msg.error : type}`);
+    else log.debug({ type }, 'voice: speech to text said');
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/speakable.ts

@@ -0,0 +1,54 @@
+const FIRST_CLAUSE_MIN = 24;
+const LONGEST = 220;
+const SENTENCE_END = /[.!?…]+["')\]]*\s+|\n+/g;
+const CLAUSE_END = /[,;:]\s+/g;
+
+function speakable(text: string): string {
+  return text
+    .replace(/https?:\/\/\S+/g, 'the link')
+    .replace(/`{1,3}[^`]*`{1,3}/g, (code) => code.replace(/`/g, ''))
+    .replace(/^\s*(?:#+|[-*•]|\d+\.)\s+/gm, '')
+    .replace(/[*_~#|<>[\]{}]/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function lastEnd(text: string, pattern: RegExp): number {
+  let end = -1;
+  for (const match of text.matchAll(pattern)) end = match.index + match[0].length;
+  return end;
+}
+
+function cutAt(text: string, first: boolean): number {
+  const sentence = lastEnd(text, SENTENCE_END);
+  if (sentence > 0) return sentence;
+  if (first && text.length >= FIRST_CLAUSE_MIN) {
+    const clause = lastEnd(text, CLAUSE_END);
+    if (clause > 0) return clause;
+  }
+  if (text.length < LONGEST) return -1;
+  const space = text.lastIndexOf(' ');
+  return space > 0 ? space + 1 : text.length;
+}
+
+export class Chunker {
+  private buffer = '';
+  private first = true;
+
+  add(delta: string): string[] {
+    this.buffer += delta;
+    const at = cutAt(this.buffer, this.first);
+    if (at < 0) return [];
+    const ready = speakable(this.buffer.slice(0, at));
+    this.buffer = this.buffer.slice(at);
+    if (ready === '') return [];
+    this.first = false;
+    return [ready];
+  }
+
+  flush(): string[] {
+    const rest = speakable(this.buffer);
+    this.buffer = '';
+    return rest === '' ? [] : [rest];
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/speech.ts

@@ -0,0 +1,104 @@
+import WebSocket from 'ws';
+import { errMsg, log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import { pcmOf, upsample2 } from './audio.js';
+import { elevenLabs, parsed, wsText } from './scribe.js';
+
+const TTS_MODEL = 'eleven_flash_v2_5';
+
+interface Voiced {
+  audio(pcm48k: Int16Array, chars: number): void;
+  done(): void;
+}
+
+function ttsUrl(voiceId: string): string {
+  const query = new URLSearchParams({
+    model_id: TTS_MODEL,
+    output_format: 'pcm_24000',
+    auto_mode: 'true',
+    inactivity_timeout: '180',
+  });
+  return `${elevenLabs()}/v1/text-to-speech/${encodeURIComponent(voiceId)}/stream-input?${query.toString()}`;
+}
+
+const charsOf = (msg: Record<string, unknown>): number => {
+  const alignment = msg.alignment;
+  return isRecord(alignment) && Array.isArray(alignment.chars) ? alignment.chars.length : 0;
+};
+
+const MAX_REASON = 300;
+
+export class Utterance {
+  private readonly ws: WebSocket;
+  private readonly outbox: string[] = [];
+  private listener: Voiced | null = null;
+  private carry = 0;
+  private closed = false;
+  private aborted = false;
+  private voiced = false;
+  private refusal: string | null = null;
+
+  constructor(apiKey: string, voiceId: string, failed: (reason: string) => void) {
+    this.ws = new WebSocket(ttsUrl(voiceId), { headers: { 'xi-api-key': apiKey } });
+    this.ws.on('open', () => {
+      this.ws.send(JSON.stringify({ text: ' ' }));
+      for (const line of this.outbox.splice(0)) this.ws.send(line);
+    });
+    this.ws.on('message', (data) => {
+      this.onMessage(wsText(data));
+    });
+    this.ws.on('error', (err) => {
+      this.refusal ??= errMsg(err);
+      log.warn({ err: errMsg(err) }, 'voice: text to speech connection error');
+    });
+    this.ws.on('close', (code, why) => {
+      this.closed = true;
+      this.listener?.done();
+      this.listener = null;
+      const reason = this.refusal ?? (code === 1000 ? null : (why.toString() || `closed (${String(code)})`));
+      if (!this.aborted && !this.voiced && reason !== null) failed(reason.slice(0, MAX_REASON));
+    });
+  }
+
+  get usable(): boolean {
+    return !this.closed && this.listener === null && this.ws.readyState <= WebSocket.OPEN;
+  }
+
+  attach(listener: Voiced): void {
+    this.listener = listener;
+  }
+
+  say(text: string): void {
+    this.post({ text: `${text} ` });
+  }
+
+  end(): void {
+    this.post({ text: '' });
+  }
+
+  abort(): void {
+    this.aborted = true;
+    this.listener = null;
+    this.ws.terminate();
+  }
+
+  private post(msg: Record<string, unknown>): void {
+    const line = JSON.stringify(msg);
+    if (this.ws.readyState === WebSocket.OPEN) this.ws.send(line);
+    else this.outbox.push(line);
+  }
+
+  private onMessage(raw: string): void {
+    const msg = parsed(raw);
+    if (typeof msg.audio === 'string' && msg.audio !== '') {
+      this.voiced = true;
+      const pcm = pcmOf(Buffer.from(msg.audio, 'base64'));
+      const up = upsample2(pcm, this.carry);
+      this.carry = pcm.at(-1) ?? this.carry;
+      this.listener?.audio(up, charsOf(msg));
+    } else if (msg.isFinal !== true) {
+      if (typeof msg.message === 'string' && msg.message !== '') this.refusal ??= msg.message;
+      log.warn({ msg: raw.slice(0, MAX_REASON) }, 'voice: text to speech said');
+    }
+  }
+}