@stage-labs/metro 0.1.0-beta.236 → 0.1.0-beta.238

package/runtime/node_modules/@metro-labs/daemon/src/voice/peer.ts

@@ -0,0 +1,84 @@
+import { RTCPeerConnection, RtpHeader, RtpPacket, type RTCDataChannel, type RTCRtpSender } from 'werift';
+import { errMsg, log } from '@metro-labs/core/log';
+
+const ICE_SERVERS = [{ urls: 'stun:stun.cloudflare.com:3478' }, { urls: 'stun:stun.l.google.com:19302' }];
+const UDP_PORTS: [number, number] = [40_000, 40_100];
+const GATHER_MS = 2_500;
+const OPUS_PT = 111;
+const MEDIA = JSON.stringify({ audio: true, video: false, screen: false });
+
+interface PeerEvents {
+  audio(opus: Buffer): void;
+  state(state: string): void;
+}
+
+function gathered(pc: RTCPeerConnection): Promise<void> {
+  if (pc.iceGatheringState === 'complete') return Promise.resolve();
+  return new Promise((resolve) => {
+    const timer = setTimeout(resolve, GATHER_MS);
+    pc.iceGatheringStateChange.subscribe((state) => {
+      if (state !== 'complete') return;
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+}
+
+export class Peer {
+  private readonly pc = new RTCPeerConnection({
+    iceServers: ICE_SERVERS,
+    bundlePolicy: 'max-bundle',
+    icePortRange: UDP_PORTS,
+    iceUseIpv6: false,
+  });
+  private readonly channel: RTCDataChannel;
+  private sender: RTCRtpSender | null = null;
+  private sequence = Math.floor(Math.random() * 0xffff);
+
+  constructor(private readonly events: PeerEvents) {
+    this.channel = this.pc.createDataChannel('media', { negotiated: true, id: 0 });
+    this.channel.stateChanged.subscribe((state) => {
+      if (state === 'open') this.channel.send(MEDIA);
+    });
+    this.pc.onTrack.subscribe((track) => {
+      if (track.kind === 'audio')
+        track.onReceiveRtp.subscribe((rtp) => {
+          this.events.audio(rtp.payload);
+        });
+    });
+    this.pc.connectionStateChange.subscribe((state) => {
+      this.events.state(state);
+    });
+  }
+
+  async answer(offer: string): Promise<string> {
+    await this.pc.setRemoteDescription({ type: 'offer', sdp: offer });
+    for (const transceiver of this.pc.getTransceivers()) {
+      if (transceiver.kind === 'audio') {
+        transceiver.setDirection('sendrecv');
+        this.sender = transceiver.sender;
+      } else transceiver.setDirection('inactive');
+    }
+    await this.pc.setLocalDescription(await this.pc.createAnswer());
+    await gathered(this.pc);
+    const sdp = this.pc.localDescription?.sdp;
+    if (sdp === undefined) throw new Error('no local description after the answer');
+    return sdp;
+  }
+
+  sendOpus(payload: Buffer, timestamp: number, marker: boolean): void {
+    const sender = this.sender;
+    if (sender === null) return;
+    this.sequence = (this.sequence + 1) & 0xffff;
+    const header = new RtpHeader({ payloadType: OPUS_PT, sequenceNumber: this.sequence, timestamp, marker });
+    sender.sendRtp(new RtpPacket(header, payload)).catch((err: unknown) => {
+      log.debug({ err: errMsg(err) }, 'voice: an audio packet was not sent');
+    });
+  }
+
+  close(): void {
+    this.pc.close().catch((err: unknown) => {
+      log.debug({ err: errMsg(err) }, 'voice: closing the call connection failed');
+    });
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/scribe.ts

@@ -0,0 +1,102 @@
+import WebSocket from 'ws';
+import { errMsg, log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import { bytesOf } from './audio.js';
+
+export const elevenLabs = (): string => process.env.METRO_ELEVENLABS_URL ?? 'wss://api.elevenlabs.io';
+const STT_RATE = 16_000;
+const CHUNK_SAMPLES = STT_RATE / 10;
+const HELD_SAMPLES = STT_RATE * 2;
+export const VAD_SILENCE_SECS = 0.5;
+
+interface Heard {
+  partial(text: string): void;
+  committed(text: string): void;
+  failed(reason: string): void;
+}
+
+const FATAL = new Set([
+  'auth_error',
+  'quota_exceeded',
+  'unaccepted_terms',
+  'rate_limited',
+  'resource_exhausted',
+  'session_time_limit_exceeded',
+  'error',
+]);
+
+export function wsText(data: WebSocket.RawData): string {
+  if (Array.isArray(data)) return Buffer.concat(data).toString('utf8');
+  return (Buffer.isBuffer(data) ? data : Buffer.from(data)).toString('utf8');
+}
+
+export function parsed(raw: string): Record<string, unknown> {
+  try {
+    const msg: unknown = JSON.parse(raw);
+    return isRecord(msg) ? msg : {};
+  } catch {
+    return {};
+  }
+}
+
+function sttUrl(): string {
+  const query = new URLSearchParams({
+    model_id: 'scribe_v2_realtime',
+    audio_format: 'pcm_16000',
+    commit_strategy: 'vad',
+    vad_silence_threshold_secs: String(VAD_SILENCE_SECS),
+  });
+  return `${elevenLabs()}/v1/speech-to-text/realtime?${query.toString()}`;
+}
+
+export class Scribe {
+  private readonly ws: WebSocket;
+  private buffer: Int16Array[] = [];
+  private buffered = 0;
+
+  constructor(apiKey: string, private readonly heard: Heard) {
+    this.ws = new WebSocket(sttUrl(), { headers: { 'xi-api-key': apiKey } });
+    this.ws.on('message', (data) => {
+      this.onMessage(wsText(data));
+    });
+    this.ws.on('error', (err) => {
+      log.warn({ err: errMsg(err) }, 'voice: speech to text connection error');
+    });
+    this.ws.on('close', (code) => {
+      if (code !== 1000) this.heard.failed(`speech to text closed (${String(code)})`);
+    });
+  }
+
+  send(pcm16k: Int16Array): void {
+    this.buffer.push(pcm16k);
+    this.buffered += pcm16k.length;
+    while (this.buffered > HELD_SAMPLES) this.buffered -= this.buffer.shift()?.length ?? this.buffered;
+    if (this.buffered < CHUNK_SAMPLES || this.ws.readyState !== WebSocket.OPEN) return;
+    const chunk = new Int16Array(this.buffered);
+    let at = 0;
+    for (const part of this.buffer) {
+      chunk.set(part, at);
+      at += part.length;
+    }
+    this.buffer = [];
+    this.buffered = 0;
+    this.ws.send(
+      JSON.stringify({ message_type: 'input_audio_chunk', audio_base_64: bytesOf(chunk).toString('base64'), commit: false, sample_rate: STT_RATE }),
+    );
+  }
+
+  close(): void {
+    this.ws.close(1000);
+  }
+
+  private onMessage(raw: string): void {
+    const msg = parsed(raw);
+    const type = typeof msg.message_type === 'string' ? msg.message_type : '';
+    const text = typeof msg.text === 'string' ? msg.text.trim() : '';
+    if (type === 'partial_transcript') this.heard.partial(text);
+    else if (type === 'committed_transcript') {
+      if (text !== '') this.heard.committed(text);
+    } else if (FATAL.has(type)) this.heard.failed(`speech to text: ${typeof msg.error === 'string' ? msg.error : type}`);
+    else log.debug({ type }, 'voice: speech to text said');
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/speakable.ts

@@ -0,0 +1,54 @@
+const FIRST_CLAUSE_MIN = 24;
+const LONGEST = 220;
+const SENTENCE_END = /[.!?…]+["')\]]*\s+|\n+/g;
+const CLAUSE_END = /[,;:]\s+/g;
+
+function speakable(text: string): string {
+  return text
+    .replace(/https?:\/\/\S+/g, 'the link')
+    .replace(/`{1,3}[^`]*`{1,3}/g, (code) => code.replace(/`/g, ''))
+    .replace(/^\s*(?:#+|[-*•]|\d+\.)\s+/gm, '')
+    .replace(/[*_~#|<>[\]{}]/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function lastEnd(text: string, pattern: RegExp): number {
+  let end = -1;
+  for (const match of text.matchAll(pattern)) end = match.index + match[0].length;
+  return end;
+}
+
+function cutAt(text: string, first: boolean): number {
+  const sentence = lastEnd(text, SENTENCE_END);
+  if (sentence > 0) return sentence;
+  if (first && text.length >= FIRST_CLAUSE_MIN) {
+    const clause = lastEnd(text, CLAUSE_END);
+    if (clause > 0) return clause;
+  }
+  if (text.length < LONGEST) return -1;
+  const space = text.lastIndexOf(' ');
+  return space > 0 ? space + 1 : text.length;
+}
+
+export class Chunker {
+  private buffer = '';
+  private first = true;
+
+  add(delta: string): string[] {
+    this.buffer += delta;
+    const at = cutAt(this.buffer, this.first);
+    if (at < 0) return [];
+    const ready = speakable(this.buffer.slice(0, at));
+    this.buffer = this.buffer.slice(at);
+    if (ready === '') return [];
+    this.first = false;
+    return [ready];
+  }
+
+  flush(): string[] {
+    const rest = speakable(this.buffer);
+    this.buffer = '';
+    return rest === '' ? [] : [rest];
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/speech.ts

@@ -0,0 +1,104 @@
+import WebSocket from 'ws';
+import { errMsg, log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import { pcmOf, upsample2 } from './audio.js';
+import { elevenLabs, parsed, wsText } from './scribe.js';
+
+const TTS_MODEL = 'eleven_flash_v2_5';
+
+interface Voiced {
+  audio(pcm48k: Int16Array, chars: number): void;
+  done(): void;
+}
+
+function ttsUrl(voiceId: string): string {
+  const query = new URLSearchParams({
+    model_id: TTS_MODEL,
+    output_format: 'pcm_24000',
+    auto_mode: 'true',
+    inactivity_timeout: '180',
+  });
+  return `${elevenLabs()}/v1/text-to-speech/${encodeURIComponent(voiceId)}/stream-input?${query.toString()}`;
+}
+
+const charsOf = (msg: Record<string, unknown>): number => {
+  const alignment = msg.alignment;
+  return isRecord(alignment) && Array.isArray(alignment.chars) ? alignment.chars.length : 0;
+};
+
+const MAX_REASON = 300;
+
+export class Utterance {
+  private readonly ws: WebSocket;
+  private readonly outbox: string[] = [];
+  private listener: Voiced | null = null;
+  private carry = 0;
+  private closed = false;
+  private aborted = false;
+  private voiced = false;
+  private refusal: string | null = null;
+
+  constructor(apiKey: string, voiceId: string, failed: (reason: string) => void) {
+    this.ws = new WebSocket(ttsUrl(voiceId), { headers: { 'xi-api-key': apiKey } });
+    this.ws.on('open', () => {
+      this.ws.send(JSON.stringify({ text: ' ' }));
+      for (const line of this.outbox.splice(0)) this.ws.send(line);
+    });
+    this.ws.on('message', (data) => {
+      this.onMessage(wsText(data));
+    });
+    this.ws.on('error', (err) => {
+      this.refusal ??= errMsg(err);
+      log.warn({ err: errMsg(err) }, 'voice: text to speech connection error');
+    });
+    this.ws.on('close', (code, why) => {
+      this.closed = true;
+      this.listener?.done();
+      this.listener = null;
+      const reason = this.refusal ?? (code === 1000 ? null : (why.toString() || `closed (${String(code)})`));
+      if (!this.aborted && !this.voiced && reason !== null) failed(reason.slice(0, MAX_REASON));
+    });
+  }
+
+  get usable(): boolean {
+    return !this.closed && this.listener === null && this.ws.readyState <= WebSocket.OPEN;
+  }
+
+  attach(listener: Voiced): void {
+    this.listener = listener;
+  }
+
+  say(text: string): void {
+    this.post({ text: `${text} ` });
+  }
+
+  end(): void {
+    this.post({ text: '' });
+  }
+
+  abort(): void {
+    this.aborted = true;
+    this.listener = null;
+    this.ws.terminate();
+  }
+
+  private post(msg: Record<string, unknown>): void {
+    const line = JSON.stringify(msg);
+    if (this.ws.readyState === WebSocket.OPEN) this.ws.send(line);
+    else this.outbox.push(line);
+  }
+
+  private onMessage(raw: string): void {
+    const msg = parsed(raw);
+    if (typeof msg.audio === 'string' && msg.audio !== '') {
+      this.voiced = true;
+      const pcm = pcmOf(Buffer.from(msg.audio, 'base64'));
+      const up = upsample2(pcm, this.carry);
+      this.carry = pcm.at(-1) ?? this.carry;
+      this.listener?.audio(up, charsOf(msg));
+    } else if (msg.isFinal !== true) {
+      if (typeof msg.message === 'string' && msg.message !== '') this.refusal ??= msg.message;
+      log.warn({ msg: raw.slice(0, MAX_REASON) }, 'voice: text to speech said');
+    }
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/store.ts

@@ -0,0 +1,97 @@
+import { join } from 'node:path';
+import { readJson, writeSecure } from '@metro-labs/core/secure-fs';
+import { isRecord } from '@metro-labs/core/is-record';
+import { ApiError } from '@metro-labs/http/api-error';
+import { agentsDir } from '../agents/files.js';
+import { readModelConfig, type ModelConfig } from '../gateway/model-config.js';
+
+const VOICE_PROVIDERS = ['elevenlabs'] as const;
+type VoiceProvider = (typeof VOICE_PROVIDERS)[number];
+
+export interface VoiceConfig {
+  provider: VoiceProvider;
+  apiKey: string;
+  voiceId: string;
+  model: string;
+  enabled: boolean;
+}
+
+const DEFAULT_VOICE_ID = 'EXAVITQu4vr4xnSDxMaL';
+const DEFAULT_VOICE_MODEL = 'anthropic:claude-sonnet-5-5';
+
+const VOICE_FILE = 'voice.json';
+const MAX_FIELD = 512;
+
+const text = (value: unknown): string => (typeof value === 'string' ? value.trim() : '');
+
+const isProvider = (value: unknown): value is VoiceProvider =>
+  typeof value === 'string' && (VOICE_PROVIDERS as readonly string[]).includes(value);
+
+export function parseVoice(raw: unknown): VoiceConfig {
+  const r = isRecord(raw) ? raw : {};
+  return {
+    provider: isProvider(r.provider) ? r.provider : 'elevenlabs',
+    apiKey: text(r.apiKey),
+    voiceId: text(r.voiceId),
+    model: text(r.model),
+    enabled: r.enabled !== false,
+  };
+}
+
+export const readVoice = (dir = agentsDir()): VoiceConfig =>
+  parseVoice(readJson<unknown>(join(dir, VOICE_FILE), null, { warn: 'voice: voice.json is not readable' }));
+
+export function writeVoice(cfg: VoiceConfig, dir = agentsDir()): void {
+  writeSecure(join(dir, VOICE_FILE), JSON.stringify(cfg, null, 2));
+}
+
+export const voiceIdOf = (cfg: VoiceConfig): string => (cfg.voiceId === '' ? DEFAULT_VOICE_ID : cfg.voiceId);
+const voiceModelOf = (cfg: VoiceConfig): string => (cfg.model === '' ? DEFAULT_VOICE_MODEL : cfg.model);
+export function brainModel(cfg: VoiceConfig, models: ModelConfig = readModelConfig()): string {
+  const model = voiceModelOf(cfg);
+  const anthropic = /^anthropic:(.+)$/.exec(model)?.[1];
+  if (anthropic === undefined) return model;
+  return models.connections.some((c) => c.provider === 'anthropic') ? model : anthropic;
+}
+
+export const voiceReady = (cfg: VoiceConfig): boolean => cfg.enabled && cfg.apiKey !== '';
+
+export function publicVoice(cfg: VoiceConfig): Record<string, unknown> {
+  return {
+    provider: cfg.provider,
+    providers: VOICE_PROVIDERS,
+    hasKey: cfg.apiKey !== '',
+    voiceId: cfg.voiceId,
+    model: cfg.model,
+    enabled: cfg.enabled,
+    defaults: { voiceId: DEFAULT_VOICE_ID, model: DEFAULT_VOICE_MODEL },
+  };
+}
+
+function field(body: Record<string, unknown>, key: string, current: string): string {
+  const value = body[key];
+  if (value === undefined) return current;
+  if (typeof value !== 'string') throw new ApiError(`${key} must be a string`, 400);
+  if (value.length > MAX_FIELD) throw new ApiError(`${key} is too long`, 400);
+  return value.trim();
+}
+
+function keyOf(body: Record<string, unknown>, current: string): string {
+  if (body.apiKey === null) return '';
+  const next = field(body, 'apiKey', current);
+  return next === '' ? current : next;
+}
+
+export function patchVoice(cfg: VoiceConfig, body: unknown): VoiceConfig {
+  if (!isRecord(body)) throw new ApiError('the body must be a JSON object', 400);
+  if (body.provider !== undefined && !isProvider(body.provider))
+    throw new ApiError(`provider must be one of ${VOICE_PROVIDERS.join(', ')}`, 400);
+  if (body.enabled !== undefined && typeof body.enabled !== 'boolean') throw new ApiError('enabled must be true or false', 400);
+  return {
+    provider: isProvider(body.provider) ? body.provider : cfg.provider,
+    apiKey: keyOf(body, cfg.apiKey),
+    voiceId: field(body, 'voiceId', cfg.voiceId),
+    model: field(body, 'model', cfg.model),
+    enabled: typeof body.enabled === 'boolean' ? body.enabled : cfg.enabled,
+  };
+}