@stage-labs/metro 0.1.0-beta.3 → 0.1.0-beta.300

package/runtime/node_modules/@metro-labs/daemon/src/voice/speakable.ts

@@ -0,0 +1,54 @@
+const FIRST_CLAUSE_MIN = 24;
+const LONGEST = 220;
+const SENTENCE_END = /[.!?…]+["')\]]*\s+|\n+/g;
+const CLAUSE_END = /[,;:]\s+/g;
+
+function speakable(text: string): string {
+  return text
+    .replace(/https?:\/\/\S+/g, 'the link')
+    .replace(/`{1,3}[^`]*`{1,3}/g, (code) => code.replace(/`/g, ''))
+    .replace(/^\s*(?:#+|[-*•]|\d+\.)\s+/gm, '')
+    .replace(/[*_~#|<>[\]{}]/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function lastEnd(text: string, pattern: RegExp): number {
+  let end = -1;
+  for (const match of text.matchAll(pattern)) end = match.index + match[0].length;
+  return end;
+}
+
+function cutAt(text: string, first: boolean): number {
+  const sentence = lastEnd(text, SENTENCE_END);
+  if (sentence > 0) return sentence;
+  if (first && text.length >= FIRST_CLAUSE_MIN) {
+    const clause = lastEnd(text, CLAUSE_END);
+    if (clause > 0) return clause;
+  }
+  if (text.length < LONGEST) return -1;
+  const space = text.lastIndexOf(' ');
+  return space > 0 ? space + 1 : text.length;
+}
+
+export class Chunker {
+  private buffer = '';
+  private first = true;
+
+  add(delta: string): string[] {
+    this.buffer += delta;
+    const at = cutAt(this.buffer, this.first);
+    if (at < 0) return [];
+    const ready = speakable(this.buffer.slice(0, at));
+    this.buffer = this.buffer.slice(at);
+    if (ready === '') return [];
+    this.first = false;
+    return [ready];
+  }
+
+  flush(): string[] {
+    const rest = speakable(this.buffer);
+    this.buffer = '';
+    return rest === '' ? [] : [rest];
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/speech-queue.ts

@@ -0,0 +1,200 @@
+import type { SpeechStatus } from '@metro-labs/core/call';
+import { errMsg, log } from '@metro-labs/core/log';
+import { Player, type Playback } from './audio.js';
+import { Utterance } from './speech.js';
+import type { SharedSpeechAction } from './shared.js';
+import { voiceIdOf, type VoiceConfig } from './store.js';
+
+const MAX_QUEUED = 8;
+const MAX_TEXT = 32_000;
+const SPEECH_MS = 60_000;
+
+type Speech = Pick<Utterance, 'attach' | 'say' | 'end' | 'abort'>;
+type MakeSpeech = (failed: (reason: string) => void) => Speech;
+
+interface Queued {
+  action: SharedSpeechAction;
+  queuedAt: number;
+}
+
+interface Playing extends Queued {
+  utterance: Speech | null;
+  timer: ReturnType<typeof setTimeout>;
+  synthesized: boolean;
+  started: boolean;
+  synthesisStartedAt: number;
+  firstAudioAt: number | null;
+  markedFrameResolvedAt: number | null;
+}
+
+export class SpeechQueue {
+  private readonly player: Player;
+  private readonly waiting: Queued[] = [];
+  private current: Playing | null = null;
+  private live = false;
+  private closed = false;
+  private onFailed: () => void = () => undefined;
+
+  constructor(
+    cfg: VoiceConfig,
+    send: Playback['send'],
+    private readonly makeSpeech: MakeSpeech = (failed) => new Utterance(cfg.apiKey, voiceIdOf(cfg), failed),
+    private readonly timeoutMs = SPEECH_MS,
+  ) {
+    this.player = new Player({
+      send: (opus, timestamp, marker) => {
+        if (this.current !== null && !this.valid(this.current)) return;
+        return send(opus, timestamp, marker);
+      },
+      played: () => undefined,
+    });
+    this.player.whenFirstPlayed(() => {
+      const current = this.current;
+      if (current === null || current.started) return;
+      current.started = true;
+      current.markedFrameResolvedAt = Date.now();
+      current.action.status('started');
+    });
+    this.player.whenIdle(() => {
+      this.drained();
+    });
+    this.player.whenFailed((reason) => {
+      this.fail(reason);
+    });
+  }
+
+  get busy(): boolean {
+    return this.player.busy;
+  }
+
+  whenFailed(failed: () => void): void {
+    this.onFailed = failed;
+  }
+
+  enqueue(action: SharedSpeechAction): boolean {
+    if (this.closed || action.text.trim() === '' || action.text.length > MAX_TEXT) return false;
+    if (this.waiting.length + (this.current === null ? 0 : 1) >= MAX_QUEUED || !action.isValid()) return false;
+    this.waiting.push({ action, queuedAt: Date.now() });
+    queueMicrotask(() => {
+      this.next();
+    });
+    return true;
+  }
+
+  connect(): void {
+    if (this.closed || this.live) return;
+    this.live = true;
+    this.player.start();
+    this.next();
+  }
+
+  cancel(): void {
+    const waiting = this.waiting.splice(0);
+    this.settle('interrupted');
+    for (const queued of waiting) {
+      this.report(queued, 'interrupted');
+      queued.action.status('interrupted');
+    }
+  }
+
+  close(): void {
+    if (this.closed) return;
+    this.closed = true;
+    this.live = false;
+    this.cancel();
+    this.player.stop();
+  }
+
+  private next(): void {
+    if (!this.live || this.closed || this.current !== null) return;
+    const queued = this.waiting.shift();
+    if (queued === undefined) return;
+    const { action } = queued;
+    if (!action.isValid()) {
+      this.report(queued, 'failed');
+      action.status('failed');
+      this.next();
+      return;
+    }
+    const playing: Playing = {
+      ...queued, utterance: null, synthesized: false, started: false,
+      synthesisStartedAt: Date.now(), firstAudioAt: null, markedFrameResolvedAt: null,
+      timer: setTimeout(() => {
+        if (this.current === playing) this.fail('speech timed out');
+      }, this.timeoutMs),
+    };
+    this.current = playing;
+    try {
+      const utterance = this.makeSpeech((reason) => {
+        if (this.current === playing) this.fail(reason);
+      });
+      if (this.current !== playing) {
+        utterance.abort();
+        return;
+      }
+      playing.utterance = utterance;
+      utterance.attach({
+        audio: (pcm, chars) => {
+          if (this.current === playing && pcm.length > 0) playing.firstAudioAt ??= Date.now();
+          if (!this.valid(playing)) return;
+          this.player.push(pcm, chars);
+        },
+        done: () => {
+          if (!this.valid(playing)) return;
+          playing.synthesized = true;
+          this.player.finish();
+          this.drained();
+        },
+      });
+      utterance.say(action.text);
+      utterance.end();
+    } catch (err) {
+      if (this.current === playing) this.fail(errMsg(err));
+    }
+  }
+
+  private valid(playing: Playing): boolean {
+    if (this.closed || this.current !== playing) return false;
+    if (playing.action.isValid()) return true;
+    this.settle('failed');
+    return false;
+  }
+
+  private drained(): void {
+    const current = this.current;
+    if (current === null || !current.synthesized || this.player.busy) return;
+    if (current.started) this.settle('completed');
+    else this.fail('no audio received');
+  }
+
+  private fail(reason: string): void {
+    log.warn({ reason }, 'voice: queued speech failed');
+    this.settle('failed');
+    this.onFailed();
+  }
+
+  private report(queued: Queued, status: SpeechStatus, playing?: Playing): void {
+    log.info({
+      actionId: queued.action.actionId, status, queuedAt: queued.queuedAt,
+      synthesisStartedAt: playing?.synthesisStartedAt ?? null,
+      firstAudioAt: playing?.firstAudioAt ?? null,
+      markedFrameResolvedAt: playing?.markedFrameResolvedAt ?? null,
+      settledAt: Date.now(),
+    }, 'voice: speech timing');
+  }
+
+  private settle(status: SpeechStatus): void {
+    const current = this.current;
+    this.current = null;
+    this.player.clear();
+    if (current !== null) {
+      clearTimeout(current.timer);
+      current.utterance?.abort();
+      this.report(current, status, current);
+      current.action.status(status);
+    }
+    queueMicrotask(() => {
+      this.next();
+    });
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/speech.ts

@@ -0,0 +1,129 @@
+import WebSocket from 'ws';
+import { errMsg, log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import { pcmOf, upsample2 } from './audio.js';
+import { elevenLabs, parsed, wsText } from './scribe.js';
+
+const TTS_MODEL = 'eleven_flash_v2_5';
+
+export interface Voiced {
+  audio(pcm48k: Int16Array, chars: number): void;
+  done(): void;
+}
+
+function ttsUrl(voiceId: string): string {
+  const query = new URLSearchParams({
+    model_id: TTS_MODEL,
+    output_format: 'pcm_24000',
+    auto_mode: 'true',
+    inactivity_timeout: '180',
+  });
+  return `${elevenLabs()}/v1/text-to-speech/${encodeURIComponent(voiceId)}/stream-input?${query.toString()}`;
+}
+
+const charsOf = (msg: Record<string, unknown>): number => {
+  const alignment = msg.alignment;
+  return isRecord(alignment) && Array.isArray(alignment.chars) ? alignment.chars.length : 0;
+};
+
+const MAX_REASON = 300;
+
+export class Utterance {
+  private readonly ws: WebSocket;
+  private readonly outbox: string[] = [];
+  private listener: Voiced | null = null;
+  private carry = 0;
+  private closed = false;
+  private aborted = false;
+  private voiced = false;
+  private refusal: string | null = null;
+  private finished = false;
+
+  constructor(apiKey: string, voiceId: string, private readonly failed: (reason: string) => void) {
+    this.ws = new WebSocket(ttsUrl(voiceId), { headers: { 'xi-api-key': apiKey } });
+    this.ws.on('open', () => {
+      if (this.aborted) return;
+      this.ws.send(JSON.stringify({ text: ' ' }));
+      for (const line of this.outbox.splice(0)) this.ws.send(line);
+    });
+    this.ws.on('message', (data) => {
+      this.onMessage(wsText(data));
+    });
+    this.ws.on('error', (err) => {
+      if (this.aborted || this.finished) return;
+      this.refusal ??= errMsg(err);
+      log.warn({ err: errMsg(err) }, 'voice: text to speech connection error');
+    });
+    this.ws.on('close', (code, why) => {
+      this.closed = true;
+      const reason = this.refusal ?? (code === 1000 ? null : (why.toString() || `closed (${String(code)})`));
+      this.complete(reason);
+    });
+  }
+
+  get usable(): boolean {
+    return !this.closed && this.listener === null && this.ws.readyState <= WebSocket.OPEN;
+  }
+
+  attach(listener: Voiced): void {
+    if (!this.aborted && !this.finished) this.listener = listener;
+  }
+
+  say(text: string): void {
+    this.post({ text: `${text} ` });
+  }
+
+  end(): void {
+    this.post({ text: '' });
+  }
+
+  abort(): void {
+    if (this.aborted) return;
+    this.aborted = true;
+    this.listener = null;
+    this.outbox.length = 0;
+    this.ws.terminate();
+  }
+
+  private complete(reason: string | null): void {
+    if (this.aborted || this.finished) return;
+    this.finished = true;
+    const listener = this.listener;
+    this.listener = null;
+    this.outbox.length = 0;
+    const failure = reason ?? (listener !== null && !this.voiced ? 'no audio received' : null);
+    if (failure !== null) this.failed(failure.slice(0, MAX_REASON));
+    else listener?.done();
+  }
+
+  private post(msg: Record<string, unknown>): void {
+    if (this.aborted || this.closed || this.finished) return;
+    const line = JSON.stringify(msg);
+    if (this.ws.readyState === WebSocket.OPEN) this.ws.send(line);
+    else this.outbox.push(line);
+  }
+
+  private refused(msg: Record<string, unknown>, raw: string): void {
+    if (typeof msg.message === 'string' && msg.message !== '') this.refusal ??= msg.message;
+    log.warn({ msg: raw.slice(0, MAX_REASON) }, 'voice: text to speech said');
+  }
+
+  private audio(encoded: string, chars: number): void {
+    this.voiced = true;
+    const pcm = pcmOf(Buffer.from(encoded, 'base64'));
+    const up = upsample2(pcm, this.carry);
+    this.carry = pcm.at(-1) ?? this.carry;
+    this.listener?.audio(up, chars);
+  }
+
+  private onMessage(raw: string): void {
+    if (this.aborted || this.finished) return;
+    const msg = parsed(raw);
+    if (typeof msg.audio === 'string' && msg.audio !== '') this.audio(msg.audio, charsOf(msg));
+    else if (msg.isFinal !== true) this.refused(msg, raw);
+    if (this.refusal !== null || msg.isFinal === true) {
+      this.complete(this.refusal);
+      this.ws.close(1000);
+    }
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/store.ts

@@ -0,0 +1,116 @@
+import { join } from 'node:path';
+import { readJson, writeSecure } from '@metro-labs/core/secure-fs';
+import { isRecord } from '@metro-labs/core/is-record';
+import { ApiError } from '@metro-labs/http/api-error';
+import { agentsDir } from '../agents/files.js';
+import { readModelConfig, type ModelConfig } from '../gateway/model-config.js';
+
+const VOICE_PROVIDERS = ['elevenlabs'] as const;
+type VoiceProvider = (typeof VOICE_PROVIDERS)[number];
+
+export interface VoiceConfig {
+  provider: VoiceProvider;
+  apiKey: string;
+  voiceId: string;
+  model: string;
+  language: string;
+  enabled: boolean;
+}
+
+const DEFAULT_LANGUAGE = 'en';
+const AUTO_LANGUAGE = 'auto';
+const LANGUAGE_RE = /^(?:[a-z]{2,3}|auto)$/;
+
+const DEFAULT_VOICE_ID = 'EXAVITQu4vr4xnSDxMaL';
+const DEFAULT_VOICE_MODEL = 'anthropic:claude-sonnet-5-5';
+
+const VOICE_FILE = 'voice.json';
+const MAX_FIELD = 512;
+
+const text = (value: unknown): string => (typeof value === 'string' ? value.trim() : '');
+
+const isProvider = (value: unknown): value is VoiceProvider =>
+  typeof value === 'string' && (VOICE_PROVIDERS as readonly string[]).includes(value);
+
+export function parseVoice(raw: unknown): VoiceConfig {
+  const r = isRecord(raw) ? raw : {};
+  return {
+    provider: isProvider(r.provider) ? r.provider : 'elevenlabs',
+    apiKey: text(r.apiKey),
+    voiceId: text(r.voiceId),
+    model: text(r.model),
+    language: LANGUAGE_RE.test(text(r.language)) ? text(r.language) : '',
+    enabled: r.enabled !== false,
+  };
+}
+
+export const readVoice = (dir = agentsDir()): VoiceConfig =>
+  parseVoice(readJson<unknown>(join(dir, VOICE_FILE), null, { warn: 'voice: voice.json is not readable' }));
+
+export function writeVoice(cfg: VoiceConfig, dir = agentsDir()): void {
+  writeSecure(join(dir, VOICE_FILE), JSON.stringify(cfg, null, 2));
+}
+
+export const voiceIdOf = (cfg: VoiceConfig): string => (cfg.voiceId === '' ? DEFAULT_VOICE_ID : cfg.voiceId);
+const voiceModelOf = (cfg: VoiceConfig): string => (cfg.model === '' ? DEFAULT_VOICE_MODEL : cfg.model);
+export function brainModel(cfg: VoiceConfig, models: ModelConfig = readModelConfig()): string {
+  const model = voiceModelOf(cfg);
+  const anthropic = /^anthropic:(.+)$/.exec(model)?.[1];
+  if (anthropic === undefined) return model;
+  return models.connections.some((c) => c.provider === 'anthropic') ? model : anthropic;
+}
+
+export function languageOf(cfg: VoiceConfig): string | null {
+  const language = cfg.language === '' ? DEFAULT_LANGUAGE : cfg.language;
+  return language === AUTO_LANGUAGE ? null : language;
+}
+
+export const voiceReady = (cfg: VoiceConfig): boolean => cfg.enabled && cfg.apiKey !== '';
+
+export function publicVoice(cfg: VoiceConfig): Record<string, unknown> {
+  return {
+    provider: cfg.provider,
+    providers: VOICE_PROVIDERS,
+    hasKey: cfg.apiKey !== '',
+    voiceId: cfg.voiceId,
+    model: cfg.model,
+    language: cfg.language,
+    enabled: cfg.enabled,
+    defaults: { voiceId: DEFAULT_VOICE_ID, model: DEFAULT_VOICE_MODEL, language: DEFAULT_LANGUAGE },
+  };
+}
+
+function field(body: Record<string, unknown>, key: string, current: string): string {
+  const value = body[key];
+  if (value === undefined) return current;
+  if (typeof value !== 'string') throw new ApiError(`${key} must be a string`, 400);
+  if (value.length > MAX_FIELD) throw new ApiError(`${key} is too long`, 400);
+  return value.trim();
+}
+
+function keyOf(body: Record<string, unknown>, current: string): string {
+  if (body.apiKey === null) return '';
+  const next = field(body, 'apiKey', current);
+  return next === '' ? current : next;
+}
+
+function languageField(body: Record<string, unknown>, current: string): string {
+  const next = field(body, 'language', current).toLowerCase();
+  if (next !== '' && !LANGUAGE_RE.test(next)) throw new ApiError("language is a two or three letter ISO 639 code, or 'auto'", 400);
+  return next;
+}
+
+export function patchVoice(cfg: VoiceConfig, body: unknown): VoiceConfig {
+  if (!isRecord(body)) throw new ApiError('the body must be a JSON object', 400);
+  if (body.provider !== undefined && !isProvider(body.provider))
+    throw new ApiError(`provider must be one of ${VOICE_PROVIDERS.join(', ')}`, 400);
+  if (body.enabled !== undefined && typeof body.enabled !== 'boolean') throw new ApiError('enabled must be true or false', 400);
+  return {
+    provider: isProvider(body.provider) ? body.provider : cfg.provider,
+    apiKey: keyOf(body, cfg.apiKey),
+    voiceId: field(body, 'voiceId', cfg.voiceId),
+    model: field(body, 'model', cfg.model),
+    language: languageField(body, cfg.language),
+    enabled: typeof body.enabled === 'boolean' ? body.enabled : cfg.enabled,
+  };
+}