@stage-labs/metro 0.1.0-beta.30 → 0.1.0-beta.301

package/runtime/node_modules/@metro-labs/daemon/src/voice/scribe.ts

@@ -0,0 +1,115 @@
+import WebSocket from 'ws';
+import { errMsg, log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import { bytesOf } from './audio.js';
+
+export const elevenLabs = (): string => process.env.METRO_ELEVENLABS_URL ?? 'wss://api.elevenlabs.io';
+const STT_RATE = 16_000;
+const CHUNK_SAMPLES = STT_RATE / 10;
+const HELD_SAMPLES = STT_RATE * 2;
+export const VAD_SILENCE_SECS = 0.5;
+
+export interface Heard {
+  partial(text: string): void;
+  committed(text: string): void;
+  failed(reason: string): void;
+}
+
+const FATAL = new Set([
+  'auth_error',
+  'quota_exceeded',
+  'unaccepted_terms',
+  'rate_limited',
+  'resource_exhausted',
+  'session_time_limit_exceeded',
+  'error',
+]);
+
+export function wsText(data: WebSocket.RawData): string {
+  if (Array.isArray(data)) return Buffer.concat(data).toString('utf8');
+  return (Buffer.isBuffer(data) ? data : Buffer.from(data)).toString('utf8');
+}
+
+export function parsed(raw: string): Record<string, unknown> {
+  try {
+    const msg: unknown = JSON.parse(raw);
+    return isRecord(msg) ? msg : {};
+  } catch {
+    return {};
+  }
+}
+
+export function sttUrl(language: string | null): string {
+  const query = new URLSearchParams({
+    model_id: 'scribe_v2_realtime',
+    audio_format: 'pcm_16000',
+    commit_strategy: 'vad',
+    vad_silence_threshold_secs: String(VAD_SILENCE_SECS),
+  });
+  if (language !== null) query.set('language_code', language);
+  return `${elevenLabs()}/v1/speech-to-text/realtime?${query.toString()}`;
+}
+
+export class Scribe {
+  private readonly ws: WebSocket;
+  private buffer: Int16Array[] = [];
+  private buffered = 0;
+  private closed = false;
+
+  constructor(apiKey: string, language: string | null, private readonly heard: Heard) {
+    this.ws = new WebSocket(sttUrl(language), { headers: { 'xi-api-key': apiKey } });
+    this.ws.on('message', (data) => {
+      this.onMessage(wsText(data));
+    });
+    this.ws.on('error', (err) => {
+      if (this.closed) return;
+      log.warn({ err: errMsg(err) }, 'voice: speech to text connection error');
+    });
+    this.ws.on('close', (code) => {
+      if (this.closed) return;
+      this.closed = true;
+      this.buffer = [];
+      this.buffered = 0;
+      this.heard.failed(`speech to text closed (${String(code)})`);
+    });
+  }
+
+  send(pcm16k: Int16Array): void {
+    if (this.closed) return;
+    this.buffer.push(pcm16k);
+    this.buffered += pcm16k.length;
+    while (this.buffered > HELD_SAMPLES) this.buffered -= this.buffer.shift()?.length ?? this.buffered;
+    if (this.buffered < CHUNK_SAMPLES || this.ws.readyState !== WebSocket.OPEN) return;
+    const chunk = new Int16Array(this.buffered);
+    let at = 0;
+    for (const part of this.buffer) {
+      chunk.set(part, at);
+      at += part.length;
+    }
+    this.buffer = [];
+    this.buffered = 0;
+    this.ws.send(
+      JSON.stringify({ message_type: 'input_audio_chunk', audio_base_64: bytesOf(chunk).toString('base64'), commit: false, sample_rate: STT_RATE }),
+    );
+  }
+
+  close(): void {
+    if (this.closed) return;
+    this.closed = true;
+    this.buffer = [];
+    this.buffered = 0;
+    this.ws.terminate();
+  }
+
+  private onMessage(raw: string): void {
+    if (this.closed) return;
+    const msg = parsed(raw);
+    const type = typeof msg.message_type === 'string' ? msg.message_type : '';
+    const text = typeof msg.text === 'string' ? msg.text.trim() : '';
+    if (type === 'partial_transcript') this.heard.partial(text);
+    else if (type === 'committed_transcript') {
+      if (text !== '') this.heard.committed(text);
+    } else if (FATAL.has(type)) this.heard.failed(`speech to text: ${typeof msg.error === 'string' ? msg.error : type}`);
+    else log.debug({ type }, 'voice: speech to text said');
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/shared-talk.ts

@@ -0,0 +1,85 @@
+import { randomUUID } from 'node:crypto';
+import { downsample3, OpusIn, type Playback } from './audio.js';
+import { Scribe, type Heard } from './scribe.js';
+import type { SharedCallTransport, SharedSpeechAction } from './shared.js';
+import { SpeechQueue } from './speech-queue.js';
+import { languageOf, type VoiceConfig } from './store.js';
+
+const BARGE_MIN_WORDS = 2;
+const wordsIn = (text: string): number => text.split(/\s+/).filter((word) => /\p{L}/u.test(word)).length;
+
+type MakeScribe = (heard: Heard) => Pick<Scribe, 'send' | 'close'>;
+
+export class SharedTalk implements SharedCallTransport {
+  private readonly opus = new OpusIn();
+  private readonly scribe: Pick<Scribe, 'send' | 'close'>;
+  private readonly speech: SpeechQueue;
+  private closed = false;
+
+  constructor(
+    cfg: VoiceConfig,
+    send: Playback['send'],
+    private readonly ended: (reason: string) => void,
+    heard: (text: string, sourceId: string) => void,
+    makeScribe: MakeScribe = (events) => new Scribe(cfg.apiKey, languageOf(cfg), events),
+    speech = new SpeechQueue(cfg, send),
+  ) {
+    this.speech = speech;
+    this.speech.whenFailed(() => {
+      this.stop('speech is unavailable');
+    });
+    this.scribe = makeScribe({
+      partial: (text) => {
+        this.barge(text);
+      },
+      committed: (text) => {
+        if (this.closed || text.trim() === '') return;
+        this.barge(text);
+        heard(text, randomUUID());
+      },
+      failed: (reason) => {
+        if (!this.closed) ended(reason);
+      },
+    });
+  }
+
+  enqueue(action: SharedSpeechAction): boolean {
+    return !this.closed && this.speech.enqueue(action);
+  }
+
+  cancel(): void {
+    this.speech.cancel();
+  }
+
+  terminate(): void {
+    this.stop('the shared conversation ended');
+  }
+
+  connect(): void {
+    if (!this.closed) this.speech.connect();
+  }
+
+  hear(packet: Buffer): void {
+    if (this.closed) return;
+    const pcm = this.opus.decode(packet);
+    if (pcm !== null) this.scribe.send(downsample3(pcm));
+  }
+
+  finish(): void {
+    if (this.closed) return;
+    this.closed = true;
+    this.speech.close();
+    this.scribe.close();
+    this.opus.close();
+  }
+
+  private stop(reason: string): void {
+    if (this.closed) return;
+    this.finish();
+    this.ended(reason);
+  }
+
+  private barge(text: string): void {
+    if (!this.closed && this.speech.busy && wordsIn(text) >= BARGE_MIN_WORDS) this.speech.cancel();
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/shared.ts

@@ -0,0 +1,200 @@
+import { randomUUID } from 'node:crypto';
+import { sameCall, type CallNotice, type CallRoute, type CallSource, type SpeechStatus, type SpeechTarget } from '@metro-labs/core/call';
+import { str } from '@metro-labs/core/str';
+import { log } from '@metro-labs/core/log';
+import { agentIdForLine, allowlistForLine, lineReceives, mayApprove, senderPermitted, accountFromLine } from '../agents/map.js';
+import { eventInScope } from '../agents/scope.js';
+import { harnessRunner } from '../claude/runner.js';
+import { policyFor, type Access } from '../policy/policy.js';
+
+export interface SharedSpeechAction {
+  actionId: string;
+  text: string;
+  isValid(): boolean;
+  status(status: SpeechStatus): void;
+}
+
+export interface SharedCallTransport {
+  enqueue(action: SharedSpeechAction): boolean;
+  terminate(): void;
+}
+
+export interface SpeechReceipt {
+  actionId: string;
+  sourceId: string;
+  status: SpeechStatus;
+  duplicate: boolean;
+}
+
+interface Action extends SpeechReceipt {
+  text: string;
+}
+
+interface Active {
+  route: CallRoute;
+  at: number;
+  transport: SharedCallTransport;
+  sources: Set<string>;
+  actions: Map<string, Action>;
+}
+
+export interface SharedCallBridge {
+  available(route: CallRoute): boolean;
+  notify(notice: CallNotice): boolean;
+  revoked(route: CallRoute): void;
+}
+
+const SOURCES_MAX = 1024;
+const TEXT_MAX = 32_000;
+const sourceValid = (sourceId: string): boolean => sourceId.length > 0 && sourceId.length <= 2048;
+
+const authorized = (route: CallRoute): boolean =>
+  agentIdForLine(route.line) === route.agentId && eventInScope(new Set([route.agentId]), route.line) && lineReceives(route.line) &&
+  mayApprove(route.line, route.from) && senderPermitted(allowlistForLine(route.line), route.from);
+
+function access(route: CallRoute): Access {
+  const account = accountFromLine(route.line);
+  return account === undefined ? 'deny' : policyFor({ kind: 'channel', station: account.station, account: account.accountId }, { name: 'send', group: 'write' });
+}
+
+export class SharedCalls {
+  private active: Active | null = null;
+  private bridge: SharedCallBridge | null = null;
+
+  constructor(private readonly permitted = authorized, private readonly policy = access) {}
+
+  connect(bridge: SharedCallBridge): void {
+    this.disconnect();
+    this.bridge = bridge;
+  }
+
+  disconnect(): void {
+    this.revoke();
+    this.bridge = null;
+  }
+
+  revoke(): void {
+    const active = this.active;
+    if (active !== null) this.end(active.route);
+  }
+
+  open(route: CallRoute, sourceId: string, transport: SharedCallTransport): boolean {
+    if (!this.permitted(route) || this.bridge?.available(route) !== true || !sourceValid(sourceId)) return false;
+    if (this.active !== null && sameCall(this.active.route, route)) return this.active.transport === transport && this.active.sources.has(sourceId);
+    if (this.active !== null) this.end(this.active.route);
+    this.active = { route, at: Date.now(), transport, sources: new Set([sourceId]), actions: new Map() };
+    if (this.bridge.notify({ type: 'started', route, sourceId })) return true;
+    this.end(route);
+    return false;
+  }
+
+  snapshot(): CallRoute | null {
+    const route = this.active?.route;
+    return route !== undefined && this.current(route) !== null ? route : null;
+  }
+
+  valid(source: CallSource): boolean {
+    return this.current(source.route)?.sources.has(source.sourceId) === true;
+  }
+
+  private current(route: CallRoute): Active | null {
+    const active = this.active;
+    return active !== null && sameCall(active.route, route) && this.permitted(route) && this.bridge?.available(route) === true ? active : null;
+  }
+
+  heard(route: CallRoute, text: string, sourceId: string): void {
+    if (text.trim() === '' || text.length > TEXT_MAX || !this.addSource(route, sourceId)) return;
+    if (this.bridge?.notify({ type: 'heard', route, text, sourceId }) !== true) this.end(route);
+  }
+
+  chat(event: Record<string, unknown>, scope: Set<string>): Record<string, string> {
+    const active = this.active;
+    if (active === null || event.senderVerified === false || !scope.has(active.route.agentId)) return {};
+    const { route } = active;
+    if (event.line !== route.line || event.from !== route.from || Date.parse(str(event.ts)) < active.at || !Number.isFinite(Date.parse(str(event.ts)))) return {};
+    const sourceId = str(event.messageId);
+    if (!this.addSource(route, sourceId, true)) return {};
+    return { call_id: route.callId, call_generation: route.generation, call_source_id: sourceId };
+  }
+
+  bindChat(event: Record<string, unknown>, scope: Set<string>): () => Record<string, string> {
+    const meta = this.chat(event, scope);
+    const route = this.active?.route;
+    const sourceId = meta.call_source_id;
+    return () => route !== undefined && sourceId !== undefined && this.valid({ route, sourceId }) ? meta : {};
+  }
+
+  source(line: string, target: SpeechTarget, scope: Set<string>): CallSource | null {
+    const route = this.active?.route;
+    if (route === undefined || route.line !== line || route.callId !== target.callId || route.generation !== target.generation || !scope.has(route.agentId)) return null;
+    const source = { route, sourceId: target.sourceId };
+    return this.valid(source) ? source : null;
+  }
+
+  speak(line: string, target: SpeechTarget, text: string, scope: Set<string>): SpeechReceipt {
+    const source = this.source(line, target, scope);
+    const active = this.active;
+    if (source === null || active === null) throw new Error('The call or its authorized source has ended or changed. Nothing was spoken.');
+    if (text.trim() === '' || text.length > TEXT_MAX) throw new Error('Call speech needs 1 to 32000 characters of text.');
+    const previous = active.actions.get(source.sourceId);
+    if (previous !== undefined) {
+      if (previous.text !== text) throw new Error('This source already has a different speech action. Nothing was repeated.');
+      return { actionId: previous.actionId, sourceId: previous.sourceId, status: previous.status, duplicate: true };
+    }
+    const accessAtAcceptance = this.policy(source.route);
+    if (accessAtAcceptance === 'deny') throw new Error('The owner blocked sending on this channel.');
+    const action: Action = { actionId: randomUUID(), sourceId: source.sourceId, text, status: 'accepted', duplicate: false };
+    active.actions.set(source.sourceId, action);
+    this.status(source.route, action, 'accepted');
+    const queued = active.transport.enqueue({
+      actionId: action.actionId,
+      text,
+      isValid: () => this.valid(source) && this.policy(source.route) === accessAtAcceptance,
+      status: (status) => { this.status(source.route, action, status); },
+    });
+    if (!queued) this.status(source.route, action, 'failed');
+    else if (action.status === 'accepted') this.status(source.route, action, 'queued');
+    return { actionId: action.actionId, sourceId: action.sourceId, status: action.status, duplicate: false };
+  }
+
+  end(route: CallRoute): void {
+    const active = this.active;
+    if (active === null || !sameCall(active.route, route)) return;
+    this.active = null;
+    this.bridge?.revoked(route);
+    active.transport.terminate();
+    for (const action of active.actions.values())
+      if (!this.terminal(action.status)) this.status(route, action, 'interrupted');
+    this.bridge?.notify({ type: 'ended', route });
+  }
+
+  private addSource(route: CallRoute, sourceId: string, duplicate = false): boolean {
+    const active = this.current(route);
+    if (active === null || !sourceValid(sourceId)) return false;
+    if (active.sources.has(sourceId)) return duplicate;
+    if (active.sources.size >= SOURCES_MAX) {
+      log.warn('sdk-call: the call source limit was reached; ending the call');
+      this.end(route);
+      return false;
+    }
+    active.sources.add(sourceId);
+    return true;
+  }
+
+  private terminal(status: SpeechStatus): boolean {
+    return status === 'completed' || status === 'interrupted' || status === 'failed';
+  }
+
+  private status(route: CallRoute, action: Action, status: SpeechStatus): void {
+    if (this.terminal(action.status) || (status === 'started' && action.status === 'started')) return;
+    action.status = status;
+    log.info({ actionId: action.actionId, status, at: Date.now() }, 'sdk-call: speech action');
+    this.bridge?.notify({ type: 'speech', route, sourceId: action.sourceId, actionId: action.actionId, status });
+  }
+}
+
+export const sharedCalls = new SharedCalls();
+export const sharedCallsSelected = (): boolean => harnessRunner() === 'sdk';
+export const openSharedCall = (route: CallRoute, sourceId: string, transport: SharedCallTransport): boolean => sharedCalls.open(route, sourceId, transport);
+export const hearSharedCall = (route: CallRoute, text: string, sourceId: string): void => { sharedCalls.heard(route, text, sourceId); };
+export const endSharedCall = (route: CallRoute): void => { sharedCalls.end(route); };

package/runtime/node_modules/@metro-labs/daemon/src/voice/speakable.ts

@@ -0,0 +1,54 @@
+const FIRST_CLAUSE_MIN = 24;
+const LONGEST = 220;
+const SENTENCE_END = /[.!?…]+["')\]]*\s+|\n+/g;
+const CLAUSE_END = /[,;:]\s+/g;
+
+function speakable(text: string): string {
+  return text
+    .replace(/https?:\/\/\S+/g, 'the link')
+    .replace(/`{1,3}[^`]*`{1,3}/g, (code) => code.replace(/`/g, ''))
+    .replace(/^\s*(?:#+|[-*•]|\d+\.)\s+/gm, '')
+    .replace(/[*_~#|<>[\]{}]/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function lastEnd(text: string, pattern: RegExp): number {
+  let end = -1;
+  for (const match of text.matchAll(pattern)) end = match.index + match[0].length;
+  return end;
+}
+
+function cutAt(text: string, first: boolean): number {
+  const sentence = lastEnd(text, SENTENCE_END);
+  if (sentence > 0) return sentence;
+  if (first && text.length >= FIRST_CLAUSE_MIN) {
+    const clause = lastEnd(text, CLAUSE_END);
+    if (clause > 0) return clause;
+  }
+  if (text.length < LONGEST) return -1;
+  const space = text.lastIndexOf(' ');
+  return space > 0 ? space + 1 : text.length;
+}
+
+export class Chunker {
+  private buffer = '';
+  private first = true;
+
+  add(delta: string): string[] {
+    this.buffer += delta;
+    const at = cutAt(this.buffer, this.first);
+    if (at < 0) return [];
+    const ready = speakable(this.buffer.slice(0, at));
+    this.buffer = this.buffer.slice(at);
+    if (ready === '') return [];
+    this.first = false;
+    return [ready];
+  }
+
+  flush(): string[] {
+    const rest = speakable(this.buffer);
+    this.buffer = '';
+    return rest === '' ? [] : [rest];
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/speech-queue.ts

@@ -0,0 +1,200 @@
+import type { SpeechStatus } from '@metro-labs/core/call';
+import { errMsg, log } from '@metro-labs/core/log';
+import { Player, type Playback } from './audio.js';
+import { Utterance } from './speech.js';
+import type { SharedSpeechAction } from './shared.js';
+import { voiceIdOf, type VoiceConfig } from './store.js';
+
+const MAX_QUEUED = 8;
+const MAX_TEXT = 32_000;
+const SPEECH_MS = 60_000;
+
+type Speech = Pick<Utterance, 'attach' | 'say' | 'end' | 'abort'>;
+type MakeSpeech = (failed: (reason: string) => void) => Speech;
+
+interface Queued {
+  action: SharedSpeechAction;
+  queuedAt: number;
+}
+
+interface Playing extends Queued {
+  utterance: Speech | null;
+  timer: ReturnType<typeof setTimeout>;
+  synthesized: boolean;
+  started: boolean;
+  synthesisStartedAt: number;
+  firstAudioAt: number | null;
+  markedFrameResolvedAt: number | null;
+}
+
+export class SpeechQueue {
+  private readonly player: Player;
+  private readonly waiting: Queued[] = [];
+  private current: Playing | null = null;
+  private live = false;
+  private closed = false;
+  private onFailed: () => void = () => undefined;
+
+  constructor(
+    cfg: VoiceConfig,
+    send: Playback['send'],
+    private readonly makeSpeech: MakeSpeech = (failed) => new Utterance(cfg.apiKey, voiceIdOf(cfg), failed),
+    private readonly timeoutMs = SPEECH_MS,
+  ) {
+    this.player = new Player({
+      send: (opus, timestamp, marker) => {
+        if (this.current !== null && !this.valid(this.current)) return;
+        return send(opus, timestamp, marker);
+      },
+      played: () => undefined,
+    });
+    this.player.whenFirstPlayed(() => {
+      const current = this.current;
+      if (current === null || current.started) return;
+      current.started = true;
+      current.markedFrameResolvedAt = Date.now();
+      current.action.status('started');
+    });
+    this.player.whenIdle(() => {
+      this.drained();
+    });
+    this.player.whenFailed((reason) => {
+      this.fail(reason);
+    });
+  }
+
+  get busy(): boolean {
+    return this.player.busy;
+  }
+
+  whenFailed(failed: () => void): void {
+    this.onFailed = failed;
+  }
+
+  enqueue(action: SharedSpeechAction): boolean {
+    if (this.closed || action.text.trim() === '' || action.text.length > MAX_TEXT) return false;
+    if (this.waiting.length + (this.current === null ? 0 : 1) >= MAX_QUEUED || !action.isValid()) return false;
+    this.waiting.push({ action, queuedAt: Date.now() });
+    queueMicrotask(() => {
+      this.next();
+    });
+    return true;
+  }
+
+  connect(): void {
+    if (this.closed || this.live) return;
+    this.live = true;
+    this.player.start();
+    this.next();
+  }
+
+  cancel(): void {
+    const waiting = this.waiting.splice(0);
+    this.settle('interrupted');
+    for (const queued of waiting) {
+      this.report(queued, 'interrupted');
+      queued.action.status('interrupted');
+    }
+  }
+
+  close(): void {
+    if (this.closed) return;
+    this.closed = true;
+    this.live = false;
+    this.cancel();
+    this.player.stop();
+  }
+
+  private next(): void {
+    if (!this.live || this.closed || this.current !== null) return;
+    const queued = this.waiting.shift();
+    if (queued === undefined) return;
+    const { action } = queued;
+    if (!action.isValid()) {
+      this.report(queued, 'failed');
+      action.status('failed');
+      this.next();
+      return;
+    }
+    const playing: Playing = {
+      ...queued, utterance: null, synthesized: false, started: false,
+      synthesisStartedAt: Date.now(), firstAudioAt: null, markedFrameResolvedAt: null,
+      timer: setTimeout(() => {
+        if (this.current === playing) this.fail('speech timed out');
+      }, this.timeoutMs),
+    };
+    this.current = playing;
+    try {
+      const utterance = this.makeSpeech((reason) => {
+        if (this.current === playing) this.fail(reason);
+      });
+      if (this.current !== playing) {
+        utterance.abort();
+        return;
+      }
+      playing.utterance = utterance;
+      utterance.attach({
+        audio: (pcm, chars) => {
+          if (this.current === playing && pcm.length > 0) playing.firstAudioAt ??= Date.now();
+          if (!this.valid(playing)) return;
+          this.player.push(pcm, chars);
+        },
+        done: () => {
+          if (!this.valid(playing)) return;
+          playing.synthesized = true;
+          this.player.finish();
+          this.drained();
+        },
+      });
+      utterance.say(action.text);
+      utterance.end();
+    } catch (err) {
+      if (this.current === playing) this.fail(errMsg(err));
+    }
+  }
+
+  private valid(playing: Playing): boolean {
+    if (this.closed || this.current !== playing) return false;
+    if (playing.action.isValid()) return true;
+    this.settle('failed');
+    return false;
+  }
+
+  private drained(): void {
+    const current = this.current;
+    if (current === null || !current.synthesized || this.player.busy) return;
+    if (current.started) this.settle('completed');
+    else this.fail('no audio received');
+  }
+
+  private fail(reason: string): void {
+    log.warn({ reason }, 'voice: queued speech failed');
+    this.settle('failed');
+    this.onFailed();
+  }
+
+  private report(queued: Queued, status: SpeechStatus, playing?: Playing): void {
+    log.info({
+      actionId: queued.action.actionId, status, queuedAt: queued.queuedAt,
+      synthesisStartedAt: playing?.synthesisStartedAt ?? null,
+      firstAudioAt: playing?.firstAudioAt ?? null,
+      markedFrameResolvedAt: playing?.markedFrameResolvedAt ?? null,
+      settledAt: Date.now(),
+    }, 'voice: speech timing');
+  }
+
+  private settle(status: SpeechStatus): void {
+    const current = this.current;
+    this.current = null;
+    this.player.clear();
+    if (current !== null) {
+      clearTimeout(current.timer);
+      current.utterance?.abort();
+      this.report(current, status, current);
+      current.action.status(status);
+    }
+    queueMicrotask(() => {
+      this.next();
+    });
+  }
+}