@stage-labs/metro 0.1.0-beta.3 → 0.1.0-beta.300

package/runtime/node_modules/@metro-labs/daemon/src/voice/audio.ts

@@ -0,0 +1,193 @@
+import OpusScript from 'opusscript';
+import { errMsg, log } from '@metro-labs/core/log';
+
+const RATE = 48_000;
+const FRAME = 960;
+const FRAME_MS = 20;
+const MAX_LATE_FRAMES = 5;
+const MAX_QUEUED_FRAMES = 3_000;
+
+interface Frame {
+  pcm: Int16Array;
+  chars: number;
+}
+
+export function downsample3(pcm: Int16Array): Int16Array {
+  const out = new Int16Array(Math.floor(pcm.length / 3));
+  for (let i = 0; i < out.length; i += 1)
+    out[i] = Math.round(((pcm[i * 3] ?? 0) + (pcm[i * 3 + 1] ?? 0) + (pcm[i * 3 + 2] ?? 0)) / 3);
+  return out;
+}
+
+export function upsample2(pcm: Int16Array, carry: number): Int16Array {
+  const out = new Int16Array(pcm.length * 2);
+  let prev = carry;
+  for (let i = 0; i < pcm.length; i += 1) {
+    const cur = pcm[i] ?? 0;
+    out[i * 2] = Math.round((prev + cur) / 2);
+    out[i * 2 + 1] = cur;
+    prev = cur;
+  }
+  return out;
+}
+
+export const pcmOf = (bytes: Buffer): Int16Array => {
+  const copy = Buffer.from(bytes);
+  return new Int16Array(copy.buffer, copy.byteOffset, Math.floor(copy.length / 2));
+};
+
+export const bytesOf = (pcm: Int16Array): Buffer => Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+
+export class OpusIn {
+  private readonly decoder = new OpusScript(RATE, 1, OpusScript.Application.VOIP);
+
+  decode(packet: Buffer): Int16Array | null {
+    try {
+      return pcmOf(this.decoder.decode(packet));
+    } catch (err) {
+      log.debug({ err: errMsg(err) }, 'voice: an audio packet could not be decoded');
+      return null;
+    }
+  }
+
+  close(): void {
+    this.decoder.delete();
+  }
+}
+
+export interface Playback {
+  send(opus: Buffer, timestamp: number, marker: boolean): void | Promise<void>;
+  played(chars: number): void;
+}
+
+export class Player {
+  private readonly encoder = new OpusScript(RATE, 1, OpusScript.Application.VOIP);
+  private readonly queue: Frame[] = [];
+  private pending = new Int16Array(0);
+  private pendingChars = 0;
+  private timer: ReturnType<typeof setTimeout> | null = null;
+  private started = 0;
+  private sent = 0;
+  private speaking = false;
+  private onIdle: (() => void) | null = null;
+  private onFirst: (() => void) | null = null;
+  private onFailed: ((reason: string) => void) | null = null;
+  private generation = 0;
+  private inFlight = 0;
+  private closed = false;
+
+  constructor(private readonly out: Playback) {}
+
+  get busy(): boolean {
+    return this.queue.length > 0 || this.pending.length > 0 || this.inFlight > 0;
+  }
+
+  whenFirstPlayed(fn: () => void): void {
+    this.onFirst = fn;
+  }
+
+  whenIdle(fn: () => void): void {
+    this.onIdle = fn;
+  }
+
+  whenFailed(fn: (reason: string) => void): void {
+    this.onFailed = fn;
+  }
+
+  push(pcm: Int16Array, chars: number): void {
+    if (this.closed) return;
+    if (this.queue.length + this.inFlight + Math.ceil((this.pending.length + pcm.length) / FRAME) > MAX_QUEUED_FRAMES) {
+      this.failed('audio queue is full');
+      return;
+    }
+    const joined = new Int16Array(this.pending.length + pcm.length);
+    joined.set(this.pending);
+    joined.set(pcm, this.pending.length);
+    let at = 0;
+    for (; at + FRAME <= joined.length; at += FRAME) this.queue.push({ pcm: joined.slice(at, at + FRAME), chars: 0 });
+    this.pending = joined.slice(at);
+    this.pendingChars += chars;
+    const last = this.queue.at(-1);
+    if (last !== undefined && this.pending.length === 0) {
+      last.chars += this.pendingChars;
+      this.pendingChars = 0;
+    }
+  }
+
+  finish(): void {
+    if (this.closed || this.pending.length === 0) return;
+    const pcm = new Int16Array(FRAME);
+    pcm.set(this.pending);
+    this.queue.push({ pcm, chars: this.pendingChars });
+    this.pending = new Int16Array(0);
+    this.pendingChars = 0;
+  }
+
+  clear(): void {
+    this.generation += 1;
+    this.inFlight = 0;
+    this.speaking = false;
+    this.queue.length = 0;
+    this.pending = new Int16Array(0);
+    this.pendingChars = 0;
+  }
+
+  start(): void {
+    if (this.closed || this.timer !== null) return;
+    this.started = performance.now();
+    this.sent = 0;
+    this.tick();
+  }
+
+  stop(): void {
+    if (this.closed) return;
+    this.closed = true;
+    if (this.timer !== null) clearTimeout(this.timer);
+    this.timer = null;
+    this.clear();
+    this.encoder.delete();
+  }
+
+  private tick(): void {
+    if (this.closed) return;
+    const due = Math.floor((performance.now() - this.started) / FRAME_MS) + 1;
+    if (due - this.sent > MAX_LATE_FRAMES) this.sent = due - 1;
+    while (!this.closed && this.sent < due) this.sendOne();
+    if (this.closed) return;
+    const next = this.started + this.sent * FRAME_MS;
+    this.timer = setTimeout(() => {
+      this.tick();
+    }, Math.max(0, next - performance.now()));
+  }
+
+  private sendOne(): void {
+    const frame = this.queue.shift();
+    const marker = frame !== undefined && !this.speaking;
+    this.speaking = frame !== undefined;
+    const pcm = frame?.pcm ?? new Int16Array(FRAME);
+    const timestamp = (this.sent * FRAME) >>> 0;
+    this.sent += 1;
+    const generation = this.generation;
+    if (frame !== undefined) this.inFlight += 1;
+    try {
+      const sent = this.out.send(this.encoder.encode(bytesOf(pcm), FRAME), timestamp, marker);
+      Promise.resolve(sent).then(() => {
+        if (this.closed || generation !== this.generation || frame === undefined) return;
+        this.inFlight -= 1;
+        if (marker) this.onFirst?.();
+        if (frame.chars > 0) this.out.played(frame.chars);
+        if (!this.busy) this.onIdle?.();
+      }).catch((err: unknown) => {
+        if (!this.closed && generation === this.generation) this.failed(errMsg(err));
+      });
+    } catch (err) {
+      if (!this.closed && generation === this.generation) this.failed(errMsg(err));
+    }
+  }
+
+  private failed(reason: string): void {
+    this.clear();
+    log.warn({ reason }, 'voice: audio playback failed');
+    this.onFailed?.(reason);
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/brain.ts

@@ -0,0 +1,146 @@
+import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process';
+import { errMsg, log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import { drainLines } from '@metro-labs/core/trains/protocol';
+import { asAgent } from '../agent-user/user.js';
+import { metroCli } from '../claude/session.js';
+
+const CLOSE_GRACE_MS = 10_000;
+const DENIED = 'The owner did not approve this.';
+
+export interface ToolAsk {
+  requestId: string;
+  tool: string;
+  input: Record<string, unknown>;
+  description: string;
+  fromWorker: boolean;
+}
+
+export interface Thinking {
+  turnStarted(): void;
+  text(delta: string): void;
+  tool(name: string): void;
+  turnEnded(): void;
+  asked(ask: ToolAsk): void;
+  exited(reason: string): void;
+}
+
+const record = (value: unknown): Record<string, unknown> => (isRecord(value) ? value : {});
+const str = (value: unknown): string => (typeof value === 'string' ? value : '');
+
+export function toolAsk(msg: Record<string, unknown>): ToolAsk | null {
+  const request = record(msg.request);
+  const requestId = str(msg.request_id);
+  if (request.subtype !== 'can_use_tool' || requestId === '' || str(request.tool_name) === '') return null;
+  return {
+    requestId,
+    tool: str(request.tool_name),
+    input: record(request.input),
+    description: str(request.description) || str(request.decision_reason),
+    fromWorker: str(request.agent_id) !== '',
+  };
+}
+
+function streamEvent(msg: Record<string, unknown>, mind: Thinking): void {
+  if (msg.parent_tool_use_id !== null && msg.parent_tool_use_id !== undefined) return;
+  const event = record(msg.event);
+  const delta = record(event.delta);
+  if (event.type === 'content_block_delta' && delta.type === 'text_delta' && typeof delta.text === 'string') mind.text(delta.text);
+  const block = record(event.content_block);
+  if (event.type === 'content_block_start' && block.type === 'tool_use') mind.tool(typeof block.name === 'string' ? block.name : 'tool');
+}
+
+function startBrain(model: string): ChildProcessWithoutNullStreams {
+  const [command = 'metro', ...args] = metroCli(['voice', '--model', model]);
+  const [file, argv] = asAgent(command, args);
+  return spawn(file, argv, { stdio: ['pipe', 'pipe', 'pipe'], cwd: '/' });
+}
+
+export class Brain {
+  private readonly child: ChildProcessWithoutNullStreams;
+  private out = '';
+  private gone = false;
+  private closing = false;
+  private killTimer: ReturnType<typeof setTimeout> | undefined;
+
+  constructor(model: string, private readonly mind: Thinking, launch = startBrain) {
+    this.child = launch(model);
+    this.child.stdout.setEncoding('utf8');
+    this.child.stdout.on('data', (chunk: string) => {
+      this.out = drainLines('voice-brain', this.out + chunk, (line) => {
+        this.read(line);
+      });
+    });
+    this.child.stderr.setEncoding('utf8');
+    this.child.stderr.on('data', (chunk: string) => {
+      log.debug({ text: chunk.slice(0, 500) }, 'voice: brain stderr');
+    });
+    this.child.stdin.on('error', (err) => {
+      log.debug({ err: errMsg(err) }, 'voice: brain stdin closed');
+    });
+    this.child.on('error', (err) => {
+      this.exit(`could not start: ${errMsg(err)}`);
+    });
+    this.child.on('exit', (code, signal) => {
+      this.exit(`exited (${String(code ?? signal)})`);
+    });
+  }
+
+  tell(text: string, now = false): void {
+    const message = { type: 'user', message: { role: 'user', content: text }, parent_tool_use_id: null, session_id: '' };
+    this.write(now ? { ...message, priority: 'now' } : message);
+  }
+
+  answer(ask: ToolAsk, behavior: 'allow' | 'deny'): void {
+    const response = behavior === 'allow' ? { behavior, updatedInput: ask.input } : { behavior, message: DENIED };
+    this.write({ type: 'control_response', response: { subtype: 'success', request_id: ask.requestId, response } });
+  }
+
+  close(): void {
+    if (this.gone || this.closing) return;
+    this.closing = true;
+    this.child.stdin.end();
+    this.child.kill('SIGTERM');
+    this.killTimer = setTimeout(() => {
+      if (!this.gone) this.child.kill('SIGKILL');
+    }, CLOSE_GRACE_MS);
+    this.killTimer.unref();
+  }
+
+  private write(message: Record<string, unknown>): void {
+    if (!this.gone && !this.closing && this.child.stdin.writable) this.child.stdin.write(`${JSON.stringify(message)}\n`);
+  }
+
+  private read(line: string): void {
+    if (this.closing || this.gone) return;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      return;
+    }
+    const msg = record(parsed);
+    if (msg.type === 'system' && msg.subtype === 'init') this.mind.turnStarted();
+    else if (msg.type === 'stream_event') streamEvent(msg, this.mind);
+    else if (msg.type === 'result') this.mind.turnEnded();
+    else if (msg.type === 'control_request') this.control(msg);
+  }
+
+  private control(msg: Record<string, unknown>): void {
+    const ask = toolAsk(msg);
+    if (ask !== null) {
+      this.mind.asked(ask);
+      return;
+    }
+    const subtype = str(record(msg.request).subtype);
+    log.warn({ subtype }, 'voice: the agent session asked for something the call does not handle');
+    this.write({ type: 'control_response', response: { subtype: 'error', request_id: str(msg.request_id), error: `not supported on a voice call: ${subtype}` } });
+  }
+
+  private exit(reason: string): void {
+    if (this.gone) return;
+    this.gone = true;
+    clearTimeout(this.killTimer);
+    if (!this.closing) this.mind.exited(reason);
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/call.ts

@@ -0,0 +1,274 @@
+import { randomUUID } from 'node:crypto';
+import type { CallRoute } from '@metro-labs/core/call';
+import { errMsg, log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import { forwardTrainCall } from '../stations/train-call.js';
+import { CallApprovals } from './approvals.js';
+import type { Playback } from './audio.js';
+import { Peer, type PeerEvents } from './peer.js';
+import { SharedTalk } from './shared-talk.js';
+import { endSharedCall, hearSharedCall, openSharedCall, sharedCallsSelected, type SharedCallTransport } from './shared.js';
+import { Talk } from './talk.js';
+import type { VoiceConfig } from './store.js';
+
+const OFFER_WAIT_MS = 45_000;
+const LONGEST_CALL_MS = 60 * 60_000;
+const CONTEXT_MESSAGES = 12;
+const CONTEXT_CHARS = 400;
+const LEAVE_RETRY_MS = 3_000;
+const SIGNAL_WAIT_MS = 2_000;
+
+export interface CallStart {
+  agentId: string;
+  sourceId: string;
+  line: string;
+  lineName: string;
+  direct: boolean;
+  from: string;
+  callerName: string;
+  callId: string;
+  callerPeer: string;
+}
+
+type CallAudio = Pick<Talk, 'connect' | 'hear' | 'finish'>;
+type CallPeer = Pick<Peer, 'answer' | 'sendOpus' | 'close'>;
+type ContextStart = Omit<CallStart, 'agentId' | 'sourceId'>;
+
+export interface CallDependencies {
+  train: typeof trainCall;
+  peer(events: PeerEvents): CallPeer;
+  cli(cfg: VoiceConfig, out: Playback['send'], ended: (reason: string) => void, approvals: CallApprovals): CallAudio & Pick<Talk, 'prime'>;
+  shared(cfg: VoiceConfig, out: Playback['send'], ended: (reason: string) => void, heard: (text: string, sourceId: string) => void): CallAudio & SharedCallTransport;
+  selected: typeof sharedCallsSelected;
+  open: typeof openSharedCall;
+  heard: typeof hearSharedCall;
+  ended: typeof endSharedCall;
+  signalWaitMs: number;
+  leaveRetryMs: number;
+}
+
+const str = (value: unknown): string => (typeof value === 'string' ? value : '');
+
+export async function trainCall(action: string, args: Record<string, unknown>): Promise<unknown> {
+  const answer = await forwardTrainCall('xmtp', action, args);
+  if (answer.error !== undefined) throw new Error(answer.error);
+  return answer.result;
+}
+
+const dependencies: CallDependencies = {
+  train: trainCall,
+  peer: (events) => new Peer(events),
+  cli: (cfg, out, ended, approvals) => new Talk(cfg, out, ended, approvals),
+  shared: (cfg, out, ended, heard) => new SharedTalk(cfg, out, ended, heard),
+  selected: sharedCallsSelected,
+  open: openSharedCall,
+  heard: hearSharedCall,
+  ended: endSharedCall,
+  signalWaitMs: SIGNAL_WAIT_MS,
+  leaveRetryMs: LEAVE_RETRY_MS,
+};
+
+function speaker(row: Record<string, unknown>, start: ContextStart): string {
+  if (row.self === true) return 'you';
+  return row.from === start.from ? start.callerName : `another member (${str(row.from)})`;
+}
+
+function contextLine(row: unknown, start: ContextStart): string | null {
+  if (!isRecord(row) || typeof row.text !== 'string' || row.text.startsWith('[call')) return null;
+  const who = speaker(row, start);
+  return `${str(row.ts).slice(11, 16)} ${who}: ${row.text.replace(/\s+/g, ' ').slice(0, CONTEXT_CHARS)}`;
+}
+
+async function recentChat(start: CallStart, train: typeof trainCall): Promise<string> {
+  try {
+    const result = await train('read', { line: start.line, limit: CONTEXT_MESSAGES });
+    const rows = isRecord(result) && Array.isArray(result.messages) ? result.messages : [];
+    return rows.map((row) => contextLine(row, start)).filter((l): l is string => l !== null).join('\n');
+  } catch (err) {
+    log.warn({ err: errMsg(err) }, 'voice: could not read the chat before the call');
+    return '';
+  }
+}
+
+function where(start: ContextStart): string {
+  if (start.lineName !== '') return `the group chat "${start.lineName}"`;
+  return start.direct ? `your direct chat with ${start.callerName}` : 'a group chat with no name';
+}
+
+export function opening(start: ContextStart, chat: string): string {
+  const recent = chat === '' ? 'No earlier messages were found there.' : `Recent messages there before the call, oldest first (times in UTC):\n${chat}`;
+  return [
+    `[A voice call with ${start.callerName} is starting on Stage, in ${where(start)}.`,
+    `Line of this chat: ${start.line} (pass it verbatim to the metro tools, e.g. read, to look at or act on this chat).`,
+    `Caller: ${start.callerName} (${start.from}).`,
+    recent,
+    `Greet ${start.callerName} in a few words.]`,
+  ].join('\n');
+}
+
+export class Call {
+  readonly route: CallRoute;
+  private readonly selfPeer = randomUUID();
+  private readonly startedAt = Date.now();
+  private readonly talk: CallAudio;
+  private readonly shared: SharedCallTransport | null;
+  private readonly prime: ((context: string) => void) | null;
+  private readonly deps: CallDependencies;
+  private peer: CallPeer | null = null;
+  private timer: ReturnType<typeof setTimeout> | null = null;
+  private readonly longest: ReturnType<typeof setTimeout>;
+  private over = false;
+  private begun = false;
+  private left: Promise<void> = Promise.resolve();
+
+  constructor(
+    readonly start: CallStart,
+    cfg: VoiceConfig,
+    private readonly onOver: () => void,
+    deps: Partial<CallDependencies> = {},
+  ) {
+    this.deps = { ...dependencies, ...deps };
+    this.route = Object.freeze({ agentId: start.agentId, line: start.line, from: start.from, callId: start.callId, generation: randomUUID() });
+    const out: Playback['send'] = (opus, timestamp, marker) => {
+      if (this.over || this.peer === null) return Promise.reject(new Error('call audio is not connected'));
+      return this.peer.sendOpus(opus, timestamp, marker);
+    };
+    const ended = (reason: string): void => {
+      this.hangUp(reason);
+    };
+    if (this.deps.selected()) {
+      const shared = this.deps.shared(cfg, out, ended, (text, sourceId) => {
+        if (!this.over) this.deps.heard(this.route, text, sourceId);
+      });
+      this.talk = shared;
+      this.shared = shared;
+      this.prime = null;
+    } else {
+      const approvals = new CallApprovals(start.line, async (text) => {
+        await this.deps.train('send', { line: start.line, text });
+      });
+      const cli = this.deps.cli(cfg, out, ended, approvals);
+      this.talk = cli;
+      this.shared = null;
+      this.prime = (context) => {
+        cli.prime(context);
+      };
+    }
+    this.longest = setTimeout(() => {
+      this.hangUp('the call reached one hour');
+    }, LONGEST_CALL_MS);
+  }
+
+  async begin(): Promise<void> {
+    if (this.over || this.begun) return;
+    this.begun = true;
+    log.info({ line: this.start.line, callId: this.start.callId }, 'voice: answering a call');
+    if (this.shared !== null) {
+      if (!this.deps.open(this.route, this.start.sourceId, this.shared)) throw new Error('the shared conversation is unavailable');
+    } else {
+      recentChat(this.start, this.deps.train).then((chat) => {
+        if (!this.over) this.prime?.(opening(this.start, chat));
+      }).catch((err: unknown) => {
+        log.warn({ err: errMsg(err) }, 'voice: could not start the agent on the call');
+      });
+    }
+    await this.signal({ kind: 'join', callId: this.start.callId, from: this.selfPeer });
+    if (this.over) {
+      await this.sendLeave();
+      return;
+    }
+    if (this.peer === null) this.timer = setTimeout(() => {
+      this.hangUp('no offer arrived');
+    }, OFFER_WAIT_MS);
+  }
+
+  signalled(raw: Record<string, unknown>): void {
+    if (raw.callId !== this.start.callId || this.over) return;
+    const fromCaller = raw.from === this.start.callerPeer;
+    if (raw.kind === 'offer' && fromCaller && raw.to === this.selfPeer && this.peer === null && typeof raw.sdp === 'string')
+      this.answer(this.start.callerPeer, raw.sdp).catch((err: unknown) => {
+        this.hangUp(`answering failed: ${errMsg(err)}`);
+      });
+    else if (raw.kind === 'leave' && fromCaller) this.hangUp('the caller hung up');
+  }
+
+  hangUp(reason: string): void {
+    this.end(reason);
+  }
+
+  leave(reason: string): Promise<void> {
+    this.end(reason);
+    return this.left;
+  }
+
+  private async answer(to: string, offer: string): Promise<void> {
+    if (this.timer !== null) clearTimeout(this.timer);
+    this.peer = this.deps.peer({
+      audio: (opus) => {
+        if (!this.over) this.talk.hear(opus);
+      },
+      state: (state) => {
+        this.stateChanged(state);
+      },
+    });
+    const sdp = await this.peer.answer(offer);
+    if (this.over) return;
+    await this.signal({ kind: 'answer', callId: this.start.callId, from: this.selfPeer, to, sdp });
+  }
+
+  private stateChanged(state: string): void {
+    if (this.over) return;
+    log.info({ state, afterMs: Date.now() - this.startedAt }, 'voice: call connection');
+    if (state === 'connected') this.talk.connect();
+    else if (state === 'failed' || state === 'closed') this.hangUp(`the connection ${state}`);
+  }
+
+  private signal(signal: Record<string, unknown>): Promise<unknown> {
+    return this.deps.train('callSignal', { line: this.start.line, signal });
+  }
+
+  private end(reason: string): void {
+    if (this.over) return;
+    this.over = true;
+    if (this.timer !== null) clearTimeout(this.timer);
+    clearTimeout(this.longest);
+    log.info({ reason, seconds: Math.round((Date.now() - this.startedAt) / 1000) }, 'voice: call over');
+    this.talk.finish();
+    this.peer?.close();
+    if (this.shared !== null) this.deps.ended(this.route);
+    this.onOver();
+    this.left = this.sendLeave().then(async () => {
+      if (this.shared !== null || reason.startsWith('the caller') || reason === 'no offer arrived') return;
+      await this.deps.train('send', { line: this.start.line, text: `The voice call ended: ${reason}.` });
+    }).catch((err: unknown) => {
+      log.warn({ err: errMsg(err) }, 'voice: wrapping up the call failed');
+    });
+  }
+
+  private async sendLeave(): Promise<void> {
+    try {
+      await this.leaveOnce();
+    } catch (err) {
+      log.warn({ err: errMsg(err) }, 'voice: could not send the hang-up, trying once more');
+      await new Promise((r) => setTimeout(r, this.deps.leaveRetryMs));
+      await this.leaveOnce().catch((again: unknown) => {
+        log.warn({ err: errMsg(again) }, 'voice: could not send the hang-up; the next start sends it');
+      });
+    }
+  }
+
+  private async leaveOnce(): Promise<void> {
+    const leave = { kind: 'leave', callId: this.start.callId, from: this.selfPeer };
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const capped = new Promise<never>((_resolve, reject) => {
+      timer = setTimeout(() => {
+        reject(new Error('hang-up signal timed out'));
+      }, this.deps.signalWaitMs);
+    });
+    try {
+      await Promise.race([this.signal(leave), capped]);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+}