@stage-labs/metro 0.1.0-beta.235 → 0.1.0-beta.237

package/runtime/node_modules/@metro-labs/daemon/src/voice/talk.ts

@@ -0,0 +1,234 @@
+import { log } from '@metro-labs/core/log';
+import { downsample3, OpusIn, Player, type Playback } from './audio.js';
+import { Brain, type Thinking } from './brain.js';
+import { Scribe, VAD_SILENCE_SECS } from './scribe.js';
+import { Utterance } from './speech.js';
+import { Chunker } from './speakable.js';
+import { brainModel, voiceIdOf, type VoiceConfig } from './store.js';
+
+const BARGE_MIN_CHARS = 4;
+const NOTES_WAIT_MS = 60_000;
+const NOTES_PROMPT =
+  '[The call has ended. Write short call notes for the chat: what was discussed, what was decided and what you will follow up on. Plain text, at most 6 short lines, no title and no greeting. Reply with the notes only.]';
+
+interface Reply {
+  utterance: Utterance;
+  said: string;
+  heard: number;
+  chunker: Chunker;
+  askedAt: number | null;
+  firstText: number | null;
+  firstAudio: number | null;
+}
+
+const since = (start: number | null, at: number | null): number | null =>
+  start === null || at === null ? null : Math.round(at - start);
+
+function heardNote(reply: Reply | null): string | null {
+  if (reply === null || reply.said.trim() === '' || reply.heard >= reply.said.trim().length - 2) return null;
+  const part = reply.said.slice(0, reply.heard).trim();
+  return part === ''
+    ? '[The caller interrupted you before hearing your reply.]'
+    : `[The caller interrupted you. They heard only: "${part}". The rest of your reply was not heard.]`;
+}
+
+export class Talk implements Thinking {
+  private readonly brain: Brain;
+  private readonly scribe: Scribe;
+  private readonly opus = new OpusIn();
+  private readonly player: Player;
+  private spare: Utterance | null = null;
+  private reply: Reply | null = null;
+  private cut: string | null = null;
+  private muted = false;
+  private busy = false;
+  private askedAt: number | null = null;
+  private notes: ((text: string) => void) | null = null;
+  private notesStarted = false;
+  private live = false;
+  private finished = false;
+  private heardCaller = false;
+
+  constructor(
+    private readonly cfg: VoiceConfig,
+    out: (opus: Buffer, timestamp: number, marker: boolean) => void,
+    private readonly ended: (reason: string) => void,
+  ) {
+    const playback: Playback = {
+      send: out,
+      played: (chars) => {
+        this.played(chars);
+      },
+    };
+    this.player = new Player(playback);
+    this.player.whenFirstPlayed(() => {
+      this.firstPlayed();
+    });
+    this.brain = new Brain(brainModel(cfg), this);
+    this.scribe = new Scribe(cfg.apiKey, {
+      partial: (text) => {
+        this.partial(text);
+      },
+      committed: (text) => {
+        this.committed(text);
+      },
+      failed: (reason) => {
+        this.ended(reason);
+      },
+    });
+    this.spare = new Utterance(cfg.apiKey, voiceIdOf(cfg));
+  }
+
+  prime(context: string): void {
+    this.busy = true;
+    this.brain.tell(context);
+  }
+
+  connect(): void {
+    this.live = true;
+    this.spare ??= new Utterance(this.cfg.apiKey, voiceIdOf(this.cfg));
+    this.player.start();
+  }
+
+  hear(packet: Buffer): void {
+    if (this.finished) return;
+    const pcm = this.opus.decode(packet);
+    if (pcm !== null) this.scribe.send(downsample3(pcm));
+  }
+
+  async finish(): Promise<string | null> {
+    if (this.finished) return null;
+    this.finished = true;
+    this.live = false;
+    this.player.stop();
+    this.scribe.close();
+    this.reply?.utterance.abort();
+    this.spare?.abort();
+    this.opus.close();
+    const notes = this.heardCaller && this.brain.alive ? await this.callNotes() : null;
+    this.brain.close();
+    return notes;
+  }
+
+  turnStarted(): void {
+    this.muted = false;
+    this.busy = true;
+    if (this.notes !== null) this.notesStarted = true;
+    if (this.notes !== null || this.finished) return;
+    this.reply = { utterance: this.takeUtterance(), said: '', heard: 0, chunker: new Chunker(), askedAt: this.askedAt, firstText: null, firstAudio: null };
+    this.askedAt = null;
+  }
+
+  text(delta: string): void {
+    const reply = this.reply;
+    if (this.muted || reply === null || this.notes !== null) return;
+    reply.firstText ??= performance.now();
+    this.speak(reply, reply.chunker.add(delta));
+  }
+
+  tool(name: string): void {
+    log.info({ tool: name }, 'voice: the agent uses a tool during the call');
+    if (this.reply !== null && !this.muted) this.speak(this.reply, this.reply.chunker.flush());
+  }
+
+  turnEnded(text: string): void {
+    this.busy = false;
+    const notes = this.notes;
+    if (notes !== null) {
+      if (!this.notesStarted) return;
+      this.notes = null;
+      notes(text);
+      return;
+    }
+    const reply = this.reply;
+    if (reply === null || this.muted) return;
+    this.speak(reply, reply.chunker.flush());
+    reply.utterance.end();
+    log.info(
+      { firstText: since(reply.askedAt, reply.firstText), firstAudio: since(reply.askedAt, reply.firstAudio), vadSilenceMs: VAD_SILENCE_SECS * 1000 },
+      'voice: reply timing in ms from the caller’s words being committed',
+    );
+  }
+
+  exited(reason: string): void {
+    log.warn({ reason }, 'voice: the agent session for the call ended');
+    this.notes?.('');
+    this.notes = null;
+    this.ended(`agent session ${reason}`);
+  }
+
+  private takeUtterance(): Utterance {
+    const ready = this.spare?.usable === true ? this.spare : new Utterance(this.cfg.apiKey, voiceIdOf(this.cfg));
+    this.spare = this.live ? new Utterance(this.cfg.apiKey, voiceIdOf(this.cfg)) : null;
+    if (ready === this.spare) this.spare = null;
+    const reply = (): Reply | null => this.reply;
+    ready.attach({
+      audio: (pcm, chars) => {
+        const current = reply();
+        if (current?.utterance !== ready) return;
+        current.firstAudio ??= performance.now();
+        this.player.push(pcm, chars);
+      },
+      done: () => {
+        this.player.finish();
+      },
+    });
+    return ready;
+  }
+
+  private speak(reply: Reply, parts: string[]): void {
+    for (const part of parts) {
+      reply.said += `${part} `;
+      reply.utterance.say(part);
+    }
+  }
+
+  private played(chars: number): void {
+    if (this.reply !== null) this.reply.heard += chars;
+  }
+
+  private firstPlayed(): void {
+    const reply = this.reply;
+    if (reply?.askedAt != null) log.info({ firstPlayed: since(reply.askedAt, performance.now()) }, 'voice: first reply audio sent to the caller');
+  }
+
+  private partial(text: string): void {
+    if (text.replace(/\s/g, '').length < BARGE_MIN_CHARS || this.muted) return;
+    if (!this.player.busy && !this.busy) return;
+    const reply = this.reply;
+    this.cut = heardNote(reply);
+    reply?.utterance.abort();
+    this.player.clear();
+    this.muted = true;
+  }
+
+  private committed(text: string): void {
+    this.heardCaller = true;
+    const note = this.cut;
+    const now = this.busy || note !== null;
+    this.cut = null;
+    if (now && !this.muted) {
+      this.reply?.utterance.abort();
+      this.player.clear();
+      this.muted = true;
+    }
+    this.askedAt = performance.now();
+    this.busy = true;
+    this.brain.tell(note === null ? text : `${note}\n${text}`, now);
+  }
+
+  private callNotes(): Promise<string | null> {
+    return new Promise((resolve) => {
+      const timer = setTimeout(() => {
+        this.notes = null;
+        resolve(null);
+      }, NOTES_WAIT_MS);
+      this.notesStarted = false;
+      this.notes = (text) => {
+        clearTimeout(timer);
+        resolve(text.trim() === '' ? null : text.trim());
+      };
+      this.brain.tell(NOTES_PROMPT, this.busy);
+    });
+  }
+}

package/runtime/node_modules/@metro-labs/xmtp/src/actions-conv.ts

@@ -15,6 +15,7 @@ import {
 } from './group.js';
 import { updateChannelMeta } from './actions-meta.js';
 import { deletedByRequests, isDeleteRequest, superAdminCheck } from './delete-requests.js';
+import { isCallSignal } from './codecs.js';
 
 type Args = Record<string, unknown>;
 type Handler = (id: string, args: Args) => Promise<void>;
@@ -63,7 +64,7 @@ async function read(id: string, args: Args): Promise<void> {
   await conv.sync().catch(() => undefined);
   const all = await conv.messages();
   const deleted = deletedByRequests(all, superAdminCheck(conv));
-  const slice = upTo(all.filter((m) => !isDeleteRequest(m)), before).slice(-lim);
+  const slice = upTo(all.filter((m) => !isDeleteRequest(m) && !isCallSignal(m)), before).slice(-lim);
   const parsed = parseLine(line);
   if (!parsed)
     throw new TrainError('NOT_FOUND', `could not parse line ${line}`);

package/runtime/node_modules/@metro-labs/xmtp/src/actions.ts

@@ -12,7 +12,7 @@ import {
 import { convOf, type Account } from './accounts.js';
 import { resolveMsgId } from './wire.js';
 import { emitOutbound } from './emit.js';
-import { FrameCodec, PollCodec, buildPollContent, encodeDeleteMessage } from './codecs.js';
+import { CallSignalCodec, FrameCodec, PollCodec, buildPollContent, encodeDeleteMessage } from './codecs.js';
 import { buildFrameContent } from './frames.js';
 import { convHandlers } from './actions-conv.js';
 import { messagingAliases } from '@metro-labs/core/stations/messaging-normalize';
@@ -142,6 +142,15 @@ async function remove(id: string, args: Args): Promise<void> {
   respond(id, { result: { messageId: sentId } });
 }
 
+async function callSignal(id: string, args: Args): Promise<void> {
+  const { line, signal } = args as { line: string; signal?: unknown };
+  if (typeof signal !== 'object' || signal === null) throw badArgs('callSignal requires a signal object');
+  const { conv } = await convOf(line);
+  if (!conv) throw noConv(line);
+  const messageId = await conv.send(new CallSignalCodec().encode(signal as Record<string, unknown>), { shouldPush: false });
+  respond(id, { result: { messageId } });
+}
+
 async function sendAttachment(id: string, args: Args): Promise<void> {
   const { line, name, mime, dataB64 } = args as {
     line: string;
@@ -223,6 +232,7 @@ const handlers: Record<string, (id: string, args: Args) => Promise<void>> = {
   react,
   reply,
   delete: remove,
+  callSignal,
   sendAttachment,
   sendImage,
   ...convHandlers,

package/runtime/node_modules/@metro-labs/xmtp/src/codecs.ts

@@ -11,17 +11,19 @@ const dec = (e: EncodedContent): unknown =>
 
 const makeJsonCodec = <T>(
   contentType: ContentTypeId,
-  fallbackFn: (c: T) => string,
+  fallbackFn: (c: T) => string | undefined,
+  push = true,
 ) =>
   class {
     get contentType() {
       return contentType;
     }
     encode(c: T): EncodedContent {
+      const fallback = fallbackFn(c);
       return {
         type: contentType,
         parameters: {},
-        fallback: fallbackFn(c),
+        ...(fallback === undefined ? {} : { fallback }),
         content: enc(c),
       };
     }
@@ -32,7 +34,7 @@ const makeJsonCodec = <T>(
       return fallbackFn(c);
     }
     shouldPush() {
-      return true;
+      return push;
     }
   };
 
@@ -235,6 +237,20 @@ export function encodeDeleteMessage(messageId: string): EncodedContent {
   };
 }
 
+export const CALL_TYPES = new Set(['callInvite', 'callSignal']);
+const callContentType = (typeId: string): ContentTypeId => ({
+  authorityId: 'stage.box',
+  typeId,
+  versionMajor: 1,
+  versionMinor: 0,
+});
+const CallInviteCodec = makeJsonCodec<{ video?: unknown }>(callContentType('callInvite'), (c) =>
+  c.video === true ? '📞 Video call' : '📞 Voice call',
+);
+export const isCallSignal = (m: { contentType?: ContentTypeId }): boolean =>
+  m.contentType?.authorityId === 'stage.box' && m.contentType.typeId === 'callSignal';
+export const CallSignalCodec = makeJsonCodec<Record<string, unknown>>(callContentType('callSignal'), () => undefined, false);
+
 export const CODECS = (): ContentCodec[] => [
   new PollCodec(),
   new SignatureRequestCodec(),
@@ -242,4 +258,6 @@ export const CODECS = (): ContentCodec[] => [
   new FrameCodec(),
   new FrameActionCodec(),
   new DeleteRequestCodec(),
+  new CallInviteCodec(),
+  new CallSignalCodec(),
 ];

package/runtime/node_modules/@metro-labs/xmtp/src/emit-payloads.ts

@@ -9,7 +9,7 @@ import {
 } from './attachments.js';
 import { reportAttachment } from '@metro-labs/core/stations/train-events';
 import { sentByMe } from './wire.js';
-import { frameActionText, frameFallback, type FrameContent } from './codecs.js';
+import { CALL_TYPES, frameActionText, frameFallback, type FrameContent } from './codecs.js';
 import { asFrameAction } from './frames.js';
 
 export interface EnvelopeCtx {
@@ -336,6 +336,7 @@ export function typedEnvelope(
   if (typeId === 'reaction') return reactionPayload(base, c as Reaction);
   if (typeId === 'reply') return replyPayload(base, typeId, c as Reply);
   if (typeId === 'poll') return pollPayload(base, typeId, c);
+  if (typeId !== undefined && CALL_TYPES.has(typeId)) return { ...base, payload: { contentType: typeId, call: c } };
   return (
     attachmentEnvelope(base, typeId ?? '', c, ctx) ??
     refEnvelope(base, typeId, c) ??

package/runtime/runtime.json

@@ -1,3 +1,3 @@
 {
-  "version": "0.1.0-beta.235"
+  "version": "0.1.0-beta.237"
 }

package/runtime/stations.json

@@ -1,8 +1,10 @@
 {
   "core": {
     "@modelcontextprotocol/sdk": "^1.12.0",
+    "opusscript": "^0.1.1",
     "pino": "^9.5.0",
     "pino-pretty": "^13.1.3",
+    "werift": "^0.24.4",
     "ws": "^8.18.0",
     "zod": "^3.23.8"
   },