@stage-labs/metro 0.1.0-beta.24 → 0.1.0-beta.240

package/runtime/node_modules/@metro-labs/daemon/src/voice/talk.ts

@@ -0,0 +1,275 @@
+import { log } from '@metro-labs/core/log';
+import { downsample3, OpusIn, Player, type Playback } from './audio.js';
+import { Brain, type Thinking } from './brain.js';
+import { Scribe, VAD_SILENCE_SECS } from './scribe.js';
+import { Utterance } from './speech.js';
+import { Chunker } from './speakable.js';
+import { brainModel, voiceIdOf, type VoiceConfig } from './store.js';
+
+const BARGE_MIN_WORDS = 2;
+const NOTES_WAIT_MS = 60_000;
+const NOTES_PROMPT =
+  '[The call has ended. Write short call notes for the chat: what was discussed, what was decided and what you will follow up on. Plain text, at most 6 short lines, no title and no greeting. Reply with the notes only.]';
+
+interface Reply {
+  utterance: Utterance;
+  said: string;
+  heard: number;
+  chunker: Chunker;
+  askedAt: number | null;
+  saidAt: number | null;
+  firstText: number | null;
+  firstAudio: number | null;
+  firstPlayed: number | null;
+}
+
+const wordsIn = (text: string): number => text.split(/\s+/).filter((word) => /\p{L}/u.test(word)).length;
+
+const since = (start: number | null, at: number | null): number | null =>
+  start === null || at === null ? null : Math.round(at - start);
+
+function heardNote(reply: Reply | null): string | null {
+  if (reply === null || reply.said.trim() === '' || reply.heard >= reply.said.trim().length - 2) return null;
+  const part = reply.said.slice(0, reply.heard).trim();
+  return part === ''
+    ? '[The caller interrupted you before hearing your reply.]'
+    : `[The caller interrupted you. They heard only: "${part}". The rest of your reply was not heard.]`;
+}
+
+export class Talk implements Thinking {
+  private readonly brain: Brain;
+  private readonly scribe: Scribe;
+  private readonly opus = new OpusIn();
+  private readonly player: Player;
+  private spare: Utterance | null = null;
+  private reply: Reply | null = null;
+  private cut: string | null = null;
+  private muted = false;
+  private busy = false;
+  private askedAt: number | null = null;
+  private saidAt: number | null = null;
+  private lastHeardAt: number | null = null;
+  private readonly delays: number[] = [];
+  private notes: ((text: string) => void) | null = null;
+  private notesStarted = false;
+  private live = false;
+  private finished = false;
+  private heardCaller = false;
+  private spoke = false;
+
+  constructor(
+    private readonly cfg: VoiceConfig,
+    out: (opus: Buffer, timestamp: number, marker: boolean) => void,
+    private readonly ended: (reason: string) => void,
+  ) {
+    const playback: Playback = {
+      send: out,
+      played: (chars) => {
+        this.played(chars);
+      },
+    };
+    this.player = new Player(playback);
+    this.player.whenFirstPlayed(() => {
+      this.firstPlayed();
+    });
+    this.brain = new Brain(brainModel(cfg), this);
+    this.scribe = new Scribe(cfg.apiKey, {
+      partial: (text) => {
+        this.partial(text);
+      },
+      committed: (text) => {
+        this.committed(text);
+      },
+      failed: (reason) => {
+        this.ended(reason);
+      },
+    });
+    this.spare = this.utterance();
+  }
+
+  prime(context: string): void {
+    this.busy = true;
+    this.brain.tell(context);
+  }
+
+  connect(): void {
+    this.live = true;
+    this.spare ??= this.utterance();
+    this.player.start();
+  }
+
+  hear(packet: Buffer): void {
+    if (this.finished) return;
+    const pcm = this.opus.decode(packet);
+    if (pcm !== null) this.scribe.send(downsample3(pcm));
+  }
+
+  async finish(): Promise<string | null> {
+    if (this.finished) return null;
+    this.finished = true;
+    this.live = false;
+    this.player.stop();
+    this.scribe.close();
+    this.reply?.utterance.abort();
+    this.spare?.abort();
+    this.opus.close();
+    const notes = this.heardCaller && this.brain.alive ? await this.callNotes() : null;
+    this.brain.close();
+    return notes;
+  }
+
+  turnStarted(): void {
+    this.muted = false;
+    this.busy = true;
+    if (this.notes !== null) this.notesStarted = true;
+    if (this.notes !== null || this.finished) return;
+    this.reply = {
+      utterance: this.takeUtterance(),
+      said: '',
+      heard: 0,
+      chunker: new Chunker(),
+      askedAt: this.askedAt,
+      saidAt: this.saidAt,
+      firstText: null,
+      firstAudio: null,
+      firstPlayed: null,
+    };
+    this.askedAt = null;
+    this.saidAt = null;
+  }
+
+  text(delta: string): void {
+    const reply = this.reply;
+    if (this.muted || reply === null || this.notes !== null) return;
+    reply.firstText ??= performance.now();
+    this.speak(reply, reply.chunker.add(delta));
+  }
+
+  tool(name: string): void {
+    log.info({ tool: name }, 'voice: the agent uses a tool during the call');
+    if (this.reply !== null && !this.muted) this.speak(this.reply, this.reply.chunker.flush());
+  }
+
+  turnEnded(text: string): void {
+    this.busy = false;
+    const notes = this.notes;
+    if (notes !== null) {
+      if (!this.notesStarted) return;
+      this.notes = null;
+      notes(text);
+      return;
+    }
+    const reply = this.reply;
+    if (reply === null || this.muted) return;
+    this.speak(reply, reply.chunker.flush());
+    reply.utterance.end();
+    log.info(
+      { firstText: since(reply.askedAt, reply.firstText), firstAudio: since(reply.askedAt, reply.firstAudio), vadSilenceMs: VAD_SILENCE_SECS * 1000 },
+      'voice: reply timing in ms from the caller’s words being committed',
+    );
+  }
+
+  exited(reason: string): void {
+    log.warn({ reason }, 'voice: the agent session for the call ended');
+    this.notes?.('');
+    this.notes = null;
+    this.ended(`agent session ${reason}`);
+  }
+
+  private takeUtterance(): Utterance {
+    const ready = this.spare?.usable === true ? this.spare : this.utterance();
+    this.spare = this.live ? this.utterance() : null;
+    if (ready === this.spare) this.spare = null;
+    const reply = (): Reply | null => this.reply;
+    ready.attach({
+      audio: (pcm, chars) => {
+        const current = reply();
+        if (current?.utterance !== ready) return;
+        this.spoke = true;
+        current.firstAudio ??= performance.now();
+        this.player.push(pcm, chars);
+      },
+      done: () => {
+        this.player.finish();
+      },
+    });
+    return ready;
+  }
+
+  private utterance(): Utterance {
+    return new Utterance(this.cfg.apiKey, voiceIdOf(this.cfg), (reason) => {
+      log.warn({ reason }, 'voice: text to speech gave no audio');
+      if (!this.spoke && !this.finished) this.ended(`ElevenLabs could not speak (${reason})`);
+    });
+  }
+
+  private speak(reply: Reply, parts: string[]): void {
+    for (const part of parts) {
+      reply.said += `${part} `;
+      reply.utterance.say(part);
+    }
+  }
+
+  private played(chars: number): void {
+    if (this.reply !== null) this.reply.heard += chars;
+  }
+
+  get replyTime(): string | null {
+    if (this.delays.length === 0) return null;
+    const sorted = [...this.delays].sort((a, b) => a - b);
+    const middle = sorted[Math.floor(sorted.length / 2)] ?? 0;
+    return `Reply time, from your last word heard to my first word sent: about ${(middle / 1000).toFixed(1)} s (middle of ${String(sorted.length)}).`;
+  }
+
+  private firstPlayed(): void {
+    const reply = this.reply;
+    if (reply?.firstPlayed !== null || reply.saidAt === null) return;
+    reply.firstPlayed = performance.now();
+    this.delays.push(reply.firstPlayed - reply.saidAt);
+    log.info({ fromLastWord: since(reply.saidAt, reply.firstPlayed), fromCommit: since(reply.askedAt, reply.firstPlayed) }, 'voice: first reply audio sent to the caller, in ms');
+  }
+
+  private interrupts(text: string): boolean {
+    return !this.muted && this.player.busy && wordsIn(text) >= BARGE_MIN_WORDS;
+  }
+
+  private cutOff(): void {
+    this.cut = heardNote(this.reply);
+    this.reply?.utterance.abort();
+    this.player.clear();
+    this.muted = true;
+  }
+
+  private partial(text: string): void {
+    if (text !== '') this.lastHeardAt = performance.now();
+    if (this.interrupts(text)) this.cutOff();
+  }
+
+  private committed(text: string): void {
+    this.heardCaller = true;
+    if (this.interrupts(text)) this.cutOff();
+    const note = this.cut;
+    const now = this.muted;
+    this.cut = null;
+    this.askedAt = performance.now();
+    this.saidAt = this.lastHeardAt ?? this.askedAt;
+    this.lastHeardAt = null;
+    this.busy = true;
+    this.brain.tell(note === null ? text : `${note}\n${text}`, now);
+  }
+
+  private callNotes(): Promise<string | null> {
+    return new Promise((resolve) => {
+      const timer = setTimeout(() => {
+        this.notes = null;
+        resolve(null);
+      }, NOTES_WAIT_MS);
+      this.notesStarted = false;
+      this.notes = (text) => {
+        clearTimeout(timer);
+        resolve(text.trim() === '' ? null : text.trim());
+      };
+      this.brain.tell(NOTES_PROMPT, this.busy);
+    });
+  }
+}

package/runtime/node_modules/@metro-labs/discord-bot/package.json

@@ -13,10 +13,8 @@
     "test": "tsc --noEmit && bun test test/"
   },
   "dependencies": {
-    "@metro-labs/mcp": "workspace:*",
-    "@discordjs/voice": "^0.19.2",
-    "discord.js": "^14.26.4",
-    "prism-media": "^1.3.5"
+    "@metro-labs/core": "workspace:*",
+    "discord.js": "^14.26.4"
   },
   "devDependencies": {
     "@stage-labs/config": "0.1.0-beta.2",

package/runtime/node_modules/@metro-labs/discord-bot/src/accounts.ts

@@ -1,37 +1,22 @@
 import { Client } from 'discord.js';
-import { homedir } from 'node:os';
-import { join } from 'node:path';
 import {
   makeAccountStore,
   resolveAccountId,
-} from '@metro-labs/mcp/stations/account-store';
-import { Line } from '@metro-labs/mcp/lines';
-import { emit } from './wire.js';
+} from '@metro-labs/core/stations/account-store';
+import { Line } from '@metro-labs/core/lines';
 import { API } from './api-base.js';
 
-const ACCOUNTS_FILE =
-  process.env.DISCORD_BOT_ACCOUNTS_FILE ??
-  join(homedir(), '.metro', 'discord-bot-accounts.json');
-
 export interface AccountConfig {
   id: string;
   token: string;
-  owner?: string;
 }
 
 export const { loadAccounts } = makeAccountStore<AccountConfig>({
   prefix: 'discord-bot',
-  file: ACCOUNTS_FILE,
-  allowlistEnv: ['DISCORD_BOT_ONLY_ACCOUNTS', 'DISCORD_BOT_ACCOUNTS'],
   validate(raw, die) {
-    const seen = new Set<string>();
-    for (const a of raw) {
-      if (!a.id) die('account missing id');
+    for (const a of raw)
       if (!a.token || typeof a.token !== 'string')
         die(`account '${a.id}' missing token`);
-      if (seen.has(a.id)) die(`duplicate account id '${a.id}'`);
-      seen.add(a.id);
-    }
   },
 });
 
@@ -71,17 +56,12 @@ export async function rest<T = unknown>(
   };
   if (body !== undefined && !isForm)
     headers['Content-Type'] = 'application/json';
-  emit({ op: 'log', text: `discord-bot[${accountId}] api ${method} ${path}` });
   const res = await fetch(`${API}${path}`, {
     method,
     headers,
     body: restBody(body, isForm),
     signal: AbortSignal.timeout(30_000),
   });
-  emit({
-    op: 'log',
-    text: `discord-bot[${accountId}] api ${method} ${path} -> ${res.status}`,
-  });
   if (!res.ok) {
     const text = await res.text().catch(() => '');
     throw new Error(`discord-bot ${method} ${path}: ${res.status} ${text}`);

package/runtime/node_modules/@metro-labs/discord-bot/src/actions.ts

@@ -1,32 +1,19 @@
+import type { Message } from 'discord.js';
 import {
-  ActivityType,
-  type Message,
-  type PresenceStatusData,
-} from 'discord.js';
-import {
-  accountFor,
   accounts,
   encodeEmoji,
-  lineOf,
   rest,
   routeOf,
 } from './accounts.js';
 import { emitOutbound, emitOutboundEdit, emitOutboundReact } from './format.js';
-import { respond } from './wire.js';
-import { normalizeDiscord } from '@metro-labs/mcp/stations/messaging-normalize';
-import { assertContentLength } from '@metro-labs/mcp/stations/attachments';
+import { messagingAliases } from '@metro-labs/core/stations/messaging-normalize';
 import {
   appendFiles,
   outgoingFiles,
   type OutgoingFile,
 } from './send-files.js';
-import {
-  makeStation,
-  type CallMsg,
-  type StationHandler,
-} from '@metro-labs/mcp/stations/station-runtime';
-import { joinVoice, leaveVoice, voiceDebug, voiceTranscribe } from './voice.js';
-import { speak } from './voice-speak.js';
+import { makeStation, respond, type CallMsg, type StationHandler } from '@metro-labs/core/stations/station-runtime';
+import { readProfile, setProfile } from './profile.js';
 import { discordMembers } from './members.js';
 import {
   groupAddHandler,
@@ -65,40 +52,17 @@ async function sendMessage(
 export type { CallMsg };
 
 async function send(id: string, args: Record<string, unknown>): Promise<void> {
-  const {
-    line,
-    text,
-    replyTo,
-    embeds,
-    stickerIds,
-    images,
-    files,
-    account,
-    attachmentKinds,
-    attachmentNames,
-  } = args as {
+  const { line, text, replyTo, account } = args as {
     line: string;
     text?: string;
     replyTo?: string;
-    embeds?: unknown[];
-    stickerIds?: string[];
-    images?: string[];
-    files?: string[];
     account?: string;
-    attachmentKinds?: string[];
-    attachmentNames?: string[];
   };
   const { accountId, channelId } = routeOf(line, account);
   const body: Record<string, unknown> = { flags: 4 };
   if (text !== undefined) body.content = text;
   if (replyTo) body.message_reference = { message_id: replyTo };
-  if (embeds) body.embeds = embeds;
-  if (stickerIds) body.sticker_ids = stickerIds;
-  const outgoing = outgoingFiles(
-    [...(images ?? []), ...(files ?? [])],
-    attachmentNames,
-    attachmentKinds,
-  );
+  const outgoing = outgoingFiles(args.attachments);
   const res = await sendMessage(accountId, channelId, body, outgoing);
   emitOutbound(accountId, line, res.id, text ?? '', replyTo);
   respond(id, {
@@ -110,38 +74,6 @@ async function send(id: string, args: Record<string, unknown>): Promise<void> {
   });
 }
 
-function presence(id: string, args: Record<string, unknown>): void {
-  const {
-    text,
-    status = 'online',
-    account,
-  } = args as {
-    text?: string;
-    status?: PresenceStatusData;
-    account?: string;
-  };
-  const accountId = accountFor({ account });
-  const acct = accounts.get(accountId);
-  if (!acct) {
-    respond(id, { error: `unknown account '${accountId}'` });
-    return;
-  }
-  const client = acct.client;
-  if (!client.user) {
-    respond(id, { error: `gateway not ready for account '${accountId}'` });
-    return;
-  }
-  client.user.setPresence({
-    status,
-    activities: text
-      ? [{ name: 'Custom Status', type: ActivityType.Custom, state: text }]
-      : [],
-  });
-  respond(id, {
-    result: { ok: true, text: text ?? null, status, account: accountId },
-  });
-}
-
 const profileUrl = (userId?: string): string | null =>
   userId === undefined ? null : `https://discord.com/users/${userId}`;
 
@@ -154,13 +86,19 @@ function listAccounts(id: string): void {
         url: profileUrl(a.client.user?.id),
         userId: a.client.user?.id ?? null,
         username: a.client.user?.username ?? null,
-        owner: a.cfg.owner ?? null,
         ready: a.client.isReady(),
       })),
     },
   });
 }
 
+async function typing(id: string, args: Record<string, unknown>): Promise<void> {
+  const { line, account } = args as { line: string; account?: string };
+  const { accountId, channelId } = routeOf(line, account);
+  await rest(accountId, 'POST', `/channels/${channelId}/typing`);
+  respond(id, { result: { ok: true, account: accountId } });
+}
+
 async function react(id: string, args: Record<string, unknown>): Promise<void> {
   const { line, messageId, emoji, account } = args as {
     line: string;
@@ -216,7 +154,7 @@ async function remove(
   respond(id, { result: { ok: true, account: accountId } });
 }
 
-async function fetchMessages(
+async function read(
   id: string,
   args: Record<string, unknown>,
 ): Promise<void> {
@@ -244,104 +182,6 @@ async function fetchMessages(
   respond(id, { result: { messages: msgs, account: accountId } });
 }
 
-async function download(
-  id: string,
-  args: Record<string, unknown>,
-): Promise<void> {
-  const {
-    line,
-    messageId,
-    outDir = '/tmp',
-    account,
-  } = args as {
-    line: string;
-    messageId: string;
-    outDir?: string;
-    account?: string;
-  };
-  const { accountId, channelId } = routeOf(line, account);
-  const msg = await rest<{
-    attachments: { url: string; content_type?: string; filename: string }[];
-  }>(accountId, 'GET', `/channels/${channelId}/messages/${messageId}`);
-  const files: { path: string; mediaType: string }[] = [];
-  for (const att of msg.attachments) {
-    const res = await fetch(att.url, { signal: AbortSignal.timeout(60_000) });
-    if (!res.ok)
-      throw new Error(`discord-bot download ${res.status} for ${att.url}`);
-    assertContentLength(res.headers.get('content-length'));
-    const buf = await res.arrayBuffer();
-    const path = `${outDir}/${messageId}-${att.filename}`;
-    await Bun.write(path, buf);
-    files.push({
-      path,
-      mediaType: att.content_type ?? 'application/octet-stream',
-    });
-  }
-  respond(id, { result: { files, account: accountId } });
-}
-
-async function threadCreate(
-  id: string,
-  args: Record<string, unknown>,
-): Promise<void> {
-  const {
-    line,
-    messageId,
-    name,
-    autoArchiveDuration = 1440,
-    account,
-  } = args as {
-    line: string;
-    messageId?: string;
-    name: string;
-    autoArchiveDuration?: number;
-    account?: string;
-  };
-  const { accountId, channelId } = routeOf(line, account);
-  const path = messageId
-    ? `/channels/${channelId}/messages/${messageId}/threads`
-    : `/channels/${channelId}/threads`;
-  const res = await rest<{ id: string }>(accountId, 'POST', path, {
-    name,
-    auto_archive_duration: autoArchiveDuration,
-  });
-  respond(id, {
-    result: {
-      threadId: res.id,
-      line: lineOf(accountId, res.id),
-      account: accountId,
-    },
-  });
-}
-
-async function pin(id: string, args: Record<string, unknown>): Promise<void> {
-  const { line, messageId, account } = args as {
-    line: string;
-    messageId: string;
-    account?: string;
-  };
-  const { accountId, channelId } = routeOf(line, account);
-  await rest(accountId, 'PUT', `/channels/${channelId}/pins/${messageId}`);
-  respond(id, { result: { ok: true, account: accountId } });
-}
-
-async function typing(id: string, args: Record<string, unknown>): Promise<void> {
-  const { line, account } = args as { line: string; account?: string };
-  const { accountId, channelId } = routeOf(line, account);
-  await rest(accountId, 'POST', `/channels/${channelId}/typing`);
-  respond(id, { result: { ok: true, account: accountId } });
-}
-
-async function channel(
-  id: string,
-  args: Record<string, unknown>,
-): Promise<void> {
-  const { line, account } = args as { line: string; account?: string };
-  const { accountId, channelId } = routeOf(line, account);
-  const res = await rest(accountId, 'GET', `/channels/${channelId}`);
-  respond(id, { result: res });
-}
-
 async function listMembers(
   id: string,
   args: Record<string, unknown>,
@@ -365,24 +205,16 @@ const HANDLERS: Record<string, StationHandler> = {
   groupRemoveMembers: groupRemoveHandler,
   send,
   react,
+  typing,
   edit,
   delete: remove,
-  fetch: fetchMessages,
-  download,
-  thread_create: threadCreate,
-  pin,
-  typing,
-  channel,
+  read,
   listMembers,
-  set_presence: presence,
-  joinVoice,
-  leaveVoice,
-  speak,
-  voiceDebug,
-  voiceTranscribe,
+  set_profile: setProfile,
+  profile: readProfile,
 };
 
 export const handleCall = makeStation({
   handlers: HANDLERS,
-  normalize: normalizeDiscord,
+  normalize: messagingAliases(),
 });

package/runtime/node_modules/@metro-labs/discord-bot/src/attachments.ts

@@ -1,8 +1,8 @@
 import {
   saveBufferToCache,
   assertContentLength,
-} from '@metro-labs/mcp/stations/attachments';
-import type { SavedAttachment } from '@metro-labs/mcp/stations/attachments';
+} from '@metro-labs/core/stations/attachments';
+import type { SavedAttachment } from '@metro-labs/core/stations/attachments';
 
 export type { SavedAttachment };