@stage-labs/metro 0.1.0-beta.3 → 0.1.0-beta.300

package/runtime/node_modules/@metro-labs/daemon/src/voice/talk.ts

@@ -0,0 +1,264 @@
+import { log } from '@metro-labs/core/log';
+import { downsample3, OpusIn, Player, type Playback } from './audio.js';
+import type { CallApprovals } from './approvals.js';
+import { Brain, type Thinking, type ToolAsk } from './brain.js';
+import { Scribe, VAD_SILENCE_SECS } from './scribe.js';
+import { Utterance } from './speech.js';
+import { Chunker } from './speakable.js';
+import { brainModel, languageOf, voiceIdOf, type VoiceConfig } from './store.js';
+
+const BARGE_MIN_WORDS = 2;
+
+interface Reply {
+  utterance: Utterance;
+  said: string;
+  heard: number;
+  chunker: Chunker;
+  askedAt: number | null;
+  saidAt: number | null;
+  firstText: number | null;
+  firstAudio: number | null;
+  firstPlayed: number | null;
+}
+
+const toolName = (tool: string): string => tool.split('__').pop() ?? tool;
+
+const approvalNote = (tool: string, id: string): string =>
+  `[The owner was asked in the chat of this call to approve ${toolName(tool)}; they answer "yes ${id}" or "no ${id}" there. Unless you already said so, tell the caller in one short sentence that it waits for their answer in the chat.]`;
+
+const wordsIn = (text: string): number => text.split(/\s+/).filter((word) => /\p{L}/u.test(word)).length;
+
+const since = (start: number | null, at: number | null): number | null =>
+  start === null || at === null ? null : Math.round(at - start);
+
+function heardNote(reply: Reply | null): string | null {
+  if (reply === null || reply.said.trim() === '' || reply.heard >= reply.said.trim().length - 2) return null;
+  const part = reply.said.slice(0, reply.heard).trim();
+  return part === ''
+    ? '[The caller interrupted you before hearing your reply.]'
+    : `[The caller interrupted you. They heard only: "${part}". The rest of your reply was not heard.]`;
+}
+
+export class Talk implements Thinking {
+  private readonly brain: Brain;
+  private readonly scribe: Scribe;
+  private readonly opus = new OpusIn();
+  private readonly player: Player;
+  private spare: Utterance | null = null;
+  private reply: Reply | null = null;
+  private cut: string | null = null;
+  private muted = false;
+  private askedAt: number | null = null;
+  private saidAt: number | null = null;
+  private lastHeardAt: number | null = null;
+  private readonly delays: number[] = [];
+  private live = false;
+  private finished = false;
+  private spoke = false;
+
+  constructor(
+    private readonly cfg: VoiceConfig,
+    out: Playback['send'],
+    private readonly ended: (reason: string) => void,
+    private readonly approvals: CallApprovals,
+  ) {
+    const playback: Playback = {
+      send: out,
+      played: (chars) => {
+        this.played(chars);
+      },
+    };
+    this.player = new Player(playback);
+    this.player.whenFirstPlayed(() => {
+      this.firstPlayed();
+    });
+    this.player.whenFailed((reason) => {
+      if (!this.finished) this.ended(`audio playback failed: ${reason}`);
+    });
+    this.brain = new Brain(brainModel(cfg), this);
+    this.scribe = new Scribe(cfg.apiKey, languageOf(cfg), {
+      partial: (text) => {
+        this.partial(text);
+      },
+      committed: (text) => {
+        this.committed(text);
+      },
+      failed: (reason) => {
+        this.ended(reason);
+      },
+    });
+    this.spare = this.utterance();
+  }
+
+  prime(context: string): void {
+    if (!this.finished) this.brain.tell(context);
+  }
+
+  connect(): void {
+    if (this.finished || this.live) return;
+    this.live = true;
+    this.spare ??= this.utterance();
+    this.player.start();
+  }
+
+  hear(packet: Buffer): void {
+    if (this.finished) return;
+    const pcm = this.opus.decode(packet);
+    if (pcm !== null) this.scribe.send(downsample3(pcm));
+  }
+
+  finish(): void {
+    if (this.finished) return;
+    this.finished = true;
+    this.live = false;
+    this.approvals.close();
+    this.player.stop();
+    this.scribe.close();
+    this.reply?.utterance.abort();
+    this.spare?.abort();
+    this.opus.close();
+    this.brain.close();
+    this.logReplyTime();
+  }
+
+  turnStarted(): void {
+    if (this.finished) return;
+    this.muted = false;
+    this.reply = {
+      utterance: this.takeUtterance(),
+      said: '',
+      heard: 0,
+      chunker: new Chunker(),
+      askedAt: this.askedAt,
+      saidAt: this.saidAt,
+      firstText: null,
+      firstAudio: null,
+      firstPlayed: null,
+    };
+    this.askedAt = null;
+    this.saidAt = null;
+  }
+
+  text(delta: string): void {
+    const reply = this.reply;
+    if (this.finished || this.muted || reply === null) return;
+    reply.firstText ??= performance.now();
+    this.speak(reply, reply.chunker.add(delta));
+  }
+
+  tool(name: string): void {
+    if (this.finished) return;
+    log.info({ tool: name }, 'voice: the agent uses a tool during the call');
+    if (this.reply !== null && !this.muted) this.speak(this.reply, this.reply.chunker.flush());
+  }
+
+  turnEnded(): void {
+    const reply = this.reply;
+    if (this.finished || reply === null || this.muted) return;
+    this.speak(reply, reply.chunker.flush());
+    reply.utterance.end();
+    log.info(
+      { firstText: since(reply.askedAt, reply.firstText), firstAudio: since(reply.askedAt, reply.firstAudio), vadSilenceMs: VAD_SILENCE_SECS * 1000 },
+      'voice: reply timing in ms from the caller’s words being committed',
+    );
+  }
+
+  asked(ask: ToolAsk): void {
+    const id = this.finished ? null : this.approvals.ask(ask, (behavior) => {
+      this.brain.answer(ask, behavior);
+    });
+    if (id === null) {
+      this.brain.answer(ask, 'deny');
+      return;
+    }
+    log.info({ tool: ask.tool, worker: ask.fromWorker }, 'voice: a tool waits for the owner’s approval in the chat of the call');
+    this.brain.tell(approvalNote(ask.tool, id));
+  }
+
+  exited(reason: string): void {
+    if (this.finished) return;
+    log.warn({ reason }, 'voice: the agent session for the call ended');
+    this.ended(`agent session ${reason}`);
+  }
+
+  private takeUtterance(): Utterance {
+    const ready = this.spare?.usable === true ? this.spare : this.utterance();
+    this.spare = this.live ? this.utterance() : null;
+    if (ready === this.spare) this.spare = null;
+    const reply = (): Reply | null => this.reply;
+    ready.attach({
+      audio: (pcm, chars) => {
+        const current = reply();
+        if (this.finished || this.muted || current?.utterance !== ready) return;
+        this.spoke = true;
+        current.firstAudio ??= performance.now();
+        this.player.push(pcm, chars);
+      },
+      done: () => {
+        if (!this.finished && !this.muted && this.reply?.utterance === ready) this.player.finish();
+      },
+    });
+    return ready;
+  }
+
+  private utterance(): Utterance {
+    return new Utterance(this.cfg.apiKey, voiceIdOf(this.cfg), (reason) => {
+      log.warn({ reason }, 'voice: text to speech gave no audio');
+      if (!this.spoke && !this.finished) this.ended(`ElevenLabs could not speak (${reason})`);
+    });
+  }
+
+  private speak(reply: Reply, parts: string[]): void {
+    for (const part of parts) {
+      reply.said += `${part} `;
+      reply.utterance.say(part);
+    }
+  }
+
+  private played(chars: number): void {
+    if (this.reply !== null) this.reply.heard += chars;
+  }
+
+  private logReplyTime(): void {
+    if (this.delays.length === 0) return;
+    const sorted = [...this.delays].sort((a, b) => a - b);
+    log.info({ middleMs: Math.round(sorted[Math.floor(sorted.length / 2)] ?? 0), replies: sorted.length }, 'voice: reply time of the call, from the caller’s last word heard to the first reply audio sent');
+  }
+
+  private firstPlayed(): void {
+    const reply = this.reply;
+    if (reply?.firstPlayed !== null || reply.saidAt === null) return;
+    reply.firstPlayed = performance.now();
+    this.delays.push(reply.firstPlayed - reply.saidAt);
+    log.info({ fromLastWord: since(reply.saidAt, reply.firstPlayed), fromCommit: since(reply.askedAt, reply.firstPlayed) }, 'voice: first reply audio sent to the caller, in ms');
+  }
+
+  private interrupts(text: string): boolean {
+    return !this.muted && this.player.busy && wordsIn(text) >= BARGE_MIN_WORDS;
+  }
+
+  private cutOff(): void {
+    this.cut = heardNote(this.reply);
+    this.reply?.utterance.abort();
+    this.player.clear();
+    this.muted = true;
+  }
+
+  private partial(text: string): void {
+    if (this.finished) return;
+    if (text !== '') this.lastHeardAt = performance.now();
+    if (this.interrupts(text)) this.cutOff();
+  }
+
+  private committed(text: string): void {
+    if (this.finished) return;
+    if (this.interrupts(text)) this.cutOff();
+    const note = this.cut;
+    const now = this.muted;
+    this.cut = null;
+    this.askedAt = performance.now();
+    this.saidAt = this.lastHeardAt ?? this.askedAt;
+    this.lastHeardAt = null;
+    this.brain.tell(note === null ? text : `${note}\n${text}`, now);
+  }
+}

package/runtime/node_modules/@metro-labs/discord-bot/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@metro-labs/discord-bot",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/station.ts",
+    "./train": "./src/index.ts",
+    "./verify": "./src/verify.ts"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "tsc --noEmit && bun test test/"
+  },
+  "dependencies": {
+    "@metro-labs/core": "workspace:*",
+    "discord.js": "^14.26.4"
+  },
+  "devDependencies": {
+    "@stage-labs/config": "0.1.0-beta.3",
+    "eslint": "^10.3.0",
+    "knip": "^6.15.0",
+    "madge": "^8.0.0",
+    "typescript": "^5"
+  }
+}

package/runtime/node_modules/@metro-labs/discord-bot/src/accounts.ts

@@ -0,0 +1,99 @@
+import { Client } from 'discord.js';
+import {
+  makeAccountStore,
+  resolveAccountId,
+} from '@metro-labs/core/stations/account-store';
+import { Line } from '@metro-labs/core/lines';
+import { API } from './api-base.js';
+
+export interface AccountConfig {
+  id: string;
+  token: string;
+}
+
+export const { loadAccounts } = makeAccountStore<AccountConfig>({
+  prefix: 'discord-bot',
+  validate(raw, die) {
+    for (const a of raw)
+      if (!a.token || typeof a.token !== 'string')
+        die(`account '${a.id}' missing token`);
+  },
+});
+
+export interface Account {
+  cfg: AccountConfig;
+  client: Client;
+}
+export const accounts = new Map<string, Account>();
+
+function restBody(
+  body: unknown,
+  isForm: boolean,
+): RequestInit['body'] | undefined {
+  if (body === undefined) return undefined;
+  return isForm ? (body as RequestInit['body']) : JSON.stringify(body);
+}
+
+async function restResult<T>(res: Response): Promise<T> {
+  if (res.status === 204) return undefined as T;
+  const ctype = res.headers.get('content-type') ?? '';
+  if (ctype.includes('application/json')) return res.json() as Promise<T>;
+  return res.arrayBuffer().then((b) => Buffer.from(b) as unknown as T);
+}
+
+export async function rest<T = unknown>(
+  accountId: string,
+  method: string,
+  path: string,
+  body?: unknown,
+  isForm = false,
+): Promise<T> {
+  const acct = accounts.get(accountId);
+  if (!acct) throw new Error(`unknown account '${accountId}'`);
+  const headers: Record<string, string> = {
+    Authorization: `Bot ${acct.cfg.token}`,
+    'User-Agent': 'metro-discord-bot-train (https://github.com/bonustrack/stage)',
+  };
+  if (body !== undefined && !isForm)
+    headers['Content-Type'] = 'application/json';
+  const res = await fetch(`${API}${path}`, {
+    method,
+    headers,
+    body: restBody(body, isForm),
+    signal: AbortSignal.timeout(30_000),
+  });
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`discord-bot ${method} ${path}: ${res.status} ${text}`);
+  }
+  return restResult<T>(res);
+}
+
+export function lineOf(accountId: string, channelId: string): string {
+  return `metro://discord-bot/${accountId}/${channelId}`;
+}
+
+export function parseLine(
+  line: string,
+): { accountId: string; channelId: string } | null {
+  const p = Line.parseDiscord(line);
+  return p ? { accountId: p.accountId, channelId: p.resource } : null;
+}
+
+export function accountFor(args: { account?: string; line?: string }): string {
+  return resolveAccountId(accounts, args, (line) => parseLine(line)?.accountId);
+}
+
+export function routeOf(
+  line: string,
+  account?: string,
+): { accountId: string; channelId: string } {
+  const parsed = parseLine(line);
+  if (!parsed) throw new Error(`bad discord-bot line: ${line}`);
+  const accountId = account ?? parsed.accountId;
+  if (!accounts.has(accountId))
+    throw new Error(`unknown account '${accountId}' in line ${line}`);
+  return { accountId, channelId: parsed.channelId };
+}
+
+export const encodeEmoji = (e: string): string => encodeURIComponent(e);

package/runtime/node_modules/@metro-labs/discord-bot/src/actions.ts

@@ -0,0 +1,222 @@
+import type { Message } from 'discord.js';
+import {
+  accounts,
+  encodeEmoji,
+  rest,
+  routeOf,
+} from './accounts.js';
+import { emitOutbound, emitOutboundEdit, emitOutboundReact } from './format.js';
+import { messagingAliases } from '@metro-labs/core/stations/messaging-normalize';
+import {
+  appendFiles,
+  outgoingFiles,
+  type OutgoingFile,
+} from './send-files.js';
+import { makeStation, respond, type CallMsg, type StationHandler } from '@metro-labs/core/stations/station-runtime';
+import { readProfile, setProfile } from './profile.js';
+import { discordMembers } from './members.js';
+import { listChannels } from './channels.js';
+import {
+  groupAddHandler,
+  groupCreateHandler,
+  groupRemoveHandler,
+} from './group-actions.js';
+
+async function sendMessage(
+  accountId: string,
+  channel: string,
+  body: Record<string, unknown>,
+  files: OutgoingFile[],
+): Promise<{ id: string; delivered: string[] }> {
+  if (files.length === 0) {
+    const plain = await rest<{ id: string }>(
+      accountId,
+      'POST',
+      `/channels/${channel}/messages`,
+      body,
+    );
+    return { id: plain.id, delivered: [] };
+  }
+  const form = new FormData();
+  form.append('payload_json', JSON.stringify(body));
+  const delivered = await appendFiles(form, files);
+  const res = await rest<{ id: string }>(
+    accountId,
+    'POST',
+    `/channels/${channel}/messages`,
+    form,
+    true,
+  );
+  return { id: res.id, delivered };
+}
+
+export type { CallMsg };
+
+async function send(id: string, args: Record<string, unknown>): Promise<void> {
+  const { line, text, replyTo, account } = args as {
+    line: string;
+    text?: string;
+    replyTo?: string;
+    account?: string;
+  };
+  const { accountId, channelId } = routeOf(line, account);
+  const body: Record<string, unknown> = { flags: 4 };
+  if (text !== undefined) body.content = text;
+  if (replyTo) body.message_reference = { message_id: replyTo };
+  const outgoing = outgoingFiles(args.attachments);
+  const res = await sendMessage(accountId, channelId, body, outgoing);
+  emitOutbound(accountId, line, res.id, text ?? '', replyTo);
+  respond(id, {
+    result: {
+      messageId: res.id,
+      account: accountId,
+      ...(res.delivered.length ? { attachments: res.delivered } : {}),
+    },
+  });
+}
+
+const profileUrl = (userId?: string): string | null =>
+  userId === undefined ? null : `https://discord.com/users/${userId}`;
+
+function listAccounts(id: string): void {
+  respond(id, {
+    result: {
+      accounts: [...accounts.values()].map((a) => ({
+        id: a.cfg.id,
+        handle: a.client.user?.username ?? null,
+        url: profileUrl(a.client.user?.id),
+        userId: a.client.user?.id ?? null,
+        username: a.client.user?.username ?? null,
+        ready: a.client.isReady(),
+      })),
+    },
+  });
+}
+
+async function typing(id: string, args: Record<string, unknown>): Promise<void> {
+  const { line, account } = args as { line: string; account?: string };
+  const { accountId, channelId } = routeOf(line, account);
+  await rest(accountId, 'POST', `/channels/${channelId}/typing`);
+  respond(id, { result: { ok: true, account: accountId } });
+}
+
+async function react(id: string, args: Record<string, unknown>): Promise<void> {
+  const { line, messageId, emoji, account } = args as {
+    line: string;
+    messageId: string;
+    emoji: string;
+    account?: string;
+  };
+  const { accountId, channelId } = routeOf(line, account);
+  if (emoji) {
+    const e = encodeEmoji(emoji);
+    await rest(
+      accountId,
+      'PUT',
+      `/channels/${channelId}/messages/${messageId}/reactions/${e}/@me`,
+    );
+    emitOutboundReact(accountId, line, messageId, emoji);
+  } else {
+    await rest(
+      accountId,
+      'DELETE',
+      `/channels/${channelId}/messages/${messageId}/reactions/@me`,
+    );
+  }
+  respond(id, { result: { ok: true, account: accountId } });
+}
+
+async function edit(id: string, args: Record<string, unknown>): Promise<void> {
+  const { line, messageId, text, account } = args as {
+    line: string;
+    messageId: string;
+    text: string;
+    account?: string;
+  };
+  const { accountId, channelId } = routeOf(line, account);
+  await rest(accountId, 'PATCH', `/channels/${channelId}/messages/${messageId}`, {
+    content: text,
+  });
+  emitOutboundEdit(accountId, line, messageId, text);
+  respond(id, { result: { ok: true, account: accountId } });
+}
+
+async function remove(
+  id: string,
+  args: Record<string, unknown>,
+): Promise<void> {
+  const { line, messageId, account } = args as {
+    line: string;
+    messageId: string;
+    account?: string;
+  };
+  const { accountId, channelId } = routeOf(line, account);
+  await rest(accountId, 'DELETE', `/channels/${channelId}/messages/${messageId}`);
+  respond(id, { result: { ok: true, account: accountId } });
+}
+
+async function read(
+  id: string,
+  args: Record<string, unknown>,
+): Promise<void> {
+  const {
+    line,
+    limit = 20,
+    before,
+    account,
+  } = args as {
+    line: string;
+    limit?: number;
+    before?: string;
+    account?: string;
+  };
+  const { accountId, channelId } = routeOf(line, account);
+  const qs = new URLSearchParams({
+    limit: String(limit),
+    ...(before ? { before } : {}),
+  });
+  const msgs = await rest<Message[]>(
+    accountId,
+    'GET',
+    `/channels/${channelId}/messages?${qs}`,
+  );
+  respond(id, { result: { messages: msgs, account: accountId } });
+}
+
+async function listMembers(
+  id: string,
+  args: Record<string, unknown>,
+): Promise<void> {
+  const { line, limit, account } = args as {
+    line: string;
+    limit?: number;
+    account?: string;
+  };
+  const { accountId, channelId } = routeOf(line, account);
+  const result = await discordMembers(accountId, channelId, limit);
+  respond(id, { result });
+}
+
+const HANDLERS: Record<string, StationHandler> = {
+  accounts: (id) => {
+    listAccounts(id);
+  },
+  groupCreate: groupCreateHandler,
+  groupAddMembers: groupAddHandler,
+  groupRemoveMembers: groupRemoveHandler,
+  send,
+  react,
+  typing,
+  edit,
+  delete: remove,
+  read,
+  listMembers,
+  listChannels,
+  set_profile: setProfile,
+  profile: readProfile,
+};
+
+export const handleCall = makeStation({
+  handlers: HANDLERS,
+  normalize: messagingAliases(),
+});

package/runtime/node_modules/@metro-labs/discord-bot/src/api-base.ts

@@ -0,0 +1 @@
+export const API = 'https://discord.com/api/v10';

package/runtime/node_modules/@metro-labs/discord-bot/src/attachments.ts

@@ -0,0 +1,29 @@
+import {
+  saveBufferToCache,
+  assertContentLength,
+} from '@metro-labs/core/stations/attachments';
+import type { SavedAttachment } from '@metro-labs/core/stations/attachments';
+
+export type { SavedAttachment };
+
+export interface DiscordAttachmentRef {
+  url: string;
+  name?: string | null;
+  contentType?: string | null;
+}
+
+export async function saveDiscordAttachment(
+  a: DiscordAttachmentRef,
+  messageId: string,
+  index = 0,
+): Promise<SavedAttachment> {
+  const res = await fetch(a.url, { signal: AbortSignal.timeout(60_000) });
+  if (!res.ok)
+    throw new Error(`discord-bot attachment fetch ${res.status} for ${a.url}`);
+  assertContentLength(res.headers.get('content-length'));
+  const data = new Uint8Array(await res.arrayBuffer());
+  return saveBufferToCache(data, messageId, index, {
+    mime: a.contentType ?? undefined,
+    name: a.name ?? undefined,
+  });
+}