niahere 0.2.90 → 0.3.0

package/src/channels/phone/relay.ts

@@ -0,0 +1,334 @@
+/**
+ * Bridges a Twilio Media Streams WebSocket to OpenAI's Realtime API.
+ *
+ * Both endpoints use g711_ulaw audio at 8kHz so no resampling is needed —
+ * we pass the base64 audio payload through after reframing JSON envelopes.
+ *
+ * Usage:
+ *   const relay = createRelay({ ... });
+ *   await relay.ready;            // session.update has been sent
+ *   relay.onTwilioMessage(raw);   // for each frame from the Twilio WS
+ *   relay.onTwilioClose();        // when the Twilio WS closes
+ *   const result = await relay.completion;
+ */
+import { log } from "../../utils/log";
+
+export interface PhoneToolDefinition {
+  name: string;
+  description: string;
+  parameters: Record<string, unknown>;
+  handler: (args: Record<string, unknown>) => Promise<string>;
+}
+
+export interface CallContext {
+  callSid: string;
+  streamSid?: string;
+  direction: "inbound" | "outbound";
+  /** Caller phone number (E.164) for inbound, callee for outbound. */
+  remoteNumber: string | null;
+  /** Owner/contact label for logging and persona context. */
+  remoteLabel: string;
+  instructions: string;
+  tools: PhoneToolDefinition[];
+  /** Whether the model should speak first (true for outbound calls). */
+  speakFirst: boolean;
+  /** Optional opener line for outbound; the model expands on it. */
+  opener?: string;
+}
+
+export interface TranscriptTurn {
+  role: "user" | "assistant";
+  text: string;
+  ts: number;
+}
+
+export type RelayEndReason = "twilio_stop" | "openai_close" | "tool_end_call" | "error";
+
+export interface RelayResult {
+  transcript: TranscriptTurn[];
+  endedReason: RelayEndReason;
+  error?: string;
+}
+
+export interface RelayOpts {
+  /** WebSocket bound to the Twilio Media Stream connection. */
+  twilioWs: WebSocketLike;
+  openAiKey: string;
+  model: string;
+  voice: string;
+  context: CallContext;
+}
+
+export interface WebSocketLike {
+  send(data: string | ArrayBufferLike | Uint8Array): void;
+  close(code?: number, reason?: string): void;
+  readyState: number;
+}
+
+export interface RelayHandle {
+  /** Resolves once the OpenAI session.update has been sent. */
+  ready: Promise<void>;
+  /** Resolves once the call has fully wound down (Twilio stop or OpenAI close). */
+  completion: Promise<RelayResult>;
+  onTwilioMessage(raw: string): void;
+  /** Call when the Twilio WebSocket closes (event or our own close). */
+  onTwilioClose(): void;
+}
+
+export function createRelay(opts: RelayOpts): RelayHandle {
+  const { twilioWs, openAiKey, model, voice, context } = opts;
+  const transcript: TranscriptTurn[] = [];
+  let endedReason: RelayEndReason = "twilio_stop";
+  let errorMsg: string | undefined;
+  let finalized = false;
+  let pendingAssistantText = "";
+
+  const openAiUrl = `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`;
+  const openAiWs = new WebSocket(openAiUrl, {
+    headers: { Authorization: `Bearer ${openAiKey}` },
+  } as any);
+
+  /** Whether a response is currently in flight on the OpenAI side. */
+  let responseActive = false;
+  /** Whether the opener response.create has been sent yet (outbound only). */
+  let openerSent = false;
+
+  let resolveReady: () => void;
+  let rejectReady: (err: Error) => void;
+  const ready = new Promise<void>((res, rej) => {
+    resolveReady = res;
+    rejectReady = rej;
+  });
+
+  let resolveCompletion: (result: RelayResult) => void;
+  const completion = new Promise<RelayResult>((res) => {
+    resolveCompletion = res;
+  });
+
+  function sendOpenAi(payload: Record<string, unknown>): void {
+    if (openAiWs.readyState !== 1) return;
+    openAiWs.send(JSON.stringify(payload));
+  }
+
+  function sendTwilio(payload: Record<string, unknown>): void {
+    if (twilioWs.readyState !== 1) return;
+    twilioWs.send(JSON.stringify(payload));
+  }
+
+  function finalize(reason?: RelayEndReason): void {
+    if (finalized) return;
+    finalized = true;
+    if (reason) endedReason = reason;
+    if (pendingAssistantText.trim()) {
+      transcript.push({ role: "assistant", text: pendingAssistantText.trim(), ts: Date.now() });
+      pendingAssistantText = "";
+    }
+    try {
+      if (openAiWs.readyState === 1) openAiWs.close();
+    } catch {}
+    resolveCompletion({ transcript, endedReason, error: errorMsg });
+  }
+
+  openAiWs.addEventListener("open", () => {
+    log.info({ callSid: context.callSid, model }, "openai realtime: connected");
+    sendOpenAi({
+      type: "session.update",
+      session: {
+        type: "realtime",
+        model,
+        // Without this explicit list, the GA session silently drops audio
+        // synthesis and only emits transcripts — verified empirically.
+        output_modalities: ["audio"],
+        instructions: context.instructions,
+        audio: {
+          input: {
+            format: { type: "audio/pcmu" },
+            transcription: { model: "whisper-1" },
+            turn_detection: { type: "server_vad", threshold: 0.5, silence_duration_ms: 300 },
+          },
+          output: {
+            format: { type: "audio/pcmu" },
+            voice,
+          },
+        },
+        tools: context.tools.map((t) => ({
+          type: "function",
+          name: t.name,
+          description: t.description,
+          parameters: t.parameters,
+        })),
+        tool_choice: "auto",
+      },
+    });
+    // For outbound calls Nia is the caller and must speak first — fire the
+    // opener immediately once the session is configured. streamSid was set
+    // synchronously in bootstrapWsMessage before this relay was created, so
+    // any audio we generate now will forward to Twilio. Anti-restart logic
+    // lives in the outbound instructions to prevent a misheard "hi" from
+    // re-triggering the greeting.
+    if (context.speakFirst) {
+      openerSent = true;
+      sendOpenAi({
+        type: "response.create",
+        response: {
+          instructions: context.opener || "Greet the caller warmly and explain what you can help with.",
+        },
+      });
+    }
+    resolveReady();
+  });
+
+  openAiWs.addEventListener("error", (ev) => {
+    const msg = (ev as ErrorEvent).message || "openai websocket error";
+    log.error({ callSid: context.callSid, err: msg }, "openai realtime: error");
+    errorMsg = msg;
+    rejectReady(new Error(msg));
+    finalize("error");
+  });
+
+  openAiWs.addEventListener("close", () => {
+    log.info({ callSid: context.callSid }, "openai realtime: closed");
+    finalize(endedReason === "twilio_stop" ? "openai_close" : endedReason);
+  });
+
+  openAiWs.addEventListener("message", async (ev) => {
+    const isText = typeof ev.data === "string";
+    let evt: any;
+    try {
+      evt = JSON.parse(isText ? ev.data : new TextDecoder().decode(ev.data as ArrayBuffer));
+    } catch {
+      log.debug(
+        {
+          callSid: context.callSid,
+          isText,
+          bytes: isText ? (ev.data as string).length : (ev.data as ArrayBuffer).byteLength,
+        },
+        "openai realtime: non-json frame dropped",
+      );
+      return;
+    }
+
+    switch (evt.type) {
+      case "response.created": {
+        responseActive = true;
+        break;
+      }
+      case "response.done":
+      case "response.cancelled": {
+        responseActive = false;
+        break;
+      }
+      // GA event names. We keep the older `response.audio.*` aliases for
+      // forward/back compatibility — both fire `delta` with base64 audio.
+      case "response.output_audio.delta":
+      case "response.audio.delta": {
+        if (!context.streamSid) return;
+        sendTwilio({ event: "media", streamSid: context.streamSid, media: { payload: evt.delta } });
+        break;
+      }
+      case "response.output_audio_transcript.delta":
+      case "response.audio_transcript.delta": {
+        pendingAssistantText += evt.delta || "";
+        break;
+      }
+      case "response.output_audio_transcript.done":
+      case "response.audio_transcript.done": {
+        if (pendingAssistantText.trim()) {
+          transcript.push({ role: "assistant", text: pendingAssistantText.trim(), ts: Date.now() });
+          pendingAssistantText = "";
+        }
+        break;
+      }
+      case "conversation.item.input_audio_transcription.completed": {
+        const text = (evt.transcript || "").trim();
+        if (text) transcript.push({ role: "user", text, ts: Date.now() });
+        break;
+      }
+      case "input_audio_buffer.speech_started": {
+        if (context.streamSid) sendTwilio({ event: "clear", streamSid: context.streamSid });
+        if (responseActive) {
+          sendOpenAi({ type: "response.cancel" });
+          responseActive = false;
+        }
+        break;
+      }
+      case "response.function_call_arguments.done": {
+        const toolName = evt.name as string;
+        const callId = evt.call_id as string;
+        let args: Record<string, unknown> = {};
+        try {
+          args = JSON.parse(evt.arguments || "{}");
+        } catch {}
+        const tool = context.tools.find((t) => t.name === toolName);
+        let output = "";
+        if (!tool) {
+          output = `tool ${toolName} not available`;
+          log.warn({ callSid: context.callSid, toolName }, "phone: unknown tool requested");
+        } else {
+          try {
+            output = await tool.handler(args);
+          } catch (err) {
+            output = `error: ${err instanceof Error ? err.message : String(err)}`;
+            log.error({ err, callSid: context.callSid, toolName }, "phone tool handler failed");
+          }
+        }
+        sendOpenAi({
+          type: "conversation.item.create",
+          item: { type: "function_call_output", call_id: callId, output },
+        });
+        sendOpenAi({ type: "response.create" });
+
+        if (toolName === "end_call") {
+          setTimeout(() => finalize("tool_end_call"), 1500);
+        }
+        break;
+      }
+      case "error": {
+        log.error({ callSid: context.callSid, evt }, "openai realtime: error event");
+        break;
+      }
+      default: {
+        // Track unrecognized event types at debug level — invaluable when the
+        // GA API evolves and we need to discover new fields without breaking
+        // production logs. Truncate the body to keep logs sane.
+        const preview = JSON.stringify(evt).slice(0, 500);
+        log.debug({ callSid: context.callSid, type: evt.type, preview }, "openai realtime: unhandled event");
+      }
+    }
+  });
+
+  function onTwilioMessage(raw: string): void {
+    let msg: any;
+    try {
+      msg = JSON.parse(raw);
+    } catch {
+      return;
+    }
+    switch (msg.event) {
+      case "connected":
+        return;
+      case "start": {
+        context.streamSid = msg.start?.streamSid || msg.streamSid;
+        log.info(
+          { callSid: context.callSid, streamSid: context.streamSid, direction: context.direction },
+          "phone: media stream started",
+        );
+        return;
+      }
+      case "media": {
+        const payload = msg.media?.payload;
+        if (payload) sendOpenAi({ type: "input_audio_buffer.append", audio: payload });
+        return;
+      }
+      case "stop":
+        finalize("twilio_stop");
+        return;
+    }
+  }
+
+  function onTwilioClose(): void {
+    finalize(endedReason);
+  }
+
+  return { ready, completion, onTwilioMessage, onTwilioClose };
+}

package/src/channels/phone/tools.ts

@@ -0,0 +1,83 @@
+/**
+ * Tool definitions exposed to the realtime voice model during a call.
+ * Each tool is a function the model can invoke mid-conversation; the result
+ * is fed back into the response so the model can speak it.
+ */
+import { log } from "../../utils/log";
+import { addMemory } from "../../utils/memory";
+import { getChannel } from "../registry";
+import { consultClaude } from "./consult";
+import type { PhoneToolDefinition } from "./relay";
+
+interface ToolContextOpts {
+  /** Display name of the remote party (owner, contact, or raw number). */
+  callerLabel: string;
+}
+
+export function buildPhoneTools(ctx: ToolContextOpts): PhoneToolDefinition[] {
+  return [
+    {
+      name: "consult_claude",
+      description:
+        "Ask Claude a question that needs memory, careful reasoning, or up-to-date context (e.g. 'what did I say about X last week?', 'should I interrupt Aman now?'). Returns a concise answer you can speak.",
+      parameters: {
+        type: "object",
+        properties: {
+          question: { type: "string", description: "The question to ask, in plain English." },
+        },
+        required: ["question"],
+      },
+      handler: async (args) => {
+        const question = String(args.question || "");
+        if (!question.trim()) return "(no question provided)";
+        return await consultClaude(question, ctx.callerLabel);
+      },
+    },
+    {
+      name: "send_telegram",
+      description:
+        "Send a short message to the owner's Telegram (e.g. a summary of the call, an action item). Use sparingly — only when something needs to land outside the call.",
+      parameters: {
+        type: "object",
+        properties: {
+          text: { type: "string", description: "Message to send. Keep it under 400 chars." },
+        },
+        required: ["text"],
+      },
+      handler: async (args) => {
+        const text = String(args.text || "").slice(0, 1000);
+        const tg = getChannel("telegram");
+        if (!tg || !tg.sendMessage) return "telegram unavailable";
+        await tg.sendMessage(`[Phone] ${text}`);
+        return "sent";
+      },
+    },
+    {
+      name: "save_memory",
+      description:
+        "Save a single concise insight to long-term memory (max 300 chars). Use for facts or preferences worth remembering across sessions.",
+      parameters: {
+        type: "object",
+        properties: {
+          entry: { type: "string", description: "One sentence, no transcripts." },
+        },
+        required: ["entry"],
+      },
+      handler: async (args) => addMemory(String(args.entry || "")),
+    },
+    {
+      name: "end_call",
+      description: "Politely end the call after a short goodbye. Use when the conversation is naturally complete.",
+      parameters: {
+        type: "object",
+        properties: {
+          reason: { type: "string", description: "Why the call is ending (logged, not spoken)." },
+        },
+      },
+      handler: async (args) => {
+        log.info({ reason: args.reason }, "phone: end_call invoked");
+        return "ending call";
+      },
+    },
+  ];
+}

package/src/channels/phone/twilio.ts

@@ -0,0 +1,125 @@
+/**
+ * Minimal Twilio REST + webhook-signature helpers.
+ * Skips the official SDK so the daemon's dependency surface stays small —
+ * the surface we use (place call, hang up, swap URL, validate signature) is
+ * a handful of well-documented endpoints.
+ */
+import { createHmac, timingSafeEqual } from "crypto";
+
+const TWILIO_BASE = "https://api.twilio.com/2010-04-01";
+
+interface TwilioCreds {
+  accountSid: string;
+  authToken: string;
+}
+
+function basicAuth({ accountSid, authToken }: TwilioCreds): string {
+  return `Basic ${Buffer.from(`${accountSid}:${authToken}`).toString("base64")}`;
+}
+
+function accountUrl({ accountSid }: TwilioCreds, suffix: string): string {
+  return `${TWILIO_BASE}/Accounts/${encodeURIComponent(accountSid)}${suffix}`;
+}
+
+/**
+ * Validate a Twilio webhook signature.
+ * Algorithm (per Twilio's webhook security docs):
+ *   1. Take the full URL Twilio sent the request to (including query string).
+ *   2. For application/x-www-form-urlencoded bodies, sort POST keys and
+ *      append each "key" + "value" to the URL string.
+ *   3. HMAC-SHA1 with the account AuthToken, base64-encode.
+ *   4. Timing-safe compare with the X-Twilio-Signature header.
+ */
+export function validateTwilioSignature(opts: {
+  authToken: string;
+  fullUrl: string;
+  params: Record<string, string>;
+  signature: string;
+}): boolean {
+  const { authToken, fullUrl, params, signature } = opts;
+  if (!signature) return false;
+
+  const sortedKeys = Object.keys(params).sort();
+  let data = fullUrl;
+  for (const key of sortedKeys) {
+    data += key + params[key];
+  }
+
+  const computed = createHmac("sha1", authToken).update(data, "utf8").digest("base64");
+  const a = Buffer.from(computed);
+  const b = Buffer.from(signature);
+  if (a.length !== b.length) return false;
+  return timingSafeEqual(a, b);
+}
+
+export interface PlaceCallOpts extends TwilioCreds {
+  to: string;
+  from: string;
+  twimlUrl: string;
+  statusCallbackUrl?: string;
+  /** Hard cap on call duration (seconds). Maps to Twilio's TimeLimit param. */
+  maxDurationSec?: number;
+}
+
+export interface PlaceCallResult {
+  callSid: string;
+  status: string;
+}
+
+export async function placeCall(opts: PlaceCallOpts): Promise<PlaceCallResult> {
+  const body = new URLSearchParams({
+    To: opts.to,
+    From: opts.from,
+    Url: opts.twimlUrl,
+    Method: "POST",
+  });
+  if (opts.statusCallbackUrl) {
+    body.set("StatusCallback", opts.statusCallbackUrl);
+    body.set("StatusCallbackMethod", "POST");
+    body.append("StatusCallbackEvent", "initiated");
+    body.append("StatusCallbackEvent", "answered");
+    body.append("StatusCallbackEvent", "completed");
+  }
+  if (opts.maxDurationSec && opts.maxDurationSec > 0) {
+    body.set("TimeLimit", String(opts.maxDurationSec));
+  }
+
+  const resp = await fetch(accountUrl(opts, "/Calls.json"), {
+    method: "POST",
+    headers: { Authorization: basicAuth(opts), "Content-Type": "application/x-www-form-urlencoded" },
+    body: body.toString(),
+  });
+  if (!resp.ok) {
+    const text = await resp.text().catch(() => "");
+    throw new Error(`Twilio placeCall failed: ${resp.status} ${text}`);
+  }
+  const data = (await resp.json()) as { sid: string; status: string };
+  return { callSid: data.sid, status: data.status };
+}
+
+/** Swap the TwiML URL on an in-flight call (used to inject the real callSid into the path). */
+export async function updateCallUrl(opts: TwilioCreds & { callSid: string; url: string }): Promise<void> {
+  const body = new URLSearchParams({ Url: opts.url, Method: "POST" });
+  const resp = await fetch(accountUrl(opts, `/Calls/${encodeURIComponent(opts.callSid)}.json`), {
+    method: "POST",
+    headers: { Authorization: basicAuth(opts), "Content-Type": "application/x-www-form-urlencoded" },
+    body: body.toString(),
+  });
+  if (!resp.ok) {
+    const text = await resp.text().catch(() => "");
+    throw new Error(`Twilio updateCallUrl failed: ${resp.status} ${text}`);
+  }
+}
+
+export async function hangupCall(opts: TwilioCreds & { callSid: string }): Promise<void> {
+  const body = new URLSearchParams({ Status: "completed" });
+  const resp = await fetch(accountUrl(opts, `/Calls/${encodeURIComponent(opts.callSid)}.json`), {
+    method: "POST",
+    headers: { Authorization: basicAuth(opts), "Content-Type": "application/x-www-form-urlencoded" },
+    body: body.toString(),
+  });
+  if (!resp.ok) {
+    const text = await resp.text().catch(() => "");
+    throw new Error(`Twilio hangupCall failed: ${resp.status} ${text}`);
+  }
+}

package/src/channels/phone/twiml.ts

@@ -0,0 +1,60 @@
+/** Helpers that emit Twilio Markup Language XML responses. */
+
+function xmlEscape(s: string): string {
+  return s.replace(/[<>&"']/g, (c) => {
+    switch (c) {
+      case "<":
+        return "&lt;";
+      case ">":
+        return "&gt;";
+      case "&":
+        return "&amp;";
+      case '"':
+        return "&quot;";
+      case "'":
+        return "&apos;";
+      default:
+        return c;
+    }
+  });
+}
+
+/**
+ * TwiML that opens a bidirectional Media Stream to our WebSocket endpoint.
+ * Twilio forwards `customParams` as `customParameters` in the WS `start` event,
+ * which we use to attach the call's pre-built context.
+ */
+export function streamTwiML(wssUrl: string, customParams: Record<string, string>): string {
+  const params = Object.entries(customParams)
+    .map(([k, v]) => `      <Parameter name="${xmlEscape(k)}" value="${xmlEscape(v)}" />`)
+    .join("\n");
+
+  return [
+    `<?xml version="1.0" encoding="UTF-8"?>`,
+    `<Response>`,
+    `  <Connect>`,
+    `    <Stream url="${xmlEscape(wssUrl)}">`,
+    params,
+    `    </Stream>`,
+    `  </Connect>`,
+    `</Response>`,
+  ]
+    .filter((line) => line.length > 0)
+    .join("\n");
+}
+
+/** Speak a short message and hang up — used when realtime isn't available. */
+export function sayAndHangupTwiML(text: string): string {
+  return [
+    `<?xml version="1.0" encoding="UTF-8"?>`,
+    `<Response>`,
+    `  <Say>${xmlEscape(text)}</Say>`,
+    `  <Hangup/>`,
+    `</Response>`,
+  ].join("\n");
+}
+
+/** Reject the call (Twilio counts this as not-answered — lower cost than answering and hanging up). */
+export function rejectTwiML(): string {
+  return [`<?xml version="1.0" encoding="UTF-8"?>`, `<Response>`, `  <Reject/>`, `</Response>`].join("\n");
+}

package/src/cli/index.ts

@@ -14,6 +14,7 @@ import { statusCommand } from "./status";
 import { activeCommand } from "./active";
 import { modelCommand } from "./model";
 import { sendCommand, telegramCommand, slackCommand } from "./channels";
+import { phoneCommand } from "./phone";
 import { rulesCommand, memoryCommand } from "./self";
 import { watchCommand } from "./watch";
 import { agentCommand } from "./agent";
@@ -381,6 +382,11 @@ switch (command) {
     break;
   }
 
+  case "phone": {
+    await phoneCommand();
+    break;
+  }
+
   case "config": {
     const configSub = process.argv[3];
     const configKey = process.argv[4];