@guava-ai/guava-sdk 0.38.0 → 0.40.0

package/src/webrtc/index.ts

@@ -1,6 +1,6 @@
-import { type ConnectingTone, JingleTone } from "./tones.ts";
+import { type ConnectingTone, JingleTone, LeavingTone, type Tone } from "./tones.ts";
 
-export { type ConnectingTone, JingleTone, RingbackTone } from "./tones.ts";
+export { type ConnectingTone, JingleTone, RingbackTone, LeavingTone, type Tone } from "./tones.ts";
 
 export type CallState =
   | "not-started" // start() hasn't been called yet.
@@ -16,7 +16,8 @@ export type CallErrorCode =
   | "AUTHENTICATION_FAILED"
   | "PROTOCOL_ERROR" // The server sent an unexpected message.
   | "SERVER_ERROR" // The server sent an error message.
-  | "REMOTE_HANGUP"; // The remote hung up before the call connected.
+  | "REMOTE_HANGUP" // The remote hung up before the call connected.
+  | "CONNECTION_TIMEOUT"; // The call didn't connect within connectionTimeoutMs.
 
 export class CallError extends Error {
   readonly code: CallErrorCode;
@@ -59,12 +60,29 @@ export class Call {
   private _error?: CallError;
   private readonly _stateHandlers = new Set<(state: CallState) => void>();
   private readonly _remoteStreamHandlers = new Set<(stream: MediaStream) => void>();
+  private readonly _messageHandlers = new Set<(payload: Record<string, unknown>) => void>();
   private readonly _webrtcCode: string;
   private readonly _sockUrl: URL;
+
   private readonly _remoteAudio: HTMLAudioElement;
   private readonly _connectingTone?: ConnectingTone;
+  private readonly _leavingTone?: Tone;
+  private readonly _connectionTimeoutMs?: number;
+  private readonly _customPayload: Record<string, unknown>;
+  private readonly _customAudioTrack?: MediaStreamTrack;
+  private _connectionTimer?: ReturnType<typeof setTimeout>;
   private _sock?: WebSocket;
+
+  private readonly _chatSockUrl: URL;
+  // Links the chat socket to this call's session on the server.
+  private readonly _chatSessionId = crypto.randomUUID();
+  private _chatSock?: WebSocket;
+  private _chatReady = false;
+  // Messages sent before the chat socket is ready.
+  private _pendingChatMessages: string[] = [];
+
   private _localStream?: MediaStream;
+  private _micMuted = false;
   private _peerConnection?: RTCPeerConnection;
   private _remoteStream?: MediaStream;
   private _iceServers?: RTCIceServer[];
@@ -75,16 +93,44 @@ export class Call {
     webrtcCode,
     baseUrl = "https://app.goguava.ai/",
     connectingTone = new JingleTone(),
+    leavingTone = new LeavingTone(),
+    connectionTimeoutMs = 30_000,
+    customPayload = {},
+    audioTrack,
   }: {
+    /** Code used to authenticate with the signaling server. */
     webrtcCode: string;
+    /** Base URL of the Guava server. Defaults to `https://app.goguava.ai/`. */
     baseUrl?: string;
-    // Played while the call is in the "connecting" state.
+    /** Played while the call is in the "connecting" state. Defaults to {@link JingleTone}. */
     connectingTone?: ConnectingTone | "disabled";
+    /** Played when the call ends. Defaults to {@link LeavingTone}. */
+    leavingTone?: Tone | "disabled";
+    /** Max time in the "connecting" state before the call fails with CONNECTION_TIMEOUT. Defaults to 30 seconds. */
+    connectionTimeoutMs?: number | "disabled";
+    /** A custom payload that is forwarded to the agent at the start of the call. */
+    customPayload?: Record<string, unknown>;
+    /**
+     * An audio track to send. Supplying this skips requesting the microphone with `getUserMedia()`.
+     * We use `track.clone()` to create a clone of this track. The caller is responsible for calling
+     * `track.stop()` on their original reference when it is no longer needed.
+     * Because we use a clone, use {@link Call.setMicrophoneMuted} to mute the call, rather than setting
+     * `enabled` on the original track.
+     */
+    audioTrack?: MediaStreamTrack;
   }) {
     this._webrtcCode = webrtcCode;
+    this._customPayload = customPayload;
+    this._customAudioTrack = audioTrack;
     if (connectingTone !== "disabled") {
       this._connectingTone = connectingTone;
     }
+    if (leavingTone !== "disabled") {
+      this._leavingTone = leavingTone;
+    }
+    if (connectionTimeoutMs !== "disabled") {
+      this._connectionTimeoutMs = connectionTimeoutMs;
+    }
 
     this._sockUrl = new URL("/webrtc/", baseUrl);
     const sockProtocol = SOCKET_PROTOCOLS[this._sockUrl.protocol];
@@ -92,6 +138,7 @@ export class Call {
       throw new Error(`Unsupported baseUrl protocol: ${this._sockUrl.protocol}`);
     }
     this._sockUrl.protocol = sockProtocol;
+    this._chatSockUrl = new URL("/webrtc-chat/", this._sockUrl);
 
     this._remoteAudio = document.createElement("audio");
     this._remoteAudio.autoplay = true;
@@ -128,15 +175,40 @@ export class Call {
     };
   }
 
+  /**
+   * Registers a handler for messages the agent sends with `call.send_webrtc_message()`.
+   * @param handler - Called with each message's payload.
+   * @returns A function that unregisters the handler.
+   */
+  onMessage(handler: (payload: Record<string, unknown>) => void): () => void {
+    this._messageHandlers.add(handler);
+    return () => {
+      this._messageHandlers.delete(handler);
+    };
+  }
+
   private _setState(state: CallState) {
     if (state === this._state) {
       return;
     }
+
+    if (state === "ended") {
+      this._leavingTone?.play();
+    }
+
     this._state = state;
     if (state === "connecting") {
       this._connectingTone?.start();
+      if (this._connectionTimeoutMs !== undefined) {
+        this._connectionTimer = setTimeout(() => {
+          this._send({ type: "hangup" });
+          this._end(new CallError("CONNECTION_TIMEOUT", "Timed out connecting the call."));
+        }, this._connectionTimeoutMs);
+      }
     } else {
       this._connectingTone?.stop();
+      clearTimeout(this._connectionTimer);
+      this._connectionTimer = undefined;
     }
     for (const handler of this._stateHandlers) {
       handler(state);
@@ -152,29 +224,40 @@ export class Call {
     // Add the audio element.
     document.body.appendChild(this._remoteAudio);
 
-    // We start by requesting device permissions first. The idea is that, after permission has been granted,
-    // it makes sence to play a ringback, but it doesn't make sense to do so during the permission prompt.
+    if (this._customAudioTrack !== undefined) {
+      this._localStream = new MediaStream([this._customAudioTrack.clone()]);
+    } else {
+      // We start by requesting device permissions first. The idea is that, after permission has been granted,
+      // it makes sence to play a ringback, but it doesn't make sense to do so during the permission prompt.
 
-    const permissionTimer = setTimeout(() => {
-      this._setState("waiting-for-permission");
-    }, 500);
+      const permissionTimer = setTimeout(() => {
+        this._setState("waiting-for-permission");
+      }, 500);
 
-    try {
-      this._localStream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
-    } catch (error) {
-      const callError = new CallError("MICROPHONE_ACCESS_FAILED", "Failed to access microphone.", {
-        cause: error,
-      });
-      this._end(callError);
-      throw callError;
-    } finally {
-      clearTimeout(permissionTimer);
-    }
+      try {
+        this._localStream = await navigator.mediaDevices.getUserMedia({
+          audio: true,
+          video: false,
+        });
+      } catch (error) {
+        const callError = new CallError(
+          "MICROPHONE_ACCESS_FAILED",
+          "Failed to access microphone.",
+          {
+            cause: error,
+          },
+        );
+        this._end(callError);
+        throw callError;
+      } finally {
+        clearTimeout(permissionTimer);
+      }
 
-    // hangup() was called during the permission prompt.
-    if (this.state === "ended") {
-      this._teardown();
-      return;
+      // hangup() was called during the permission prompt.
+      if (this.state === "ended") {
+        this._teardown();
+        return;
+      }
     }
 
     // Now open and authenticate the signaling socket.
@@ -185,7 +268,13 @@ export class Call {
 
     this._sock = new WebSocket(this._sockUrl);
     this._sock.onopen = () => {
-      this._send({ type: "auth", body: this._webrtcCode });
+      this._send({
+        type: "auth",
+        body: JSON.stringify({
+          webrtc_code: this._webrtcCode,
+          chat_session_id: this._chatSessionId,
+        }),
+      });
     };
     this._sock.onmessage = (event) => {
       this._onMessage(JSON.parse(event.data) as SignalingMessage);
@@ -197,6 +286,70 @@ export class Call {
     await started;
   }
 
+  /**
+   * Mutes or unmutes the microphone. Can be called before the call starts.
+   * @param muted - Whether the microphone should be muted.
+   */
+  setMicrophoneMuted(muted: boolean) {
+    this._micMuted = muted;
+    for (const track of this._localStream?.getAudioTracks() ?? []) {
+      track.enabled = !muted;
+    }
+  }
+
+  /**
+   * Mutes or unmutes playback of the remote audio. Can be called before the call starts.
+   * @param muted - Whether the remote audio should be muted.
+   */
+  setRemoteMuted(muted: boolean) {
+    this._remoteAudio.muted = muted;
+  }
+
+  /**
+   * Sends a message to the agent, delivered to its `on_webrtc_message` handler.
+   * Messages sent before the message channel is ready are queued.
+   * @param payload - A JSON-serializable object.
+   */
+  sendMessage(payload: Record<string, unknown>) {
+    if (this._state === "not-started" || this._state === "ended") {
+      throw new Error(`sendMessage() called while the call is ${this._state}.`);
+    }
+    const message = JSON.stringify({ type: "webrtc-message", payload });
+    if (this._chatReady && this._chatSock !== undefined) {
+      this._chatSock.send(message);
+    } else {
+      this._pendingChatMessages.push(message);
+    }
+  }
+
+  private _openChatSocket() {
+    this._chatSock = new WebSocket(this._chatSockUrl);
+    this._chatSock.onopen = () => {
+      this._chatSock?.send(
+        JSON.stringify({ webrtc_code: this._webrtcCode, chat_session_id: this._chatSessionId }),
+      );
+    };
+    this._chatSock.onmessage = (event) => {
+      const message = JSON.parse(event.data) as {
+        type?: string;
+        command_type?: string;
+        payload?: Record<string, unknown>;
+      };
+      // The server sends "ready" once the call session is found.
+      if (message.type === "ready" && !this._chatReady) {
+        this._chatReady = true;
+        for (const pending of this._pendingChatMessages) {
+          this._chatSock?.send(pending);
+        }
+        this._pendingChatMessages = [];
+      } else if (message.command_type === "send-webrtc-message") {
+        for (const handler of this._messageHandlers) {
+          handler(message.payload ?? {});
+        }
+      }
+    };
+  }
+
   async hangup() {
     if (this._state === "not-started") {
       throw new Error(`hangup() called before start().`);
@@ -224,7 +377,8 @@ export class Call {
           return;
         }
         this._iceServers = message.ice_servers;
-        this._send({ type: "connect", body: {} });
+        this._openChatSocket();
+        this._send({ type: "connect", body: { custom_payload: this._customPayload } });
         void this._connectPeer();
         break;
       case "desc":
@@ -314,6 +468,7 @@ export class Call {
       }
     };
     for (const track of this._localStream.getAudioTracks()) {
+      track.enabled = !this._micMuted;
       this._peerConnection.addTrack(track);
     }
 
@@ -351,12 +506,23 @@ export class Call {
   }
 
   private _teardown() {
+    // Close the signaling socket.
     if (this._sock) {
       // Don't fire close handlers for a socket we closed ourselves.
       this._sock.onclose = null;
       this._sock.close();
       this._sock = undefined;
     }
+
+    // Close the chat socket.
+    if (this._chatSock) {
+      this._chatSock.close();
+      this._chatSock = undefined;
+    }
+    this._chatReady = false;
+    this._pendingChatMessages = [];
+
+    // Close the peer connection.
     if (this._peerConnection !== undefined) {
       this._peerConnection.close();
       this._peerConnection = undefined;
@@ -366,6 +532,8 @@ export class Call {
     }
     this._localStream = undefined;
     this._remoteStream = undefined;
+
+    // Remove the audio element.
     this._remoteAudio.srcObject = null;
     this._remoteAudio.remove();
   }

package/src/webrtc/tones.ts

@@ -68,7 +68,7 @@ export class JingleTone implements ConnectingTone {
   private nextBarStart = 0;
 
   // Configuration
-  private readonly volume = 0.25;
+  private readonly volume = 0.3;
   private readonly bpm = 120;
   private readonly beatsPerBar = 4;
   // Delay before the first note, so near-instant connections stay silent.
@@ -198,3 +198,87 @@ export class JingleTone implements ConnectingTone {
     };
   }
 }
+
+export interface Tone {
+  play(): void;
+}
+
+export class LeavingTone implements Tone {
+  public play(): void {
+    const ctx = new AudioContext();
+
+    if (ctx.state === "suspended") {
+      void ctx.resume();
+    }
+
+    const now = ctx.currentTime;
+
+    // First note: D4
+    const first = this.playNote(ctx, {
+      frequency: 293.665,
+      startTime: now,
+      attack: 0.065,
+      decay: 0.085,
+      volume: 0.2,
+    });
+
+    // Second note: A3, starts 139ms later
+    const second = this.playNote(ctx, {
+      frequency: 220.0,
+      startTime: now + 0.139,
+      attack: 0.018,
+      decay: 0.075,
+      volume: 0.2,
+    });
+
+    void Promise.all([first, second]).then(() => ctx.close());
+  }
+
+  // Resolves once the note has finished playing.
+  private playNote(
+    ctx: AudioContext,
+    {
+      frequency,
+      startTime,
+      attack,
+      decay,
+      volume,
+    }: {
+      frequency: number;
+      startTime: number;
+      attack: number;
+      decay: number;
+      volume: number;
+    },
+  ): Promise<void> {
+    const oscillator = ctx.createOscillator();
+    const gain = ctx.createGain();
+
+    oscillator.type = "sine";
+    oscillator.frequency.setValueAtTime(frequency, startTime);
+
+    // Smooth onset
+    gain.gain.setValueAtTime(0, startTime);
+    gain.gain.linearRampToValueAtTime(volume, startTime + attack);
+
+    // Exponential decay
+    const peakTime = startTime + attack;
+    const endTime = peakTime + decay * 5;
+
+    gain.gain.setTargetAtTime(0.00001, peakTime, decay);
+
+    oscillator.connect(gain);
+    gain.connect(ctx.destination);
+
+    oscillator.start(startTime);
+    oscillator.stop(endTime);
+
+    return new Promise((resolve) => {
+      oscillator.onended = () => {
+        oscillator.disconnect();
+        gain.disconnect();
+        resolve();
+      };
+    });
+  }
+}