@applica-software-guru/persona-chat-sdk 0.1.102

package/src/protocol/webrtc.ts

@@ -0,0 +1,379 @@
+import { PersonaProtocolBase } from './base';
+import { PersonaPacket, PersonaProtocolBaseConfig, ProtocolStatus, Session } from '../types';
+
+type AudioAnalysisData = {
+  localAmplitude: number;
+  remoteAmplitude: number;
+};
+
+type AudioVisualizerCallback = (data: AudioAnalysisData) => void;
+
+type PersonaWebRTCMessageCallback = (data: MessageEvent) => void;
+
+type PersonaWebRTCErrorCallback = (error: string) => void;
+
+type PersonaWebRTCConfig = PersonaProtocolBaseConfig & {
+  webrtcUrl: string;
+  iceServers?: RTCIceServer[];
+};
+
+class PersonaWebRTCClient {
+  private config: PersonaWebRTCConfig;
+  private pc: RTCPeerConnection | null = null;
+  private ws: WebSocket | null = null;
+  private localStream: MediaStream | null = null;
+  private remoteStream: MediaStream = new MediaStream();
+  private audioCtx: AudioContext | null = null;
+
+  private localAnalyser: AnalyserNode | null = null;
+  private remoteAnalyser: AnalyserNode | null = null;
+  private analyzerFrame: number | null = null;
+  private dataChannel: RTCDataChannel | null = null;
+
+  private isConnected: boolean = false;
+  private visualizerCallbacks: AudioVisualizerCallback[] = [];
+  private messageCallbacks: PersonaWebRTCMessageCallback[] = [];
+  private errorCallbacks: PersonaWebRTCErrorCallback[] = [];
+  private queuedMessages: PersonaPacket[] = [];
+
+  constructor(config: PersonaWebRTCConfig) {
+    this.config = config;
+  }
+
+  public async connect(session: Session): Promise<Session> {
+    if (this.isConnected) return;
+
+    this.isConnected = true;
+
+    try {
+      this.localStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    } catch (err) {
+      this.config.logger?.error('Error accessing microphone:', err);
+      return;
+    }
+
+    this.pc = new RTCPeerConnection({
+      iceServers: this.config.iceServers || [
+        {
+          urls: 'stun:34.38.108.251:3478',
+        },
+        {
+          urls: 'turn:34.38.108.251:3478',
+          username: 'webrtc',
+          credential: 'webrtc',
+        },
+      ],
+    });
+
+    this.localStream.getTracks().forEach((track) => {
+      this.pc!.addTrack(track, this.localStream!);
+    });
+
+    this.pc.ontrack = (event) => {
+      event.streams[0].getTracks().forEach((track) => {
+        this.remoteStream.addTrack(track);
+      });
+
+      if (!this.audioCtx) {
+        this._startAnalyzers();
+      }
+
+      const remoteAudio = new Audio();
+      remoteAudio.srcObject = this.remoteStream;
+      remoteAudio.play().catch((e) => {
+        this.config.logger?.error('Error playing remote audio:', e);
+      });
+    };
+
+    this.pc.onicecandidate = (event) => {
+      if (event.candidate && this.ws?.readyState === WebSocket.OPEN) {
+        this.ws.send(
+          JSON.stringify({
+            type: 'CANDIDATE',
+            src: 'client',
+            payload: { candidate: event.candidate },
+          }),
+        );
+      }
+    };
+
+    this.pc.ondatachannel = (event) => {
+      const channel = event.channel;
+      channel.onmessage = (msg) => {
+        this.messageCallbacks.forEach((callback) => {
+          callback(msg);
+        });
+      };
+      channel.onopen = () => {
+        while (this.queuedMessages.length > 0) {
+          const packet = this.queuedMessages.shift();
+          if (packet) {
+            channel.send(JSON.stringify(packet));
+            this.config.logger?.info('Sent queued message:', packet);
+          }
+        }
+      };
+    };
+
+    const url = this.config.webrtcUrl || 'wss://persona.applica.guru/api/webrtc';
+    this.ws = new WebSocket(`${url}?apiKey=${encodeURIComponent(this.config.apiKey)}`);
+    this.ws.onopen = async () => {
+      const offer = await this.pc!.createOffer();
+      await this.pc!.setLocalDescription(offer);
+
+      const metadata = {
+        apiKey: this.config.apiKey,
+        agentId: this.config.agentId,
+        userId: this.config.userId || 'anonymous',
+        sessionCode: session as string,
+      };
+      this.config.logger?.debug('Opening connection to WebRTC server: ', metadata);
+
+      const offerMessage = {
+        type: 'OFFER',
+        src: crypto.randomUUID?.() || 'client_' + Date.now(),
+        payload: {
+          sdp: {
+            sdp: offer.sdp,
+            type: offer.type,
+          },
+          connectionId: (Date.now() % 1000000).toString(),
+          metadata,
+        },
+      };
+
+      this.ws!.send(JSON.stringify(offerMessage));
+    };
+
+    this.ws.onmessage = async (event) => {
+      const data = JSON.parse(event.data);
+      if (data.type === 'ANSWER') {
+        await this.pc!.setRemoteDescription(new RTCSessionDescription(data.payload.sdp));
+      } else if (data.type === 'CANDIDATE') {
+        try {
+          await this.pc!.addIceCandidate(new RTCIceCandidate(data.payload.candidate));
+        } catch (err) {
+          this.config.logger?.error('Error adding ICE candidate:', err);
+        }
+      }
+    };
+
+    this.ws.onclose = (event: CloseEvent) => {
+      if (event.code !== 1000) {
+        this.errorCallbacks.forEach((callback) => {
+          callback('Oops! The connection to the server was lost. Please try again later.');
+        });
+      }
+      this._stopAnalyzers();
+    };
+  }
+
+  public async disconnect(): Promise<void> {
+    if (!this.isConnected) return;
+
+    this.isConnected = false;
+
+    if (this.ws?.readyState === WebSocket.OPEN) this.ws.close();
+    if (this.pc) this.pc.close();
+    if (this.localStream) {
+      this.localStream.getTracks().forEach((track) => track.stop());
+    }
+
+    this.remoteStream = new MediaStream();
+    if (this.audioCtx) {
+      await this.audioCtx.close();
+      this.audioCtx = null;
+    }
+
+    this._stopAnalyzers();
+  }
+
+  public addVisualizerCallback(callback: AudioVisualizerCallback): void {
+    this.visualizerCallbacks.push(callback);
+  }
+  public addMessageCallback(callback: PersonaWebRTCMessageCallback): void {
+    this.messageCallbacks.push(callback);
+  }
+
+  public addErrorCallback(callback: PersonaWebRTCErrorCallback): void {
+    this.errorCallbacks.push(callback);
+  }
+
+  public createDataChannel(label = 'messages'): void {
+    if (!this.pc) return;
+    this.dataChannel = this.pc.createDataChannel(label);
+    this.dataChannel.onopen = () => {
+      this.config.logger?.info('Data channel opened');
+      while (this.queuedMessages.length > 0) {
+        const packet = this.queuedMessages.shift();
+        if (packet) {
+          this.dataChannel!.send(JSON.stringify(packet));
+          this.config.logger?.info('Sent queued message:', packet);
+        }
+      }
+    };
+    this.dataChannel.onmessage = (msg: MessageEvent) => {
+      this.messageCallbacks.forEach((callback) => {
+        callback(msg);
+      });
+    };
+  }
+
+  public sendPacket(packet: PersonaPacket): void {
+    if (!this.dataChannel || this.dataChannel.readyState !== 'open') {
+      this.queuedMessages.push(packet);
+      return;
+    }
+
+    this.dataChannel.send(JSON.stringify(packet));
+    this.config.logger?.info('Sent message:', packet);
+  }
+
+  private _startAnalyzers(): void {
+    if (!this.localStream || !this.remoteStream || this.visualizerCallbacks.length === 0) {
+      return;
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    this.audioCtx = new (window.AudioContext || (window as any).webkitAudioContext)();
+
+    const localSource = this.audioCtx.createMediaStreamSource(this.localStream);
+    const remoteSource = this.audioCtx.createMediaStreamSource(this.remoteStream);
+
+    this.localAnalyser = this.audioCtx.createAnalyser();
+    this.remoteAnalyser = this.audioCtx.createAnalyser();
+    this.localAnalyser.fftSize = 256;
+    this.remoteAnalyser.fftSize = 256;
+
+    localSource.connect(this.localAnalyser);
+    remoteSource.connect(this.remoteAnalyser);
+
+    const loop = () => {
+      if (!this.localAnalyser || !this.remoteAnalyser || this.visualizerCallbacks.length === 0) {
+        return;
+      }
+
+      const localArray = new Uint8Array(this.localAnalyser.frequencyBinCount);
+      const remoteArray = new Uint8Array(this.remoteAnalyser.frequencyBinCount);
+
+      this.localAnalyser.getByteFrequencyData(localArray);
+      this.remoteAnalyser.getByteFrequencyData(remoteArray);
+
+      const localAmp = localArray.reduce((a, b) => a + b, 0) / localArray.length;
+      const remoteAmp = remoteArray.reduce((a, b) => a + b, 0) / remoteArray.length;
+
+      if (this.visualizerCallbacks.length > 0) {
+        this.visualizerCallbacks.forEach((callback) => {
+          callback({
+            localAmplitude: localAmp,
+            remoteAmplitude: remoteAmp,
+          });
+        });
+      }
+
+      this.analyzerFrame = requestAnimationFrame(loop);
+    };
+
+    this.analyzerFrame = requestAnimationFrame(loop);
+  }
+
+  private _stopAnalyzers(): void {
+    if (this.analyzerFrame) {
+      cancelAnimationFrame(this.analyzerFrame);
+      this.analyzerFrame = null;
+    }
+    this.localAnalyser = null;
+    this.remoteAnalyser = null;
+  }
+}
+
+type PersonaWebRTCProtocolConfig = PersonaWebRTCConfig & {
+  autostart?: boolean;
+};
+
+class PersonaWebRTCProtocol extends PersonaProtocolBase {
+  status: ProtocolStatus;
+  session: Session;
+  autostart: boolean;
+  config: PersonaWebRTCProtocolConfig;
+  webRTCClient: PersonaWebRTCClient;
+
+  constructor(config: PersonaWebRTCProtocolConfig) {
+    super();
+    this.config = config;
+    this.status = 'disconnected';
+    this.session = null;
+    this.autostart = config?.autostart ?? false;
+    this.webRTCClient = new PersonaWebRTCClient(config);
+    this.webRTCClient.addMessageCallback((msg: MessageEvent) => {
+      const data = JSON.parse(msg.data) as PersonaPacket;
+      this.notifyPacket(data);
+    });
+    this.webRTCClient.addErrorCallback((error: string) => {
+      this.config.logger?.error('WebRTC error:', error);
+      this.notifyPacket({
+        type: 'message',
+        payload: {
+          type: 'text',
+          role: 'assistant',
+          text: error,
+        },
+      });
+    });
+  }
+
+  public getName(): string {
+    return 'webrtc';
+  }
+  public getPriority(): number {
+    return 1000;
+  }
+
+  public async syncSession(session: Session): Promise<void> {
+    super.syncSession(session);
+    if (this.status === 'connected') {
+      await this.disconnect();
+      await this.connect(session);
+    }
+  }
+
+  public async connect(session?: Session): Promise<Session> {
+    if (this.status === 'connected') {
+      return Promise.resolve(this.session);
+    }
+    this.session = session || this.session || 'new';
+    this.setStatus('connecting');
+
+    this.config.logger?.debug('Connecting to WebRTC with sessionId:', this.session);
+    await this.webRTCClient.connect(this.session);
+    this.setStatus('connected');
+
+    await this.webRTCClient.createDataChannel();
+
+    return this.session;
+  }
+
+  public async disconnect(): Promise<void> {
+    if (this.status === 'disconnected') {
+      this.config.logger?.warn('Already disconnected');
+      return Promise.resolve();
+    }
+
+    await this.webRTCClient.disconnect();
+
+    this.setStatus('disconnected');
+    this.config?.logger?.debug('Disconnected from WebRTC');
+  }
+
+  public sendPacket(packet: PersonaPacket): Promise<void> {
+    if (this.status !== 'connected') {
+      return Promise.reject(new Error('Not connected'));
+    }
+
+    this.webRTCClient.sendPacket(packet);
+    return Promise.resolve();
+  }
+}
+
+export { PersonaWebRTCProtocol };
+export type { PersonaWebRTCProtocolConfig, AudioVisualizerCallback, AudioAnalysisData };

package/src/protocol/websocket.ts

@@ -0,0 +1,111 @@
+import { PersonaPacket, PersonaProtocolBaseConfig, ProtocolStatus, Session } from '../types';
+import { PersonaProtocolBase } from './base';
+
+type PersonaWebSocketProtocolConfig = PersonaProtocolBaseConfig & {
+  webSocketUrl: string;
+};
+
+class PersonaWebSocketProtocol extends PersonaProtocolBase {
+  status: ProtocolStatus;
+  autostart: boolean;
+  session: Session;
+  config: PersonaWebSocketProtocolConfig;
+  webSocket: WebSocket | null;
+
+  constructor(config: PersonaWebSocketProtocolConfig) {
+    super();
+    this.config = config;
+    this.status = 'disconnected';
+    this.autostart = true;
+    this.session = null;
+    this.webSocket = null;
+  }
+
+  public getName(): string {
+    return 'websocket';
+  }
+
+  public getPriority(): number {
+    return 500;
+  }
+
+  public async syncSession(session: Session): Promise<void> {
+    this.config.logger?.debug('Syncing session with WebSocket protocol:', session);
+    this.session = session;
+    if (this.webSocket && this.status === 'connected') {
+      this.disconnect();
+      this.connect(session);
+    }
+  }
+
+  public connect(session?: Session): Promise<Session> {
+    if (this.webSocket !== null && this.status === 'connected') {
+      return Promise.resolve(this.session);
+    }
+
+    const sid = session || this.session || 'new';
+
+    this.config.logger?.debug('Connecting to WebSocket with sessionId:', sid);
+
+    const apiKey = encodeURIComponent(this.config.apiKey);
+    const agentId = this.config.agentId;
+    const userId = this.config.userId || 'anonymous';
+    const webSocketUrl = `${this.config.webSocketUrl}?sessionCode=${sid}&agentId=${agentId}&apiKey=${apiKey}&userId=${encodeURIComponent(
+      userId,
+    )}`;
+    this.setStatus('connecting');
+    this.webSocket = new WebSocket(webSocketUrl);
+    this.webSocket.addEventListener('open', () => {
+      this.setStatus('connected');
+    });
+    this.webSocket.addEventListener('message', (event) => {
+      const data = JSON.parse(event.data) as PersonaPacket;
+      this.notifyPacket(data);
+    });
+    this.webSocket.addEventListener('close', (event: CloseEvent) => {
+      this.setStatus('disconnected');
+      this.webSocket = null;
+      if (event.code !== 1000) {
+        this.notifyPacket({
+          type: 'message',
+          payload: {
+            role: 'assistant',
+            type: 'text',
+            text: 'Oops! The connection to the server was lost. Please try again later.',
+          },
+        });
+        this.config.logger?.warn('WebSocket connection closed');
+      }
+    });
+
+    this.webSocket.addEventListener('error', () => {
+      this.setStatus('disconnected');
+      this.webSocket = null;
+      this.config.logger?.error('WebSocket connection error');
+    });
+
+    return Promise.resolve(sid);
+  }
+
+  public disconnect(): Promise<void> {
+    this.config.logger?.debug('Disconnecting WebSocket');
+    if (this.webSocket && this.status === 'connected') {
+      this.webSocket.close(1000, 'Normal closure');
+      this.setStatus('disconnected');
+      this.webSocket = null;
+    }
+    return Promise.resolve();
+  }
+
+  public sendPacket(packet: PersonaPacket): Promise<void> {
+    if (this.webSocket && this.status === 'connected') {
+      this.webSocket.send(JSON.stringify(packet));
+      return Promise.resolve();
+    } else {
+      return Promise.reject(new Error('WebSocket is not connected'));
+    }
+  }
+}
+
+export { PersonaWebSocketProtocol };
+export type { PersonaWebSocketProtocolConfig };

package/src/runtime/context.ts

@@ -0,0 +1,88 @@
+import { createContext, useContext } from 'react';
+import { PersonaProtocol, ProtocolStatus } from '../types';
+import { PersonaMessage } from '../types';
+import { PersonaWebRTCProtocol } from '../protocol';
+import { getFirstAvailableProtocolEndpoint } from './utils';
+
+export type PersonaRuntimeContextType = {
+  protocols: PersonaProtocol[];
+  protocolsStatus: Map<string, ProtocolStatus>;
+  getMessages: () => PersonaMessage[];
+};
+
+export const PersonaRuntimeContext = createContext<PersonaRuntimeContextType | undefined>(undefined);
+
+/**
+ * Hook to access the PersonaRuntime context
+ */
+export function usePersonaRuntime(): PersonaRuntimeContextType {
+  const context = useContext(PersonaRuntimeContext);
+  if (!context) {
+    throw new Error('usePersonaRuntime must be used within a PersonaRuntimeProvider');
+  }
+  return context;
+}
+
+/**
+ * Retrieves a specific protocol instance from the PersonaRuntimeContext.
+ *
+ * @param protocol - The name of the protocol to use.
+ * @returns {PersonaProtocol | null} - The protocol instance or null if not found.
+ * @throws {Error} - If the hook is used outside of a PersonaRuntimeProvider.
+ */
+export function usePersonaRuntimeProtocol(protocol: string): PersonaProtocol | null {
+  const context = useContext(PersonaRuntimeContext);
+  if (!context) {
+    throw new Error('usePersonaRuntimeProtocol must be used within a PersonaRuntimeProvider');
+  }
+
+  const protocolInstance = context.protocols.find((p) => p.getName() === protocol);
+  if (!protocolInstance) {
+    return null;
+  }
+
+  const status = context.protocolsStatus.get(protocolInstance.getName());
+
+  return {
+    ...protocolInstance,
+    connect: protocolInstance.connect.bind(protocolInstance),
+    disconnect: protocolInstance.disconnect.bind(protocolInstance),
+    sendPacket: protocolInstance.sendPacket.bind(protocolInstance),
+    setSession: protocolInstance.setSession.bind(protocolInstance),
+    addStatusChangeListener: protocolInstance.addStatusChangeListener.bind(protocolInstance),
+    addPacketListener: protocolInstance.addPacketListener.bind(protocolInstance),
+    getName: protocolInstance.getName.bind(protocolInstance),
+    getPriority: protocolInstance.getPriority.bind(protocolInstance),
+    status: status || protocolInstance.status,
+  };
+}
+
+/**
+ * Gets the endpoint URL of the first available protocol
+ */
+export function usePersonaRuntimeEndpoint(): string | null {
+  const context = useContext(PersonaRuntimeContext);
+  if (!context) {
+    throw new Error('usePersonaRuntimeEndpoint must be used within a PersonaRuntimeProvider');
+  }
+
+  return getFirstAvailableProtocolEndpoint(context.protocols);
+}
+
+/**
+ * Gets the WebRTC protocol instance
+ */
+export function usePersonaRuntimeWebRTCProtocol(): PersonaWebRTCProtocol | null {
+  return usePersonaRuntimeProtocol('webrtc') as PersonaWebRTCProtocol;
+}
+
+/**
+ * Gets all messages from the current conversation
+ */
+export function usePersonaRuntimeMessages(): PersonaMessage[] {
+  const context = useContext(PersonaRuntimeContext);
+  if (!context) {
+    throw new Error('usePersonaRuntimeMessages must be used within a PersonaRuntimeProvider');
+  }
+  return context.getMessages();
+}

package/src/runtime/file-attachment-adapter.ts

@@ -0,0 +1,48 @@
+import type { AttachmentAdapter } from '@assistant-ui/react';
+import type { PendingAttachment, CompleteAttachment, Attachment } from '@assistant-ui/react';
+
+function getFileDataURL(file: File): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => resolve(reader.result as string);
+    reader.onerror = (error) => reject(error);
+    reader.readAsDataURL(file);
+  });
+}
+
+/**
+ * Generic file attachment adapter that accepts any file type.
+ * Files are read as base64 data URLs and sent with their original MIME type.
+ */
+export class SimpleFileAttachmentAdapter implements AttachmentAdapter {
+  accept = '*';
+
+  async add(state: { file: File }): Promise<PendingAttachment> {
+    return {
+      id: state.file.name,
+      type: 'file',
+      name: state.file.name,
+      contentType: state.file.type || 'application/octet-stream',
+      file: state.file,
+      status: { type: 'requires-action', reason: 'composer-send' },
+    };
+  }
+
+  async send(attachment: PendingAttachment): Promise<CompleteAttachment> {
+    const dataUrl = await getFileDataURL(attachment.file);
+    const data = dataUrl.split(';base64,')[1];
+    return {
+      ...attachment,
+      status: { type: 'complete' },
+      content: [
+        {
+          type: 'file' as const,
+          data,
+          mimeType: attachment.contentType || 'application/octet-stream',
+        },
+      ],
+    };
+  }
+
+  async remove(_attachment: Attachment): Promise<void> {}
+}