@unith-ai/core-client 1.3.0 → 1.4.0

package/README.md

@@ -1,10 +1,15 @@
 # Unith Core Client Typescript SDK
 
-An SDK library for building complex digital human experiences using javascript/typescript.
+An SDK library for building complex digital human experiences using javascript/typescript that run on [Unith AI](https://www.unith.ai/).
+
+## Prerequisite
+
+Before proceeding with using this library, you're expected to have an account on [Unith AI](https://www.unith.ai/), create a digital human and take note of your API key. You can create an account [here](https://app.unith.ai/) in minutes!
 
 ## Installation
 
 Install the package in your project through package manager.
+
 ```shell
 npm install @unith-ai/core-client
 # or
@@ -20,33 +25,25 @@ This library is designed for use in plain JavaScript applications or to serve as
 ### Initialize Digital Human
 
 First, initialize the Conversation instance:
+
 ```js
 const conversation = await Conversation.startDigitalHuman(options);
 ```
 
-This will establish a WebSocket connection and initialize the digital human avatar with audio/video streaming capabilities.
+This will establish a WebSocket connection and initialize the digital human with realtime audio & video streaming capabilities.
 
 #### Session Configuration
 
 The options passed to `startDigitalHuman` specify how the session is established:
+
 ```js
 const conversation = await Conversation.startDigitalHuman({
   orgId: "your-org-id",
   headId: "your-head-id",
-  username: "anonymous",
-  password: "Password1",
-  environment: "production", // or "development"
   element: document.getElementById("video-container"), // HTML element for video output
   apiKey: "your-api-key",
-  mode: "default",
-  frameRate: 30,
-  streamType: "jpg", // or "vp8"
-  quality: "high",
-  crop: false,
-  showIdle: false,
-  language: "en",
   allowWakeLock: true,
-  fadeTransitionsType: VideoTransitionType.NONE,
+  ...callbacks,
 });
 ```
 
@@ -54,106 +51,52 @@ const conversation = await Conversation.startDigitalHuman({
 
 - **orgId** - Your organization ID
 - **headId** - The digital human head ID to use
+- **apiKey** - API key for authentication (default: "")
 - **element** - HTML element where the video will be rendered
-- **username** - Authentication username (default: "anonymous")
-- **password** - Authentication password (default: "Password1")
 
 #### Optional Parameters
 
-- **environment** - API environment ("production" or "development", default: "production")
-- **apiKey** - API key for authentication (default: "")
 - **mode** - Conversation mode (default: "default")
-- **frameRate** - Video frame rate (default: 30)
-- **streamType** - Video stream format ("jpg" or "vp8", default: "jpg")
-- **quality** - Video quality ("high", "medium", or "low", default: "high")
-- **crop** - Whether to crop the video (default: false)
-- **showIdle** - Whether to show idle video when not speaking (default: false)
 - **language** - Language code for the conversation (default: browser language)
 - **allowWakeLock** - Prevent screen from sleeping during conversation (default: true)
-- **fadeTransitionsType** - Video transition type (default: VideoTransitionType.NONE)
 
-#### Optional Callbacks
+#### Callbacks
 
 Register callbacks to handle various events:
-```js
-const conversation = await Conversation.startDigitalHuman({
-  // ... required options
-  onConnect: ({ userId, headInfo, microphoneAccess }) => {
-    console.log("Connected:", userId);
-  },
-  onDisconnect: (details) => {
-    console.log("Disconnected:", details.reason);
-  },
-  onStatusChange: ({ status }) => {
-    console.log("Status changed:", status); // "connecting", "connected", "disconnecting", "disconnected"
-  },
-  onText: (event) => {
-    console.log("User message:", event.text);
-  },
-  onResponse: (event) => {
-    console.log("AI response:", event.text);
-  },
-  onJoin: (event) => {
-    console.log("Joined conversation:", event);
-  },
-  onStreaming: (event) => {
-    console.log("Streaming event:", event.type); // "audio_frame", "video_frame", "metadata", "cache", "error"
-  },
-  onMuteStatusChange: ({ isMuted }) => {
-    console.log("Mute status:", isMuted);
-  },
-  onSpeakingStart: () => {
-    console.log("Digital human started speaking");
-  },
-  onSpeakingEnd: () => {
-    console.log("Digital human stopped speaking");
-  },
-  onStoppingEnd: () => {
-    console.log("Response stopped");
-  },
-  onTimeout: () => {
-    console.log("Session timed out");
-  },
-  onTimeoutWarning: () => {
-    console.log("Session will timeout soon");
-  },
-  onKeepSession: ({ granted }) => {
-    console.log("Keep session request:", granted);
-  },
-  onError: ({ message, endConversation, type }) => {
-    console.error("Error:", message);
-    // type: "toast" or "modal"
-    // endConversation: true if session should be restarted
-  },
-});
-```
-
-#### Event Types
 
-- **onConnect** - Called when the WebSocket connection is established
-- **onDisconnect** - Called when the connection is closed
-- **onStatusChange** - Called when connection status changes
-- **onText** - Called when a user text message is received
-- **onResponse** - Called when the AI generates a response
-- **onJoin** - Called when successfully joining the conversation
-- **onStreaming** - Called for audio/video frame events
+- **onConnect ({userId, headInfo, microphoneAccess})** - Called when the WebSocket connection is established
+  - **userId** `Boolean` Unique Identifier for the users session.
+  - **headInfo** `ConnectHeadType` Object with data about the digital human.
+    - **name** `String` Digital human head name
+    - **phrases** `String[]` Array with phrases set during digital human creation.
+    - **language** `String` Language code setup during digital human creation.
+    - **avatar** `String` Static image url for digital human.
+  - **microphoneAccess** `Boolean` True if microphone access was granted, False otherwise.
+- **onDisconnect ()** - Called when the connection is closed
+- **onStatusChange ({status})** - Called when connection status changes
+  - **status** `"connecting" | "connected" | "disconnecting" | "disconnected"` Shows current websocket connection status.
+- **onMessage ({ timestamp, speaker, text, visible })** - Called when websocket receives a message or sends a response.
+  - **timestamp** `Date` Timestamp when message was received/sent
+  - **sender** `"user" | "ai"` Shows who the message came from.
+  - **text** `String` Message text
+  - **visible** `Boolean` Flag that you can use to control visibility of message. Sometimes, message comes before the video response starts playing. In such cases, this is usually `false`. Listen the `onSpeakingStart` event to change visibility when the video response starts playing.
 - **onMuteStatusChange** - Called when mute status changes
 - **onSpeakingStart** - Called when the digital human starts speaking
 - **onSpeakingEnd** - Called when the digital human finishes speaking
 - **onStoppingEnd** - Called when a response is manually stopped
 - **onTimeout** - Called when the session times out due to inactivity
-- **onTimeoutWarning** - Called before the session times out
+- **onTimeoutWarning** - Called before the session times out. This event warns you that the customers session is going to end in a bit. You can call the `keepSession` method to extend the customers session.
 - **onKeepSession** - Called when a keep-alive request is processed
 - **onError** - Called when an error occurs
 
 ### Getting Background Video
 
 Retrieve the idle background video URL for use in welcome screens or widget mode:
+
 ```js
 const videoUrl = await Conversation.getBackgroundVideo({
   orgId: "your-org-id",
   headId: "your-head-id",
-  environment: "production",
 });
 ```
 
@@ -161,7 +104,8 @@ const videoUrl = await Conversation.getBackgroundVideo({
 
 #### startSession()
 
-Start the conversation session and begin audio playback:
+Start the conversation session and begin audio & video playback:
+
 ```js
 await conversation.startSession();
 ```
@@ -171,40 +115,23 @@ This method should be called after user interaction to ensure audio context is p
 #### sendMessage(message)
 
 Send a text message to the digital human:
+
 ```js
-conversation.sendMessage({
-  id: 1,
-  timestamp: new Date().toISOString(),
-  speaker: "user",
-  text: "Hello, how are you?",
-  isSent: false,
-  user_id: "user-123",
-  username: "John Doe",
-  event: EventType.TEXT,
-  visible: true,
-});
+conversation.sendMessage("Hello, how are you?");
 ```
 
-#### keepSession(message)
+#### keepSession()
+
+Sends keep-alive event to prevent session timeout:
 
-Send a keep-alive message to prevent session timeout:
 ```js
-conversation.keepSession({
-  id: 1,
-  timestamp: new Date().toISOString(),
-  speaker: "user",
-  text: "",
-  isSent: false,
-  user_id: "user-123",
-  username: "John Doe",
-  event: EventType.KEEP_SESSION,
-  visible: true,
-});
+conversation.keepSession();
 ```
 
 #### stopCurrentResponse()
 
 Stop the current response from the digital human:
+
 ```js
 conversation.stopCurrentResponse();
 ```
@@ -214,6 +141,7 @@ This clears both audio and video queues and returns the digital human to idle st
 #### toggleMuteStatus()
 
 Toggle the mute status of the audio output:
+
 ```js
 const volume = await conversation.toggleMuteStatus();
 console.log("New volume:", volume); // 0 for muted, 1 for unmuted
@@ -222,6 +150,7 @@ console.log("New volume:", volume); // 0 for muted, 1 for unmuted
 #### getUserId()
 
 Get the current user's ID:
+
 ```js
 const userId = conversation.getUserId();
 ```
@@ -229,76 +158,30 @@ const userId = conversation.getUserId();
 #### endSession()
 
 End the conversation session and clean up resources:
+
 ```js
 await conversation.endSession();
 ```
 
 This closes the WebSocket connection, releases the wake lock, and destroys audio/video outputs.
 
-#### initializeMicrophone()
-
-Initialize microphone for speech recognition (ASR):
-```js
-const asrToken = await conversation.initializeMicrophone(
-  (result) => {
-    console.log("Speech result:", result);
-  },
-  (error) => {
-    console.error("Microphone error:", error);
-  },
-  (status) => {
-    console.log("Microphone status:", status);
-  }
-);
-```
-
 ### Message Structure
 
 Messages sent to and from the digital human follow this structure:
+
 ```typescript
 interface Message {
-  id: number;
-  timestamp: string; // ISO format
-  speaker: "user" | "backend";
+  timestamp: Date;
+  sender: SpeakerType;
   text: string;
-  isSent: boolean;
-  user_id: string;
-  username: string;
-  event: EventType.TEXT | EventType.KEEP_SESSION;
   visible: boolean;
-  session_id?: string; // Auto-generated
 }
 ```
 
-### Event Types
-
-The SDK defines the following event types:
-```typescript
-enum EventType {
-  TEXT = "text",
-  RESPONSE = "response",
-  JOIN = "join",
-  STREAMING = "streaming",
-  BINARY = "binary",
-  TIMEOUT_WARNING = "timeout_warning",
-  TIME_OUT = "timeout",
-  KEEP_SESSION = "keep_session",
-}
-```
-
-### Streaming Event Types
-
-Streaming events can have the following types:
-
-- **audio_frame** - Audio data for playback
-- **video_frame** - Video frame data
-- **metadata** - Stream control metadata (start/end)
-- **cache** - Cached video response
-- **error** - Streaming error occurred
-
 ### Error Handling
 
 Always handle errors appropriately:
+
 ```js
 try {
   const conversation = await Conversation.startDigitalHuman({
@@ -324,106 +207,10 @@ try {
 }
 ```
 
-### Common Error Types
-
-The SDK handles several error scenarios:
-
-- **resource_exhausted** - Server at capacity
-- **deadline_exceeded** - Request timeout
-- **inactivity_timeout** - Session inactive for too long
-- **connection** - WebSocket connection failed
-
-## Framework Integration Examples
-
-### React/Preact Example
-```jsx
-import { useEffect, useRef, useState } from "react";
-import { Conversation, EventType } from "@unith-ai/core-client";
-
-function DigitalHuman() {
-  const videoRef = useRef(null);
-  const conversationRef = useRef(null);
-  const [status, setStatus] = useState("disconnected");
-  const [messages, setMessages] = useState([]);
-
-  useEffect(() => {
-    const startConversation = async () => {
-      try {
-        const conversation = await Conversation.startDigitalHuman({
-          orgId: "your-org-id",
-          headId: "your-head-id",
-          element: videoRef.current,
-          onStatusChange: ({ status }) => setStatus(status),
-          onText: (event) => {
-            setMessages((prev) => [...prev, event]);
-          },
-          onResponse: (event) => {
-            setMessages((prev) => [...prev, event]);
-          },
-        });
-
-        conversationRef.current = conversation;
-        await conversation.startSession();
-      } catch (error) {
-        console.error("Failed to start:", error);
-      }
-    };
-
-    startConversation();
-
-    return () => {
-      conversationRef.current?.endSession();
-    };
-  }, []);
-
-  const sendMessage = async (text) => {
-    if (!conversationRef.current) return;
-
-    await conversationRef.current.sendMessage({
-      id: messages.length,
-      timestamp: new Date().toISOString(),
-      speaker: "user",
-      text,
-      isSent: false,
-      user_id: conversationRef.current.getUserId(),
-      username: "User",
-      event: EventType.TEXT,
-      visible: true,
-    });
-  };
-
-  return (
-    <div>
-      <div ref={videoRef} style={{ width: "100%", height: "600px" }} />
-      <div>Status: {status}</div>
-      <button onClick={() => sendMessage("Hello!")}>Send Message</button>
-    </div>
-  );
-}
-```
-
 ## TypeScript Support
 
 Full TypeScript types are included:
-```typescript
-import {
-  Conversation,
-  EventType,
-  Status,
-  HeadType,
-  VideoTransitionType,
-  type ConversationOptions,
-  type Message,
-  type IncomingSocketEvent,
-} from "@unith-ai/core-client";
-```
 
 ## Development
 
 Please refer to the README.md file in the root of this repository.
-
-## Contributing
-
-Please create an issue first to discuss proposed changes. Any contributions are welcome!
-
-Remember, if merged, your code will be used as part of a MIT licensed project. By submitting a Pull Request, you are giving your consent for your code to be integrated into this library.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@unith-ai/core-client",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "",
   "main": "./dist/lib.umd.js",
   "module": "./dist/lib.module.js",

package/dist/index.d.ts

@@ -1,91 +0,0 @@
-import { AudioOutput } from "./modules/audio";
-import { AVController } from "./modules/av";
-import { Connection } from "./modules/connection";
-import { IdleVideo } from "./modules/idle-video";
-import { SyncController } from "./modules/sync";
-import { User } from "./modules/user";
-import { VideoOutput } from "./modules/video";
-import { ResultType, STATUS_TYPES } from "./types/audio";
-import type { ConversationOptions, ConversationEvents, DigitalHumanOptions, HeadOptions, VideoHtmlElement } from "./types/Conversation";
-import { Environment } from "./types/environment";
-import { Message } from "./types/event";
-import { HeadType } from "./types/User";
-export * from "./types/event";
-export * from "./types/Conversation";
-export * from "./types/connection";
-export * from "./types/User";
-export * from "./types/environment";
-export { VideoTransitionType } from "./types/vp8";
-export type Options = ConversationOptions & ConversationEvents;
-export type HeadConfigOptions = HeadOptions & Partial<DigitalHumanOptions>;
-export type PartialOptions = HeadConfigOptions & VideoHtmlElement & Partial<ConversationEvents>;
-export declare class Conversation {
-    options: Options;
-    microphoneAccess: boolean;
-    connection: Connection;
-    idleVideo: IdleVideo;
-    wakeLock: WakeLockSentinel | null;
-    user: User;
-    audioOutput: AudioOutput;
-    videoOutput: VideoOutput;
-    headInfo: HeadType;
-    private status;
-    protected volume: number;
-    private sessionStarted;
-    syncController: SyncController;
-    avController: AVController;
-    private monitor;
-    private videoFrameQueue;
-    private cachedResponseQueue;
-    private static getFullOptions;
-    /**
-     * Starts a digital human conversation.
-     * @param options - The options for the conversation.
-     * @returns A promise that resolves to a Conversation instance.
-     */
-    static startDigitalHuman(options: PartialOptions): Promise<Conversation>;
-    /**
-     * This retrieves the background video to use for widget mode & welcome screen
-     * @param options PartialOptions
-     * @returns Video link to use as widget background
-     */
-    static getBackgroundVideo(options: {
-        orgId: string;
-        headId: string;
-        environment: Environment;
-    }): Promise<string>;
-    constructor(options: Options, microphoneAccess: boolean, connection: Connection, idleVideo: IdleVideo, wakeLock: WakeLockSentinel | null, user: User, audioOutput: AudioOutput, videoOutput: VideoOutput, headInfo: HeadType);
-    private startLatencyMonitoring;
-    private handleIOSSilentMode;
-    private onOutputWorkletMessage;
-    private handleJoinEvent;
-    private handleTextEvent;
-    private handleResponseEvent;
-    private handleStreamError;
-    private handleStreamingEvent;
-    private handleVideoFrame;
-    private handleBinaryData;
-    private handleAudioFrame;
-    private onMessage;
-    private endSessionWithDetails;
-    getUserId(): string;
-    private handleEndSession;
-    private updateStatus;
-    /**
-     * To stop current response, we'll do the following:
-     * 1. set a flag that'll prevent adding new audio & video events to their respective queues
-     * 2. clear video queue & switch state to idle
-     * 3. clear audio queue
-     * 4. send an event if all expected response from BE has been received. If not, FE will keep status in 'stopping' mode
-     */
-    stopCurrentResponse(): void;
-    toggleMuteStatus(): Promise<number>;
-    startSession(): Promise<Connection>;
-    initializeMicrophone(onSpeechResult: (result: ResultType) => void, onError: (e: string) => void, onStatusChange: (status: STATUS_TYPES) => void): Promise<{
-        token: string | null | undefined;
-        region: string | null | undefined;
-    }>;
-    endSession(): Promise<void>;
-    sendMessage(message: Message): void;
-    keepSession(message: Message): void;
-}