@rqmii_immersive/agent-react-sdk 1.0.0

package/README.simple.md

@@ -0,0 +1,114 @@
+# Rqmii React Agent SDK (Simple Guide)
+
+Minimal guide to integrate quickly.
+
+## Install
+
+```bash
+npm install @rqmii_immersive/agent-react-sdk
+```
+
+## Required Config
+
+You need:
+
+- `restUrl` -> FastAPI backend URL
+- `wsUrl` -> realtime Socket.IO URL
+
+Example env:
+
+```env
+VITE_FASTAPI_BACKEND_URL=https://api.example.com
+VITE_WS_AUDIO_URL=wss://realtime.example.com
+```
+
+## Basic Usage
+
+```jsx
+import {
+  RqmiiAgentRoom,
+  useRqmiiConnection,
+  useRqmiiMessages,
+  useRqmiiPressToTalk,
+} from "@rqmii_immersive/agent-react-sdk";
+
+function ChatUI() {
+  const { connectionState, isSessionReady, error } = useRqmiiConnection();
+  const { messages, sendTextMessage } = useRqmiiMessages();
+  const { eventHandlers, isRecording } = useRqmiiPressToTalk();
+
+  return (
+    <div>
+      <div>{connectionState}</div>
+      <div>ready: {String(isSessionReady)}</div>
+      {error && <div style={{ color: "red" }}>{error}</div>}
+
+      <button onClick={() => sendTextMessage("Hello")}>Send</button>
+      <button {...eventHandlers}>
+        {isRecording ? "Recording..." : "Hold to Talk"}
+      </button>
+      <pre>{JSON.stringify(messages, null, 2)}</pre>
+    </div>
+  );
+}
+
+export default function App() {
+  return (
+    <RqmiiAgentRoom
+      wsUrl={import.meta.env.VITE_WS_AUDIO_URL}
+      restUrl={import.meta.env.VITE_FASTAPI_BACKEND_URL}
+      session={{ mode: "initialize", apiKey: "YOUR_API_KEY", language: "en" }}
+      connect
+    >
+      <ChatUI />
+    </RqmiiAgentRoom>
+  );
+}
+```
+
+## Cloud Avatar
+
+```jsx
+import {
+  RqmiiCloudAvatarPlayer,
+  useRqmiiAvatar,
+} from "@rqmii_immersive/agent-react-sdk";
+
+function CloudAvatarUI() {
+  const { isAvatarSpeaking, cloudStreamState } = useRqmiiAvatar();
+
+  return (
+    <>
+      <div>speaking: {String(isAvatarSpeaking)}</div>
+      <div>stream: {cloudStreamState}</div>
+      <RqmiiCloudAvatarPlayer muted={false} />
+    </>
+  );
+}
+```
+
+## Desktop Avatar Join
+
+```jsx
+<RqmiiAgentRoom
+  wsUrl={WS_URL}
+  restUrl={REST_URL}
+  session={{ mode: "join_desktop", uniqueId: "A1B2C3" }}
+  connect
+>
+  <YourUI />
+</RqmiiAgentRoom>
+```
+
+## Voice Input / Output
+
+- send mic chunks: `sendAudioChunk(chunk)`
+- end utterance: `endAudioStream()`
+- interrupt: `interrupt()`
+- SDK plays output audio internally using `pcm-stream-player`
+
+## Notes
+
+- Reconnection is built-in and follows backend reconnect credential flow.
+- Avatar speaking state comes from backend avatar events.
+- For full docs, see `README.md`.

package/dist/index.d.ts

@@ -0,0 +1,285 @@
+import * as React from 'react';
+
+export type AgentType =
+  | 'text'
+  | 'voice'
+  | 'text_and_voice'
+  | 'cloud_avatar'
+  | 'desktop_avatar'
+  | (string & {});
+
+export type ConnectionState =
+  | 'idle'
+  | 'initializing'
+  | 'connecting'
+  | 'connected'
+  | 'reconnecting'
+  | 'failed'
+  | 'ended'
+  | (string & {});
+
+export type CloudStreamState =
+  | 'idle'
+  | 'connecting'
+  | 'loading'
+  | 'connected'
+  | 'disconnected'
+  | 'error'
+  | (string & {});
+
+export interface RqmiiSessionInfo {
+  sessionId: string | null;
+  role: string | null;
+  agentType: AgentType | null;
+  streamingUrl: string | null;
+  uniqueId: string | null;
+}
+
+export interface RqmiiMessage {
+  id: string;
+  role: 'assistant' | 'user' | string;
+  text: string;
+  streaming?: boolean;
+  timestamp?: string;
+}
+
+export interface RqmiiReconnectOptions {
+  enabled?: boolean;
+  maxAttempts?: number;
+  initialDelayMs?: number;
+  maxDelayMs?: number;
+  timeoutMs?: number;
+}
+
+export interface RqmiiPCMOptions {
+  encoding?: string;
+  channels?: number;
+  sampleRate?: number;
+}
+
+export interface InitializeSessionArgs {
+  apiKey?: string;
+  publicChatId?: string;
+  agentId?: string;
+  language?: string;
+}
+
+export interface JoinDesktopSessionArgs {
+  uniqueId: string;
+}
+
+export interface RqmiiAgentProviderProps {
+  children?: React.ReactNode;
+  wsUrl: string;
+  restUrl: string;
+  reconnectOptions?: RqmiiReconnectOptions;
+  enableAudio?: boolean;
+  pcmOptions?: RqmiiPCMOptions;
+  debug?: boolean;
+}
+
+export interface RqmiiAgentRoomSessionInitialize {
+  mode?: 'initialize';
+  apiKey?: string;
+  publicChatId?: string;
+  agentId?: string;
+  language?: string;
+}
+
+export interface RqmiiAgentRoomSessionJoinDesktop {
+  mode: 'join_desktop';
+  uniqueId: string;
+}
+
+export type RqmiiAgentRoomSession =
+  | RqmiiAgentRoomSessionInitialize
+  | RqmiiAgentRoomSessionJoinDesktop;
+
+export interface RqmiiAgentRoomProps extends RqmiiAgentProviderProps {
+  session?: RqmiiAgentRoomSession;
+  connect?: boolean;
+  onError?: (error: unknown) => void;
+  onInitialized?: (data: any) => void;
+}
+
+export interface RqmiiCloudAvatarPlayerProps {
+  className?: string;
+  style?: React.CSSProperties;
+  autoConnect?: boolean;
+  muted?: boolean;
+  onConnected?: () => void;
+  onDisconnected?: () => void;
+  onError?: (error: unknown) => void;
+}
+
+export interface EpicPixelStreamingPlayerProps {
+  signallingUrl?: string;
+  autoConnect?: boolean;
+  width?: string | number;
+  height?: string | number;
+  onConnecting?: () => void;
+  onLoading?: () => void;
+  onConnected?: () => void;
+  onDisconnected?: () => void;
+  onError?: (error: unknown) => void;
+  muted?: boolean;
+}
+
+export interface UseRqmiiConnectionReturn {
+  connectionState: ConnectionState;
+  isConnected: boolean;
+  isSessionReady: boolean;
+  error: string | null;
+  reconnectAttempt: number;
+  session: RqmiiSessionInfo;
+  initializeSession: (args?: InitializeSessionArgs) => Promise<any>;
+  joinDesktopSession: (args: JoinDesktopSessionArgs) => Promise<any>;
+  disconnect: () => void;
+}
+
+export interface UseRqmiiMessagesReturn {
+  messages: RqmiiMessage[];
+  sendTextMessage: (text: string) => boolean;
+  clearMessages: () => void;
+}
+
+export interface UseRqmiiVoiceReturn {
+  sendAudioChunk: (audio: ArrayBuffer | Uint8Array | unknown) => boolean;
+  endAudioStream: () => boolean;
+  isAudioPlaying: boolean;
+  transcript: unknown;
+  partialTranscript: unknown;
+  interrupt: () => boolean;
+  resumeAudioPlayback: () => Promise<void>;
+  attachAudioElement: (element: HTMLAudioElement | null) => void;
+}
+
+export interface UseRqmiiAvatarReturn {
+  streamingUrl: string | null;
+  isAvatarSpeaking: boolean;
+  speakingSource: 'avatar_events';
+  isCloudAvatar: boolean;
+  isDesktopAvatar: boolean;
+  cloudStreamState: CloudStreamState;
+  cloudStreamError: string | null;
+  isCloudStreamConnected: boolean;
+  isCloudStreamLoading: boolean;
+  setCloudStreamState: (nextState: CloudStreamState, error?: unknown) => void;
+}
+
+export interface UseRqmiiPressToTalkOptions {
+  onAudioChunk?: (chunk: ArrayBuffer) => void;
+  onAudioEnd?: () => void;
+  onError?: (error: unknown) => void;
+  holdDelayMs?: number;
+  timesliceMs?: number;
+  interruptOnHoldStart?: boolean;
+  audioConstraints?: MediaTrackConstraints;
+}
+
+export interface UseRqmiiPressToTalkReturn {
+  isPressing: boolean;
+  isRecording: boolean;
+  micError: string | null;
+  startRecording: () => Promise<boolean>;
+  stopRecording: (args?: { discard?: boolean }) => boolean;
+  startPress: () => void;
+  stopPress: () => void;
+  eventHandlers: React.HTMLAttributes<HTMLElement>;
+}
+
+export interface UseRqmiiVADOptions {
+  onAudioChunk?: (chunk: ArrayBuffer) => void;
+  onAudioEnd?: () => void;
+  onSpeechStart?: () => void;
+  onSpeechEnd?: () => void;
+  onVADMisfire?: () => void;
+  onError?: (error: unknown) => void;
+  interruptOnSpeechStart?: boolean;
+  forceAudioMp4?: boolean;
+}
+
+export interface UseRqmiiVADReturn {
+  isListening: boolean;
+  isLoading: boolean;
+  isSpeaking: boolean;
+  micError: string | null;
+  toggle: () => void;
+  start: () => void;
+  pause: () => void;
+  userSpeaking: boolean;
+}
+
+export interface RqmiiAgentContextValue extends UseRqmiiConnectionReturn {
+  isAudioPlaying: boolean;
+  isAvatarSpeaking: boolean;
+  cloudStream: {
+    state: CloudStreamState;
+    error: string | null;
+  };
+  transcript: unknown;
+  partialTranscript: unknown;
+  messages: RqmiiMessage[];
+  initializeSession: (args?: InitializeSessionArgs) => Promise<any>;
+  joinDesktopSession: (args: JoinDesktopSessionArgs) => Promise<any>;
+  disconnect: () => void;
+  emit: (event: string, payload?: unknown) => boolean;
+  sendTextMessage: (text: string) => boolean;
+  sendAudioChunk: (audio: ArrayBuffer | Uint8Array | unknown) => boolean;
+  endAudioStream: () => boolean;
+  interrupt: () => boolean;
+  refreshSessionActivity: () => boolean;
+  clearSessionHistory: () => boolean;
+  clearMessages: () => void;
+  resumeAudioPlayback: () => Promise<void>;
+  attachAudioElement: (element: HTMLAudioElement | null) => void;
+  setCloudStreamState: (nextState: CloudStreamState, error?: unknown) => void;
+  audioElement: HTMLAudioElement | null;
+  socket: any;
+}
+
+export const AGENT_TYPES: {
+  readonly TEXT: 'text';
+  readonly VOICE: 'voice';
+  readonly TEXT_AND_VOICE: 'text_and_voice';
+  readonly CLOUD_AVATAR: 'cloud_avatar';
+  readonly DESKTOP_AVATAR: 'desktop_avatar';
+};
+
+export const SOCKET_EVENTS: Record<string, string>;
+
+export const CONNECTION_STATE: {
+  readonly IDLE: 'idle';
+  readonly INITIALIZING: 'initializing';
+  readonly CONNECTING: 'connecting';
+  readonly CONNECTED: 'connected';
+  readonly RECONNECTING: 'reconnecting';
+  readonly FAILED: 'failed';
+  readonly ENDED: 'ended';
+};
+
+export const CLOUD_STREAM_STATE: {
+  readonly IDLE: 'idle';
+  readonly CONNECTING: 'connecting';
+  readonly LOADING: 'loading';
+  readonly CONNECTED: 'connected';
+  readonly DISCONNECTED: 'disconnected';
+  readonly ERROR: 'error';
+};
+
+export function RqmiiAgentProvider(props: RqmiiAgentProviderProps): React.JSX.Element;
+export function useRqmiiAgentContext(): RqmiiAgentContextValue;
+
+export function useRqmiiAgent(): RqmiiAgentContextValue;
+export function useRqmiiConnection(): UseRqmiiConnectionReturn;
+export function useRqmiiMessages(): UseRqmiiMessagesReturn;
+export function useRqmiiVoice(): UseRqmiiVoiceReturn;
+export function useRqmiiAvatar(): UseRqmiiAvatarReturn;
+export function useRqmiiAudioElement(audioRef: React.RefObject<HTMLAudioElement | null>): void;
+export function useRqmiiPressToTalk(options?: UseRqmiiPressToTalkOptions): UseRqmiiPressToTalkReturn;
+export function useRqmiiVAD(options?: UseRqmiiVADOptions): UseRqmiiVADReturn;
+
+export function RqmiiAgentRoom(props: RqmiiAgentRoomProps): React.JSX.Element;
+export function RqmiiCloudAvatarPlayer(props: RqmiiCloudAvatarPlayerProps): React.JSX.Element | null;
+export const EpicPixelStreamingPlayer: React.ComponentType<EpicPixelStreamingPlayerProps>;
+