@rqmii_immersive/agent-react-sdk 1.0.0

package/README.frameworks.md

@@ -0,0 +1,244 @@
+# Rqmii React Agent SDK - Framework Recipes
+
+This guide shows integration patterns for Vite, Next.js, and CRA.
+
+## 1. Vite
+
+### Install
+
+```bash
+npm install @rqmii_immersive/agent-react-sdk
+```
+
+### Env
+
+`.env`:
+
+```env
+VITE_FASTAPI_BACKEND_URL=https://api.example.com
+VITE_WS_AUDIO_URL=wss://realtime.example.com
+```
+
+### App Example
+
+```jsx
+import {
+  RqmiiAgentRoom,
+  useRqmiiMessages,
+} from "@rqmii_immersive/agent-react-sdk";
+
+function UI() {
+  const { sendTextMessage } = useRqmiiMessages();
+  return <button onClick={() => sendTextMessage("Hi")}>Send</button>;
+}
+
+export default function App() {
+  return (
+    <RqmiiAgentRoom
+      wsUrl={import.meta.env.VITE_WS_AUDIO_URL}
+      restUrl={import.meta.env.VITE_FASTAPI_BACKEND_URL}
+      session={{ mode: "initialize", apiKey: "YOUR_API_KEY", language: "en" }}
+      connect
+    >
+      <UI />
+    </RqmiiAgentRoom>
+  );
+}
+```
+
+## 2. Next.js (App Router)
+
+Use client components for SDK usage.
+
+### Install
+
+```bash
+npm install @rqmii_immersive/agent-react-sdk
+```
+
+### Env
+
+`.env.local`:
+
+```env
+NEXT_PUBLIC_FASTAPI_BACKEND_URL=https://api.example.com
+NEXT_PUBLIC_WS_AUDIO_URL=wss://realtime.example.com
+```
+
+### Client Component
+
+`app/agent/page.jsx`:
+
+```jsx
+"use client";
+
+import {
+  RqmiiAgentRoom,
+  useRqmiiConnection,
+  useRqmiiMessages,
+} from "@rqmii_immersive/agent-react-sdk";
+
+function UI() {
+  const { connectionState } = useRqmiiConnection();
+  const { messages, sendTextMessage } = useRqmiiMessages();
+
+  return (
+    <div>
+      <div>{connectionState}</div>
+      <button onClick={() => sendTextMessage("Hello")}>Send</button>
+      <pre>{JSON.stringify(messages, null, 2)}</pre>
+    </div>
+  );
+}
+
+export default function AgentPage() {
+  return (
+    <RqmiiAgentRoom
+      wsUrl={process.env.NEXT_PUBLIC_WS_AUDIO_URL}
+      restUrl={process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}
+      session={{ mode: "initialize", apiKey: "YOUR_API_KEY", language: "en" }}
+      connect
+    >
+      <UI />
+    </RqmiiAgentRoom>
+  );
+}
+```
+
+### Next.js Notes
+
+- Do not use SDK in server components.
+- Keep all SDK consumers under `'use client'`.
+- For cloud avatar player, render it only on client.
+
+## 3. Create React App (CRA)
+
+### Install
+
+```bash
+npm install @rqmii_immersive/agent-react-sdk
+```
+
+### Env
+
+`.env`:
+
+```env
+REACT_APP_FASTAPI_BACKEND_URL=https://api.example.com
+REACT_APP_WS_AUDIO_URL=wss://realtime.example.com
+```
+
+### App Example
+
+```jsx
+import {
+  RqmiiAgentRoom,
+  useRqmiiMessages,
+} from "@rqmii_immersive/agent-react-sdk";
+
+function UI() {
+  const { sendTextMessage } = useRqmiiMessages();
+  return <button onClick={() => sendTextMessage("Hello")}>Send</button>;
+}
+
+export default function App() {
+  return (
+    <RqmiiAgentRoom
+      wsUrl={process.env.REACT_APP_WS_AUDIO_URL}
+      restUrl={process.env.REACT_APP_FASTAPI_BACKEND_URL}
+      session={{ mode: "initialize", apiKey: "YOUR_API_KEY", language: "en" }}
+      connect
+    >
+      <UI />
+    </RqmiiAgentRoom>
+  );
+}
+```
+
+## 4. Common Recipes
+
+### Cloud Avatar
+
+```jsx
+import {
+  RqmiiCloudAvatarPlayer,
+  useRqmiiAvatar,
+} from "@rqmii_immersive/agent-react-sdk";
+
+function AvatarPanel() {
+  const { cloudStreamState, isAvatarSpeaking } = useRqmiiAvatar();
+  return (
+    <>
+      <div>stream: {cloudStreamState}</div>
+      <div>speaking: {String(isAvatarSpeaking)}</div>
+      <RqmiiCloudAvatarPlayer muted={false} />
+    </>
+  );
+}
+```
+
+### Desktop Avatar Join
+
+```jsx
+<RqmiiAgentRoom
+  wsUrl={WS_URL}
+  restUrl={REST_URL}
+  session={{ mode: "join_desktop", uniqueId: "A1B2C3" }}
+  connect
+>
+  <UI />
+</RqmiiAgentRoom>
+```
+
+### Manual Provider Control
+
+```jsx
+import {
+  RqmiiAgentProvider,
+  useRqmiiConnection,
+} from "@rqmii_immersive/agent-react-sdk";
+
+function Controls() {
+  const { initializeSession, disconnect } = useRqmiiConnection();
+  return (
+    <>
+      <button
+        onClick={() => initializeSession({ apiKey: "KEY", language: "en" })}
+      >
+        Init
+      </button>
+      <button onClick={disconnect}>Disconnect</button>
+    </>
+  );
+}
+
+export default function App() {
+  return (
+    <RqmiiAgentProvider wsUrl={WS_URL} restUrl={REST_URL}>
+      <Controls />
+    </RqmiiAgentProvider>
+  );
+}
+```
+
+## 5. Troubleshooting by Framework
+
+### Vite
+
+- Ensure env keys start with `VITE_`.
+- Restart dev server after env changes.
+
+### Next.js
+
+- Ensure page/component is client-side (`'use client'`).
+- Use `NEXT_PUBLIC_` env keys.
+
+### CRA
+
+- Ensure env keys start with `REACT_APP_`.
+- Restart `npm start` after env changes.
+
+## 6. Where to Read More
+
+- Full technical docs: `README.md`
+- Quick docs: `README.simple.md`

package/README.md

@@ -0,0 +1,430 @@
+# Rqmii React Agent SDK
+
+Standalone React SDK for integrating React apps with:
+
+- `RqmiiAgentBackendAPI` REST endpoints (`/sessions/init`, `/sessions/join`)
+- `RqmiiAgentBackendJs` realtime Socket.IO server
+
+Quick navigation:
+
+- Full guide: `README.md` (this file)
+- Simple guide: `README.simple.md`
+- Framework recipes (Vite/Next.js/CRA): `README.frameworks.md`
+
+This SDK gives you a LiveKit-style integration model:
+
+- room wrapper component (`RqmiiAgentRoom`)
+- provider (`RqmiiAgentProvider`)
+- hooks usable by any child component (`useRqmiiConnection`, `useRqmiiMessages`, `useRqmiiVoice`, `useRqmiiAvatar`)
+
+## 1. Installation
+
+```bash
+npm install @rqmii_immersive/agent-react-sdk
+```
+
+If you are testing locally from this mono-repo:
+
+```bash
+npm install file:../RqmiiAgentReactSDK
+```
+
+## 2. Required Backend Configuration
+
+You must provide both URLs to the SDK:
+
+- `restUrl`: FastAPI backend base URL (for session init/join)
+- `wsUrl`: realtime coordinator Socket.IO URL
+
+Typical env values:
+
+```env
+VITE_FASTAPI_BACKEND_URL=https://api.example.com
+VITE_WS_AUDIO_URL=wss://realtime.example.com
+```
+
+## 3. Import Surface
+
+From the package root:
+
+```js
+import {
+  RqmiiAgentProvider,
+  RqmiiAgentRoom,
+  RqmiiCloudAvatarPlayer,
+  EpicPixelStreamingPlayer,
+  useRqmiiAgent,
+  useRqmiiConnection,
+  useRqmiiMessages,
+  useRqmiiVoice,
+  useRqmiiAvatar,
+  useRqmiiAudioElement,
+  useRqmiiPressToTalk,
+  useRqmiiVAD,
+  AGENT_TYPES,
+  CONNECTION_STATE,
+  CLOUD_STREAM_STATE,
+  SOCKET_EVENTS,
+} from "@rqmii_immersive/agent-react-sdk";
+```
+
+If your bundler requires explicit style import, also import:
+
+```js
+import "@rqmii_immersive/agent-react-sdk/style.css";
+```
+
+## 4. Quick Start (Room Wrapper)
+
+```jsx
+import {
+  RqmiiAgentRoom,
+  RqmiiCloudAvatarPlayer,
+  useRqmiiConnection,
+  useRqmiiMessages,
+  useRqmiiAvatar,
+} from "@rqmii_immersive/agent-react-sdk";
+
+function ChatUI() {
+  const { messages, sendTextMessage } = useRqmiiMessages();
+  const { connectionState, isSessionReady, error } = useRqmiiConnection();
+  const { isAvatarSpeaking, cloudStreamState } = useRqmiiAvatar();
+
+  return (
+    <div>
+      <div>Conn: {connectionState}</div>
+      <div>Ready: {String(isSessionReady)}</div>
+      <div>Avatar Speaking: {String(isAvatarSpeaking)}</div>
+      <div>Cloud Stream: {cloudStreamState}</div>
+      {error && <div style={{ color: "red" }}>{error}</div>}
+
+      <button onClick={() => sendTextMessage("Hello")}>Send</button>
+      <pre>{JSON.stringify(messages, null, 2)}</pre>
+
+      <RqmiiCloudAvatarPlayer muted={false} />
+    </div>
+  );
+}
+
+export default function App() {
+  return (
+    <RqmiiAgentRoom
+      wsUrl={import.meta.env.VITE_WS_AUDIO_URL}
+      restUrl={import.meta.env.VITE_FASTAPI_BACKEND_URL}
+      session={{ mode: "initialize", apiKey: "YOUR_API_KEY", language: "en" }}
+      connect
+      onError={(e) => console.error(e)}
+      onInitialized={(data) => console.log("session initialized", data)}
+    >
+      <ChatUI />
+    </RqmiiAgentRoom>
+  );
+}
+```
+
+## 5. Agent-Type Integration Flows
+
+### 5.1 Cloud Avatar
+
+Use `session.mode = 'initialize'` with `apiKey` (or `publicChatId` / `agentId`).
+
+- backend returns `streaming_url`
+- render `RqmiiCloudAvatarPlayer`
+- speaking state comes from avatar events, not audio chunks
+
+```jsx
+<RqmiiAgentRoom
+  wsUrl={WS_URL}
+  restUrl={REST_URL}
+  session={{ mode: "initialize", apiKey: API_KEY, language: "en" }}
+  connect
+>
+  <RqmiiCloudAvatarPlayer muted={false} />
+</RqmiiAgentRoom>
+```
+
+### 5.2 Desktop Avatar (User Join)
+
+User joins existing desktop avatar session by `uniqueId`:
+
+```jsx
+<RqmiiAgentRoom
+  wsUrl={WS_URL}
+  restUrl={REST_URL}
+  session={{ mode: "join_desktop", uniqueId: "A1B2C3" }}
+  connect
+>
+  <DesktopUI />
+</RqmiiAgentRoom>
+```
+
+### 5.3 Text Agent
+
+- initialize session
+- send text using `sendTextMessage`
+- read assistant chunks/final from `messages`
+
+### 5.4 Voice / Text-and-Voice Agent
+
+- send microphone chunks via `sendAudioChunk`
+- signal utterance end via `endAudioStream`
+- audio playback handled internally via `pcm-stream-player`
+
+## 6. Avatar State Model (Important)
+
+`isAvatarSpeaking` is server-event driven only:
+
+- starts on `avatar_speaking`
+- ends on `avatar_speaking_done`
+
+This means avatar speaking state is independent from PCM playback.
+
+For cloud avatar rendering lifecycle, `useRqmiiAvatar()` exposes:
+
+- `cloudStreamState`
+- `cloudStreamError`
+- `isCloudStreamConnected`
+- `isCloudStreamLoading`
+
+`cloudStreamState` values:
+
+- `idle`
+- `connecting`
+- `loading`
+- `connected`
+- `disconnected`
+- `error`
+
+## 7. Reconnection Model
+
+The backend uses one-time session JWTs, so default Socket.IO auto-reconnect with the same token is not enough.
+
+This SDK implements the coordinator reconnect contract:
+
+1. connect using initial `session_token`
+2. server emits `reconnect_credential` with `{ sessionId, reconnectToken, role }`
+3. on disconnect, SDK reconnects with reconnect handshake auth
+4. on success, server emits `reconnect_success` and rotates token
+
+Configurable with `reconnectOptions` on provider/room:
+
+```jsx
+<RqmiiAgentRoom
+  wsUrl={WS_URL}
+  restUrl={REST_URL}
+  reconnectOptions={{
+    enabled: true,
+    maxAttempts: 8,
+    initialDelayMs: 500,
+    maxDelayMs: 5000,
+    timeoutMs: 8000,
+  }}
+  session={{ mode: "initialize", apiKey: API_KEY }}
+  connect
+>
+  <UI />
+</RqmiiAgentRoom>
+```
+
+## 8. Provider-Only Integration (No Room Wrapper)
+
+Use this when you need full manual control:
+
+```jsx
+import {
+  RqmiiAgentProvider,
+  useRqmiiConnection,
+} from "@rqmii_immersive/agent-react-sdk";
+
+function Controller() {
+  const { initializeSession, joinDesktopSession, disconnect } =
+    useRqmiiConnection();
+
+  return (
+    <div>
+      <button
+        onClick={() => initializeSession({ apiKey: API_KEY, language: "en" })}
+      >
+        Init
+      </button>
+      <button onClick={() => joinDesktopSession({ uniqueId: "A1B2C3" })}>
+        Join Desktop
+      </button>
+      <button onClick={disconnect}>Disconnect</button>
+    </div>
+  );
+}
+
+export default function App() {
+  return (
+    <RqmiiAgentProvider wsUrl={WS_URL} restUrl={REST_URL}>
+      <Controller />
+    </RqmiiAgentProvider>
+  );
+}
+```
+
+## 9. Hook Reference
+
+### `useRqmiiConnection()`
+
+Returns:
+
+- `connectionState`
+- `isConnected`
+- `isSessionReady`
+- `error`
+- `reconnectAttempt`
+- `session`
+- `initializeSession(args)`
+- `joinDesktopSession(args)`
+- `disconnect()`
+
+### `useRqmiiMessages()`
+
+Returns:
+
+- `messages`
+- `sendTextMessage(text)`
+- `clearMessages()`
+
+Message stream behavior:
+
+- server chunks update streaming assistant message (`agent_response_chunk`)
+- final response completes that message (`agent_response`)
+
+### `useRqmiiVoice()`
+
+Returns:
+
+- `sendAudioChunk(audio)`
+- `endAudioStream()`
+- `isAudioPlaying`
+- `transcript`
+- `partialTranscript`
+- `interrupt()`
+- `resumeAudioPlayback()`
+- `attachAudioElement(element)`
+
+### `useRqmiiAvatar()`
+
+Returns:
+
+- `streamingUrl`
+- `isAvatarSpeaking`
+- `speakingSource` (`'avatar_events'`)
+- `isCloudAvatar`
+- `isDesktopAvatar`
+- `cloudStreamState`
+- `cloudStreamError`
+- `isCloudStreamConnected`
+- `isCloudStreamLoading`
+- `setCloudStreamState(nextState, error?)`
+
+### `useRqmiiAudioElement(audioRef)`
+
+Optional helper to bind an audio element and improve autoplay unlock UX.
+
+Note: voice playback does not require HTML audio element because SDK uses WebAudio (`pcm-stream-player`).
+
+### `useRqmiiPressToTalk(options?)`
+
+Built-in press-and-release voice input mode.
+
+- streams mic chunks to backend (`audio_chunk`)
+- sends `audio_end` automatically on release
+- exposes ready-to-spread button handlers via `eventHandlers`
+
+Minimal usage:
+
+```jsx
+const { eventHandlers, isRecording } = useRqmiiPressToTalk();
+<button {...eventHandlers}>
+  {isRecording ? "Recording..." : "Hold to Talk"}
+</button>;
+```
+
+### `useRqmiiVAD(options?)`
+
+Built-in VAD (voice activity detection) mode using `@ricky0123/vad-react`.
+
+- toggle VAD listening with `toggle()`
+- auto interrupts on speech start (configurable)
+- handles pre-speech padding and streaming chunks/end events
+
+Minimal usage:
+
+```jsx
+const { toggle, isListening, isSpeaking } = useRqmiiVAD();
+```
+
+## 10. Cloud Player Wrapper API
+
+`RqmiiCloudAvatarPlayer` props:
+
+- `className`
+- `style`
+- `autoConnect` (default `true`)
+- `muted` (default `false`)
+- `onConnected()`
+- `onDisconnected()`
+- `onError(error)`
+
+Internally it forwards cloud stream lifecycle to context state.
+
+## 11. Error Handling Strategy
+
+- fatal session/socket errors populate `error` in connection state
+- `TTS_ERROR` socket errors are treated as non-fatal by default
+- reconnect failures transition to `connectionState = failed`
+
+Recommended UI behavior:
+
+- show persistent banner for `error`
+- show reconnect indicator while `connectionState === reconnecting`
+- offer retry button that calls `initializeSession` again
+
+## 12. Production Checklist
+
+- use secure `https/wss` URLs in production
+- ensure CORS allows browser origins for REST and Socket.IO
+- pass correct API key/public chat id per environment
+- monitor reconnect failures and auth errors in logs
+- handle browser autoplay policies with user gesture before first audio
+
+## 13. Troubleshooting
+
+### Session initializes but socket fails auth
+
+Check:
+
+- `wsUrl` points to `RqmiiAgentBackendJs`
+- token is passed via SDK flow (do not manually override connect handshake)
+- backend clocks are synchronized for JWT expiry
+
+### Cloud avatar not visible
+
+Check:
+
+- `/sessions/init` response includes `streaming_url`
+- `RqmiiCloudAvatarPlayer` is rendered
+- network allows signaling/WebRTC paths
+
+### Avatar speaking UI not updating
+
+Check:
+
+- server emits `avatar_speaking` and `avatar_speaking_done`
+- your UI reads `isAvatarSpeaking` from `useRqmiiAvatar()`
+
+### Voice playback missing for text-and-voice
+
+Check:
+
+- agent type is not avatar (`cloud_avatar`/`desktop_avatar`)
+- browser has user interaction to unlock audio context
+- `agent_audio_chunk` events are arriving from server
+
+## 14. Current Scope
+
+This SDK currently targets React integration and browser runtime. If you need Next.js SSR-specific wrappers or TypeScript declaration generation (`.d.ts`), add them in your package pipeline as a next step.