@touchcastllc/napster-companion-api 1.0.0-alpha.34 → 1.0.0-alpha.36

package/README.md

@@ -20,6 +20,7 @@
 - **Zero External Dependencies:** Drop in a script tag and go
 - **TypeScript Support:** Fully typed APIs
 - **Tree-Shakeable:** Import only what you need
+- **Screen Sharing:** Let the avatar see your screen in real time
 - **Production Ready:** Minified builds with type definitions
 
 ---
@@ -377,6 +378,7 @@ async function initAdvancedAvatar() {
     features: {
       inactiveTimeout: { enabled: true, duration: 120000, countdown: 15 },
       showSDKLoader: { enabled: true, bgColor: "#f0f0f0" },
+      screenShare: { enabled: true },
     },
 
     style: {
@@ -523,7 +525,149 @@ const instance = await NapsterCompanionApiSdk.init(token, {
 });
 ```
 
-### 3.2 Position & Layout
+### 3.2 Face Tracking
+
+Real-time face detection using TensorFlow.js + MediaPipe Face Mesh. Detects the user's face via their camera, extracts 468 facial landmarks, and determines talking state via mouth-opening ratio. Fully client-side — no backend involved.
+
+#### Prerequisites
+
+Install the optional peer dependencies (only needed when face tracking is enabled):
+
+```bash
+npm install @tensorflow/tfjs @tensorflow-models/face-landmarks-detection
+```
+
+These libraries (~2MB) are lazy-loaded at runtime — they are not included in the main SDK bundle.
+
+#### Enable & Configure
+
+```typescript
+const instance = await NapsterCompanionApiSdk.init(token, {
+  features: {
+    faceTracking: {
+      enabled: true,
+      fps: 15,                // Detection rate: 1–30 FPS (default: 15)
+      talkingThreshold: 0.015, // Mouth opening ratio threshold (default: 0.015)
+    },
+  },
+  onFaceTrackingData: (data) => {
+    if (data) {
+      console.log("Talking:", data.isTalking);
+      console.log("Landmarks:", data.landmarks.length); // 468
+      console.log("Mouth:", data.mouthLandmarks);
+    } else {
+      console.log("No face detected");
+    }
+  },
+  onFaceTrackingError: (error) => {
+    console.error("Face tracking error:", error.message);
+  },
+});
+```
+
+#### Start & Stop
+
+```typescript
+// Start detection (opens camera, loads model on first call)
+await instance.startFaceTracking();
+
+// Pause detection (stops camera, keeps model loaded for fast restart)
+instance.stopFaceTracking();
+
+// Restart — reuses model, only reopens camera
+await instance.startFaceTracking();
+```
+
+#### Face Tracking Data
+
+Each detection frame emits a `FaceTrackingData` object (or `null` if no face is detected):
+
+```typescript
+interface FaceTrackingData {
+  landmarks: Array<{ x: number; y: number; z: number }>; // 468 points, 0–1 normalized
+  mouthLandmarks: {
+    topLip:      { x: number; y: number };
+    bottomLip:   { x: number; y: number };
+    leftCorner:  { x: number; y: number };
+    rightCorner: { x: number; y: number };
+  } | null;
+  isTalking: boolean; // smoothed mouth-opening ratio > threshold
+}
+```
+
+#### Error Codes
+
+| Code                       | When                                      |
+| -------------------------- | ----------------------------------------- |
+| `CAMERA_PERMISSION_DENIED` | User denied camera access                 |
+| `CAMERA_NOT_FOUND`         | No camera available                       |
+| `MODEL_LOAD_FAILED`        | TensorFlow.js / MediaPipe failed to load  |
+| `DETECTION_FAILED`         | Runtime detection error                   |
+
+#### Cleanup
+
+Face tracking is automatically destroyed when `instance.destroy()` is called. The model is disposed and camera tracks are stopped.
+
+### 3.3 Screen Sharing
+
+Share the user's screen with the avatar in real time. The SDK captures the display at 1 FPS, renders each frame onto an internal canvas, and streams the canvas track over WebRTC so the avatar can "see" what the user sees. Fully browser-based — no plugins or extensions required.
+
+#### Enable & Configure
+
+```typescript
+const instance = await NapsterCompanionApiSdk.init(token, {
+  features: {
+    screenShare: {
+      enabled: true,
+    },
+  },
+});
+```
+
+> **Note:** `features.screenShare.enabled` must be `true` for the screen share controls to appear and for `isScreenShareSupported` to return `true`.
+
+#### Start & Stop
+
+```typescript
+// Start sharing (opens browser's screen picker)
+await instance.startScreenShare();
+
+// Stop sharing
+instance.stopScreenShare();
+
+// Or toggle on/off
+await instance.toggleScreenShare();
+```
+
+The SDK sends `start_video` / `stop_video` commands to the avatar server automatically. If the user stops sharing via the browser's native "Stop sharing" button, the SDK detects this and cleans up.
+
+#### Checking State
+
+```typescript
+// Whether screen sharing is currently active
+instance.isScreenSharing; // boolean
+
+// Whether the browser supports screen sharing AND the feature is enabled
+instance.isScreenShareSupported; // boolean
+```
+
+#### How It Works
+
+1. **`getDisplayMedia`** captures the user's screen at max 1 FPS, 1280×720.
+2. **`MediaCapture`** renders each frame onto a hidden `<canvas>` via a Web Worker timer.
+3. **`canvas.captureStream(1)`** produces a video track that is added to the WebRTC peer connection at setup time.
+4. When the user starts sharing, frames begin flowing on the already-connected track and the SDK sends `start_video` through the data channel.
+5. When the user stops sharing, the display stream tracks are stopped and `stop_video` is sent.
+
+#### Cleanup
+
+Screen sharing is automatically stopped when:
+
+- The user calls `instance.destroy()`
+- The WebRTC connection closes (network failure, session end)
+- The user clicks the browser's native "Stop sharing" button
+
+### 3.4 Position & Layout
 
 Customize the avatar's position on the screen using predefined positions or custom styles. Also you can override the position using style properties.
 
@@ -685,7 +829,75 @@ if (instance.isFeatureEnabled("disclaimer")) {
 }
 ```
 
-### 5.4 Communication Methods
+### 5.4 Face Tracking Methods
+
+#### `startFaceTracking(): Promise<void>`
+
+Start face tracking. Opens the user's camera and begins the detection loop. The model is loaded on the first call and reused on subsequent calls.
+
+```typescript
+await instance.startFaceTracking();
+```
+
+> **Note:** Requires `features.faceTracking.enabled = true` in the init config. Throws a `FeatureError` if not enabled.
+
+#### `stopFaceTracking(): void`
+
+Pause face tracking. Stops the camera and detection loop but keeps the model loaded for fast restart.
+
+```typescript
+instance.stopFaceTracking();
+```
+
+### 5.5 Screen Sharing Methods
+
+#### `startScreenShare(): Promise<void>`
+
+Start screen sharing. Opens the browser's screen picker and begins streaming frames to the avatar.
+
+```typescript
+await instance.startScreenShare();
+```
+
+> **Note:** Requires `features.screenShare.enabled = true` in the init config. Does nothing if already sharing.
+
+#### `stopScreenShare(): void`
+
+Stop screen sharing. Releases the display stream and notifies the server.
+
+```typescript
+instance.stopScreenShare();
+```
+
+#### `toggleScreenShare(): Promise<void>`
+
+Toggle screen sharing on or off.
+
+```typescript
+await instance.toggleScreenShare();
+```
+
+#### `isScreenSharing: boolean` _(read-only)_
+
+Whether screen sharing is currently active.
+
+```typescript
+if (instance.isScreenSharing) {
+  console.log("User is sharing their screen");
+}
+```
+
+#### `isScreenShareSupported: boolean` _(read-only)_
+
+Whether screen sharing is supported by the browser **and** enabled in the feature config.
+
+```typescript
+if (instance.isScreenShareSupported) {
+  showScreenShareButton();
+}
+```
+
+### 5.6 Communication Methods
 
 #### `sendCommand(command: { type: string; data?: object }): void`
 
@@ -736,6 +948,14 @@ interface NapsterCompanionApiConfig {
     };
     disclaimer?: { enabled: boolean; text?: string };
     showSDKLoader?: { enabled: boolean; bgColor?: string };
+    screenShare?: {
+      enabled: boolean;
+    };
+    faceTracking?: {
+      enabled: boolean;
+      fps?: number;              // 1–30, default 15
+      talkingThreshold?: number; // default 0.015
+    };
   };
 
   // Configuration
@@ -749,6 +969,8 @@ interface NapsterCompanionApiConfig {
   onInactivityStatusChange?: (isInactive: boolean) => void;
   onDestroy?: () => void;
   onFeaturesUpdate?: (features: FeatureConfig) => void;
+  onFaceTrackingData?: (data: FaceTrackingData | null) => void;
+  onFaceTrackingError?: (error: Error) => void;
 }
 ```
 
@@ -767,6 +989,9 @@ npm install @touchcastllc/napster-companion-api
 
 # Install peer dependencies
 npm install @reduxjs/toolkit
+
+# If using face tracking, also install:
+npm install @tensorflow/tfjs @tensorflow-models/face-landmarks-detection
 ```
 
 **Problem: Invalid or expired token**  

package/lib/components/Avatar/index.d.ts

@@ -10,6 +10,9 @@ export interface AvatarOptions {
     onError?: (error: Error) => void;
     onInactivityStatusChange?: (isInactive: boolean) => void;
     features?: Partial<FeatureConfig>;
+    onScreenShareToggle?: () => void;
+    isScreenSharing?: boolean;
+    isScreenShareSupported?: boolean;
 }
 export interface StreamsUpdate {
     videoStream?: MediaStream | null;
@@ -22,6 +25,7 @@ export interface AvatarController extends BaseController<AvatarOptions> {
     updateFeatures: (features: Partial<FeatureConfig>) => void;
     setAvatarReady: (ready: boolean) => void;
     handleMessage: (data: EventMessage) => void;
+    updateScreenShareState: (isSharing: boolean) => void;
 }
 /**
  * Vanilla Avatar implementation.

package/lib/components/Embed/index.d.ts

@@ -10,10 +10,14 @@ export interface EmbedOptions {
     onError?: (error: Error) => void;
     onInactivityStatusChange?: (isInactive: boolean) => void;
     features?: Partial<FeatureConfig>;
+    onScreenShareToggle?: () => void;
+    isScreenSharing?: boolean;
+    isScreenShareSupported?: boolean;
 }
 export interface EmbedController extends BaseController<EmbedOptions> {
     updateStreams: (streams: StreamsUpdate) => void;
     handleMessage: (data: EventMessage) => void;
+    updateScreenShareState: (isSharing: boolean) => void;
 }
 /**
  * Vanilla Embed implementation that renders a simple live container with Avatar

package/lib/components/WaveForm/index.d.ts

@@ -11,6 +11,9 @@ export interface WaveFormOptions {
     onStopTalking?: () => void;
     features?: Partial<NapsterCompanionApiConfig["features"]>;
     onVolumeChange?: (volume: number) => void;
+    onScreenShareToggle?: () => void;
+    isScreenSharing?: boolean;
+    isScreenShareSupported?: boolean;
 }
 export type WaveFormController = BaseController<WaveFormOptions>;
 export declare function createWaveForm(mount: HTMLElement, options: WaveFormOptions): WaveFormController;

package/lib/constants/events.d.ts

@@ -46,5 +46,7 @@ export declare enum WebSocketMessageType {
 export declare enum DataChannelMessageType {
     SEND_MESSAGE = "send_message",
     CANCEL = "cancel",
-    SET_SETTINGS = "set_settings"
+    SET_SETTINGS = "set_settings",
+    START_VIDEO = "start_video",
+    STOP_VIDEO = "stop_video"
 }

package/lib/index.d.ts

@@ -11,11 +11,15 @@ declare class NapsterCompanionApiSdkVanilla implements NapsterCompanionApiSDK {
     private rootEl;
     private embed;
     private webrtcController;
+    private screenShareController;
     private faceTrackingController;
     private config;
     private isInitialized;
+    private unloadHandler;
     static getInstance(): NapsterCompanionApiSdkVanilla;
+    constructor();
     private renderApp;
+    private handleScreenShareToggle;
     private initializeWebRTC;
     private destroyWebRTC;
     private destroyInstance;