@mentra/sdk 2.1.1 → 2.1.2-alpha.1

package/dist/app/session/modules/audio.js

@@ -0,0 +1,317 @@
+"use strict";
+/**
+ * 🔊 Audio Module
+ *
+ * Audio functionality for App Sessions.
+ * Handles audio playback on connected glasses.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AudioManager = void 0;
+const types_1 = require("../../../types");
+/**
+ * 🔊 Audio Module Implementation
+ *
+ * Audio management for App Sessions.
+ * Provides methods for:
+ * - 🎵 Playing audio on glasses
+ * - ⏹️ Stopping audio playback
+ * - 🔍 Monitoring audio request status
+ * - 🧹 Cleanup and cancellation
+ *
+ * @example
+ * ```typescript
+ * // Play audio
+ * const result = await session.audio.playAudio({
+ *   audioUrl: 'https://example.com/sound.mp3',
+ *   volume: 0.8
+ * });
+ *
+ * // Stop all audio
+ * session.audio.stopAudio();
+ * ```
+ */
+class AudioManager {
+    /**
+     * Create a new AudioManager
+     *
+     * @param packageName - The App package name
+     * @param sessionId - The current session ID
+     * @param send - Function to send messages to the cloud
+     * @param session - Reference to the parent AppSession (optional)
+     * @param logger - Logger instance for debugging
+     */
+    constructor(packageName, sessionId, send, session, logger) {
+        /** Map to store pending audio play request promises */
+        this.pendingAudioRequests = new Map();
+        this.packageName = packageName;
+        this.sessionId = sessionId;
+        this.send = send;
+        this.session = session;
+        this.logger = logger || console;
+    }
+    // =====================================
+    // 🎵 Audio Playback Functionality
+    // =====================================
+    /**
+     * 🔊 Play audio on the connected glasses
+     * @param options - Audio playback configuration
+     * @returns Promise that resolves with playback result
+     *
+     * @example
+     * ```typescript
+     * // Play audio from URL
+     * const result = await session.audio.playAudio({
+     *   audioUrl: 'https://example.com/sound.mp3',
+     *   volume: 0.8
+     * });
+     * ```
+     */
+    async playAudio(options) {
+        return new Promise((resolve, reject) => {
+            try {
+                // Validate input
+                if (!options.audioUrl) {
+                    reject(new Error('audioUrl must be provided'));
+                    return;
+                }
+                // Generate unique request ID
+                const requestId = `audio_req_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
+                // Store promise resolvers for when we get the response
+                this.pendingAudioRequests.set(requestId, { resolve, reject });
+                // Create audio play request message
+                const message = {
+                    type: types_1.AppToCloudMessageType.AUDIO_PLAY_REQUEST,
+                    packageName: this.packageName,
+                    sessionId: this.sessionId,
+                    requestId,
+                    timestamp: new Date(),
+                    audioUrl: options.audioUrl,
+                    volume: options.volume ?? 1.0,
+                    stopOtherAudio: options.stopOtherAudio ?? true
+                };
+                // Send request to cloud
+                this.send(message);
+                // Set timeout to avoid hanging promises
+                const timeoutMs = 60000; // 60 seconds
+                if (this.session && this.session.resources) {
+                    // Use session's resource tracker for automatic cleanup
+                    this.session.resources.setTimeout(() => {
+                        if (this.pendingAudioRequests.has(requestId)) {
+                            this.pendingAudioRequests.get(requestId).reject(new Error('Audio play request timed out'));
+                            this.pendingAudioRequests.delete(requestId);
+                            this.logger.warn({ requestId }, `🔊 Audio play request timed out`);
+                        }
+                    }, timeoutMs);
+                }
+                else {
+                    // Fallback to regular setTimeout if session not available
+                    setTimeout(() => {
+                        if (this.pendingAudioRequests.has(requestId)) {
+                            this.pendingAudioRequests.get(requestId).reject(new Error('Audio play request timed out'));
+                            this.pendingAudioRequests.delete(requestId);
+                            this.logger.warn({ requestId }, `🔊 Audio play request timed out`);
+                        }
+                    }, timeoutMs);
+                }
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                reject(new Error(`Failed to play audio: ${errorMessage}`));
+            }
+        });
+    }
+    /**
+     * 🔇 Stop audio playback on the connected glasses
+     *
+     * @example
+     * ```typescript
+     * // Stop all currently playing audio
+     * session.audio.stopAudio();
+     * ```
+     */
+    stopAudio() {
+        try {
+            // Create audio stop request message
+            const message = {
+                type: types_1.AppToCloudMessageType.AUDIO_STOP_REQUEST,
+                packageName: this.packageName,
+                sessionId: this.sessionId,
+                timestamp: new Date()
+            };
+            // Send request to cloud (one-way, no response expected)
+            this.send(message);
+            this.logger.info(`🔇 Audio stop request sent`);
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            this.logger.error(`Failed to stop audio: ${errorMessage}`);
+        }
+    }
+    /**
+     * 🗣️ Convert text to speech and play it on the connected glasses
+     * @param text - Text to convert to speech (required)
+     * @param options - Text-to-speech configuration (optional)
+     * @returns Promise that resolves with playback result
+     *
+     * @example
+     * ```typescript
+     * // Basic text-to-speech
+     * const result = await session.audio.speak('Hello, world!');
+     *
+     * // With custom voice settings
+     * const result = await session.audio.speak('Hello, world!', {
+     *   voice_id: 'your_voice_id',
+     *   voice_settings: {
+     *     stability: 0.5,
+     *     speed: 1.2
+     *   },
+     *   volume: 0.8
+     * });
+     * ```
+     */
+    async speak(text, options = {}) {
+        // Validate input
+        if (!text) {
+            throw new Error('text must be provided');
+        }
+        // Get the HTTPS server URL from the session
+        const baseUrl = this.session?.getHttpsServerUrl?.();
+        if (!baseUrl) {
+            throw new Error('Cannot determine server URL for TTS endpoint');
+        }
+        // Build query parameters for the TTS endpoint
+        const queryParams = new URLSearchParams();
+        queryParams.append('text', text);
+        if (options.voice_id) {
+            queryParams.append('voice_id', options.voice_id);
+        }
+        if (options.model_id) {
+            queryParams.append('model_id', options.model_id);
+        }
+        if (options.voice_settings) {
+            queryParams.append('voice_settings', JSON.stringify(options.voice_settings));
+        }
+        // Construct the TTS URL
+        const ttsUrl = `${baseUrl}/api/tts?${queryParams.toString()}`;
+        this.logger.info({ text, ttsUrl }, `🗣️ Generating speech from text`);
+        // Use the existing playAudio method to play the TTS audio
+        return this.playAudio({
+            audioUrl: ttsUrl,
+            volume: options.volume
+        });
+    }
+    // =====================================
+    // 📥 Response Handling
+    // =====================================
+    /**
+     * 📥 Handle audio play response from cloud
+     *
+     * This method is called internally when an audio play response is received.
+     * It resolves the corresponding pending promise with the response data.
+     *
+     * @param response - The audio play response received
+     * @internal This method is used internally by AppSession
+     */
+    handleAudioPlayResponse(response) {
+        const pendingRequest = this.pendingAudioRequests.get(response.requestId);
+        if (pendingRequest) {
+            // Resolve the promise with the response data
+            pendingRequest.resolve({
+                success: response.success,
+                error: response.error,
+                duration: response.duration
+            });
+            // Clean up
+            this.pendingAudioRequests.delete(response.requestId);
+            this.logger.info({
+                requestId: response.requestId,
+                success: response.success,
+                duration: response.duration
+            }, `🔊 Audio play response received`);
+        }
+        else {
+            this.logger.warn({ requestId: response.requestId }, `🔊 Received audio play response for unknown request ID`);
+        }
+    }
+    // =====================================
+    // 🔍 Status and Management
+    // =====================================
+    /**
+     * 🔍 Check if there are pending audio requests
+     * @param requestId - Optional specific request ID to check
+     * @returns True if there are pending requests (or specific request exists)
+     */
+    hasPendingRequest(requestId) {
+        if (requestId) {
+            return this.pendingAudioRequests.has(requestId);
+        }
+        return this.pendingAudioRequests.size > 0;
+    }
+    /**
+     * 📊 Get the number of pending audio requests
+     * @returns Number of pending requests
+     */
+    getPendingRequestCount() {
+        return this.pendingAudioRequests.size;
+    }
+    /**
+     * 📋 Get all pending request IDs
+     * @returns Array of pending request IDs
+     */
+    getPendingRequestIds() {
+        return Array.from(this.pendingAudioRequests.keys());
+    }
+    /**
+     * ❌ Cancel a specific audio request
+     * @param requestId - The request ID to cancel
+     * @returns True if the request was found and cancelled
+     */
+    cancelAudioRequest(requestId) {
+        const pendingRequest = this.pendingAudioRequests.get(requestId);
+        if (pendingRequest) {
+            pendingRequest.reject(new Error('Audio request cancelled'));
+            this.pendingAudioRequests.delete(requestId);
+            this.logger.info({ requestId }, `🔊 Audio request cancelled`);
+            return true;
+        }
+        return false;
+    }
+    /**
+     * 🧹 Cancel all pending audio requests
+     * @returns Number of requests that were cancelled
+     */
+    cancelAllAudioRequests() {
+        const count = this.pendingAudioRequests.size;
+        this.pendingAudioRequests.forEach((request, requestId) => {
+            request.reject(new Error('Audio request cancelled due to cleanup'));
+            this.logger.debug({ requestId }, `🔊 Audio request cancelled during cleanup`);
+        });
+        this.pendingAudioRequests.clear();
+        if (count > 0) {
+            this.logger.info({ cancelledCount: count }, `🧹 Cancelled all pending audio requests`);
+        }
+        return count;
+    }
+    // =====================================
+    // 🔧 Internal Management
+    // =====================================
+    /**
+     * 🔄 Update the session ID when reconnecting
+     * @param newSessionId - The new session ID
+     * @internal Used by AppSession during reconnection
+     */
+    updateSessionId(newSessionId) {
+        this.sessionId = newSessionId;
+        this.logger.debug({ newSessionId }, `🔄 Audio module session ID updated`);
+    }
+    /**
+     * 🧹 Cancel all pending requests (cleanup)
+     * @returns Object with count of cancelled requests
+     * @internal Used by AppSession during cleanup
+     */
+    cancelAllRequests() {
+        const audioRequests = this.cancelAllAudioRequests();
+        return { audioRequests };
+    }
+}
+exports.AudioManager = AudioManager;

package/dist/app/session/modules/camera-managed-extension.d.ts

@@ -0,0 +1,154 @@
+/**
+ * 📷 Camera Module Managed Streaming Extension
+ *
+ * Extends the camera module with managed streaming capabilities.
+ * Apps can request managed streams and receive HLS/DASH URLs without managing RTMP endpoints.
+ */
+import { ManagedStreamStatus, RestreamDestination } from '../../../types';
+import { VideoConfig, AudioConfig, StreamConfig } from '../../../types/rtmp-stream';
+import { Logger } from 'pino';
+/**
+ * Configuration options for a managed stream
+ */
+export interface ManagedStreamOptions {
+    /** Stream quality preset */
+    quality?: '720p' | '1080p';
+    /** Enable WebRTC for ultra-low latency viewing */
+    enableWebRTC?: boolean;
+    /** Optional video configuration settings */
+    video?: VideoConfig;
+    /** Optional audio configuration settings */
+    audio?: AudioConfig;
+    /** Optional stream configuration settings */
+    stream?: StreamConfig;
+    /** Optional RTMP destinations to re-stream to (YouTube, Twitch, etc) */
+    restreamDestinations?: RestreamDestination[];
+}
+/**
+ * Result returned when starting a managed stream
+ */
+export interface ManagedStreamResult {
+    /** HLS URL for viewing the stream */
+    hlsUrl: string;
+    /** DASH URL for viewing the stream */
+    dashUrl: string;
+    /** WebRTC URL if enabled */
+    webrtcUrl?: string;
+    /** Internal stream ID */
+    streamId: string;
+}
+/**
+ * 📹 Managed Streaming Extension for Camera Module
+ *
+ * Provides managed streaming capabilities where the cloud handles
+ * RTMP endpoints and returns HLS/DASH URLs for viewing.
+ *
+ * @example
+ * ```typescript
+ * // Start a managed stream
+ * const urls = await session.camera.startManagedStream({
+ *   quality: '720p',
+ *   enableWebRTC: true
+ * });
+ * console.log('HLS URL:', urls.hlsUrl);
+ * console.log('DASH URL:', urls.dashUrl);
+ *
+ * // Monitor managed stream status
+ * session.camera.onManagedStreamStatus((status) => {
+ *   console.log('Managed stream status:', status.status);
+ * });
+ *
+ * // Stop managed stream
+ * await session.camera.stopManagedStream();
+ * ```
+ */
+export declare class CameraManagedExtension {
+    private send;
+    private packageName;
+    private sessionId;
+    private logger;
+    private session?;
+    private isManagedStreaming;
+    private currentManagedStreamId?;
+    private currentManagedStreamUrls?;
+    private managedStreamStatus?;
+    private pendingManagedStreamRequest?;
+    constructor(packageName: string, sessionId: string, send: (message: any) => void, logger: Logger, session?: any);
+    /**
+     * 📹 Start a managed stream
+     *
+     * The cloud will handle the RTMP endpoint and return HLS/DASH URLs for viewing.
+     * Multiple apps can consume the same managed stream simultaneously.
+     *
+     * @param options - Configuration options for the managed stream
+     * @returns Promise that resolves with viewing URLs when the stream is ready
+     *
+     * @example
+     * ```typescript
+     * const urls = await session.camera.startManagedStream({
+     *   quality: '1080p',
+     *   enableWebRTC: true,
+     *   video: { fps: 30 },
+     *   audio: { sampleRate: 48000 }
+     * });
+     * ```
+     */
+    startManagedStream(options?: ManagedStreamOptions): Promise<ManagedStreamResult>;
+    /**
+     * 🛑 Stop the current managed stream
+     *
+     * This will stop streaming for this app only. If other apps are consuming
+     * the same managed stream, it will continue for them.
+     *
+     * @returns Promise that resolves when the stop request is sent
+     */
+    stopManagedStream(): Promise<void>;
+    /**
+     * 📊 Check if currently managed streaming
+     *
+     * @returns true if a managed stream is active
+     */
+    isManagedStreamActive(): boolean;
+    /**
+     * 🔗 Get current managed stream URLs
+     *
+     * @returns Current stream URLs or undefined if not streaming
+     */
+    getManagedStreamUrls(): ManagedStreamResult | undefined;
+    /**
+     * 📊 Get current managed stream status
+     *
+     * @returns Current stream status or undefined
+     */
+    getManagedStreamStatus(): ManagedStreamStatus | undefined;
+    /**
+     * 🔔 Register a handler for managed stream status updates
+     *
+     * @param handler - Function to call when stream status changes
+     * @returns Cleanup function to unregister the handler
+     *
+     * @example
+     * ```typescript
+     * const cleanup = session.camera.onManagedStreamStatus((status) => {
+     *   console.log('Status:', status.status);
+     *   if (status.status === 'active') {
+     *     console.log('Stream is live!');
+     *   }
+     * });
+     *
+     * // Later, unregister the handler
+     * cleanup();
+     * ```
+     */
+    onManagedStreamStatus(handler: (status: ManagedStreamStatus) => void): () => void;
+    /**
+     * Handle incoming managed stream status messages
+     * Called by the parent AppSession when messages are received
+     */
+    handleManagedStreamStatus(status: ManagedStreamStatus): void;
+    /**
+     * 🧹 Clean up all managed streaming state
+     */
+    cleanup(): void;
+}
+//# sourceMappingURL=camera-managed-extension.d.ts.map

package/dist/app/session/modules/camera-managed-extension.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"camera-managed-extension.d.ts","sourceRoot":"","sources":["../../../../src/app/session/modules/camera-managed-extension.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAGL,mBAAmB,EAKnB,mBAAmB,EACpB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AACpF,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAE9B;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3B,kDAAkD;IAClD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,4CAA4C;IAC5C,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,4CAA4C;IAC5C,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,6CAA6C;IAC7C,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,wEAAwE;IACxE,oBAAoB,CAAC,EAAE,mBAAmB,EAAE,CAAC;CAC9C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,IAAI,CAAyB;IACrC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAC,CAAM;IAGtB,OAAO,CAAC,kBAAkB,CAAkB;IAC5C,OAAO,CAAC,sBAAsB,CAAC,CAAS;IACxC,OAAO,CAAC,wBAAwB,CAAC,CAAsB;IACvD,OAAO,CAAC,mBAAmB,CAAC,CAAsB;IAGlD,OAAO,CAAC,2BAA2B,CAAC,CAGlC;gBAGA,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,IAAI,EAC5B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,GAAG;IASf;;;;;;;;;;;;;;;;;;OAkBG;IACG,kBAAkB,CAAC,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAyC1F;;;;;;;OAOG;IACG,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IAmBxC;;;;OAIG;IACH,qBAAqB,IAAI,OAAO;IAIhC;;;;OAIG;IACH,oBAAoB,IAAI,mBAAmB,GAAG,SAAS;IAIvD;;;;OAIG;IACH,sBAAsB,IAAI,mBAAmB,GAAG,SAAS;IAIzD;;;;;;;;;;;;;;;;;;OAkBG;IACH,qBAAqB,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,mBAAmB,KAAK,IAAI,GAAG,MAAM,IAAI;IAYjF;;;OAGG;IACH,yBAAyB,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAwD5D;;OAEG;IACH,OAAO,IAAI,IAAI;CAahB"}