@mentra/sdk 2.1.26 → 2.1.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/dist/app/session/api-client.d.ts.map +1 -1
  2. package/dist/app/session/dashboard.d.ts +5 -8
  3. package/dist/app/session/dashboard.d.ts.map +1 -1
  4. package/dist/app/session/events.d.ts +5 -2
  5. package/dist/app/session/events.d.ts.map +1 -1
  6. package/dist/app/session/index.d.ts +62 -3
  7. package/dist/app/session/index.d.ts.map +1 -1
  8. package/dist/app/session/modules/audio.d.ts +33 -4
  9. package/dist/app/session/modules/audio.d.ts.map +1 -1
  10. package/dist/app/session/modules/camera-managed-extension.d.ts +2 -3
  11. package/dist/app/session/modules/camera-managed-extension.d.ts.map +1 -1
  12. package/dist/app/session/modules/camera.d.ts +5 -5
  13. package/dist/app/session/modules/camera.d.ts.map +1 -1
  14. package/dist/app/session/modules/led.d.ts +141 -0
  15. package/dist/app/session/modules/led.d.ts.map +1 -0
  16. package/dist/app/session/modules/location.d.ts +1 -2
  17. package/dist/app/session/modules/location.d.ts.map +1 -1
  18. package/dist/app/session/modules/simple-storage.d.ts.map +1 -1
  19. package/dist/constants/log-messages/color.d.ts +5 -0
  20. package/dist/constants/log-messages/color.d.ts.map +1 -0
  21. package/dist/constants/log-messages/logos.d.ts +4 -0
  22. package/dist/constants/log-messages/logos.d.ts.map +1 -0
  23. package/dist/constants/{messages.d.ts → log-messages/updates.d.ts} +2 -3
  24. package/dist/constants/log-messages/updates.d.ts.map +1 -0
  25. package/dist/constants/log-messages/warning.d.ts +8 -0
  26. package/dist/constants/log-messages/warning.d.ts.map +1 -0
  27. package/dist/index.d.ts +7 -7
  28. package/dist/index.d.ts.map +1 -1
  29. package/dist/index.js +5347 -112
  30. package/dist/index.js.map +45 -0
  31. package/dist/logging/logger.d.ts +2 -1
  32. package/dist/logging/logger.d.ts.map +1 -1
  33. package/dist/types/capabilities.d.ts +3 -0
  34. package/dist/types/capabilities.d.ts.map +1 -1
  35. package/dist/types/index.d.ts +4 -14
  36. package/dist/types/index.d.ts.map +1 -1
  37. package/dist/types/message-types.d.ts +12 -3
  38. package/dist/types/message-types.d.ts.map +1 -1
  39. package/dist/types/messages/app-to-cloud.d.ts +48 -2
  40. package/dist/types/messages/app-to-cloud.d.ts.map +1 -1
  41. package/dist/types/messages/cloud-to-app.d.ts +25 -6
  42. package/dist/types/messages/cloud-to-app.d.ts.map +1 -1
  43. package/dist/types/messages/cloud-to-glasses.d.ts +43 -1
  44. package/dist/types/messages/cloud-to-glasses.d.ts.map +1 -1
  45. package/dist/types/messages/glasses-to-cloud.d.ts +32 -1
  46. package/dist/types/messages/glasses-to-cloud.d.ts.map +1 -1
  47. package/dist/types/rtmp-stream.d.ts +1 -1
  48. package/dist/types/rtmp-stream.d.ts.map +1 -1
  49. package/dist/types/streams.d.ts +28 -1
  50. package/dist/types/streams.d.ts.map +1 -1
  51. package/dist/utils/permissions-utils.d.ts +8 -0
  52. package/dist/utils/permissions-utils.d.ts.map +1 -0
  53. package/package.json +12 -3
  54. package/dist/app/index.js +0 -24
  55. package/dist/app/server/index.js +0 -658
  56. package/dist/app/session/api-client.js +0 -101
  57. package/dist/app/session/dashboard.js +0 -149
  58. package/dist/app/session/events.js +0 -305
  59. package/dist/app/session/index.js +0 -1571
  60. package/dist/app/session/layouts.js +0 -372
  61. package/dist/app/session/modules/audio.js +0 -321
  62. package/dist/app/session/modules/camera-managed-extension.js +0 -310
  63. package/dist/app/session/modules/camera.js +0 -603
  64. package/dist/app/session/modules/index.js +0 -19
  65. package/dist/app/session/modules/location.js +0 -58
  66. package/dist/app/session/modules/simple-storage.js +0 -232
  67. package/dist/app/session/settings.js +0 -358
  68. package/dist/app/token/index.js +0 -22
  69. package/dist/app/token/utils.js +0 -144
  70. package/dist/app/webview/index.js +0 -382
  71. package/dist/constants/index.js +0 -16
  72. package/dist/constants/messages.d.ts.map +0 -1
  73. package/dist/constants/messages.js +0 -57
  74. package/dist/examples/managed-rtmp-streaming-example.js +0 -158
  75. package/dist/examples/managed-rtmp-streaming-with-restream-example.js +0 -124
  76. package/dist/examples/rtmp-streaming-example.js +0 -102
  77. package/dist/logging/logger.js +0 -79
  78. package/dist/types/capabilities.js +0 -9
  79. package/dist/types/dashboard/index.js +0 -12
  80. package/dist/types/enums.js +0 -75
  81. package/dist/types/index.js +0 -101
  82. package/dist/types/layouts.js +0 -3
  83. package/dist/types/message-types.js +0 -207
  84. package/dist/types/messages/app-to-cloud.js +0 -95
  85. package/dist/types/messages/base.js +0 -3
  86. package/dist/types/messages/cloud-to-app.js +0 -78
  87. package/dist/types/messages/cloud-to-glasses.js +0 -68
  88. package/dist/types/messages/glasses-to-cloud.js +0 -139
  89. package/dist/types/models.js +0 -101
  90. package/dist/types/photo-data.js +0 -2
  91. package/dist/types/rtmp-stream.js +0 -3
  92. package/dist/types/streams.js +0 -306
  93. package/dist/types/token.js +0 -7
  94. package/dist/types/webhooks.js +0 -28
  95. package/dist/utils/animation-utils.js +0 -340
  96. package/dist/utils/bitmap-utils.js +0 -475
  97. package/dist/utils/resource-tracker.js +0 -153
@@ -1,1571 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.SimpleStorage = exports.AudioManager = exports.CameraModule = exports.TpaSession = exports.AppSession = void 0;
7
- /**
8
- * 🎯 App Session Module
9
- *
10
- * Manages an active Third Party App session with MentraOS Cloud.
11
- * Handles real-time communication, event subscriptions, and display management.
12
- */
13
- const ws_1 = require("ws");
14
- const events_1 = require("./events");
15
- const layouts_1 = require("./layouts");
16
- const settings_1 = require("./settings");
17
- const location_1 = require("./modules/location");
18
- const camera_1 = require("./modules/camera");
19
- const audio_1 = require("./modules/audio");
20
- const resource_tracker_1 = require("../../utils/resource-tracker");
21
- const types_1 = require("../../types");
22
- const axios_1 = __importDefault(require("axios"));
23
- const events_2 = __importDefault(require("events"));
24
- const node_fetch_1 = __importDefault(require("node-fetch"));
25
- // Import the cloud-to-app specific type guards
26
- const cloud_to_app_1 = require("../../types/messages/cloud-to-app");
27
- const simple_storage_1 = require("./modules/simple-storage");
28
- // List of event types that should never be subscribed to as streams
29
- const APP_TO_APP_EVENT_TYPES = [
30
- "app_message_received",
31
- "app_user_joined",
32
- "app_user_left",
33
- "app_room_updated",
34
- "app_direct_message_response",
35
- ];
36
- /**
37
- * 🚀 App Session Implementation
38
- *
39
- * Manages a live connection between your App and MentraOS Cloud.
40
- * Provides interfaces for:
41
- * - 🎮 Event handling (transcription, head position, etc.)
42
- * - 📱 Display management in AR view
43
- * - 🔌 Connection lifecycle
44
- * - 🔄 Automatic reconnection
45
- *
46
- * @example
47
- * ```typescript
48
- * const session = new AppSession({
49
- * packageName: 'org.example.myapp',
50
- * apiKey: 'your_api_key'
51
- * });
52
- *
53
- * // Handle events
54
- * session.onTranscription((data) => {
55
- * session.layouts.showTextWall(data.text);
56
- * });
57
- *
58
- * // Connect to cloud
59
- * await session.connect('session_123');
60
- * ```
61
- */
62
- class AppSession {
63
- constructor(config) {
64
- this.config = config;
65
- /** WebSocket connection to MentraOS Cloud */
66
- this.ws = null;
67
- /** Current session identifier */
68
- this.sessionId = null;
69
- /** Number of reconnection attempts made */
70
- this.reconnectAttempts = 0;
71
- /** Active event subscriptions */
72
- this.subscriptions = new Set();
73
- /** Map to store rate options for streams */
74
- this.streamRates = new Map();
75
- /** Resource tracker for automatic cleanup */
76
- this.resources = new resource_tracker_1.ResourceTracker();
77
- /** Internal settings storage - use public settings API instead */
78
- this.settingsData = [];
79
- /** App configuration loaded from app_config.json */
80
- this.appConfig = null;
81
- /** Whether to update subscriptions when settings change */
82
- this.shouldUpdateSubscriptionsOnSettingsChange = false;
83
- /** Settings that should trigger subscription updates when changed */
84
- this.subscriptionUpdateTriggers = [];
85
- /** Pending user discovery requests waiting for responses */
86
- this.pendingUserDiscoveryRequests = new Map();
87
- /** Pending direct message requests waiting for responses */
88
- this.pendingDirectMessages = new Map();
89
- /** 🔧 Device capabilities available for this session */
90
- this.capabilities = null;
91
- /** Dedicated emitter for App-to-App events */
92
- this.appEvents = new events_2.default();
93
- // Set defaults and merge with provided config
94
- this.config = {
95
- mentraOSWebsocketUrl: `ws://localhost:8002/app-ws`, // Use localhost as default
96
- autoReconnect: true, // Enable auto-reconnection by default for better reliability
97
- maxReconnectAttempts: 3, // Default to 3 reconnection attempts for better resilience
98
- reconnectDelay: 1000, // Start with 1 second delay (uses exponential backoff)
99
- ...config,
100
- };
101
- this.appServer = this.config.appServer;
102
- this.logger = this.appServer.logger.child({
103
- userId: this.config.userId,
104
- service: "app-session",
105
- });
106
- this.userId = this.config.userId;
107
- // Make sure the URL is correctly formatted to prevent double protocol issues
108
- if (this.config.mentraOSWebsocketUrl) {
109
- try {
110
- const url = new URL(this.config.mentraOSWebsocketUrl);
111
- if (!["ws:", "wss:"].includes(url.protocol)) {
112
- // Fix URLs with incorrect protocol (e.g., 'ws://http://host')
113
- const fixedUrl = this.config.mentraOSWebsocketUrl.replace(/^ws:\/\/http:\/\//, "ws://");
114
- this.config.mentraOSWebsocketUrl = fixedUrl;
115
- this.logger.warn(`⚠️ [${this.config.packageName}] Fixed malformed WebSocket URL: ${fixedUrl}`);
116
- }
117
- }
118
- catch (error) {
119
- this.logger.error(error, `⚠️ [${this.config.packageName}] Invalid WebSocket URL format: ${this.config.mentraOSWebsocketUrl}`);
120
- }
121
- }
122
- // Log initialization
123
- this.logger.debug(`🚀 [${this.config.packageName}] App Session initialized`);
124
- this.logger.debug(`🚀 [${this.config.packageName}] WebSocket URL: ${this.config.mentraOSWebsocketUrl}`);
125
- // Validate URL format - give early warning for obvious issues
126
- // Check URL format but handle undefined case
127
- if (this.config.mentraOSWebsocketUrl) {
128
- try {
129
- const url = new URL(this.config.mentraOSWebsocketUrl);
130
- if (!["ws:", "wss:"].includes(url.protocol)) {
131
- this.logger.error({ config: this.config }, `⚠️ [${this.config.packageName}] Invalid WebSocket URL protocol: ${url.protocol}. Should be ws: or wss:`);
132
- }
133
- }
134
- catch (error) {
135
- this.logger.error(error, `⚠️ [${this.config.packageName}] Invalid WebSocket URL format: ${this.config.mentraOSWebsocketUrl}`);
136
- }
137
- }
138
- this.events = new events_1.EventManager(this.subscribe.bind(this), this.unsubscribe.bind(this));
139
- this.layouts = new layouts_1.LayoutManager(config.packageName, this.send.bind(this));
140
- // Initialize settings manager with all necessary parameters, including subscribeFn for MentraOS settings
141
- this.settings = new settings_1.SettingsManager(this.settingsData, this.config.packageName, this.config.mentraOSWebsocketUrl, this.sessionId ?? undefined, async (streams) => {
142
- this.logger.debug({ streams: JSON.stringify(streams) }, `[AppSession] subscribeFn called for streams`);
143
- streams.forEach((stream) => {
144
- if (!this.subscriptions.has(stream)) {
145
- this.subscriptions.add(stream);
146
- this.logger.debug(`[AppSession] Auto-subscribed to stream '${stream}' for MentraOS setting.`);
147
- }
148
- else {
149
- this.logger.debug(`[AppSession] Already subscribed to stream '${stream}'.`);
150
- }
151
- });
152
- this.logger.debug({ subscriptions: JSON.stringify(Array.from(this.subscriptions)) }, `[AppSession] Current subscriptions after subscribeFn`);
153
- if (this.ws?.readyState === 1) {
154
- this.updateSubscriptions();
155
- this.logger.debug(`[AppSession] Sent updated subscriptions to cloud after auto-subscribing to MentraOS setting.`);
156
- }
157
- else {
158
- this.logger.debug(`[AppSession] WebSocket not open, will send subscriptions when connected.`);
159
- }
160
- });
161
- // Initialize dashboard API with this session instance
162
- // Import DashboardManager dynamically to avoid circular dependency
163
- const { DashboardManager } = require("./dashboard");
164
- this.dashboard = new DashboardManager(this, this.send.bind(this));
165
- // Initialize camera module with session reference
166
- this.camera = new camera_1.CameraModule(this.config.packageName, this.sessionId || "unknown-session-id", this.send.bind(this), this, // Pass session reference
167
- this.logger.child({ module: "camera" }));
168
- // Initialize audio module with session reference
169
- this.audio = new audio_1.AudioManager(this.config.packageName, this.sessionId || "unknown-session-id", this.send.bind(this), this, // Pass session reference
170
- this.logger.child({ module: "audio" }));
171
- this.simpleStorage = new simple_storage_1.SimpleStorage(this);
172
- this.location = new location_1.LocationManager(this, this.send.bind(this));
173
- }
174
- /**
175
- * Get the current session ID
176
- * @returns The current session ID or 'unknown-session-id' if not connected
177
- */
178
- getSessionId() {
179
- return this.sessionId || "unknown-session-id";
180
- }
181
- /**
182
- * Get the package name for this App
183
- * @returns The package name
184
- */
185
- getPackageName() {
186
- return this.config.packageName;
187
- }
188
- // =====================================
189
- // 🎮 Direct Event Handling Interface
190
- // =====================================
191
- /**
192
- * @deprecated Use session.events.onTranscription() instead
193
- */
194
- onTranscription(handler) {
195
- return this.events.onTranscription(handler);
196
- }
197
- /**
198
- * 🌐 Listen for speech transcription events in a specific language
199
- * @param language - Language code (e.g., "en-US")
200
- * @param handler - Function to handle transcription data
201
- * @returns Cleanup function to remove the handler
202
- * @throws Error if language code is invalid
203
- * @deprecated Use session.events.onTranscriptionForLanguage() instead
204
- */
205
- onTranscriptionForLanguage(language, handler, disableLanguageIdentification = false) {
206
- return this.events.onTranscriptionForLanguage(language, handler, disableLanguageIdentification);
207
- }
208
- /**
209
- * 🌐 Listen for speech translation events for a specific language pair
210
- * @param sourceLanguage - Source language code (e.g., "es-ES")
211
- * @param targetLanguage - Target language code (e.g., "en-US")
212
- * @param handler - Function to handle translation data
213
- * @returns Cleanup function to remove the handler
214
- * @throws Error if language codes are invalid
215
- * @deprecated Use session.events.onTranslationForLanguage() instead
216
- */
217
- onTranslationForLanguage(sourceLanguage, targetLanguage, handler) {
218
- return this.events.ontranslationForLanguage(sourceLanguage, targetLanguage, handler);
219
- }
220
- /**
221
- * 👤 Listen for head position changes
222
- * @param handler - Function to handle head position updates
223
- * @returns Cleanup function to remove the handler
224
- * @deprecated Use session.events.onHeadPosition() instead
225
- */
226
- onHeadPosition(handler) {
227
- return this.events.onHeadPosition(handler);
228
- }
229
- /**
230
- * 🔘 Listen for hardware button press events
231
- * @param handler - Function to handle button events
232
- * @returns Cleanup function to remove the handler
233
- * @deprecated Use session.events.onButtonPress() instead
234
- */
235
- onButtonPress(handler) {
236
- return this.events.onButtonPress(handler);
237
- }
238
- /**
239
- * 📱 Listen for phone notification events
240
- * @param handler - Function to handle notifications
241
- * @returns Cleanup function to remove the handler
242
- * @deprecated Use session.events.onPhoneNotifications() instead
243
- */
244
- onPhoneNotifications(handler) {
245
- return this.events.onPhoneNotifications(handler);
246
- }
247
- /**
248
- * 📱 Listen for phone notification dismissed events
249
- * @param handler - Function to handle notification dismissal data
250
- * @returns Cleanup function to remove the handler
251
- * @deprecated Use session.events.onPhoneNotificationDismissed() instead
252
- */
253
- onPhoneNotificationDismissed(handler) {
254
- return this.events.onPhoneNotificationDismissed(handler);
255
- }
256
- /**
257
- * 📡 Listen for VPS coordinates updates
258
- * @param handler - Function to handle VPS coordinates
259
- * @returns Cleanup function to remove the handler
260
- * @deprecated Use session.events.onVpsCoordinates() instead
261
- */
262
- onVpsCoordinates(handler) {
263
- this.subscribe(types_1.StreamType.VPS_COORDINATES);
264
- return this.events.onVpsCoordinates(handler);
265
- }
266
- /**
267
- * 📸 Listen for photo responses
268
- * @param handler - Function to handle photo response data
269
- * @returns Cleanup function to remove the handler
270
- * @deprecated Use session.events.onPhotoTaken() instead
271
- */
272
- onPhotoTaken(handler) {
273
- this.subscribe(types_1.StreamType.PHOTO_TAKEN);
274
- return this.events.onPhotoTaken(handler);
275
- }
276
- // =====================================
277
- // 📡 Pub/Sub Interface
278
- // =====================================
279
- /**
280
- * 📬 Subscribe to a specific event stream
281
- * @param sub - A string or a rich subscription object
282
- */
283
- subscribe(sub) {
284
- let type;
285
- let rate;
286
- if (typeof sub === "string") {
287
- type = sub;
288
- }
289
- else {
290
- // it's a LocationStreamRequest object
291
- type = sub.stream;
292
- rate = sub.rate;
293
- }
294
- if (APP_TO_APP_EVENT_TYPES.includes(type)) {
295
- this.logger.warn(`[AppSession] Attempted to subscribe to App-to-App event type '${type}', which is not a valid stream. Use the event handler (e.g., onAppMessage) instead.`);
296
- return;
297
- }
298
- this.subscriptions.add(type);
299
- if (rate) {
300
- this.streamRates.set(type, rate);
301
- }
302
- if (this.ws?.readyState === 1) {
303
- this.updateSubscriptions();
304
- }
305
- }
306
- /**
307
- * 📭 Unsubscribe from a specific event stream
308
- * @param sub - The subscription to remove
309
- */
310
- unsubscribe(sub) {
311
- let type;
312
- if (typeof sub === "string") {
313
- type = sub;
314
- }
315
- else {
316
- type = sub.stream;
317
- }
318
- if (APP_TO_APP_EVENT_TYPES.includes(type)) {
319
- this.logger.warn(`[AppSession] Attempted to unsubscribe from App-to-App event type '${type}', which is not a valid stream.`);
320
- return;
321
- }
322
- this.subscriptions.delete(type);
323
- this.streamRates.delete(type); // also remove from our rate map
324
- if (this.ws?.readyState === 1) {
325
- this.updateSubscriptions();
326
- }
327
- }
328
- /**
329
- * 🎯 Generic event listener (pub/sub style)
330
- * @param event - Event name to listen for
331
- * @param handler - Event handler function
332
- */
333
- on(event, handler) {
334
- return this.events.on(event, handler);
335
- }
336
- // =====================================
337
- // 🔌 Connection Management
338
- // =====================================
339
- /**
340
- * 🚀 Connect to MentraOS Cloud
341
- * @param sessionId - Unique session identifier
342
- * @returns Promise that resolves when connected
343
- */
344
- async connect(sessionId) {
345
- this.sessionId = sessionId;
346
- // Configure settings API client with the WebSocket URL and session ID
347
- // This allows settings to be fetched from the correct server
348
- this.settings.configureApiClient(this.config.packageName, this.config.mentraOSWebsocketUrl || "", sessionId);
349
- // Update the sessionId in the camera module
350
- if (this.camera) {
351
- this.camera.updateSessionId(sessionId);
352
- }
353
- // Update the sessionId in the audio module
354
- if (this.audio) {
355
- this.audio.updateSessionId(sessionId);
356
- }
357
- return new Promise((resolve, reject) => {
358
- try {
359
- // Clear previous resources if reconnecting
360
- if (this.ws) {
361
- // Don't call full dispose() as that would clear subscriptions
362
- if (this.ws.readyState !== 3) {
363
- // 3 = CLOSED
364
- this.ws.close();
365
- }
366
- this.ws = null;
367
- }
368
- // Validate WebSocket URL before attempting connection
369
- if (!this.config.mentraOSWebsocketUrl) {
370
- this.logger.error("WebSocket URL is missing or undefined");
371
- reject(new Error("WebSocket URL is required"));
372
- return;
373
- }
374
- // Add debug logging for connection attempts
375
- this.logger.info(`🔌🔌🔌 [${this.config.packageName}] Attempting to connect to: ${this.config.mentraOSWebsocketUrl} for session ${this.sessionId}`);
376
- // Create connection with error handling
377
- this.ws = new ws_1.WebSocket(this.config.mentraOSWebsocketUrl);
378
- // Track WebSocket for automatic cleanup
379
- this.resources.track(() => {
380
- if (this.ws && this.ws.readyState !== 3) {
381
- // 3 = CLOSED
382
- this.ws.close();
383
- }
384
- });
385
- this.ws.on("open", () => {
386
- try {
387
- this.sendConnectionInit();
388
- }
389
- catch (error) {
390
- this.logger.error(error, "Error during connection initialization");
391
- const errorMessage = error instanceof Error ? error.message : String(error);
392
- this.events.emit("error", new Error(`Connection initialization failed: ${errorMessage}`));
393
- reject(error);
394
- }
395
- });
396
- // Message handler with comprehensive error recovery
397
- const messageHandler = async (data, isBinary) => {
398
- try {
399
- // Handle binary messages (typically audio data)
400
- if (isBinary && Buffer.isBuffer(data)) {
401
- try {
402
- // Validate buffer before processing
403
- if (data.length === 0) {
404
- this.events.emit("error", new Error("Received empty binary data"));
405
- return;
406
- }
407
- // Convert Node.js Buffer to ArrayBuffer safely
408
- const arrayBuf = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
409
- // Create AUDIO_CHUNK event message with validation
410
- const audioChunk = {
411
- type: types_1.StreamType.AUDIO_CHUNK,
412
- arrayBuffer: arrayBuf,
413
- timestamp: new Date(), // Ensure timestamp is present
414
- };
415
- this.handleMessage(audioChunk);
416
- return;
417
- }
418
- catch (error) {
419
- this.logger.error(error, "Error processing binary message:");
420
- const errorMessage = error instanceof Error ? error.message : String(error);
421
- this.events.emit("error", new Error(`Failed to process binary message: ${errorMessage}`));
422
- return;
423
- }
424
- }
425
- // Handle ArrayBuffer data type directly
426
- if (data instanceof ArrayBuffer) {
427
- return;
428
- }
429
- // Handle JSON messages with validation
430
- try {
431
- // Convert string data to JSON safely
432
- let jsonData;
433
- if (typeof data === "string") {
434
- jsonData = data;
435
- }
436
- else if (Buffer.isBuffer(data)) {
437
- jsonData = data.toString("utf8");
438
- }
439
- else {
440
- throw new Error("Unknown message format");
441
- }
442
- // Validate JSON before parsing
443
- if (!jsonData || jsonData.trim() === "") {
444
- this.events.emit("error", new Error("Received empty JSON message"));
445
- return;
446
- }
447
- // Parse JSON with error handling
448
- const message = JSON.parse(jsonData);
449
- // Basic schema validation
450
- if (!message ||
451
- typeof message !== "object" ||
452
- !("type" in message)) {
453
- this.events.emit("error", new Error("Malformed message: missing type property"));
454
- return;
455
- }
456
- // Process the validated message
457
- this.handleMessage(message);
458
- }
459
- catch (error) {
460
- this.logger.error(error, "JSON parsing error");
461
- const errorMessage = error instanceof Error ? error.message : String(error);
462
- this.events.emit("error", new Error(`Failed to parse JSON message: ${errorMessage}`));
463
- }
464
- }
465
- catch (error) {
466
- // Final catch - should never reach here if individual handlers work correctly
467
- this.logger.error({ error }, "Unhandled message processing error");
468
- const errorMessage = error instanceof Error ? error.message : String(error);
469
- this.events.emit("error", new Error(`Unhandled message error: ${errorMessage}`));
470
- }
471
- };
472
- this.ws.on("message", messageHandler);
473
- // Track event handler removal for automatic cleanup
474
- this.resources.track(() => {
475
- if (this.ws) {
476
- this.ws.off("message", messageHandler);
477
- }
478
- });
479
- // Connection closure handler
480
- const closeHandler = (code, reason) => {
481
- const reasonStr = reason ? `: ${reason}` : "";
482
- const closeInfo = `Connection closed (code: ${code})${reasonStr}`;
483
- // Emit the disconnected event with structured data for better handling
484
- this.events.emit("disconnected", {
485
- message: closeInfo,
486
- code: code,
487
- reason: reason || "",
488
- wasClean: code === 1000 || code === 1001,
489
- });
490
- // Only attempt reconnection for abnormal closures
491
- // https://developer.mozilla.org/en-US/docs/Web/API/CloseEvent/code
492
- // 1000 (Normal Closure) and 1001 (Going Away) are normal
493
- // 1002-1015 are abnormal, and reason "App stopped" means intentional closure
494
- // 1008 usually when the userSession no longer exists on server. i.e user disconnected from cloud.
495
- const isNormalClosure = code === 1000 || code === 1001 || code === 1008;
496
- const isManualStop = reason && reason.includes("App stopped");
497
- const isUserSessionEnded = reason && reason.includes("User session ended");
498
- // Log closure details for diagnostics
499
- this.logger.debug(`🔌 [${this.config.packageName}] WebSocket closed with code ${code}${reasonStr}`);
500
- this.logger.debug(`🔌 [${this.config.packageName}] isNormalClosure: ${isNormalClosure}, isManualStop: ${isManualStop}, isUserSessionEnded: ${isUserSessionEnded}`);
501
- if (!isNormalClosure && !isManualStop) {
502
- this.logger.warn(`🔌 [${this.config.packageName}] Abnormal closure detected, attempting reconnection`);
503
- this.handleReconnection();
504
- }
505
- else {
506
- this.logger.debug(`🔌 [${this.config.packageName}] Normal closure detected, not attempting reconnection`);
507
- }
508
- // if user session ended, then trigger onStop.
509
- if (isUserSessionEnded) {
510
- this.logger.info(`🛑 [${this.config.packageName}] User session ended - emitting disconnected event with sessionEnded flag`);
511
- // Emit a disconnected event with a special flag to indicate session end
512
- // This will be caught by AppServer which will call the onStop callback
513
- const disconnectInfo = {
514
- message: "User session ended",
515
- code: 1000, // Normal closure
516
- reason: "User session ended",
517
- wasClean: true,
518
- permanent: true, // This is permanent - no reconnection
519
- sessionEnded: true, // Special flag to indicate session disposal
520
- };
521
- this.events.emit("disconnected", disconnectInfo);
522
- }
523
- };
524
- this.ws.on("close", closeHandler);
525
- // Track event handler removal
526
- this.resources.track(() => {
527
- if (this.ws) {
528
- this.ws.off("close", closeHandler);
529
- }
530
- });
531
- // Connection error handler
532
- const errorHandler = (error) => {
533
- this.logger.error(error, "WebSocket error");
534
- this.events.emit("error", error);
535
- };
536
- // Enhanced error handler with detailed logging
537
- this.ws.on("error", (error) => {
538
- this.logger.error(error, `⛔️⛔️⛔️ [${this.config.packageName}] WebSocket connection error: ${error.message}`);
539
- // Try to provide more context
540
- const errMsg = error.message || "";
541
- if (errMsg.includes("ECONNREFUSED")) {
542
- this.logger.error(`⛔️⛔️⛔️ [${this.config.packageName}] Connection refused - Check if the server is running at the specified URL`);
543
- }
544
- else if (errMsg.includes("ETIMEDOUT")) {
545
- this.logger.error(`⛔️⛔️⛔️ [${this.config.packageName}] Connection timed out - Check network connectivity and firewall rules`);
546
- }
547
- errorHandler(error);
548
- });
549
- // Track event handler removal
550
- this.resources.track(() => {
551
- if (this.ws) {
552
- this.ws.off("error", errorHandler);
553
- }
554
- });
555
- // Set up connection success handler
556
- const connectedCleanup = this.events.onConnected(() => resolve());
557
- // Track event handler removal
558
- this.resources.track(connectedCleanup);
559
- // Connection timeout with configurable duration
560
- const timeoutMs = 5000; // 5 seconds default
561
- const connectionTimeout = this.resources.setTimeout(() => {
562
- // Use tracked timeout that will be auto-cleared
563
- this.logger.error({
564
- config: this.config,
565
- sessionId: this.sessionId,
566
- timeoutMs,
567
- }, `⏱️⏱️⏱️ [${this.config.packageName}] Connection timeout after ${timeoutMs}ms`);
568
- this.events.emit("error", new Error(`Connection timeout after ${timeoutMs}ms`));
569
- reject(new Error("Connection timeout"));
570
- }, timeoutMs);
571
- // Clear timeout on successful connection
572
- const timeoutCleanup = this.events.onConnected(() => {
573
- clearTimeout(connectionTimeout);
574
- resolve();
575
- });
576
- // Track event handler removal
577
- this.resources.track(timeoutCleanup);
578
- }
579
- catch (error) {
580
- this.logger.error(error, "Connection setup error");
581
- const errorMessage = error instanceof Error ? error.message : String(error);
582
- reject(new Error(`Failed to setup connection: ${errorMessage}`));
583
- }
584
- });
585
- }
586
- /**
587
- * 👋 Disconnect from MentraOS Cloud
588
- */
589
- disconnect() {
590
- // Clean up camera module first
591
- if (this.camera) {
592
- this.camera.cancelAllRequests();
593
- }
594
- // Clean up audio module
595
- if (this.audio) {
596
- this.audio.cancelAllRequests();
597
- }
598
- // Use the resource tracker to clean up everything
599
- this.resources.dispose();
600
- // Clean up additional resources not handled by the tracker
601
- this.ws = null;
602
- this.sessionId = null;
603
- this.subscriptions.clear();
604
- this.reconnectAttempts = 0;
605
- }
606
- /**
607
- * 🛠️ Get all current user settings
608
- * @returns A copy of the current settings array
609
- * @deprecated Use session.settings.getAll() instead
610
- */
611
- getSettings() {
612
- return this.settings.getAll();
613
- }
614
- /**
615
- * 🔍 Get a specific setting value by key
616
- * @param key The setting key to look for
617
- * @returns The setting's value, or undefined if not found
618
- * @deprecated Use session.settings.get(key) instead
619
- */
620
- getSetting(key) {
621
- return this.settings.get(key);
622
- }
623
- /**
624
- * ⚙️ Configure settings-based subscription updates
625
- * This allows Apps to automatically update their subscriptions when certain settings change
626
- * @param options Configuration options for settings-based subscriptions
627
- */
628
- setSubscriptionSettings(options) {
629
- this.shouldUpdateSubscriptionsOnSettingsChange = true;
630
- this.subscriptionUpdateTriggers = options.updateOnChange;
631
- this.subscriptionSettingsHandler = options.handler;
632
- // If we already have settings, update subscriptions immediately
633
- if (this.settingsData.length > 0) {
634
- this.updateSubscriptionsFromSettings();
635
- }
636
- }
637
- /**
638
- * 🔄 Update subscriptions based on current settings
639
- * Called automatically when relevant settings change
640
- */
641
- updateSubscriptionsFromSettings() {
642
- if (!this.subscriptionSettingsHandler)
643
- return;
644
- try {
645
- // Get new subscriptions from handler
646
- const newSubscriptions = this.subscriptionSettingsHandler(this.settingsData);
647
- // Update all subscriptions at once
648
- this.subscriptions.clear();
649
- newSubscriptions.forEach((subscription) => {
650
- this.subscriptions.add(subscription);
651
- });
652
- // Send subscription update to cloud if connected
653
- if (this.ws && this.ws.readyState === 1) {
654
- this.updateSubscriptions();
655
- }
656
- }
657
- catch (error) {
658
- this.logger.error(error, "Error updating subscriptions from settings");
659
- const errorMessage = error instanceof Error ? error.message : String(error);
660
- this.events.emit("error", new Error(`Failed to update subscriptions: ${errorMessage}`));
661
- }
662
- }
663
- /**
664
- * 🧪 For testing: Update settings locally
665
- * In normal operation, settings come from the cloud
666
- * @param newSettings The new settings to apply
667
- */
668
- updateSettingsForTesting(newSettings) {
669
- this.settingsData = newSettings;
670
- // Update the settings manager with the new settings
671
- this.settings.updateSettings(newSettings);
672
- // Emit update event for backwards compatibility
673
- this.events.emit("settings_update", this.settingsData);
674
- // Check if we should update subscriptions
675
- if (this.shouldUpdateSubscriptionsOnSettingsChange) {
676
- this.updateSubscriptionsFromSettings();
677
- }
678
- }
679
- /**
680
- * 📝 Load configuration from a JSON file
681
- * @param jsonData JSON string containing App configuration
682
- * @returns The loaded configuration
683
- * @throws Error if the configuration is invalid
684
- */
685
- loadConfigFromJson(jsonData) {
686
- try {
687
- const parsedConfig = JSON.parse(jsonData);
688
- if ((0, types_1.validateAppConfig)(parsedConfig)) {
689
- this.appConfig = parsedConfig;
690
- return parsedConfig;
691
- }
692
- else {
693
- throw new Error("Invalid App configuration format");
694
- }
695
- }
696
- catch (error) {
697
- const errorMessage = error instanceof Error ? error.message : String(error);
698
- throw new Error(`Failed to load App configuration: ${errorMessage}`);
699
- }
700
- }
701
- /**
702
- * 📋 Get the loaded App configuration
703
- * @returns The current App configuration or null if not loaded
704
- */
705
- getConfig() {
706
- return this.appConfig;
707
- }
708
- /**
709
- * 🔌 Get the WebSocket server URL for this session
710
- * @returns The WebSocket server URL used by this session
711
- */
712
- getServerUrl() {
713
- return this.config.mentraOSWebsocketUrl;
714
- }
715
- getHttpsServerUrl() {
716
- if (!this.config.mentraOSWebsocketUrl) {
717
- return undefined;
718
- }
719
- return AppSession.convertToHttps(this.config.mentraOSWebsocketUrl);
720
- }
721
- static convertToHttps(rawUrl) {
722
- if (!rawUrl)
723
- return "";
724
- // Remove ws:// or wss://
725
- let url = rawUrl.replace(/^wss?:\/\//, "");
726
- // Remove trailing /app-ws
727
- url = url.replace(/\/app-ws$/, "");
728
- // Prepend https://
729
- return `https://${url}`;
730
- }
731
- /**
732
- * 🔍 Get default settings from the App configuration
733
- * @returns Array of settings with default values
734
- * @throws Error if configuration is not loaded
735
- */
736
- getDefaultSettings() {
737
- if (!this.appConfig) {
738
- throw new Error("App configuration not loaded. Call loadConfigFromJson first.");
739
- }
740
- return this.appConfig.settings
741
- .filter((s) => s.type !== "group")
742
- .map((s) => ({
743
- ...s,
744
- value: s.defaultValue, // Set value to defaultValue
745
- }));
746
- }
747
- /**
748
- * 🔍 Get setting schema from configuration
749
- * @param key Setting key to look up
750
- * @returns The setting schema or undefined if not found
751
- */
752
- getSettingSchema(key) {
753
- if (!this.appConfig)
754
- return undefined;
755
- const setting = this.appConfig.settings.find((s) => s.type !== "group" && "key" in s && s.key === key);
756
- return setting;
757
- }
758
- // =====================================
759
- // 🔧 Private Methods
760
- // =====================================
761
- /**
762
- * 📨 Handle incoming messages from cloud
763
- */
764
- handleMessage(message) {
765
- try {
766
- // Validate message before processing
767
- if (!this.validateMessage(message)) {
768
- this.events.emit("error", new Error("Invalid message format received"));
769
- return;
770
- }
771
- // Handle binary data (audio or video)
772
- if (message instanceof ArrayBuffer) {
773
- this.handleBinaryMessage(message);
774
- return;
775
- }
776
- // Using type guards to determine message type and safely handle each case
777
- try {
778
- if ((0, types_1.isAppConnectionAck)(message)) {
779
- // Get settings from connection acknowledgment
780
- const receivedSettings = message.settings || [];
781
- this.settingsData = receivedSettings;
782
- // Store config if provided
783
- if (message.config && (0, types_1.validateAppConfig)(message.config)) {
784
- this.appConfig = message.config;
785
- }
786
- // Use default settings from config if no settings were provided
787
- if (receivedSettings.length === 0 && this.appConfig) {
788
- try {
789
- this.settingsData = this.getDefaultSettings();
790
- }
791
- catch (error) {
792
- this.logger.warn(error, "Failed to load default settings from config:");
793
- }
794
- }
795
- // Update the settings manager with the new settings
796
- this.settings.updateSettings(this.settingsData);
797
- // Handle MentraOS system settings if provided
798
- this.logger.debug({ mentraosSettings: JSON.stringify(message.mentraosSettings) }, `[AppSession] CONNECTION_ACK mentraosSettings}`);
799
- if (message.mentraosSettings) {
800
- this.logger.info({ mentraosSettings: JSON.stringify(message.mentraosSettings) }, `[AppSession] Calling updatementraosSettings with`);
801
- this.settings.updateMentraosSettings(message.mentraosSettings);
802
- }
803
- else {
804
- this.logger.warn(`[AppSession] CONNECTION_ACK message missing mentraosSettings field`);
805
- }
806
- // Handle device capabilities if provided
807
- if (message.capabilities) {
808
- this.capabilities = message.capabilities;
809
- this.logger.info(`[AppSession] Device capabilities loaded for model: ${message.capabilities.modelName}`);
810
- }
811
- else {
812
- this.logger.debug(`[AppSession] No capabilities provided in CONNECTION_ACK`);
813
- }
814
- // Emit connected event with settings
815
- this.events.emit("connected", this.settingsData);
816
- // Update subscriptions (normal flow)
817
- this.updateSubscriptions();
818
- // If settings-based subscriptions are enabled, update those too
819
- if (this.shouldUpdateSubscriptionsOnSettingsChange &&
820
- this.settingsData.length > 0) {
821
- this.updateSubscriptionsFromSettings();
822
- }
823
- }
824
- else if ((0, types_1.isAppConnectionError)(message) ||
825
- message.type === "connection_error") {
826
- // Handle both App-specific connection_error and standard connection_error
827
- const errorMessage = message.message || "Unknown connection error";
828
- this.events.emit("error", new Error(errorMessage));
829
- }
830
- else if (message.type === types_1.StreamType.AUDIO_CHUNK) {
831
- if (this.subscriptions.has(types_1.StreamType.AUDIO_CHUNK)) {
832
- // Only process if we're subscribed to avoid unnecessary processing
833
- this.events.emit(types_1.StreamType.AUDIO_CHUNK, message);
834
- }
835
- }
836
- else if ((0, types_1.isDataStream)(message)) {
837
- // Ensure streamType exists before emitting the event
838
- const messageStreamType = message.streamType;
839
- // if (message.streamType === StreamType.TRANSCRIPTION) {
840
- // const transcriptionData = message.data as TranscriptionData;
841
- // if (transcriptionData.transcribeLanguage) {
842
- // messageStreamType = createTranscriptionStream(transcriptionData.transcribeLanguage) as ExtendedStreamType;
843
- // }
844
- // } else if (message.streamType === StreamType.TRANSLATION) {
845
- // const translationData = message.data as TranslationData;
846
- // if (translationData.transcribeLanguage && translationData.translateLanguage) {
847
- // messageStreamType = createTranslationStream(translationData.transcribeLanguage, translationData.translateLanguage) as ExtendedStreamType;
848
- // }
849
- // }
850
- if (messageStreamType && this.subscriptions.has(messageStreamType)) {
851
- const sanitizedData = this.sanitizeEventData(messageStreamType, message.data);
852
- this.events.emit(messageStreamType, sanitizedData);
853
- }
854
- }
855
- else if ((0, cloud_to_app_1.isRtmpStreamStatus)(message)) {
856
- // Emit as a standard stream event if subscribed
857
- if (this.subscriptions.has(types_1.StreamType.RTMP_STREAM_STATUS)) {
858
- this.events.emit(types_1.StreamType.RTMP_STREAM_STATUS, message);
859
- }
860
- // Update camera module's internal stream state
861
- this.camera.updateStreamState(message);
862
- }
863
- else if ((0, cloud_to_app_1.isManagedStreamStatus)(message)) {
864
- // Emit as a standard stream event if subscribed
865
- if (this.subscriptions.has(types_1.StreamType.MANAGED_STREAM_STATUS)) {
866
- this.events.emit(types_1.StreamType.MANAGED_STREAM_STATUS, message);
867
- }
868
- // Update camera module's managed stream state
869
- this.camera.handleManagedStreamStatus(message);
870
- }
871
- else if ((0, cloud_to_app_1.isStreamStatusCheckResponse)(message)) {
872
- // Handle stream status check response
873
- // This is a direct response, not a subscription-based event
874
- this.camera.handleStreamCheckResponse(message);
875
- }
876
- else if ((0, types_1.isSettingsUpdate)(message)) {
877
- // Store previous settings to check for changes
878
- const _prevSettings = [...this.settingsData];
879
- // Update internal settings storage
880
- this.settingsData = message.settings || [];
881
- // Update the settings manager with the new settings
882
- const changes = this.settings.updateSettings(this.settingsData);
883
- // Emit settings update event (for backwards compatibility)
884
- this.events.emit("settings_update", this.settingsData);
885
- // --- MentraOS settings update logic ---
886
- // If the message.settings looks like MentraOS settings (object with known keys), update mentraosSettings
887
- if (message.settings && typeof message.settings === "object") {
888
- this.settings.updateMentraosSettings(message.settings);
889
- }
890
- // Check if we should update subscriptions
891
- if (this.shouldUpdateSubscriptionsOnSettingsChange) {
892
- // Check if any subscription trigger settings changed
893
- const shouldUpdateSubs = this.subscriptionUpdateTriggers.some((key) => {
894
- return key in changes;
895
- });
896
- if (shouldUpdateSubs) {
897
- this.updateSubscriptionsFromSettings();
898
- }
899
- }
900
- }
901
- else if ((0, types_1.isCapabilitiesUpdate)(message)) {
902
- // Update device capabilities
903
- const capabilitiesMessage = message;
904
- this.capabilities = capabilitiesMessage.capabilities;
905
- this.logger.info(capabilitiesMessage.capabilities, `[AppSession] Capabilities updated for model: ${capabilitiesMessage.modelName}`);
906
- // Emit capabilities update event for applications to handle
907
- this.events.emit("capabilities_update", {
908
- capabilities: capabilitiesMessage.capabilities,
909
- modelName: capabilitiesMessage.modelName,
910
- timestamp: capabilitiesMessage.timestamp,
911
- });
912
- }
913
- else if ((0, types_1.isAppStopped)(message)) {
914
- const reason = message.reason || "unknown";
915
- const displayReason = `App stopped: ${reason}`;
916
- // Don't emit disconnected event here - let the WebSocket close handler do it
917
- // This prevents duplicate disconnected events when the session is disposed
918
- this.logger.info(`📤 [${this.config.packageName}] Received APP_STOPPED message: ${displayReason}`);
919
- // Clear reconnection state
920
- this.reconnectAttempts = 0;
921
- }
922
- // Handle dashboard mode changes
923
- else if ((0, types_1.isDashboardModeChanged)(message)) {
924
- try {
925
- // Use proper type
926
- const mode = message.mode || "none";
927
- // Update dashboard state in the API
928
- if (this.dashboard && "content" in this.dashboard) {
929
- this.dashboard.content.setCurrentMode(mode);
930
- }
931
- }
932
- catch (error) {
933
- this.logger.error(error, "Error handling dashboard mode change");
934
- }
935
- }
936
- // Handle always-on dashboard state changes
937
- else if ((0, types_1.isDashboardAlwaysOnChanged)(message)) {
938
- try {
939
- // Use proper type
940
- const enabled = !!message.enabled;
941
- // Update dashboard state in the API
942
- if (this.dashboard && "content" in this.dashboard) {
943
- this.dashboard.content.setAlwaysOnEnabled(enabled);
944
- }
945
- }
946
- catch (error) {
947
- this.logger.error(error, "Error handling dashboard always-on change");
948
- }
949
- }
950
- // Handle custom messages
951
- else if (message.type === types_1.CloudToAppMessageType.CUSTOM_MESSAGE) {
952
- this.events.emit("custom_message", message);
953
- return;
954
- }
955
- // Handle App-to-App communication messages
956
- else if (message.type === "app_message_received") {
957
- this.appEvents.emit("app_message_received", message);
958
- }
959
- else if (message.type === "app_user_joined") {
960
- this.appEvents.emit("app_user_joined", message);
961
- }
962
- else if (message.type === "app_user_left") {
963
- this.appEvents.emit("app_user_left", message);
964
- }
965
- else if (message.type === "app_room_updated") {
966
- this.appEvents.emit("app_room_updated", message);
967
- }
968
- else if (message.type === "app_direct_message_response") {
969
- const response = message;
970
- if (response.messageId &&
971
- this.pendingDirectMessages.has(response.messageId)) {
972
- const { resolve } = this.pendingDirectMessages.get(response.messageId);
973
- resolve(response.success);
974
- this.pendingDirectMessages.delete(response.messageId);
975
- }
976
- }
977
- else if (message.type === "augmentos_settings_update") {
978
- const mentraosMsg = message;
979
- if (mentraosMsg.settings &&
980
- typeof mentraosMsg.settings === "object") {
981
- this.settings.updateMentraosSettings(mentraosMsg.settings);
982
- }
983
- }
984
- // Handle 'connection_error' as a specific case if cloud sends this string literal
985
- else if (message.type === "connection_error") {
986
- // Treat 'connection_error' (string literal) like AppConnectionError
987
- // This handles cases where the cloud might send the type as a direct string
988
- // instead of the enum's 'tpa_connection_error' value.
989
- const errorMessage = message.message ||
990
- "Unknown connection error (type: connection_error)";
991
- this.logger.warn(`Received 'connection_error' type directly. Consider aligning cloud to send 'tpa_connection_error'. Message: ${errorMessage}`);
992
- this.events.emit("error", new Error(errorMessage));
993
- }
994
- else if (message.type === "permission_error") {
995
- // Handle permission errors from cloud
996
- this.logger.warn({
997
- message: message.message,
998
- details: message.details,
999
- detailsCount: message.details?.length || 0,
1000
- rejectedStreams: message.details?.map((d) => d.stream) || [],
1001
- }, "Permission error received:");
1002
- // Emit permission error event for application handling
1003
- this.events.emit("permission_error", {
1004
- message: message.message,
1005
- details: message.details,
1006
- timestamp: message.timestamp,
1007
- });
1008
- // Optionally emit individual permission denied events for each stream
1009
- message.details?.forEach((detail) => {
1010
- this.events.emit("permission_denied", {
1011
- stream: detail.stream,
1012
- requiredPermission: detail.requiredPermission,
1013
- message: detail.message,
1014
- });
1015
- });
1016
- }
1017
- else if ((0, types_1.isAudioPlayResponse)(message)) {
1018
- // Delegate audio play response handling to the audio module
1019
- if (this.audio) {
1020
- this.audio.handleAudioPlayResponse(message);
1021
- }
1022
- }
1023
- else if ((0, cloud_to_app_1.isPhotoResponse)(message)) {
1024
- // Legacy photo response handling - now photos come directly via webhook
1025
- // This branch can be removed in the future as all photos now go through /photo-upload
1026
- this.logger.warn({ message }, "Received legacy photo response - photos should now come via /photo-upload webhook");
1027
- }
1028
- // Handle unrecognized message types gracefully
1029
- else {
1030
- this.logger.warn(`Unrecognized message type: ${message.type}`);
1031
- this.events.emit("error", new Error(`Unrecognized message type: ${message.type}`));
1032
- }
1033
- }
1034
- catch (processingError) {
1035
- // Catch any errors during message processing to prevent App crashes
1036
- this.logger.error(processingError, "Error processing message:");
1037
- const errorMessage = processingError instanceof Error
1038
- ? processingError.message
1039
- : String(processingError);
1040
- this.events.emit("error", new Error(`Error processing message: ${errorMessage}`));
1041
- }
1042
- }
1043
- catch (error) {
1044
- // Final safety net to ensure the App doesn't crash on any unexpected errors
1045
- this.logger.error(error, "Unexpected error in message handler");
1046
- const errorMessage = error instanceof Error ? error.message : String(error);
1047
- this.events.emit("error", new Error(`Unexpected error in message handler: ${errorMessage}`));
1048
- }
1049
- }
1050
- /**
1051
- * 🧪 Validate incoming message structure
1052
- * @param message - Message to validate
1053
- * @returns boolean indicating if the message is valid
1054
- */
1055
- validateMessage(message) {
1056
- // Handle ArrayBuffer case separately
1057
- if (message instanceof ArrayBuffer) {
1058
- return true; // ArrayBuffers are always considered valid at this level
1059
- }
1060
- // Check if message is null or undefined
1061
- if (!message) {
1062
- return false;
1063
- }
1064
- // Check if message has a type property
1065
- if (!("type" in message)) {
1066
- return false;
1067
- }
1068
- // All other message types should be objects with a type property
1069
- return true;
1070
- }
1071
- /**
1072
- * 📦 Handle binary message data (audio or video)
1073
- * @param buffer - Binary data as ArrayBuffer
1074
- */
1075
- handleBinaryMessage(buffer) {
1076
- try {
1077
- // Safety check - only process if we're subscribed to avoid unnecessary work
1078
- if (!this.subscriptions.has(types_1.StreamType.AUDIO_CHUNK)) {
1079
- return;
1080
- }
1081
- // Validate buffer has content before processing
1082
- if (!buffer || buffer.byteLength === 0) {
1083
- this.events.emit("error", new Error("Received empty binary message"));
1084
- return;
1085
- }
1086
- // Create a safety wrapped audio chunk with proper defaults
1087
- const audioChunk = {
1088
- type: types_1.StreamType.AUDIO_CHUNK,
1089
- timestamp: new Date(),
1090
- arrayBuffer: buffer,
1091
- sampleRate: 16000, // Default sample rate
1092
- };
1093
- // Emit to subscribers
1094
- this.events.emit(types_1.StreamType.AUDIO_CHUNK, audioChunk);
1095
- }
1096
- catch (error) {
1097
- this.logger.error(error, "Error processing binary message");
1098
- const errorMessage = error instanceof Error ? error.message : String(error);
1099
- this.events.emit("error", new Error(`Error processing binary message: ${errorMessage}`));
1100
- }
1101
- }
1102
- /**
1103
- * 🧹 Sanitize event data to prevent crashes from malformed data
1104
- * @param streamType - The type of stream data
1105
- * @param data - The potentially unsafe data to sanitize
1106
- * @returns Sanitized data safe for processing
1107
- */
1108
- sanitizeEventData(streamType, data) {
1109
- try {
1110
- // If data is null or undefined, return an empty object to prevent crashes
1111
- if (data === null || data === undefined) {
1112
- return {};
1113
- }
1114
- // For specific stream types, perform targeted sanitization
1115
- switch (streamType) {
1116
- case types_1.StreamType.TRANSCRIPTION: {
1117
- // Ensure text field exists and is a string
1118
- if (typeof data.text !== "string") {
1119
- return {
1120
- text: "",
1121
- isFinal: true,
1122
- startTime: Date.now(),
1123
- endTime: Date.now(),
1124
- };
1125
- }
1126
- break;
1127
- }
1128
- case types_1.StreamType.HEAD_POSITION: {
1129
- // Ensure position data has required numeric fields
1130
- // Handle HeadPosition - Note the property position instead of x,y,z
1131
- const pos = data;
1132
- if (typeof pos?.position !== "string") {
1133
- return { position: "up", timestamp: new Date() };
1134
- }
1135
- break;
1136
- }
1137
- case types_1.StreamType.BUTTON_PRESS: {
1138
- // Ensure button type is valid
1139
- const btn = data;
1140
- if (!btn.buttonId || !btn.pressType) {
1141
- return {
1142
- buttonId: "unknown",
1143
- pressType: "short",
1144
- timestamp: new Date(),
1145
- };
1146
- }
1147
- break;
1148
- }
1149
- }
1150
- return data;
1151
- }
1152
- catch (error) {
1153
- this.logger.error(error, `Error sanitizing ${streamType} data`);
1154
- // Return a safe empty object if something goes wrong
1155
- return {};
1156
- }
1157
- }
1158
- /**
1159
- * 🔐 Send connection initialization message
1160
- */
1161
- sendConnectionInit() {
1162
- const message = {
1163
- type: types_1.AppToCloudMessageType.CONNECTION_INIT,
1164
- sessionId: this.sessionId,
1165
- packageName: this.config.packageName,
1166
- apiKey: this.config.apiKey,
1167
- timestamp: new Date(),
1168
- };
1169
- this.send(message);
1170
- }
1171
- /**
1172
- * 📝 Update subscription list with cloud
1173
- */
1174
- updateSubscriptions() {
1175
- this.logger.info({ subscriptions: JSON.stringify(Array.from(this.subscriptions)) }, `[AppSession] updateSubscriptions: sending subscriptions to cloud`);
1176
- // [MODIFIED] builds the array of SubscriptionRequest objects to send to the cloud
1177
- const subscriptionPayload = Array.from(this.subscriptions).map((stream) => {
1178
- const rate = this.streamRates.get(stream);
1179
- if (rate && stream === types_1.StreamType.LOCATION_STREAM) {
1180
- return { stream: "location_stream", rate: rate };
1181
- }
1182
- return stream;
1183
- });
1184
- const message = {
1185
- type: types_1.AppToCloudMessageType.SUBSCRIPTION_UPDATE,
1186
- packageName: this.config.packageName,
1187
- subscriptions: subscriptionPayload, // [MODIFIED]
1188
- sessionId: this.sessionId,
1189
- timestamp: new Date(),
1190
- };
1191
- this.send(message);
1192
- }
1193
- /**
1194
- * 🔄 Handle reconnection with exponential backoff
1195
- */
1196
- async handleReconnection() {
1197
- // Check if reconnection is allowed
1198
- if (!this.config.autoReconnect || !this.sessionId) {
1199
- this.logger.debug(`🔄 Reconnection skipped: autoReconnect=${this.config.autoReconnect}, sessionId=${this.sessionId ? "valid" : "invalid"}`);
1200
- return;
1201
- }
1202
- // Check if we've exceeded the maximum attempts
1203
- const maxAttempts = this.config.maxReconnectAttempts || 3;
1204
- if (this.reconnectAttempts >= maxAttempts) {
1205
- this.logger.info(`🔄 Maximum reconnection attempts (${maxAttempts}) reached, giving up`);
1206
- // Emit a permanent disconnection event to trigger onStop in the App server
1207
- this.events.emit("disconnected", {
1208
- message: `Connection permanently lost after ${maxAttempts} failed reconnection attempts`,
1209
- code: 4000, // Custom code for max reconnection attempts exhausted
1210
- reason: "Maximum reconnection attempts exceeded",
1211
- wasClean: false,
1212
- permanent: true, // Flag this as a permanent disconnection
1213
- });
1214
- return;
1215
- }
1216
- // Calculate delay with exponential backoff
1217
- const baseDelay = this.config.reconnectDelay || 1000;
1218
- const delay = baseDelay * Math.pow(2, this.reconnectAttempts);
1219
- this.reconnectAttempts++;
1220
- this.logger.debug(`🔄 [${this.config.packageName}] Reconnection attempt ${this.reconnectAttempts}/${maxAttempts} in ${delay}ms`);
1221
- // Use the resource tracker for the timeout
1222
- await new Promise((resolve) => {
1223
- this.resources.setTimeout(() => resolve(), delay);
1224
- });
1225
- try {
1226
- this.logger.debug(`🔄 [${this.config.packageName}] Attempting to reconnect...`);
1227
- await this.connect(this.sessionId);
1228
- this.logger.debug(`✅ [${this.config.packageName}] Reconnection successful!`);
1229
- this.reconnectAttempts = 0;
1230
- }
1231
- catch (error) {
1232
- const errorMessage = error instanceof Error ? error.message : String(error);
1233
- this.logger.error(error, `❌ [${this.config.packageName}] Reconnection failed for user ${this.userId}`);
1234
- this.events.emit("error", new Error(`Reconnection failed: ${errorMessage}`));
1235
- // Check if this was the last attempt
1236
- if (this.reconnectAttempts >= maxAttempts) {
1237
- this.logger.debug(`🔄 [${this.config.packageName}] Final reconnection attempt failed, emitting permanent disconnection`);
1238
- // Emit permanent disconnection event after the last failed attempt
1239
- this.events.emit("disconnected", {
1240
- message: `Connection permanently lost after ${maxAttempts} failed reconnection attempts`,
1241
- code: 4000, // Custom code for max reconnection attempts exhausted
1242
- reason: "Maximum reconnection attempts exceeded",
1243
- wasClean: false,
1244
- permanent: true, // Flag this as a permanent disconnection
1245
- });
1246
- }
1247
- }
1248
- }
1249
- /**
1250
- * 📤 Send message to cloud with validation and error handling
1251
- * @throws {Error} If WebSocket is not connected
1252
- */
1253
- send(message) {
1254
- try {
1255
- // Verify WebSocket connection is valid
1256
- if (!this.ws) {
1257
- throw new Error("WebSocket connection not established");
1258
- }
1259
- if (this.ws.readyState !== 1) {
1260
- const stateMap = {
1261
- 0: "CONNECTING",
1262
- 1: "OPEN",
1263
- 2: "CLOSING",
1264
- 3: "CLOSED",
1265
- };
1266
- const stateName = stateMap[this.ws.readyState] || "UNKNOWN";
1267
- throw new Error(`WebSocket not connected (current state: ${stateName})`);
1268
- }
1269
- // Validate message before sending
1270
- if (!message || typeof message !== "object") {
1271
- throw new Error("Invalid message: must be an object");
1272
- }
1273
- if (!("type" in message)) {
1274
- throw new Error('Invalid message: missing "type" property');
1275
- }
1276
- // Ensure message format is consistent
1277
- if (!("timestamp" in message) || !(message.timestamp instanceof Date)) {
1278
- message.timestamp = new Date();
1279
- }
1280
- // Try to send with error handling
1281
- try {
1282
- const serializedMessage = JSON.stringify(message);
1283
- this.ws.send(serializedMessage);
1284
- }
1285
- catch (sendError) {
1286
- const errorMessage = sendError instanceof Error ? sendError.message : String(sendError);
1287
- throw new Error(`Failed to send message: ${errorMessage}`);
1288
- }
1289
- }
1290
- catch (error) {
1291
- // Log the error and emit an event so App developers are aware
1292
- this.logger.error(error, "Message send error");
1293
- // Ensure we always emit an Error object
1294
- if (error instanceof Error) {
1295
- this.events.emit("error", error);
1296
- }
1297
- else {
1298
- this.events.emit("error", new Error(String(error)));
1299
- }
1300
- // Re-throw to maintain the original function behavior
1301
- throw error;
1302
- }
1303
- }
1304
- /**
1305
- * Fetch the onboarding instructions for this session from the backend.
1306
- * @returns Promise resolving to the instructions string or null
1307
- */
1308
- async getInstructions() {
1309
- try {
1310
- const baseUrl = this.getServerUrl();
1311
- const response = await axios_1.default.get(`${baseUrl}/api/instructions`, {
1312
- params: { userId: this.userId },
1313
- });
1314
- return response.data.instructions || null;
1315
- }
1316
- catch (err) {
1317
- this.logger.error(err, `Error fetching instructions from backend`);
1318
- return null;
1319
- }
1320
- }
1321
- // =====================================
1322
- // 👥 App-to-App Communication Interface
1323
- // =====================================
1324
- /**
1325
- * 👥 Discover other users currently using the same App
1326
- * @param includeProfiles - Whether to include user profile information
1327
- * @returns Promise that resolves with list of active users
1328
- */
1329
- async discoverAppUsers(domain, includeProfiles = false) {
1330
- // Use the domain argument as the base URL if provided
1331
- if (!domain) {
1332
- throw new Error("Domain (API base URL) is required for user discovery");
1333
- }
1334
- const url = `${domain}/api/app-communication/discover-users`;
1335
- // Use the user's core token for authentication
1336
- const appApiKey = this.config.apiKey; // This may need to be updated if you store the core token elsewhere
1337
- if (!appApiKey) {
1338
- throw new Error("Core token (apiKey) is required for user discovery");
1339
- }
1340
- const body = {
1341
- packageName: this.config.packageName,
1342
- userId: this.userId,
1343
- includeUserProfiles: includeProfiles,
1344
- };
1345
- const response = await (0, node_fetch_1.default)(url, {
1346
- method: "POST",
1347
- headers: {
1348
- Authorization: `Bearer ${appApiKey}`,
1349
- "Content-Type": "application/json",
1350
- },
1351
- body: JSON.stringify(body),
1352
- });
1353
- if (!response.ok) {
1354
- const errorText = await response.text();
1355
- throw new Error(`Failed to discover users: ${response.status} ${response.statusText} - ${errorText}`);
1356
- }
1357
- return await response.json();
1358
- }
1359
- /**
1360
- * 🔍 Check if a specific user is currently active
1361
- * @param userId - User ID to check for
1362
- * @returns Promise that resolves with boolean indicating if user is active
1363
- */
1364
- async isUserActive(userId) {
1365
- try {
1366
- const userList = await this.discoverAppUsers("", false);
1367
- return userList.users.some((user) => user.userId === userId);
1368
- }
1369
- catch (error) {
1370
- this.logger.error({ error, userId }, "Error checking if user is active");
1371
- return false;
1372
- }
1373
- }
1374
- /**
1375
- * 📊 Get user count for this App
1376
- * @returns Promise that resolves with number of active users
1377
- */
1378
- async getUserCount(domain) {
1379
- try {
1380
- const userList = await this.discoverAppUsers(domain, false);
1381
- return userList.totalUsers;
1382
- }
1383
- catch (error) {
1384
- this.logger.error(error, "Error getting user count");
1385
- return 0;
1386
- }
1387
- }
1388
- /**
1389
- * 📢 Send broadcast message to all users with same App active
1390
- * @param payload - Message payload to send
1391
- * @param roomId - Optional room ID for room-based messaging
1392
- * @returns Promise that resolves when message is sent
1393
- */
1394
- async broadcastToAppUsers(payload, _roomId) {
1395
- try {
1396
- const messageId = this.generateMessageId();
1397
- const message = {
1398
- type: "app_broadcast_message",
1399
- packageName: this.config.packageName,
1400
- sessionId: this.sessionId,
1401
- payload,
1402
- messageId,
1403
- senderUserId: this.userId,
1404
- timestamp: new Date(),
1405
- };
1406
- this.send(message);
1407
- }
1408
- catch (error) {
1409
- const errorMessage = error instanceof Error ? error.message : String(error);
1410
- throw new Error(`Failed to broadcast message: ${errorMessage}`);
1411
- }
1412
- }
1413
- /**
1414
- * 📤 Send direct message to specific user
1415
- * @param targetUserId - User ID to send message to
1416
- * @param payload - Message payload to send
1417
- * @returns Promise that resolves with success status
1418
- */
1419
- async sendDirectMessage(targetUserId, payload) {
1420
- return new Promise((resolve, reject) => {
1421
- try {
1422
- const messageId = this.generateMessageId();
1423
- // Store promise resolver
1424
- this.pendingDirectMessages.set(messageId, { resolve, reject });
1425
- const message = {
1426
- type: "app_direct_message",
1427
- packageName: this.config.packageName,
1428
- sessionId: this.sessionId,
1429
- targetUserId,
1430
- payload,
1431
- messageId,
1432
- senderUserId: this.userId,
1433
- timestamp: new Date(),
1434
- };
1435
- this.send(message);
1436
- // Set timeout to avoid hanging promises
1437
- const timeoutMs = 15000; // 15 seconds
1438
- this.resources.setTimeout(() => {
1439
- if (this.pendingDirectMessages.has(messageId)) {
1440
- this.pendingDirectMessages
1441
- .get(messageId)
1442
- .reject(new Error("Direct message timed out"));
1443
- this.pendingDirectMessages.delete(messageId);
1444
- }
1445
- }, timeoutMs);
1446
- }
1447
- catch (error) {
1448
- const errorMessage = error instanceof Error ? error.message : String(error);
1449
- reject(new Error(`Failed to send direct message: ${errorMessage}`));
1450
- }
1451
- });
1452
- }
1453
- /**
1454
- * 🏠 Join a communication room for group messaging
1455
- * @param roomId - Room ID to join
1456
- * @param roomConfig - Optional room configuration
1457
- * @returns Promise that resolves when room is joined
1458
- */
1459
- async joinAppRoom(roomId, roomConfig) {
1460
- try {
1461
- const message = {
1462
- type: "app_room_join",
1463
- packageName: this.config.packageName,
1464
- sessionId: this.sessionId,
1465
- roomId,
1466
- roomConfig,
1467
- timestamp: new Date(),
1468
- };
1469
- this.send(message);
1470
- }
1471
- catch (error) {
1472
- const errorMessage = error instanceof Error ? error.message : String(error);
1473
- throw new Error(`Failed to join room: ${errorMessage}`);
1474
- }
1475
- }
1476
- /**
1477
- * 🚪 Leave a communication room
1478
- * @param roomId - Room ID to leave
1479
- * @returns Promise that resolves when room is left
1480
- */
1481
- async leaveAppRoom(roomId) {
1482
- try {
1483
- const message = {
1484
- type: "app_room_leave",
1485
- packageName: this.config.packageName,
1486
- sessionId: this.sessionId,
1487
- roomId,
1488
- timestamp: new Date(),
1489
- };
1490
- this.send(message);
1491
- }
1492
- catch (error) {
1493
- const errorMessage = error instanceof Error ? error.message : String(error);
1494
- throw new Error(`Failed to leave room: ${errorMessage}`);
1495
- }
1496
- }
1497
- /**
1498
- * 📨 Listen for messages from other App users
1499
- * @param handler - Function to handle incoming messages
1500
- * @returns Cleanup function to remove the handler
1501
- */
1502
- onAppMessage(handler) {
1503
- this.appEvents.on("app_message_received", handler);
1504
- return () => this.appEvents.off("app_message_received", handler);
1505
- }
1506
- /**
1507
- * 👋 Listen for user join events
1508
- * @param handler - Function to handle user join events
1509
- * @returns Cleanup function to remove the handler
1510
- */
1511
- onAppUserJoined(handler) {
1512
- this.appEvents.on("app_user_joined", handler);
1513
- return () => this.appEvents.off("app_user_joined", handler);
1514
- }
1515
- /**
1516
- * 🚪 Listen for user leave events
1517
- * @param handler - Function to handle user leave events
1518
- * @returns Cleanup function to remove the handler
1519
- */
1520
- onAppUserLeft(handler) {
1521
- this.appEvents.on("app_user_left", handler);
1522
- return () => this.appEvents.off("app_user_left", handler);
1523
- }
1524
- /**
1525
- * 🏠 Listen for room update events
1526
- * @param handler - Function to handle room updates
1527
- * @returns Cleanup function to remove the handler
1528
- */
1529
- onAppRoomUpdated(handler) {
1530
- this.appEvents.on("app_room_updated", handler);
1531
- return () => this.appEvents.off("app_room_updated", handler);
1532
- }
1533
- /**
1534
- * 🔧 Generate unique message ID
1535
- * @returns Unique message identifier
1536
- */
1537
- generateMessageId() {
1538
- return `msg_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
1539
- }
1540
- }
1541
- exports.AppSession = AppSession;
1542
- /**
1543
- * @deprecated Use `AppSession` instead. `TpaSession` is deprecated and will be removed in a future version.
1544
- * This is an alias for backward compatibility only.
1545
- *
1546
- * @example
1547
- * ```typescript
1548
- * // ❌ Deprecated - Don't use this
1549
- * const session = new TpaSession(config);
1550
- *
1551
- * // ✅ Use this instead
1552
- * const session = new AppSession(config);
1553
- * ```
1554
- */
1555
- class TpaSession extends AppSession {
1556
- constructor(config) {
1557
- super(config);
1558
- // Emit a deprecation warning to help developers migrate
1559
- console.warn("⚠️ DEPRECATION WARNING: TpaSession is deprecated and will be removed in a future version. " +
1560
- "Please use AppSession instead. " +
1561
- 'Simply replace "TpaSession" with "AppSession" in your code.');
1562
- }
1563
- }
1564
- exports.TpaSession = TpaSession;
1565
- // Export module types for developers
1566
- var camera_2 = require("./modules/camera");
1567
- Object.defineProperty(exports, "CameraModule", { enumerable: true, get: function () { return camera_2.CameraModule; } });
1568
- var audio_2 = require("./modules/audio");
1569
- Object.defineProperty(exports, "AudioManager", { enumerable: true, get: function () { return audio_2.AudioManager; } });
1570
- var simple_storage_2 = require("./modules/simple-storage");
1571
- Object.defineProperty(exports, "SimpleStorage", { enumerable: true, get: function () { return simple_storage_2.SimpleStorage; } });