@mentra/sdk 2.1.27 → 2.1.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/dist/app/session/api-client.d.ts.map +1 -1
  2. package/dist/app/session/dashboard.d.ts +5 -8
  3. package/dist/app/session/dashboard.d.ts.map +1 -1
  4. package/dist/app/session/events.d.ts +2 -1
  5. package/dist/app/session/events.d.ts.map +1 -1
  6. package/dist/app/session/index.d.ts +62 -3
  7. package/dist/app/session/index.d.ts.map +1 -1
  8. package/dist/app/session/modules/audio.d.ts +33 -4
  9. package/dist/app/session/modules/audio.d.ts.map +1 -1
  10. package/dist/app/session/modules/camera-managed-extension.d.ts +2 -3
  11. package/dist/app/session/modules/camera-managed-extension.d.ts.map +1 -1
  12. package/dist/app/session/modules/camera.d.ts +5 -5
  13. package/dist/app/session/modules/camera.d.ts.map +1 -1
  14. package/dist/app/session/modules/led.d.ts +141 -0
  15. package/dist/app/session/modules/led.d.ts.map +1 -0
  16. package/dist/app/session/modules/location.d.ts +1 -2
  17. package/dist/app/session/modules/location.d.ts.map +1 -1
  18. package/dist/app/session/modules/simple-storage.d.ts.map +1 -1
  19. package/dist/index.d.ts +7 -7
  20. package/dist/index.d.ts.map +1 -1
  21. package/dist/index.js +5347 -112
  22. package/dist/index.js.map +45 -0
  23. package/dist/logging/logger.d.ts +1 -1
  24. package/dist/logging/logger.d.ts.map +1 -1
  25. package/dist/types/capabilities.d.ts +3 -0
  26. package/dist/types/capabilities.d.ts.map +1 -1
  27. package/dist/types/index.d.ts +4 -14
  28. package/dist/types/index.d.ts.map +1 -1
  29. package/dist/types/message-types.d.ts +8 -1
  30. package/dist/types/message-types.d.ts.map +1 -1
  31. package/dist/types/messages/app-to-cloud.d.ts +48 -2
  32. package/dist/types/messages/app-to-cloud.d.ts.map +1 -1
  33. package/dist/types/messages/cloud-to-app.d.ts +16 -6
  34. package/dist/types/messages/cloud-to-app.d.ts.map +1 -1
  35. package/dist/types/messages/cloud-to-glasses.d.ts +29 -1
  36. package/dist/types/messages/cloud-to-glasses.d.ts.map +1 -1
  37. package/dist/types/messages/glasses-to-cloud.d.ts +24 -1
  38. package/dist/types/messages/glasses-to-cloud.d.ts.map +1 -1
  39. package/dist/types/rtmp-stream.d.ts +1 -1
  40. package/dist/types/rtmp-stream.d.ts.map +1 -1
  41. package/dist/types/streams.d.ts +28 -1
  42. package/dist/types/streams.d.ts.map +1 -1
  43. package/package.json +9 -3
  44. package/dist/app/index.js +0 -24
  45. package/dist/app/server/index.js +0 -658
  46. package/dist/app/session/api-client.js +0 -101
  47. package/dist/app/session/dashboard.js +0 -149
  48. package/dist/app/session/events.js +0 -315
  49. package/dist/app/session/index.js +0 -1573
  50. package/dist/app/session/layouts.js +0 -372
  51. package/dist/app/session/modules/audio.js +0 -321
  52. package/dist/app/session/modules/camera-managed-extension.js +0 -310
  53. package/dist/app/session/modules/camera.js +0 -607
  54. package/dist/app/session/modules/index.js +0 -19
  55. package/dist/app/session/modules/location.js +0 -61
  56. package/dist/app/session/modules/simple-storage.js +0 -232
  57. package/dist/app/session/settings.js +0 -358
  58. package/dist/app/token/index.js +0 -22
  59. package/dist/app/token/utils.js +0 -144
  60. package/dist/app/webview/index.js +0 -382
  61. package/dist/constants/index.js +0 -16
  62. package/dist/constants/log-messages/color.js +0 -14
  63. package/dist/constants/log-messages/logos.js +0 -48
  64. package/dist/constants/log-messages/updates.js +0 -55
  65. package/dist/constants/log-messages/warning.js +0 -89
  66. package/dist/examples/managed-rtmp-streaming-example.js +0 -158
  67. package/dist/examples/managed-rtmp-streaming-with-restream-example.js +0 -124
  68. package/dist/examples/rtmp-streaming-example.js +0 -102
  69. package/dist/logging/logger.js +0 -79
  70. package/dist/types/capabilities.js +0 -9
  71. package/dist/types/dashboard/index.js +0 -12
  72. package/dist/types/enums.js +0 -75
  73. package/dist/types/index.js +0 -101
  74. package/dist/types/layouts.js +0 -3
  75. package/dist/types/message-types.js +0 -212
  76. package/dist/types/messages/app-to-cloud.js +0 -95
  77. package/dist/types/messages/base.js +0 -3
  78. package/dist/types/messages/cloud-to-app.js +0 -78
  79. package/dist/types/messages/cloud-to-glasses.js +0 -68
  80. package/dist/types/messages/glasses-to-cloud.js +0 -140
  81. package/dist/types/models.js +0 -101
  82. package/dist/types/photo-data.js +0 -2
  83. package/dist/types/rtmp-stream.js +0 -3
  84. package/dist/types/streams.js +0 -306
  85. package/dist/types/token.js +0 -7
  86. package/dist/types/webhooks.js +0 -28
  87. package/dist/utils/animation-utils.js +0 -340
  88. package/dist/utils/bitmap-utils.js +0 -475
  89. package/dist/utils/permissions-utils.js +0 -263
  90. package/dist/utils/resource-tracker.js +0 -153
@@ -1,1573 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.SimpleStorage = exports.AudioManager = exports.CameraModule = exports.TpaSession = exports.AppSession = void 0;
7
- /**
8
- * 🎯 App Session Module
9
- *
10
- * Manages an active Third Party App session with MentraOS Cloud.
11
- * Handles real-time communication, event subscriptions, and display management.
12
- */
13
- const ws_1 = require("ws");
14
- const events_1 = require("./events");
15
- const layouts_1 = require("./layouts");
16
- const settings_1 = require("./settings");
17
- const location_1 = require("./modules/location");
18
- const camera_1 = require("./modules/camera");
19
- const audio_1 = require("./modules/audio");
20
- const resource_tracker_1 = require("../../utils/resource-tracker");
21
- const types_1 = require("../../types");
22
- const axios_1 = __importDefault(require("axios"));
23
- const events_2 = __importDefault(require("events"));
24
- const node_fetch_1 = __importDefault(require("node-fetch"));
25
- // Import the cloud-to-app specific type guards
26
- const cloud_to_app_1 = require("../../types/messages/cloud-to-app");
27
- const simple_storage_1 = require("./modules/simple-storage");
28
- const permissions_utils_1 = require("../../utils/permissions-utils");
29
- // List of event types that should never be subscribed to as streams
30
- const APP_TO_APP_EVENT_TYPES = [
31
- "app_message_received",
32
- "app_user_joined",
33
- "app_user_left",
34
- "app_room_updated",
35
- "app_direct_message_response",
36
- ];
37
- /**
38
- * 🚀 App Session Implementation
39
- *
40
- * Manages a live connection between your App and MentraOS Cloud.
41
- * Provides interfaces for:
42
- * - 🎮 Event handling (transcription, head position, etc.)
43
- * - 📱 Display management in AR view
44
- * - 🔌 Connection lifecycle
45
- * - 🔄 Automatic reconnection
46
- *
47
- * @example
48
- * ```typescript
49
- * const session = new AppSession({
50
- * packageName: 'org.example.myapp',
51
- * apiKey: 'your_api_key'
52
- * });
53
- *
54
- * // Handle events
55
- * session.onTranscription((data) => {
56
- * session.layouts.showTextWall(data.text);
57
- * });
58
- *
59
- * // Connect to cloud
60
- * await session.connect('session_123');
61
- * ```
62
- */
63
- class AppSession {
64
- constructor(config) {
65
- this.config = config;
66
- /** WebSocket connection to MentraOS Cloud */
67
- this.ws = null;
68
- /** Current session identifier */
69
- this.sessionId = null;
70
- /** Number of reconnection attempts made */
71
- this.reconnectAttempts = 0;
72
- /** Active event subscriptions */
73
- this.subscriptions = new Set();
74
- /** Map to store rate options for streams */
75
- this.streamRates = new Map();
76
- /** Resource tracker for automatic cleanup */
77
- this.resources = new resource_tracker_1.ResourceTracker();
78
- /** Internal settings storage - use public settings API instead */
79
- this.settingsData = [];
80
- /** App configuration loaded from app_config.json */
81
- this.appConfig = null;
82
- /** Whether to update subscriptions when settings change */
83
- this.shouldUpdateSubscriptionsOnSettingsChange = false;
84
- /** Settings that should trigger subscription updates when changed */
85
- this.subscriptionUpdateTriggers = [];
86
- /** Pending user discovery requests waiting for responses */
87
- this.pendingUserDiscoveryRequests = new Map();
88
- /** Pending direct message requests waiting for responses */
89
- this.pendingDirectMessages = new Map();
90
- /** 🔧 Device capabilities available for this session */
91
- this.capabilities = null;
92
- /** Dedicated emitter for App-to-App events */
93
- this.appEvents = new events_2.default();
94
- // Set defaults and merge with provided config
95
- this.config = {
96
- mentraOSWebsocketUrl: `ws://localhost:8002/app-ws`, // Use localhost as default
97
- autoReconnect: true, // Enable auto-reconnection by default for better reliability
98
- maxReconnectAttempts: 3, // Default to 3 reconnection attempts for better resilience
99
- reconnectDelay: 1000, // Start with 1 second delay (uses exponential backoff)
100
- ...config,
101
- };
102
- this.appServer = this.config.appServer;
103
- this.logger = this.appServer.logger.child({
104
- userId: this.config.userId,
105
- service: "app-session",
106
- });
107
- this.userId = this.config.userId;
108
- // Make sure the URL is correctly formatted to prevent double protocol issues
109
- if (this.config.mentraOSWebsocketUrl) {
110
- try {
111
- const url = new URL(this.config.mentraOSWebsocketUrl);
112
- if (!["ws:", "wss:"].includes(url.protocol)) {
113
- // Fix URLs with incorrect protocol (e.g., 'ws://http://host')
114
- const fixedUrl = this.config.mentraOSWebsocketUrl.replace(/^ws:\/\/http:\/\//, "ws://");
115
- this.config.mentraOSWebsocketUrl = fixedUrl;
116
- this.logger.warn(`⚠️ [${this.config.packageName}] Fixed malformed WebSocket URL: ${fixedUrl}`);
117
- }
118
- }
119
- catch (error) {
120
- this.logger.error(error, `⚠️ [${this.config.packageName}] Invalid WebSocket URL format: ${this.config.mentraOSWebsocketUrl}`);
121
- }
122
- }
123
- // Log initialization
124
- this.logger.debug(`🚀 [${this.config.packageName}] App Session initialized`);
125
- this.logger.debug(`🚀 [${this.config.packageName}] WebSocket URL: ${this.config.mentraOSWebsocketUrl}`);
126
- // Validate URL format - give early warning for obvious issues
127
- // Check URL format but handle undefined case
128
- if (this.config.mentraOSWebsocketUrl) {
129
- try {
130
- const url = new URL(this.config.mentraOSWebsocketUrl);
131
- if (!["ws:", "wss:"].includes(url.protocol)) {
132
- this.logger.error({ config: this.config }, `⚠️ [${this.config.packageName}] Invalid WebSocket URL protocol: ${url.protocol}. Should be ws: or wss:`);
133
- }
134
- }
135
- catch (error) {
136
- this.logger.error(error, `⚠️ [${this.config.packageName}] Invalid WebSocket URL format: ${this.config.mentraOSWebsocketUrl}`);
137
- }
138
- }
139
- this.events = new events_1.EventManager(this.subscribe.bind(this), this.unsubscribe.bind(this), this.config.packageName, this.getHttpsServerUrl() || "");
140
- this.layouts = new layouts_1.LayoutManager(config.packageName, this.send.bind(this));
141
- // Initialize settings manager with all necessary parameters, including subscribeFn for MentraOS settings
142
- this.settings = new settings_1.SettingsManager(this.settingsData, this.config.packageName, this.config.mentraOSWebsocketUrl, this.sessionId ?? undefined, async (streams) => {
143
- this.logger.debug({ streams: JSON.stringify(streams) }, `[AppSession] subscribeFn called for streams`);
144
- streams.forEach((stream) => {
145
- if (!this.subscriptions.has(stream)) {
146
- this.subscriptions.add(stream);
147
- this.logger.debug(`[AppSession] Auto-subscribed to stream '${stream}' for MentraOS setting.`);
148
- }
149
- else {
150
- this.logger.debug(`[AppSession] Already subscribed to stream '${stream}'.`);
151
- }
152
- });
153
- this.logger.debug({ subscriptions: JSON.stringify(Array.from(this.subscriptions)) }, `[AppSession] Current subscriptions after subscribeFn`);
154
- if (this.ws?.readyState === 1) {
155
- this.updateSubscriptions();
156
- this.logger.debug(`[AppSession] Sent updated subscriptions to cloud after auto-subscribing to MentraOS setting.`);
157
- }
158
- else {
159
- this.logger.debug(`[AppSession] WebSocket not open, will send subscriptions when connected.`);
160
- }
161
- });
162
- // Initialize dashboard API with this session instance
163
- // Import DashboardManager dynamically to avoid circular dependency
164
- const { DashboardManager } = require("./dashboard");
165
- this.dashboard = new DashboardManager(this, this.send.bind(this));
166
- // Initialize camera module with session reference
167
- this.camera = new camera_1.CameraModule(this.config.packageName, this.sessionId || "unknown-session-id", this.send.bind(this), this, // Pass session reference
168
- this.logger.child({ module: "camera" }));
169
- // Initialize audio module with session reference
170
- this.audio = new audio_1.AudioManager(this.config.packageName, this.sessionId || "unknown-session-id", this.send.bind(this), this, // Pass session reference
171
- this.logger.child({ module: "audio" }));
172
- this.simpleStorage = new simple_storage_1.SimpleStorage(this);
173
- this.location = new location_1.LocationManager(this, this.send.bind(this));
174
- }
175
- /**
176
- * Get the current session ID
177
- * @returns The current session ID or 'unknown-session-id' if not connected
178
- */
179
- getSessionId() {
180
- return this.sessionId || "unknown-session-id";
181
- }
182
- /**
183
- * Get the package name for this App
184
- * @returns The package name
185
- */
186
- getPackageName() {
187
- return this.config.packageName;
188
- }
189
- // =====================================
190
- // 🎮 Direct Event Handling Interface
191
- // =====================================
192
- /**
193
- * @deprecated Use session.events.onTranscription() instead
194
- */
195
- onTranscription(handler) {
196
- return this.events.onTranscription(handler);
197
- }
198
- /**
199
- * 🌐 Listen for speech transcription events in a specific language
200
- * @param language - Language code (e.g., "en-US")
201
- * @param handler - Function to handle transcription data
202
- * @returns Cleanup function to remove the handler
203
- * @throws Error if language code is invalid
204
- * @deprecated Use session.events.onTranscriptionForLanguage() instead
205
- */
206
- onTranscriptionForLanguage(language, handler, disableLanguageIdentification = false) {
207
- return this.events.onTranscriptionForLanguage(language, handler, disableLanguageIdentification);
208
- }
209
- /**
210
- * 🌐 Listen for speech translation events for a specific language pair
211
- * @param sourceLanguage - Source language code (e.g., "es-ES")
212
- * @param targetLanguage - Target language code (e.g., "en-US")
213
- * @param handler - Function to handle translation data
214
- * @returns Cleanup function to remove the handler
215
- * @throws Error if language codes are invalid
216
- * @deprecated Use session.events.onTranslationForLanguage() instead
217
- */
218
- onTranslationForLanguage(sourceLanguage, targetLanguage, handler) {
219
- return this.events.ontranslationForLanguage(sourceLanguage, targetLanguage, handler);
220
- }
221
- /**
222
- * 👤 Listen for head position changes
223
- * @param handler - Function to handle head position updates
224
- * @returns Cleanup function to remove the handler
225
- * @deprecated Use session.events.onHeadPosition() instead
226
- */
227
- onHeadPosition(handler) {
228
- return this.events.onHeadPosition(handler);
229
- }
230
- /**
231
- * 🔘 Listen for hardware button press events
232
- * @param handler - Function to handle button events
233
- * @returns Cleanup function to remove the handler
234
- * @deprecated Use session.events.onButtonPress() instead
235
- */
236
- onButtonPress(handler) {
237
- return this.events.onButtonPress(handler);
238
- }
239
- /**
240
- * 📱 Listen for phone notification events
241
- * @param handler - Function to handle notifications
242
- * @returns Cleanup function to remove the handler
243
- * @deprecated Use session.events.onPhoneNotifications() instead
244
- */
245
- onPhoneNotifications(handler) {
246
- (0, permissions_utils_1.readNotificationWarnLog)(this.getHttpsServerUrl() || "", this.getPackageName(), "onPhoneNotifications");
247
- return this.events.onPhoneNotifications(handler);
248
- }
249
- /**
250
- * 📱 Listen for phone notification dismissed events
251
- * @param handler - Function to handle notification dismissal data
252
- * @returns Cleanup function to remove the handler
253
- * @deprecated Use session.events.onPhoneNotificationDismissed() instead
254
- */
255
- onPhoneNotificationDismissed(handler) {
256
- return this.events.onPhoneNotificationDismissed(handler);
257
- }
258
- /**
259
- * 📡 Listen for VPS coordinates updates
260
- * @param handler - Function to handle VPS coordinates
261
- * @returns Cleanup function to remove the handler
262
- * @deprecated Use session.events.onVpsCoordinates() instead
263
- */
264
- onVpsCoordinates(handler) {
265
- this.subscribe(types_1.StreamType.VPS_COORDINATES);
266
- return this.events.onVpsCoordinates(handler);
267
- }
268
- /**
269
- * 📸 Listen for photo responses
270
- * @param handler - Function to handle photo response data
271
- * @returns Cleanup function to remove the handler
272
- * @deprecated Use session.events.onPhotoTaken() instead
273
- */
274
- onPhotoTaken(handler) {
275
- this.subscribe(types_1.StreamType.PHOTO_TAKEN);
276
- return this.events.onPhotoTaken(handler);
277
- }
278
- // =====================================
279
- // 📡 Pub/Sub Interface
280
- // =====================================
281
- /**
282
- * 📬 Subscribe to a specific event stream
283
- * @param sub - A string or a rich subscription object
284
- */
285
- subscribe(sub) {
286
- let type;
287
- let rate;
288
- if (typeof sub === "string") {
289
- type = sub;
290
- }
291
- else {
292
- // it's a LocationStreamRequest object
293
- type = sub.stream;
294
- rate = sub.rate;
295
- }
296
- if (APP_TO_APP_EVENT_TYPES.includes(type)) {
297
- this.logger.warn(`[AppSession] Attempted to subscribe to App-to-App event type '${type}', which is not a valid stream. Use the event handler (e.g., onAppMessage) instead.`);
298
- return;
299
- }
300
- this.subscriptions.add(type);
301
- if (rate) {
302
- this.streamRates.set(type, rate);
303
- }
304
- if (this.ws?.readyState === 1) {
305
- this.updateSubscriptions();
306
- }
307
- }
308
- /**
309
- * 📭 Unsubscribe from a specific event stream
310
- * @param sub - The subscription to remove
311
- */
312
- unsubscribe(sub) {
313
- let type;
314
- if (typeof sub === "string") {
315
- type = sub;
316
- }
317
- else {
318
- type = sub.stream;
319
- }
320
- if (APP_TO_APP_EVENT_TYPES.includes(type)) {
321
- this.logger.warn(`[AppSession] Attempted to unsubscribe from App-to-App event type '${type}', which is not a valid stream.`);
322
- return;
323
- }
324
- this.subscriptions.delete(type);
325
- this.streamRates.delete(type); // also remove from our rate map
326
- if (this.ws?.readyState === 1) {
327
- this.updateSubscriptions();
328
- }
329
- }
330
- /**
331
- * 🎯 Generic event listener (pub/sub style)
332
- * @param event - Event name to listen for
333
- * @param handler - Event handler function
334
- */
335
- on(event, handler) {
336
- return this.events.on(event, handler);
337
- }
338
- // =====================================
339
- // 🔌 Connection Management
340
- // =====================================
341
- /**
342
- * 🚀 Connect to MentraOS Cloud
343
- * @param sessionId - Unique session identifier
344
- * @returns Promise that resolves when connected
345
- */
346
- async connect(sessionId) {
347
- this.sessionId = sessionId;
348
- // Configure settings API client with the WebSocket URL and session ID
349
- // This allows settings to be fetched from the correct server
350
- this.settings.configureApiClient(this.config.packageName, this.config.mentraOSWebsocketUrl || "", sessionId);
351
- // Update the sessionId in the camera module
352
- if (this.camera) {
353
- this.camera.updateSessionId(sessionId);
354
- }
355
- // Update the sessionId in the audio module
356
- if (this.audio) {
357
- this.audio.updateSessionId(sessionId);
358
- }
359
- return new Promise((resolve, reject) => {
360
- try {
361
- // Clear previous resources if reconnecting
362
- if (this.ws) {
363
- // Don't call full dispose() as that would clear subscriptions
364
- if (this.ws.readyState !== 3) {
365
- // 3 = CLOSED
366
- this.ws.close();
367
- }
368
- this.ws = null;
369
- }
370
- // Validate WebSocket URL before attempting connection
371
- if (!this.config.mentraOSWebsocketUrl) {
372
- this.logger.error("WebSocket URL is missing or undefined");
373
- reject(new Error("WebSocket URL is required"));
374
- return;
375
- }
376
- // Add debug logging for connection attempts
377
- this.logger.info(`🔌🔌🔌 [${this.config.packageName}] Attempting to connect to: ${this.config.mentraOSWebsocketUrl} for session ${this.sessionId}`);
378
- // Create connection with error handling
379
- this.ws = new ws_1.WebSocket(this.config.mentraOSWebsocketUrl);
380
- // Track WebSocket for automatic cleanup
381
- this.resources.track(() => {
382
- if (this.ws && this.ws.readyState !== 3) {
383
- // 3 = CLOSED
384
- this.ws.close();
385
- }
386
- });
387
- this.ws.on("open", () => {
388
- try {
389
- this.sendConnectionInit();
390
- }
391
- catch (error) {
392
- this.logger.error(error, "Error during connection initialization");
393
- const errorMessage = error instanceof Error ? error.message : String(error);
394
- this.events.emit("error", new Error(`Connection initialization failed: ${errorMessage}`));
395
- reject(error);
396
- }
397
- });
398
- // Message handler with comprehensive error recovery
399
- const messageHandler = async (data, isBinary) => {
400
- try {
401
- // Handle binary messages (typically audio data)
402
- if (isBinary && Buffer.isBuffer(data)) {
403
- try {
404
- // Validate buffer before processing
405
- if (data.length === 0) {
406
- this.events.emit("error", new Error("Received empty binary data"));
407
- return;
408
- }
409
- // Convert Node.js Buffer to ArrayBuffer safely
410
- const arrayBuf = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
411
- // Create AUDIO_CHUNK event message with validation
412
- const audioChunk = {
413
- type: types_1.StreamType.AUDIO_CHUNK,
414
- arrayBuffer: arrayBuf,
415
- timestamp: new Date(), // Ensure timestamp is present
416
- };
417
- this.handleMessage(audioChunk);
418
- return;
419
- }
420
- catch (error) {
421
- this.logger.error(error, "Error processing binary message:");
422
- const errorMessage = error instanceof Error ? error.message : String(error);
423
- this.events.emit("error", new Error(`Failed to process binary message: ${errorMessage}`));
424
- return;
425
- }
426
- }
427
- // Handle ArrayBuffer data type directly
428
- if (data instanceof ArrayBuffer) {
429
- return;
430
- }
431
- // Handle JSON messages with validation
432
- try {
433
- // Convert string data to JSON safely
434
- let jsonData;
435
- if (typeof data === "string") {
436
- jsonData = data;
437
- }
438
- else if (Buffer.isBuffer(data)) {
439
- jsonData = data.toString("utf8");
440
- }
441
- else {
442
- throw new Error("Unknown message format");
443
- }
444
- // Validate JSON before parsing
445
- if (!jsonData || jsonData.trim() === "") {
446
- this.events.emit("error", new Error("Received empty JSON message"));
447
- return;
448
- }
449
- // Parse JSON with error handling
450
- const message = JSON.parse(jsonData);
451
- // Basic schema validation
452
- if (!message ||
453
- typeof message !== "object" ||
454
- !("type" in message)) {
455
- this.events.emit("error", new Error("Malformed message: missing type property"));
456
- return;
457
- }
458
- // Process the validated message
459
- this.handleMessage(message);
460
- }
461
- catch (error) {
462
- this.logger.error(error, "JSON parsing error");
463
- const errorMessage = error instanceof Error ? error.message : String(error);
464
- this.events.emit("error", new Error(`Failed to parse JSON message: ${errorMessage}`));
465
- }
466
- }
467
- catch (error) {
468
- // Final catch - should never reach here if individual handlers work correctly
469
- this.logger.error({ error }, "Unhandled message processing error");
470
- const errorMessage = error instanceof Error ? error.message : String(error);
471
- this.events.emit("error", new Error(`Unhandled message error: ${errorMessage}`));
472
- }
473
- };
474
- this.ws.on("message", messageHandler);
475
- // Track event handler removal for automatic cleanup
476
- this.resources.track(() => {
477
- if (this.ws) {
478
- this.ws.off("message", messageHandler);
479
- }
480
- });
481
- // Connection closure handler
482
- const closeHandler = (code, reason) => {
483
- const reasonStr = reason ? `: ${reason}` : "";
484
- const closeInfo = `Connection closed (code: ${code})${reasonStr}`;
485
- // Emit the disconnected event with structured data for better handling
486
- this.events.emit("disconnected", {
487
- message: closeInfo,
488
- code: code,
489
- reason: reason || "",
490
- wasClean: code === 1000 || code === 1001,
491
- });
492
- // Only attempt reconnection for abnormal closures
493
- // https://developer.mozilla.org/en-US/docs/Web/API/CloseEvent/code
494
- // 1000 (Normal Closure) and 1001 (Going Away) are normal
495
- // 1002-1015 are abnormal, and reason "App stopped" means intentional closure
496
- // 1008 usually when the userSession no longer exists on server. i.e user disconnected from cloud.
497
- const isNormalClosure = code === 1000 || code === 1001 || code === 1008;
498
- const isManualStop = reason && reason.includes("App stopped");
499
- const isUserSessionEnded = reason && reason.includes("User session ended");
500
- // Log closure details for diagnostics
501
- this.logger.debug(`🔌 [${this.config.packageName}] WebSocket closed with code ${code}${reasonStr}`);
502
- this.logger.debug(`🔌 [${this.config.packageName}] isNormalClosure: ${isNormalClosure}, isManualStop: ${isManualStop}, isUserSessionEnded: ${isUserSessionEnded}`);
503
- if (!isNormalClosure && !isManualStop) {
504
- this.logger.warn(`🔌 [${this.config.packageName}] Abnormal closure detected, attempting reconnection`);
505
- this.handleReconnection();
506
- }
507
- else {
508
- this.logger.debug(`🔌 [${this.config.packageName}] Normal closure detected, not attempting reconnection`);
509
- }
510
- // if user session ended, then trigger onStop.
511
- if (isUserSessionEnded) {
512
- this.logger.info(`🛑 [${this.config.packageName}] User session ended - emitting disconnected event with sessionEnded flag`);
513
- // Emit a disconnected event with a special flag to indicate session end
514
- // This will be caught by AppServer which will call the onStop callback
515
- const disconnectInfo = {
516
- message: "User session ended",
517
- code: 1000, // Normal closure
518
- reason: "User session ended",
519
- wasClean: true,
520
- permanent: true, // This is permanent - no reconnection
521
- sessionEnded: true, // Special flag to indicate session disposal
522
- };
523
- this.events.emit("disconnected", disconnectInfo);
524
- }
525
- };
526
- this.ws.on("close", closeHandler);
527
- // Track event handler removal
528
- this.resources.track(() => {
529
- if (this.ws) {
530
- this.ws.off("close", closeHandler);
531
- }
532
- });
533
- // Connection error handler
534
- const errorHandler = (error) => {
535
- this.logger.error(error, "WebSocket error");
536
- this.events.emit("error", error);
537
- };
538
- // Enhanced error handler with detailed logging
539
- this.ws.on("error", (error) => {
540
- this.logger.error(error, `⛔️⛔️⛔️ [${this.config.packageName}] WebSocket connection error: ${error.message}`);
541
- // Try to provide more context
542
- const errMsg = error.message || "";
543
- if (errMsg.includes("ECONNREFUSED")) {
544
- this.logger.error(`⛔️⛔️⛔️ [${this.config.packageName}] Connection refused - Check if the server is running at the specified URL`);
545
- }
546
- else if (errMsg.includes("ETIMEDOUT")) {
547
- this.logger.error(`⛔️⛔️⛔️ [${this.config.packageName}] Connection timed out - Check network connectivity and firewall rules`);
548
- }
549
- errorHandler(error);
550
- });
551
- // Track event handler removal
552
- this.resources.track(() => {
553
- if (this.ws) {
554
- this.ws.off("error", errorHandler);
555
- }
556
- });
557
- // Set up connection success handler
558
- const connectedCleanup = this.events.onConnected(() => resolve());
559
- // Track event handler removal
560
- this.resources.track(connectedCleanup);
561
- // Connection timeout with configurable duration
562
- const timeoutMs = 5000; // 5 seconds default
563
- const connectionTimeout = this.resources.setTimeout(() => {
564
- // Use tracked timeout that will be auto-cleared
565
- this.logger.error({
566
- config: this.config,
567
- sessionId: this.sessionId,
568
- timeoutMs,
569
- }, `⏱️⏱️⏱️ [${this.config.packageName}] Connection timeout after ${timeoutMs}ms`);
570
- this.events.emit("error", new Error(`Connection timeout after ${timeoutMs}ms`));
571
- reject(new Error("Connection timeout"));
572
- }, timeoutMs);
573
- // Clear timeout on successful connection
574
- const timeoutCleanup = this.events.onConnected(() => {
575
- clearTimeout(connectionTimeout);
576
- resolve();
577
- });
578
- // Track event handler removal
579
- this.resources.track(timeoutCleanup);
580
- }
581
- catch (error) {
582
- this.logger.error(error, "Connection setup error");
583
- const errorMessage = error instanceof Error ? error.message : String(error);
584
- reject(new Error(`Failed to setup connection: ${errorMessage}`));
585
- }
586
- });
587
- }
588
- /**
589
- * 👋 Disconnect from MentraOS Cloud
590
- */
591
- disconnect() {
592
- // Clean up camera module first
593
- if (this.camera) {
594
- this.camera.cancelAllRequests();
595
- }
596
- // Clean up audio module
597
- if (this.audio) {
598
- this.audio.cancelAllRequests();
599
- }
600
- // Use the resource tracker to clean up everything
601
- this.resources.dispose();
602
- // Clean up additional resources not handled by the tracker
603
- this.ws = null;
604
- this.sessionId = null;
605
- this.subscriptions.clear();
606
- this.reconnectAttempts = 0;
607
- }
608
- /**
609
- * 🛠️ Get all current user settings
610
- * @returns A copy of the current settings array
611
- * @deprecated Use session.settings.getAll() instead
612
- */
613
- getSettings() {
614
- return this.settings.getAll();
615
- }
616
- /**
617
- * 🔍 Get a specific setting value by key
618
- * @param key The setting key to look for
619
- * @returns The setting's value, or undefined if not found
620
- * @deprecated Use session.settings.get(key) instead
621
- */
622
- getSetting(key) {
623
- return this.settings.get(key);
624
- }
625
- /**
626
- * ⚙️ Configure settings-based subscription updates
627
- * This allows Apps to automatically update their subscriptions when certain settings change
628
- * @param options Configuration options for settings-based subscriptions
629
- */
630
- setSubscriptionSettings(options) {
631
- this.shouldUpdateSubscriptionsOnSettingsChange = true;
632
- this.subscriptionUpdateTriggers = options.updateOnChange;
633
- this.subscriptionSettingsHandler = options.handler;
634
- // If we already have settings, update subscriptions immediately
635
- if (this.settingsData.length > 0) {
636
- this.updateSubscriptionsFromSettings();
637
- }
638
- }
639
- /**
640
- * 🔄 Update subscriptions based on current settings
641
- * Called automatically when relevant settings change
642
- */
643
- updateSubscriptionsFromSettings() {
644
- if (!this.subscriptionSettingsHandler)
645
- return;
646
- try {
647
- // Get new subscriptions from handler
648
- const newSubscriptions = this.subscriptionSettingsHandler(this.settingsData);
649
- // Update all subscriptions at once
650
- this.subscriptions.clear();
651
- newSubscriptions.forEach((subscription) => {
652
- this.subscriptions.add(subscription);
653
- });
654
- // Send subscription update to cloud if connected
655
- if (this.ws && this.ws.readyState === 1) {
656
- this.updateSubscriptions();
657
- }
658
- }
659
- catch (error) {
660
- this.logger.error(error, "Error updating subscriptions from settings");
661
- const errorMessage = error instanceof Error ? error.message : String(error);
662
- this.events.emit("error", new Error(`Failed to update subscriptions: ${errorMessage}`));
663
- }
664
- }
665
- /**
666
- * 🧪 For testing: Update settings locally
667
- * In normal operation, settings come from the cloud
668
- * @param newSettings The new settings to apply
669
- */
670
- updateSettingsForTesting(newSettings) {
671
- this.settingsData = newSettings;
672
- // Update the settings manager with the new settings
673
- this.settings.updateSettings(newSettings);
674
- // Emit update event for backwards compatibility
675
- this.events.emit("settings_update", this.settingsData);
676
- // Check if we should update subscriptions
677
- if (this.shouldUpdateSubscriptionsOnSettingsChange) {
678
- this.updateSubscriptionsFromSettings();
679
- }
680
- }
681
- /**
682
- * 📝 Load configuration from a JSON file
683
- * @param jsonData JSON string containing App configuration
684
- * @returns The loaded configuration
685
- * @throws Error if the configuration is invalid
686
- */
687
- loadConfigFromJson(jsonData) {
688
- try {
689
- const parsedConfig = JSON.parse(jsonData);
690
- if ((0, types_1.validateAppConfig)(parsedConfig)) {
691
- this.appConfig = parsedConfig;
692
- return parsedConfig;
693
- }
694
- else {
695
- throw new Error("Invalid App configuration format");
696
- }
697
- }
698
- catch (error) {
699
- const errorMessage = error instanceof Error ? error.message : String(error);
700
- throw new Error(`Failed to load App configuration: ${errorMessage}`);
701
- }
702
- }
703
- /**
704
- * 📋 Get the loaded App configuration
705
- * @returns The current App configuration or null if not loaded
706
- */
707
- getConfig() {
708
- return this.appConfig;
709
- }
710
- /**
711
- * 🔌 Get the WebSocket server URL for this session
712
- * @returns The WebSocket server URL used by this session
713
- */
714
- getServerUrl() {
715
- return this.config.mentraOSWebsocketUrl;
716
- }
717
- getHttpsServerUrl() {
718
- if (!this.config.mentraOSWebsocketUrl) {
719
- return undefined;
720
- }
721
- return AppSession.convertToHttps(this.config.mentraOSWebsocketUrl);
722
- }
723
- static convertToHttps(rawUrl) {
724
- if (!rawUrl)
725
- return "";
726
- // Remove ws:// or wss://
727
- let url = rawUrl.replace(/^wss?:\/\//, "");
728
- // Remove trailing /app-ws
729
- url = url.replace(/\/app-ws$/, "");
730
- // Prepend https://
731
- return `https://${url}`;
732
- }
733
- /**
734
- * 🔍 Get default settings from the App configuration
735
- * @returns Array of settings with default values
736
- * @throws Error if configuration is not loaded
737
- */
738
- getDefaultSettings() {
739
- if (!this.appConfig) {
740
- throw new Error("App configuration not loaded. Call loadConfigFromJson first.");
741
- }
742
- return this.appConfig.settings
743
- .filter((s) => s.type !== "group")
744
- .map((s) => ({
745
- ...s,
746
- value: s.defaultValue, // Set value to defaultValue
747
- }));
748
- }
749
- /**
750
- * 🔍 Get setting schema from configuration
751
- * @param key Setting key to look up
752
- * @returns The setting schema or undefined if not found
753
- */
754
- getSettingSchema(key) {
755
- if (!this.appConfig)
756
- return undefined;
757
- const setting = this.appConfig.settings.find((s) => s.type !== "group" && "key" in s && s.key === key);
758
- return setting;
759
- }
760
- // =====================================
761
- // 🔧 Private Methods
762
- // =====================================
763
- /**
764
- * 📨 Handle incoming messages from cloud
765
- */
766
- handleMessage(message) {
767
- try {
768
- // Validate message before processing
769
- if (!this.validateMessage(message)) {
770
- this.events.emit("error", new Error("Invalid message format received"));
771
- return;
772
- }
773
- // Handle binary data (audio or video)
774
- if (message instanceof ArrayBuffer) {
775
- this.handleBinaryMessage(message);
776
- return;
777
- }
778
- // Using type guards to determine message type and safely handle each case
779
- try {
780
- if ((0, types_1.isAppConnectionAck)(message)) {
781
- // Get settings from connection acknowledgment
782
- const receivedSettings = message.settings || [];
783
- this.settingsData = receivedSettings;
784
- // Store config if provided
785
- if (message.config && (0, types_1.validateAppConfig)(message.config)) {
786
- this.appConfig = message.config;
787
- }
788
- // Use default settings from config if no settings were provided
789
- if (receivedSettings.length === 0 && this.appConfig) {
790
- try {
791
- this.settingsData = this.getDefaultSettings();
792
- }
793
- catch (error) {
794
- this.logger.warn(error, "Failed to load default settings from config:");
795
- }
796
- }
797
- // Update the settings manager with the new settings
798
- this.settings.updateSettings(this.settingsData);
799
- // Handle MentraOS system settings if provided
800
- this.logger.debug({ mentraosSettings: JSON.stringify(message.mentraosSettings) }, `[AppSession] CONNECTION_ACK mentraosSettings}`);
801
- if (message.mentraosSettings) {
802
- this.logger.info({ mentraosSettings: JSON.stringify(message.mentraosSettings) }, `[AppSession] Calling updatementraosSettings with`);
803
- this.settings.updateMentraosSettings(message.mentraosSettings);
804
- }
805
- else {
806
- this.logger.warn(`[AppSession] CONNECTION_ACK message missing mentraosSettings field`);
807
- }
808
- // Handle device capabilities if provided
809
- if (message.capabilities) {
810
- this.capabilities = message.capabilities;
811
- this.logger.info(`[AppSession] Device capabilities loaded for model: ${message.capabilities.modelName}`);
812
- }
813
- else {
814
- this.logger.debug(`[AppSession] No capabilities provided in CONNECTION_ACK`);
815
- }
816
- // Emit connected event with settings
817
- this.events.emit("connected", this.settingsData);
818
- // Update subscriptions (normal flow)
819
- this.updateSubscriptions();
820
- // If settings-based subscriptions are enabled, update those too
821
- if (this.shouldUpdateSubscriptionsOnSettingsChange &&
822
- this.settingsData.length > 0) {
823
- this.updateSubscriptionsFromSettings();
824
- }
825
- }
826
- else if ((0, types_1.isAppConnectionError)(message) ||
827
- message.type === "connection_error") {
828
- // Handle both App-specific connection_error and standard connection_error
829
- const errorMessage = message.message || "Unknown connection error";
830
- this.events.emit("error", new Error(errorMessage));
831
- }
832
- else if (message.type === types_1.StreamType.AUDIO_CHUNK) {
833
- if (this.subscriptions.has(types_1.StreamType.AUDIO_CHUNK)) {
834
- // Only process if we're subscribed to avoid unnecessary processing
835
- this.events.emit(types_1.StreamType.AUDIO_CHUNK, message);
836
- }
837
- }
838
- else if ((0, types_1.isDataStream)(message)) {
839
- // Ensure streamType exists before emitting the event
840
- const messageStreamType = message.streamType;
841
- // if (message.streamType === StreamType.TRANSCRIPTION) {
842
- // const transcriptionData = message.data as TranscriptionData;
843
- // if (transcriptionData.transcribeLanguage) {
844
- // messageStreamType = createTranscriptionStream(transcriptionData.transcribeLanguage) as ExtendedStreamType;
845
- // }
846
- // } else if (message.streamType === StreamType.TRANSLATION) {
847
- // const translationData = message.data as TranslationData;
848
- // if (translationData.transcribeLanguage && translationData.translateLanguage) {
849
- // messageStreamType = createTranslationStream(translationData.transcribeLanguage, translationData.translateLanguage) as ExtendedStreamType;
850
- // }
851
- // }
852
- if (messageStreamType && this.subscriptions.has(messageStreamType)) {
853
- const sanitizedData = this.sanitizeEventData(messageStreamType, message.data);
854
- this.events.emit(messageStreamType, sanitizedData);
855
- }
856
- }
857
- else if ((0, cloud_to_app_1.isRtmpStreamStatus)(message)) {
858
- // Emit as a standard stream event if subscribed
859
- if (this.subscriptions.has(types_1.StreamType.RTMP_STREAM_STATUS)) {
860
- this.events.emit(types_1.StreamType.RTMP_STREAM_STATUS, message);
861
- }
862
- // Update camera module's internal stream state
863
- this.camera.updateStreamState(message);
864
- }
865
- else if ((0, cloud_to_app_1.isManagedStreamStatus)(message)) {
866
- // Emit as a standard stream event if subscribed
867
- if (this.subscriptions.has(types_1.StreamType.MANAGED_STREAM_STATUS)) {
868
- this.events.emit(types_1.StreamType.MANAGED_STREAM_STATUS, message);
869
- }
870
- // Update camera module's managed stream state
871
- this.camera.handleManagedStreamStatus(message);
872
- }
873
- else if ((0, cloud_to_app_1.isStreamStatusCheckResponse)(message)) {
874
- // Handle stream status check response
875
- // This is a direct response, not a subscription-based event
876
- this.camera.handleStreamCheckResponse(message);
877
- }
878
- else if ((0, types_1.isSettingsUpdate)(message)) {
879
- // Store previous settings to check for changes
880
- const _prevSettings = [...this.settingsData];
881
- // Update internal settings storage
882
- this.settingsData = message.settings || [];
883
- // Update the settings manager with the new settings
884
- const changes = this.settings.updateSettings(this.settingsData);
885
- // Emit settings update event (for backwards compatibility)
886
- this.events.emit("settings_update", this.settingsData);
887
- // --- MentraOS settings update logic ---
888
- // If the message.settings looks like MentraOS settings (object with known keys), update mentraosSettings
889
- if (message.settings && typeof message.settings === "object") {
890
- this.settings.updateMentraosSettings(message.settings);
891
- }
892
- // Check if we should update subscriptions
893
- if (this.shouldUpdateSubscriptionsOnSettingsChange) {
894
- // Check if any subscription trigger settings changed
895
- const shouldUpdateSubs = this.subscriptionUpdateTriggers.some((key) => {
896
- return key in changes;
897
- });
898
- if (shouldUpdateSubs) {
899
- this.updateSubscriptionsFromSettings();
900
- }
901
- }
902
- }
903
- else if ((0, types_1.isCapabilitiesUpdate)(message)) {
904
- // Update device capabilities
905
- const capabilitiesMessage = message;
906
- this.capabilities = capabilitiesMessage.capabilities;
907
- this.logger.info(capabilitiesMessage.capabilities, `[AppSession] Capabilities updated for model: ${capabilitiesMessage.modelName}`);
908
- // Emit capabilities update event for applications to handle
909
- this.events.emit("capabilities_update", {
910
- capabilities: capabilitiesMessage.capabilities,
911
- modelName: capabilitiesMessage.modelName,
912
- timestamp: capabilitiesMessage.timestamp,
913
- });
914
- }
915
- else if ((0, types_1.isAppStopped)(message)) {
916
- const reason = message.reason || "unknown";
917
- const displayReason = `App stopped: ${reason}`;
918
- // Don't emit disconnected event here - let the WebSocket close handler do it
919
- // This prevents duplicate disconnected events when the session is disposed
920
- this.logger.info(`📤 [${this.config.packageName}] Received APP_STOPPED message: ${displayReason}`);
921
- // Clear reconnection state
922
- this.reconnectAttempts = 0;
923
- }
924
- // Handle dashboard mode changes
925
- else if ((0, types_1.isDashboardModeChanged)(message)) {
926
- try {
927
- // Use proper type
928
- const mode = message.mode || "none";
929
- // Update dashboard state in the API
930
- if (this.dashboard && "content" in this.dashboard) {
931
- this.dashboard.content.setCurrentMode(mode);
932
- }
933
- }
934
- catch (error) {
935
- this.logger.error(error, "Error handling dashboard mode change");
936
- }
937
- }
938
- // Handle always-on dashboard state changes
939
- else if ((0, types_1.isDashboardAlwaysOnChanged)(message)) {
940
- try {
941
- // Use proper type
942
- const enabled = !!message.enabled;
943
- // Update dashboard state in the API
944
- if (this.dashboard && "content" in this.dashboard) {
945
- this.dashboard.content.setAlwaysOnEnabled(enabled);
946
- }
947
- }
948
- catch (error) {
949
- this.logger.error(error, "Error handling dashboard always-on change");
950
- }
951
- }
952
- // Handle custom messages
953
- else if (message.type === types_1.CloudToAppMessageType.CUSTOM_MESSAGE) {
954
- this.events.emit("custom_message", message);
955
- return;
956
- }
957
- // Handle App-to-App communication messages
958
- else if (message.type === "app_message_received") {
959
- this.appEvents.emit("app_message_received", message);
960
- }
961
- else if (message.type === "app_user_joined") {
962
- this.appEvents.emit("app_user_joined", message);
963
- }
964
- else if (message.type === "app_user_left") {
965
- this.appEvents.emit("app_user_left", message);
966
- }
967
- else if (message.type === "app_room_updated") {
968
- this.appEvents.emit("app_room_updated", message);
969
- }
970
- else if (message.type === "app_direct_message_response") {
971
- const response = message;
972
- if (response.messageId &&
973
- this.pendingDirectMessages.has(response.messageId)) {
974
- const { resolve } = this.pendingDirectMessages.get(response.messageId);
975
- resolve(response.success);
976
- this.pendingDirectMessages.delete(response.messageId);
977
- }
978
- }
979
- else if (message.type === "augmentos_settings_update") {
980
- const mentraosMsg = message;
981
- if (mentraosMsg.settings &&
982
- typeof mentraosMsg.settings === "object") {
983
- this.settings.updateMentraosSettings(mentraosMsg.settings);
984
- }
985
- }
986
- // Handle 'connection_error' as a specific case if cloud sends this string literal
987
- else if (message.type === "connection_error") {
988
- // Treat 'connection_error' (string literal) like AppConnectionError
989
- // This handles cases where the cloud might send the type as a direct string
990
- // instead of the enum's 'tpa_connection_error' value.
991
- const errorMessage = message.message ||
992
- "Unknown connection error (type: connection_error)";
993
- this.logger.warn(`Received 'connection_error' type directly. Consider aligning cloud to send 'tpa_connection_error'. Message: ${errorMessage}`);
994
- this.events.emit("error", new Error(errorMessage));
995
- }
996
- else if (message.type === "permission_error") {
997
- // Handle permission errors from cloud
998
- this.logger.warn({
999
- message: message.message,
1000
- details: message.details,
1001
- detailsCount: message.details?.length || 0,
1002
- rejectedStreams: message.details?.map((d) => d.stream) || [],
1003
- }, "Permission error received:");
1004
- // Emit permission error event for application handling
1005
- this.events.emit("permission_error", {
1006
- message: message.message,
1007
- details: message.details,
1008
- timestamp: message.timestamp,
1009
- });
1010
- // Optionally emit individual permission denied events for each stream
1011
- message.details?.forEach((detail) => {
1012
- this.events.emit("permission_denied", {
1013
- stream: detail.stream,
1014
- requiredPermission: detail.requiredPermission,
1015
- message: detail.message,
1016
- });
1017
- });
1018
- }
1019
- else if ((0, types_1.isAudioPlayResponse)(message)) {
1020
- // Delegate audio play response handling to the audio module
1021
- if (this.audio) {
1022
- this.audio.handleAudioPlayResponse(message);
1023
- }
1024
- }
1025
- else if ((0, cloud_to_app_1.isPhotoResponse)(message)) {
1026
- // Legacy photo response handling - now photos come directly via webhook
1027
- // This branch can be removed in the future as all photos now go through /photo-upload
1028
- this.logger.warn({ message }, "Received legacy photo response - photos should now come via /photo-upload webhook");
1029
- }
1030
- // Handle unrecognized message types gracefully
1031
- else {
1032
- this.logger.warn(`Unrecognized message type: ${message.type}`);
1033
- this.events.emit("error", new Error(`Unrecognized message type: ${message.type}`));
1034
- }
1035
- }
1036
- catch (processingError) {
1037
- // Catch any errors during message processing to prevent App crashes
1038
- this.logger.error(processingError, "Error processing message:");
1039
- const errorMessage = processingError instanceof Error
1040
- ? processingError.message
1041
- : String(processingError);
1042
- this.events.emit("error", new Error(`Error processing message: ${errorMessage}`));
1043
- }
1044
- }
1045
- catch (error) {
1046
- // Final safety net to ensure the App doesn't crash on any unexpected errors
1047
- this.logger.error(error, "Unexpected error in message handler");
1048
- const errorMessage = error instanceof Error ? error.message : String(error);
1049
- this.events.emit("error", new Error(`Unexpected error in message handler: ${errorMessage}`));
1050
- }
1051
- }
1052
- /**
1053
- * 🧪 Validate incoming message structure
1054
- * @param message - Message to validate
1055
- * @returns boolean indicating if the message is valid
1056
- */
1057
- validateMessage(message) {
1058
- // Handle ArrayBuffer case separately
1059
- if (message instanceof ArrayBuffer) {
1060
- return true; // ArrayBuffers are always considered valid at this level
1061
- }
1062
- // Check if message is null or undefined
1063
- if (!message) {
1064
- return false;
1065
- }
1066
- // Check if message has a type property
1067
- if (!("type" in message)) {
1068
- return false;
1069
- }
1070
- // All other message types should be objects with a type property
1071
- return true;
1072
- }
1073
- /**
1074
- * 📦 Handle binary message data (audio or video)
1075
- * @param buffer - Binary data as ArrayBuffer
1076
- */
1077
- handleBinaryMessage(buffer) {
1078
- try {
1079
- // Safety check - only process if we're subscribed to avoid unnecessary work
1080
- if (!this.subscriptions.has(types_1.StreamType.AUDIO_CHUNK)) {
1081
- return;
1082
- }
1083
- // Validate buffer has content before processing
1084
- if (!buffer || buffer.byteLength === 0) {
1085
- this.events.emit("error", new Error("Received empty binary message"));
1086
- return;
1087
- }
1088
- // Create a safety wrapped audio chunk with proper defaults
1089
- const audioChunk = {
1090
- type: types_1.StreamType.AUDIO_CHUNK,
1091
- timestamp: new Date(),
1092
- arrayBuffer: buffer,
1093
- sampleRate: 16000, // Default sample rate
1094
- };
1095
- // Emit to subscribers
1096
- this.events.emit(types_1.StreamType.AUDIO_CHUNK, audioChunk);
1097
- }
1098
- catch (error) {
1099
- this.logger.error(error, "Error processing binary message");
1100
- const errorMessage = error instanceof Error ? error.message : String(error);
1101
- this.events.emit("error", new Error(`Error processing binary message: ${errorMessage}`));
1102
- }
1103
- }
1104
- /**
1105
- * 🧹 Sanitize event data to prevent crashes from malformed data
1106
- * @param streamType - The type of stream data
1107
- * @param data - The potentially unsafe data to sanitize
1108
- * @returns Sanitized data safe for processing
1109
- */
1110
- sanitizeEventData(streamType, data) {
1111
- try {
1112
- // If data is null or undefined, return an empty object to prevent crashes
1113
- if (data === null || data === undefined) {
1114
- return {};
1115
- }
1116
- // For specific stream types, perform targeted sanitization
1117
- switch (streamType) {
1118
- case types_1.StreamType.TRANSCRIPTION: {
1119
- // Ensure text field exists and is a string
1120
- if (typeof data.text !== "string") {
1121
- return {
1122
- text: "",
1123
- isFinal: true,
1124
- startTime: Date.now(),
1125
- endTime: Date.now(),
1126
- };
1127
- }
1128
- break;
1129
- }
1130
- case types_1.StreamType.HEAD_POSITION: {
1131
- // Ensure position data has required numeric fields
1132
- // Handle HeadPosition - Note the property position instead of x,y,z
1133
- const pos = data;
1134
- if (typeof pos?.position !== "string") {
1135
- return { position: "up", timestamp: new Date() };
1136
- }
1137
- break;
1138
- }
1139
- case types_1.StreamType.BUTTON_PRESS: {
1140
- // Ensure button type is valid
1141
- const btn = data;
1142
- if (!btn.buttonId || !btn.pressType) {
1143
- return {
1144
- buttonId: "unknown",
1145
- pressType: "short",
1146
- timestamp: new Date(),
1147
- };
1148
- }
1149
- break;
1150
- }
1151
- }
1152
- return data;
1153
- }
1154
- catch (error) {
1155
- this.logger.error(error, `Error sanitizing ${streamType} data`);
1156
- // Return a safe empty object if something goes wrong
1157
- return {};
1158
- }
1159
- }
1160
- /**
1161
- * 🔐 Send connection initialization message
1162
- */
1163
- sendConnectionInit() {
1164
- const message = {
1165
- type: types_1.AppToCloudMessageType.CONNECTION_INIT,
1166
- sessionId: this.sessionId,
1167
- packageName: this.config.packageName,
1168
- apiKey: this.config.apiKey,
1169
- timestamp: new Date(),
1170
- };
1171
- this.send(message);
1172
- }
1173
- /**
1174
- * 📝 Update subscription list with cloud
1175
- */
1176
- updateSubscriptions() {
1177
- this.logger.info({ subscriptions: JSON.stringify(Array.from(this.subscriptions)) }, `[AppSession] updateSubscriptions: sending subscriptions to cloud`);
1178
- // [MODIFIED] builds the array of SubscriptionRequest objects to send to the cloud
1179
- const subscriptionPayload = Array.from(this.subscriptions).map((stream) => {
1180
- const rate = this.streamRates.get(stream);
1181
- if (rate && stream === types_1.StreamType.LOCATION_STREAM) {
1182
- return { stream: "location_stream", rate: rate };
1183
- }
1184
- return stream;
1185
- });
1186
- const message = {
1187
- type: types_1.AppToCloudMessageType.SUBSCRIPTION_UPDATE,
1188
- packageName: this.config.packageName,
1189
- subscriptions: subscriptionPayload, // [MODIFIED]
1190
- sessionId: this.sessionId,
1191
- timestamp: new Date(),
1192
- };
1193
- this.send(message);
1194
- }
1195
- /**
1196
- * 🔄 Handle reconnection with exponential backoff
1197
- */
1198
- async handleReconnection() {
1199
- // Check if reconnection is allowed
1200
- if (!this.config.autoReconnect || !this.sessionId) {
1201
- this.logger.debug(`🔄 Reconnection skipped: autoReconnect=${this.config.autoReconnect}, sessionId=${this.sessionId ? "valid" : "invalid"}`);
1202
- return;
1203
- }
1204
- // Check if we've exceeded the maximum attempts
1205
- const maxAttempts = this.config.maxReconnectAttempts || 3;
1206
- if (this.reconnectAttempts >= maxAttempts) {
1207
- this.logger.info(`🔄 Maximum reconnection attempts (${maxAttempts}) reached, giving up`);
1208
- // Emit a permanent disconnection event to trigger onStop in the App server
1209
- this.events.emit("disconnected", {
1210
- message: `Connection permanently lost after ${maxAttempts} failed reconnection attempts`,
1211
- code: 4000, // Custom code for max reconnection attempts exhausted
1212
- reason: "Maximum reconnection attempts exceeded",
1213
- wasClean: false,
1214
- permanent: true, // Flag this as a permanent disconnection
1215
- });
1216
- return;
1217
- }
1218
- // Calculate delay with exponential backoff
1219
- const baseDelay = this.config.reconnectDelay || 1000;
1220
- const delay = baseDelay * Math.pow(2, this.reconnectAttempts);
1221
- this.reconnectAttempts++;
1222
- this.logger.debug(`🔄 [${this.config.packageName}] Reconnection attempt ${this.reconnectAttempts}/${maxAttempts} in ${delay}ms`);
1223
- // Use the resource tracker for the timeout
1224
- await new Promise((resolve) => {
1225
- this.resources.setTimeout(() => resolve(), delay);
1226
- });
1227
- try {
1228
- this.logger.debug(`🔄 [${this.config.packageName}] Attempting to reconnect...`);
1229
- await this.connect(this.sessionId);
1230
- this.logger.debug(`✅ [${this.config.packageName}] Reconnection successful!`);
1231
- this.reconnectAttempts = 0;
1232
- }
1233
- catch (error) {
1234
- const errorMessage = error instanceof Error ? error.message : String(error);
1235
- this.logger.error(error, `❌ [${this.config.packageName}] Reconnection failed for user ${this.userId}`);
1236
- this.events.emit("error", new Error(`Reconnection failed: ${errorMessage}`));
1237
- // Check if this was the last attempt
1238
- if (this.reconnectAttempts >= maxAttempts) {
1239
- this.logger.debug(`🔄 [${this.config.packageName}] Final reconnection attempt failed, emitting permanent disconnection`);
1240
- // Emit permanent disconnection event after the last failed attempt
1241
- this.events.emit("disconnected", {
1242
- message: `Connection permanently lost after ${maxAttempts} failed reconnection attempts`,
1243
- code: 4000, // Custom code for max reconnection attempts exhausted
1244
- reason: "Maximum reconnection attempts exceeded",
1245
- wasClean: false,
1246
- permanent: true, // Flag this as a permanent disconnection
1247
- });
1248
- }
1249
- }
1250
- }
1251
- /**
1252
- * 📤 Send message to cloud with validation and error handling
1253
- * @throws {Error} If WebSocket is not connected
1254
- */
1255
- send(message) {
1256
- try {
1257
- // Verify WebSocket connection is valid
1258
- if (!this.ws) {
1259
- throw new Error("WebSocket connection not established");
1260
- }
1261
- if (this.ws.readyState !== 1) {
1262
- const stateMap = {
1263
- 0: "CONNECTING",
1264
- 1: "OPEN",
1265
- 2: "CLOSING",
1266
- 3: "CLOSED",
1267
- };
1268
- const stateName = stateMap[this.ws.readyState] || "UNKNOWN";
1269
- throw new Error(`WebSocket not connected (current state: ${stateName})`);
1270
- }
1271
- // Validate message before sending
1272
- if (!message || typeof message !== "object") {
1273
- throw new Error("Invalid message: must be an object");
1274
- }
1275
- if (!("type" in message)) {
1276
- throw new Error('Invalid message: missing "type" property');
1277
- }
1278
- // Ensure message format is consistent
1279
- if (!("timestamp" in message) || !(message.timestamp instanceof Date)) {
1280
- message.timestamp = new Date();
1281
- }
1282
- // Try to send with error handling
1283
- try {
1284
- const serializedMessage = JSON.stringify(message);
1285
- this.ws.send(serializedMessage);
1286
- }
1287
- catch (sendError) {
1288
- const errorMessage = sendError instanceof Error ? sendError.message : String(sendError);
1289
- throw new Error(`Failed to send message: ${errorMessage}`);
1290
- }
1291
- }
1292
- catch (error) {
1293
- // Log the error and emit an event so App developers are aware
1294
- this.logger.error(error, "Message send error");
1295
- // Ensure we always emit an Error object
1296
- if (error instanceof Error) {
1297
- this.events.emit("error", error);
1298
- }
1299
- else {
1300
- this.events.emit("error", new Error(String(error)));
1301
- }
1302
- // Re-throw to maintain the original function behavior
1303
- throw error;
1304
- }
1305
- }
1306
- /**
1307
- * Fetch the onboarding instructions for this session from the backend.
1308
- * @returns Promise resolving to the instructions string or null
1309
- */
1310
- async getInstructions() {
1311
- try {
1312
- const baseUrl = this.getServerUrl();
1313
- const response = await axios_1.default.get(`${baseUrl}/api/instructions`, {
1314
- params: { userId: this.userId },
1315
- });
1316
- return response.data.instructions || null;
1317
- }
1318
- catch (err) {
1319
- this.logger.error(err, `Error fetching instructions from backend`);
1320
- return null;
1321
- }
1322
- }
1323
- // =====================================
1324
- // 👥 App-to-App Communication Interface
1325
- // =====================================
1326
- /**
1327
- * 👥 Discover other users currently using the same App
1328
- * @param includeProfiles - Whether to include user profile information
1329
- * @returns Promise that resolves with list of active users
1330
- */
1331
- async discoverAppUsers(domain, includeProfiles = false) {
1332
- // Use the domain argument as the base URL if provided
1333
- if (!domain) {
1334
- throw new Error("Domain (API base URL) is required for user discovery");
1335
- }
1336
- const url = `${domain}/api/app-communication/discover-users`;
1337
- // Use the user's core token for authentication
1338
- const appApiKey = this.config.apiKey; // This may need to be updated if you store the core token elsewhere
1339
- if (!appApiKey) {
1340
- throw new Error("Core token (apiKey) is required for user discovery");
1341
- }
1342
- const body = {
1343
- packageName: this.config.packageName,
1344
- userId: this.userId,
1345
- includeUserProfiles: includeProfiles,
1346
- };
1347
- const response = await (0, node_fetch_1.default)(url, {
1348
- method: "POST",
1349
- headers: {
1350
- Authorization: `Bearer ${appApiKey}`,
1351
- "Content-Type": "application/json",
1352
- },
1353
- body: JSON.stringify(body),
1354
- });
1355
- if (!response.ok) {
1356
- const errorText = await response.text();
1357
- throw new Error(`Failed to discover users: ${response.status} ${response.statusText} - ${errorText}`);
1358
- }
1359
- return await response.json();
1360
- }
1361
- /**
1362
- * 🔍 Check if a specific user is currently active
1363
- * @param userId - User ID to check for
1364
- * @returns Promise that resolves with boolean indicating if user is active
1365
- */
1366
- async isUserActive(userId) {
1367
- try {
1368
- const userList = await this.discoverAppUsers("", false);
1369
- return userList.users.some((user) => user.userId === userId);
1370
- }
1371
- catch (error) {
1372
- this.logger.error({ error, userId }, "Error checking if user is active");
1373
- return false;
1374
- }
1375
- }
1376
- /**
1377
- * 📊 Get user count for this App
1378
- * @returns Promise that resolves with number of active users
1379
- */
1380
- async getUserCount(domain) {
1381
- try {
1382
- const userList = await this.discoverAppUsers(domain, false);
1383
- return userList.totalUsers;
1384
- }
1385
- catch (error) {
1386
- this.logger.error(error, "Error getting user count");
1387
- return 0;
1388
- }
1389
- }
1390
- /**
1391
- * 📢 Send broadcast message to all users with same App active
1392
- * @param payload - Message payload to send
1393
- * @param roomId - Optional room ID for room-based messaging
1394
- * @returns Promise that resolves when message is sent
1395
- */
1396
- async broadcastToAppUsers(payload, _roomId) {
1397
- try {
1398
- const messageId = this.generateMessageId();
1399
- const message = {
1400
- type: "app_broadcast_message",
1401
- packageName: this.config.packageName,
1402
- sessionId: this.sessionId,
1403
- payload,
1404
- messageId,
1405
- senderUserId: this.userId,
1406
- timestamp: new Date(),
1407
- };
1408
- this.send(message);
1409
- }
1410
- catch (error) {
1411
- const errorMessage = error instanceof Error ? error.message : String(error);
1412
- throw new Error(`Failed to broadcast message: ${errorMessage}`);
1413
- }
1414
- }
1415
- /**
1416
- * 📤 Send direct message to specific user
1417
- * @param targetUserId - User ID to send message to
1418
- * @param payload - Message payload to send
1419
- * @returns Promise that resolves with success status
1420
- */
1421
- async sendDirectMessage(targetUserId, payload) {
1422
- return new Promise((resolve, reject) => {
1423
- try {
1424
- const messageId = this.generateMessageId();
1425
- // Store promise resolver
1426
- this.pendingDirectMessages.set(messageId, { resolve, reject });
1427
- const message = {
1428
- type: "app_direct_message",
1429
- packageName: this.config.packageName,
1430
- sessionId: this.sessionId,
1431
- targetUserId,
1432
- payload,
1433
- messageId,
1434
- senderUserId: this.userId,
1435
- timestamp: new Date(),
1436
- };
1437
- this.send(message);
1438
- // Set timeout to avoid hanging promises
1439
- const timeoutMs = 15000; // 15 seconds
1440
- this.resources.setTimeout(() => {
1441
- if (this.pendingDirectMessages.has(messageId)) {
1442
- this.pendingDirectMessages
1443
- .get(messageId)
1444
- .reject(new Error("Direct message timed out"));
1445
- this.pendingDirectMessages.delete(messageId);
1446
- }
1447
- }, timeoutMs);
1448
- }
1449
- catch (error) {
1450
- const errorMessage = error instanceof Error ? error.message : String(error);
1451
- reject(new Error(`Failed to send direct message: ${errorMessage}`));
1452
- }
1453
- });
1454
- }
1455
- /**
1456
- * 🏠 Join a communication room for group messaging
1457
- * @param roomId - Room ID to join
1458
- * @param roomConfig - Optional room configuration
1459
- * @returns Promise that resolves when room is joined
1460
- */
1461
- async joinAppRoom(roomId, roomConfig) {
1462
- try {
1463
- const message = {
1464
- type: "app_room_join",
1465
- packageName: this.config.packageName,
1466
- sessionId: this.sessionId,
1467
- roomId,
1468
- roomConfig,
1469
- timestamp: new Date(),
1470
- };
1471
- this.send(message);
1472
- }
1473
- catch (error) {
1474
- const errorMessage = error instanceof Error ? error.message : String(error);
1475
- throw new Error(`Failed to join room: ${errorMessage}`);
1476
- }
1477
- }
1478
- /**
1479
- * 🚪 Leave a communication room
1480
- * @param roomId - Room ID to leave
1481
- * @returns Promise that resolves when room is left
1482
- */
1483
- async leaveAppRoom(roomId) {
1484
- try {
1485
- const message = {
1486
- type: "app_room_leave",
1487
- packageName: this.config.packageName,
1488
- sessionId: this.sessionId,
1489
- roomId,
1490
- timestamp: new Date(),
1491
- };
1492
- this.send(message);
1493
- }
1494
- catch (error) {
1495
- const errorMessage = error instanceof Error ? error.message : String(error);
1496
- throw new Error(`Failed to leave room: ${errorMessage}`);
1497
- }
1498
- }
1499
- /**
1500
- * 📨 Listen for messages from other App users
1501
- * @param handler - Function to handle incoming messages
1502
- * @returns Cleanup function to remove the handler
1503
- */
1504
- onAppMessage(handler) {
1505
- this.appEvents.on("app_message_received", handler);
1506
- return () => this.appEvents.off("app_message_received", handler);
1507
- }
1508
- /**
1509
- * 👋 Listen for user join events
1510
- * @param handler - Function to handle user join events
1511
- * @returns Cleanup function to remove the handler
1512
- */
1513
- onAppUserJoined(handler) {
1514
- this.appEvents.on("app_user_joined", handler);
1515
- return () => this.appEvents.off("app_user_joined", handler);
1516
- }
1517
- /**
1518
- * 🚪 Listen for user leave events
1519
- * @param handler - Function to handle user leave events
1520
- * @returns Cleanup function to remove the handler
1521
- */
1522
- onAppUserLeft(handler) {
1523
- this.appEvents.on("app_user_left", handler);
1524
- return () => this.appEvents.off("app_user_left", handler);
1525
- }
1526
- /**
1527
- * 🏠 Listen for room update events
1528
- * @param handler - Function to handle room updates
1529
- * @returns Cleanup function to remove the handler
1530
- */
1531
- onAppRoomUpdated(handler) {
1532
- this.appEvents.on("app_room_updated", handler);
1533
- return () => this.appEvents.off("app_room_updated", handler);
1534
- }
1535
- /**
1536
- * 🔧 Generate unique message ID
1537
- * @returns Unique message identifier
1538
- */
1539
- generateMessageId() {
1540
- return `msg_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
1541
- }
1542
- }
1543
- exports.AppSession = AppSession;
1544
- /**
1545
- * @deprecated Use `AppSession` instead. `TpaSession` is deprecated and will be removed in a future version.
1546
- * This is an alias for backward compatibility only.
1547
- *
1548
- * @example
1549
- * ```typescript
1550
- * // ❌ Deprecated - Don't use this
1551
- * const session = new TpaSession(config);
1552
- *
1553
- * // ✅ Use this instead
1554
- * const session = new AppSession(config);
1555
- * ```
1556
- */
1557
- class TpaSession extends AppSession {
1558
- constructor(config) {
1559
- super(config);
1560
- // Emit a deprecation warning to help developers migrate
1561
- console.warn("⚠️ DEPRECATION WARNING: TpaSession is deprecated and will be removed in a future version. " +
1562
- "Please use AppSession instead. " +
1563
- 'Simply replace "TpaSession" with "AppSession" in your code.');
1564
- }
1565
- }
1566
- exports.TpaSession = TpaSession;
1567
- // Export module types for developers
1568
- var camera_2 = require("./modules/camera");
1569
- Object.defineProperty(exports, "CameraModule", { enumerable: true, get: function () { return camera_2.CameraModule; } });
1570
- var audio_2 = require("./modules/audio");
1571
- Object.defineProperty(exports, "AudioManager", { enumerable: true, get: function () { return audio_2.AudioManager; } });
1572
- var simple_storage_2 = require("./modules/simple-storage");
1573
- Object.defineProperty(exports, "SimpleStorage", { enumerable: true, get: function () { return simple_storage_2.SimpleStorage; } });