homebridge-nest-accfactory 0.3.1 → 0.3.2

package/dist/nexustalk.js

@@ -5,7 +5,7 @@
 //
 // Credit to https://github.com/Brandawg93/homebridge-nest-cam for the work on the Nest Camera comms code on which this is based
 //
-// Code version 2025.06.15
+// Code version 2025.07.30
 // Mark Hulskamp
 'use strict';
 
@@ -19,15 +19,14 @@ import fs from 'node:fs';
 import path from 'node:path';
 import tls from 'tls';
 import crypto from 'crypto';
-import { fileURLToPath } from 'node:url';
 
 // Define our modules
 import Streamer from './streamer.js';
 
 // Define constants
+import { USER_AGENT, __dirname } from './consts.js';
+
 const PING_INTERVAL = 15000; // Ping interval to nexus server while stream active
-const USER_AGENT = 'Nest/5.78.0 (iOScom.nestlabs.jasper.release) os=18.0'; // User Agent string
-const __dirname = path.dirname(fileURLToPath(import.meta.url)); // Make a defined for JS __dirname
 
 const PACKET_TYPE = {
   PING: 1,
@@ -63,11 +62,9 @@ const AAC_MONO_48000_BLANK = Buffer.from([
 
 // nexusTalk object
 export default class NexusTalk extends Streamer {
+  streaming_host = undefined; // Main nexustalk streaming host
   token = undefined;
-  tokenType = undefined;
-  pingTimer = undefined; // Timer object for ping interval
-  stalledTimer = undefined; // Timer object for no received data
-  host = ''; // Host to connect to or connected too
+  useGoogleAuth = false; // Nest vs google auth
   blankAudio = AAC_MONO_48000_BLANK;
   video = {}; // Video stream details once connected
   audio = {}; // Audio stream details once connected
@@ -79,11 +76,23 @@ export default class NexusTalk extends Streamer {
   #messages = []; // Incoming messages
   #authorised = false; // Have we been authorised
   #id = undefined; // Session ID
+  #host = undefined; // Current host connected to
+  #pingTimer = undefined; // Timer object for ping interval
+  #stalledTimer = undefined; // Timer object for no received data
+
+  // Codecs being used for video, audio and talking
+  get codecs() {
+    return {
+      video: Streamer.CODEC_TYPE.H264, // Video codec
+      audio: Streamer.CODEC_TYPE.AAC, // Audio codec
+      talkback: Streamer.CODEC_TYPE.SPEEX, // Talkback codec
+    };
+  }
 
-  constructor(deviceData, options) {
-    super(deviceData, options);
+  constructor(uuid, deviceData, options) {
+    super(uuid, deviceData, options);
 
-    if (fs.existsSync(path.resolve(__dirname + '/protobuf/nest/nexustalk.proto')) === true) {
+    if (fs.existsSync(path.join(__dirname, 'protobuf/nest/nexustalk.proto')) === true) {
       protobuf.util.Long = null;
       protobuf.configure();
       this.#protobufNexusTalk = protobuf.loadSync(path.resolve(__dirname + '/protobuf/nest/nexustalk.proto'));
@@ -91,85 +100,86 @@ export default class NexusTalk extends Streamer {
 
     // Store data we need from the device data passed it
     this.token = deviceData?.apiAccess?.token;
-    this.tokenType = deviceData?.apiAccess?.oauth2 !== undefined ? 'google' : 'nest';
-    this.host = deviceData?.streaming_host; // Host we'll connect to
-
-    // Set our streamer codec types
-    this.codecs = {
-      video: 'h264',
-      audio: 'aac',
-      talk: 'speex',
-    };
-
-    // If specified option to start buffering, kick off
-    if (options?.buffer === true) {
-      this.startBuffering();
-    }
+    this.streaming_host = deviceData?.streaming_host; // Host we'll connect to
+    this.useGoogleAuth = typeof deviceData?.apiAccess?.oauth2 === 'string' && deviceData?.apiAccess?.oauth2 !== '';
   }
 
   // Class functions
-  connect(host) {
+  async connect(host) {
     // Clear any timers we have running
-    clearInterval(this.pingTimer);
-    clearTimeout(this.stalledTimer);
-    this.pingTimer = undefined;
-    this.stalledTimer = undefined;
+    clearInterval(this.#pingTimer);
+    clearTimeout(this.#stalledTimer);
+    this.#pingTimer = undefined;
+    this.#stalledTimer = undefined;
     this.#id = undefined; // No session ID yet
 
     if (this.online === true && this.videoEnabled === true) {
       if (typeof host === 'undefined' || host === null) {
         // No host parameter passed in, so we'll set this to our internally stored host
-        host = this.host;
+        host = this.streaming_host;
       }
 
       this.connected = false; // Starting connection
       this?.log?.debug?.('Connection started to "%s"', host);
+      this.#host = host; // Update internal host name since we’re about to connect
+
+      // Wrap tls.connect() in a Promise so we can await the TLS handshake
+      try {
+        await new Promise((resolve, reject) => {
+          this.#socket = tls.connect({ host: host, port: 1443 }, () => {
+            // Opened connection to Nexus server, so now need to authenticate ourselves
+            this?.log?.debug?.('Connection established to "%s"', host);
+
+            this.#socket.setKeepAlive(true); // Keep socket connection alive
+            this.connected = true;
+            this.#Authenticate(false); // Send authentication request
+            resolve(); // Allow await connect() to continue
+          });
 
-      this.#socket = tls.connect({ host: host, port: 1443 }, () => {
-        // Opened connection to Nexus server, so now need to authenticate ourselves
-        this?.log?.debug?.('Connection established to "%s"', host);
-
-        this.#socket.setKeepAlive(true); // Keep socket connection alive
-        this.host = host; // update internal host name since we've connected
-        this.connected = true;
-        this.#Authenticate(false);
-      });
-
-      this.#socket.on('error', () => {});
+          this.#socket.on('error', (err) => {
+            // TLS error (could be refused, timeout, etc.)
+            this?.log?.warn?.('TLS error on connect to "%s": %s', host, err?.message || err);
+            this.connected = undefined;
+            reject(err);
+          });
 
-      this.#socket.on('end', () => {});
+          this.#socket.on('end', () => {});
 
-      this.#socket.on('data', (data) => {
-        this.#handleNexusData(data);
-      });
+          this.#socket.on('data', (data) => {
+            this.#handleNexusData(data);
+          });
 
-      this.#socket.on('close', (hadError) => {
-        this?.log?.debug?.('Connection closed to "%s"', host);
-
-        clearInterval(this.pingTimer);
-        clearTimeout(this.stalledTimer);
-        this.pingTimer = undefined;
-        this.stalledTimer = undefined;
-        this.#authorised = false; // Since connection close, we can't be authorised anymore
-        this.#socket = undefined; // Clear socket object
-        this.connected = undefined;
-        this.#id = undefined; // Not an active session anymore
-
-        if (hadError === true && this.haveOutputs() === true) {
-          // We still have either active buffering occuring or output streams running
-          // so attempt to restart connection to existing host
-          this.connect(host);
-        }
-      });
+          this.#socket.on('close', (hadError) => {
+            this?.log?.debug?.('Connection closed to "%s"', host);
+
+            clearInterval(this.#pingTimer);
+            clearTimeout(this.#stalledTimer);
+            this.#pingTimer = undefined;
+            this.#stalledTimer = undefined;
+            this.#authorised = false; // Since connection closed, we can't be authorised anymore
+            this.#socket = undefined; // Clear socket object
+            this.connected = undefined;
+            this.#id = undefined; // Not an active session anymore
+
+            if (hadError === true && (this.isStreaming() === true || this.isBuffering() === true)) {
+              // We still have either active buffering occurring or output streams running
+              // so attempt to restart connection to existing host
+              this.connect(host);
+            }
+          });
+        });
+      } catch (error) {
+        this?.log?.error?.('Failed to connect to "%s": %s', host, String(error));
+      }
     }
   }
 
-  close(stopStreamFirst) {
+  async close(stopStreamFirst) {
     // Close an authenicated socket stream gracefully
     if (this.#socket !== undefined) {
       if (stopStreamFirst === true) {
         // Send a notifcation to nexus we're finished playback
-        this.#stopNexusData();
+        await this.#stopNexusData();
       }
       this.#socket.destroy();
     }
@@ -181,15 +191,17 @@ export default class NexusTalk extends Streamer {
     this.#messages = [];
     this.video = {};
     this.audio = {};
+    this.#host = undefined; // No longer connected to this host
   }
 
-  update(deviceData) {
+  async onUpdate(deviceData) {
     if (typeof deviceData !== 'object') {
       return;
     }
 
-    if (deviceData.apiAccess.token !== this.token) {
+    if (deviceData?.apiAccess?.token !== undefined && deviceData.apiAccess.token !== this.token) {
       // access token has changed so re-authorise
+      this?.log?.debug?.('Access token has changed for uuid "%s". Updating token', this.nest_google_uuid);
       this.token = deviceData.apiAccess.token;
 
       if (this.#socket !== undefined) {
@@ -197,30 +209,38 @@ export default class NexusTalk extends Streamer {
       }
     }
 
-    if (this.host !== deviceData.streaming_host) {
-      this.host = deviceData.streaming_host;
-      this?.log?.debug?.('New host has been requested for connection. Host requested is "%s"', this.host);
-    }
+    if (deviceData?.streaming_host !== undefined && this.streaming_host !== deviceData.streaming_host) {
+      this.streaming_host = deviceData.streaming_host;
 
-    // Let our parent handle the remaining updates
-    super.update(deviceData);
+      if (this.isStreaming() === true || this.isBuffering() === true) {
+        this?.log?.debug?.('New host has been requested for connection. Host requested is "%s"', this.streaming_host);
+
+        // Setup listener for socket close event. Once socket is closed, we'll perform the redirect
+        this.#socket?.on?.('close', () => {
+          this.connect(this.streaming_host); // Connect to new host
+        });
+        this.close(true); // Close existing socket
+      }
+    }
   }
 
-  talkingAudio(talkingData) {
+  sendTalkback(talkingBuffer) {
+    if (Buffer.isBuffer(talkingBuffer) === false || this.#protobufNexusTalk === undefined || this.#id === undefined) {
+      return;
+    }
+
     // Encode audio packet for sending to camera
-    if (typeof talkingData === 'object' && this.#protobufNexusTalk !== undefined) {
-      let TraitMap = this.#protobufNexusTalk.lookup('nest.nexustalk.v1.AudioPayload');
-      if (TraitMap !== null) {
-        let encodedData = TraitMap.encode(
-          TraitMap.fromObject({
-            payload: talkingData,
-            sessionId: this.#id,
-            codec: this.codecs.talk.toUpperCase(),
-            sampleRate: 16000,
-          }),
-        ).finish();
-        this.#sendMessage(PACKET_TYPE.AUDIO_PAYLOAD, encodedData);
-      }
+    let TraitMap = this.#protobufNexusTalk.lookup('nest.nexustalk.v1.AudioPayload');
+    if (TraitMap !== null) {
+      let encodedData = TraitMap.encode(
+        TraitMap.fromObject({
+          payload: talkingBuffer,
+          sessionId: this.#id,
+          codec: Streamer.CODEC_TYPE.SPEEX,
+          sampleRate: 16000,
+        }),
+      ).finish();
+      this.#sendMessage(PACKET_TYPE.AUDIO_PAYLOAD, encodedData);
     }
   }
 
@@ -300,15 +320,13 @@ export default class NexusTalk extends Streamer {
       let TraitMap = this.#protobufNexusTalk.lookup('nest.nexustalk.v1.AuthoriseRequest');
       if (TraitMap !== null) {
         authoriseRequest = TraitMap.encode(
-          TraitMap.fromObject(
-            this.tokenType === 'nest' ? { sessionToken: this.token } : this.tokenType === 'google' ? { oliveToken: this.token } : {},
-          ),
+          TraitMap.fromObject(this.useGoogleAuth === true ? { oliveToken: this.token } : { sessionToken: this.token }),
         ).finish();
       }
 
       if (reauthorise === true && authoriseRequest !== null) {
         // Request to re-authorise only
-        this?.log?.debug?.('Re-authentication requested to "%s"', this.host);
+        this?.log?.debug?.('Re-authentication requested to "%s"', this.#host);
         this.#sendMessage(PACKET_TYPE.AUTHORIZE_REQUEST, authoriseRequest);
       }
 
@@ -316,12 +334,12 @@ export default class NexusTalk extends Streamer {
         // This isn't a re-authorise request, so perform 'Hello' packet
         let TraitMap = this.#protobufNexusTalk.lookup('nest.nexustalk.v1.Hello');
         if (TraitMap !== null) {
-          this?.log?.debug?.('Performing authentication to "%s"', this.host);
+          this?.log?.debug?.('Performing authentication to "%s"', this.#host);
 
           let encodedData = TraitMap.encode(
             TraitMap.fromObject({
               protocolVersion: 'VERSION_3',
-              uuid: this.uuid.split(/[._]+/)[1],
+              uuid: this.nest_google_uuid.split(/[._]+/)[1],
               requireConnectedCamera: false,
               USER_AGENT: USER_AGENT,
               deviceId: crypto.randomUUID(),
@@ -350,13 +368,12 @@ export default class NexusTalk extends Streamer {
       return;
     }
 
-    this?.log?.debug?.('Redirect requested from "%s" to "%s"', this.host, redirectToHost);
+    this?.log?.debug?.('Redirect requested from "%s" to "%s"', this.#host, redirectToHost);
 
     // Setup listener for socket close event. Once socket is closed, we'll perform the redirect
-    this.#socket &&
-      this.#socket.on('close', () => {
-        this.connect(redirectToHost); // Connect to new host
-      });
+    this.#socket?.on?.('close', () => {
+      this.connect(redirectToHost); // Connect to new host
+    });
     this.close(true); // Close existing socket
   }
 
@@ -368,18 +385,17 @@ export default class NexusTalk extends Streamer {
         if (stream.codecType === this.codecs.video.toUpperCase()) {
           this.video = {
             id: stream.channelId,
-            startTime: Date.now() + stream.startTime,
+            baseTimestamp: stream.startTimestamp,
+            baseTime: Date.now(),
             sampleRate: stream.sampleRate,
-            timeStamp: 0,
           };
         }
         if (stream.codecType === this.codecs.audio.toUpperCase()) {
           this.audio = {
             id: stream.channelId,
-            startTime: Date.now() + stream.startTime,
+            baseTimestamp: stream.startTimestamp,
+            baseTime: Date.now(),
             sampleRate: stream.sampleRate,
-            timeStamp: 0,
-            talking: false,
           };
         }
       });
@@ -389,11 +405,26 @@ export default class NexusTalk extends Streamer {
       this.#packets = [];
       this.#messages = [];
 
-      this?.log?.debug?.('Playback started from "%s" with session ID "%s"', this.host, this.#id);
+      this?.log?.debug?.('Playback started from "%s" with session ID "%s"', this.#host, this.#id);
     }
   }
 
   #handlePlaybackPacket(payload) {
+    const calculateTimestamp = (delta, stream) => {
+      if (
+        typeof delta !== 'number' ||
+        typeof stream?.sampleRate !== 'number' ||
+        stream?.baseTime === undefined ||
+        stream?.baseTimestamp === undefined
+      ) {
+        return Date.now();
+      }
+
+      let deltaTicks = stream.baseTimestamp + delta - stream.baseTimestamp;
+      let deltaMs = (deltaTicks / stream.sampleRate) * 1000;
+      return stream.baseTime + deltaMs;
+    };
+
     // Decode playback packet
     if (typeof payload === 'object' && this.#protobufNexusTalk !== undefined) {
       let decodedMessage = this.#protobufNexusTalk.lookup('nest.nexustalk.v1.PlaybackPacket').decode(payload).toJSON();
@@ -401,30 +432,33 @@ export default class NexusTalk extends Streamer {
       // Setup up a timeout to monitor for no packets recieved in a certain period
       // If its trigger, we'll attempt to restart the stream and/or connection
       // <-- testing to see how often this occurs first
-      clearTimeout(this.stalledTimer);
-      this.stalledTimer = setTimeout(() => {
+      clearTimeout(this.#stalledTimer);
+      this.#stalledTimer = setTimeout(() => {
         this?.log?.debug?.(
           'We have not received any data from nexus in the past "%s" seconds for uuid "%s". Attempting restart',
           10,
-          this.uuid,
+          this.nest_google_uuid,
         );
 
         // Setup listener for socket close event. Once socket is closed, we'll perform the re-connection
-        this.#socket &&
-          this.#socket.on('close', () => {
-            this.connect(); // try reconnection
-          });
+        this.#socket?.on?.('close', () => {
+          this.connect(); // try reconnection
+        });
         this.close(false); // Close existing socket
       }, 10000);
 
+      // Timestamps are rolling — incremented from startTime using timestampDelta per packet
+
       // Handle video packet
       if (decodedMessage?.channelId !== undefined && decodedMessage.channelId === this.video?.id) {
-        this.addToOutput('video', Buffer.from(decodedMessage.payload, 'base64'));
+        let ts = calculateTimestamp(decodedMessage.timestampDelta, this.video);
+        this.add(Streamer.PACKET_TYPE.VIDEO, Buffer.from(decodedMessage.payload, 'base64'), ts);
       }
 
       // Handle audio packet
       if (decodedMessage?.channelId !== undefined && decodedMessage.channelId === this.audio?.id) {
-        this.addToOutput('audio', Buffer.from(decodedMessage.payload, 'base64'));
+        let ts = calculateTimestamp(decodedMessage.timestampDelta, this.audio);
+        this.add(Streamer.PACKET_TYPE.AUDIO, Buffer.from(decodedMessage.payload, 'base64'), ts);
       }
     }
   }
@@ -436,18 +470,17 @@ export default class NexusTalk extends Streamer {
 
       if (this.#id !== undefined && decodedMessage.reason === 'USER_ENDED_SESSION') {
         // Normal playback ended ie: when we stopped playback
-        this?.log?.debug?.('Playback ended on "%s"', this.host);
+        this?.log?.debug?.('Playback ended on "%s"', this.#host);
       }
 
       if (decodedMessage.reason !== 'USER_ENDED_SESSION') {
         // Error during playback, so we'll attempt to restart by reconnection to host
-        this?.log?.debug?.('Playback ended on "%s" with error "%s". Attempting reconnection', this.host, decodedMessage.reason);
+        this?.log?.debug?.('Playback ended on "%s" with error "%s". Attempting reconnection', this.#host, decodedMessage.reason);
 
         // Setup listener for socket close event. Once socket is closed, we'll perform the re-connection
-        this.#socket &&
-          this.#socket.on('close', () => {
-            this.connect(); // try reconnection to existing host
-          });
+        this.#socket?.on?.('close', () => {
+          this.connect(); // try reconnection to existing host
+        });
         this.close(false); // Close existing socket
       }
     }
@@ -472,7 +505,7 @@ export default class NexusTalk extends Streamer {
     if (typeof payload === 'object' && this.#protobufNexusTalk !== undefined) {
       //let decodedMessage = this.#protobufNexusTalk.lookup('nest.nexustalk.v1.TalkbackBegin').decode(payload).toJSON();
       this.audio.talking = true;
-      this?.log?.debug?.('Talking started on uuid "%s"', this.uuid);
+      this?.log?.debug?.('Talking started on uuid "%s"', this.nest_google_uuid);
     }
   }
 
@@ -481,7 +514,7 @@ export default class NexusTalk extends Streamer {
     if (typeof payload === 'object' && this.#protobufNexusTalk !== undefined) {
       //let decodedMessage = this.#protobufNexusTalk.lookup('nest.nexustalk.v1.TalkbackEnd').decode(payload).toJSON();
       this.audio.talking = false;
-      this?.log?.debug?.('Talking ended on uuid "%s"', this.uuid);
+      this?.log?.debug?.('Talking ended on uuid "%s"', this.nest_google_uuid);
     }
   }
 
@@ -521,8 +554,8 @@ export default class NexusTalk extends Streamer {
           }
 
           // Periodically send PING message to keep stream alive
-          clearInterval(this.pingTimer);
-          this.pingTimer = setInterval(() => {
+          clearInterval(this.#pingTimer);
+          this.#pingTimer = setInterval(() => {
             this.#sendMessage(PACKET_TYPE.PING, Buffer.alloc(0));
           }, PING_INTERVAL);