@alwaysai/device-agent 1.3.0-1 → 1.3.1-1

package/src/cloud-connection/device-agent-cloud-connection.ts

@@ -1,74 +1,75 @@
 // eslint-disable-next-line
 const awsIot = require('aws-iot-device-sdk');
-import { getIoTCoreEndpointUrl } from '../infrastructure/urls';
-import { existsSync } from 'fs';
-import {
-  BOOTSTRAP_PRIVATE_KEY_FILE_PATH,
-  AWS_ROOT_CERTIFICATE_FILE_PATH,
-  BOOTSTRAP_CERTIFICATES_DIR_PATH,
-  DEVICE_PRIVATE_KEY_FILE_PATH,
-  DEVICE_CERTIFICATE_FILE_PATH
-} from '../util/directories';
 import {
-  keyMirrors,
-  validateToClientMessage,
-  SignedUrlsRequestPayload,
-  getToDeviceTopic,
   AppInstallResponsePayload,
-  validateToDeviceAgentMessage,
-  ToDeviceAgentMessage,
-  ToCloudMessage,
   AppStateControlPayload,
   AppVersionControlInstallPayload,
   AppVersionControlUninstallPayload,
-  ToClientMessage,
-  DeviceActionPayload
+  DeviceActionPayload,
+  ModelsInstallResponsePayload,
+  SignedUrlsRequestPayload,
+  ToCloudMessage,
+  ToDeviceAgentMessage,
+  getToDeviceTopic,
+  buildSignedUrlsRequestMessage,
+  buildToClientStatusResponseMessage,
+  StatusResponsePayload,
+  keyMirrors,
+  validateToDeviceAgentMessage
 } from '@alwaysai/device-agent-schemas';
-import { getDeviceUuid } from '../util/get-device-id';
-import { logger } from '../util/logger';
-import { cloudModeReady } from '../util/cloud-mode-ready';
-import { AgentConfigFile } from '../infrastructure/agent-config';
+import { existsSync } from 'fs';
 import {
+  installApp,
+  restartApp,
+  setEnv,
   startApp,
   stopApp,
-  restartApp,
-  updateModelsWithPresignedUrls,
-  installApp,
   uninstallApp,
   updateAppCfg,
-  setEnv
+  updateModelsWithPresignedUrls
 } from '../application-control';
-import { ShadowHandler, ShadowTopics, ShadowUpdate } from './shadow-handler';
-import { secureTunnelNotifyHandler } from '../secure-tunneling/index';
-import { Publisher } from './publisher';
-import { LiveUpdatesHandler } from './live-updates-handler';
-import { bootstrapProvision } from './bootstrap-provision';
-import { CmdStatusManager } from './cmd-status';
-import { PassthroughHandler, runChannel } from './passthrough-handler';
+import { createAppBackup, rollbackApp } from '../application-control/backup';
+import { reboot } from '../device-control/device-control';
 import { ALWAYSAI_ANALYTICS_PASSTHROUGH } from '../environment';
-import { getStatusResponsePayload } from './messages';
-import { ModelsInstallResponsePayload } from '@alwaysai/device-agent-schemas';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+import { getIoTCoreEndpointUrl } from '../infrastructure/urls';
+import { SecureTunnelHandlerSingleton } from '../secure-tunneling/secure-tunneling';
+import { cloudModeReady } from '../util/cloud-mode-ready';
+import {
+  AWS_ROOT_CERTIFICATE_FILE_PATH,
+  BOOTSTRAP_CERTIFICATES_DIR_PATH,
+  BOOTSTRAP_PRIVATE_KEY_FILE_PATH,
+  DEVICE_CERTIFICATE_FILE_PATH,
+  DEVICE_PRIVATE_KEY_FILE_PATH
+} from '../util/directories';
+import { getDeviceUuid } from '../util/get-device-id';
+import { logger } from '../util/logger';
 import sleep from '../util/sleep';
-import { createAppBackup, rollbackApp } from '../application-control/backup';
+import { bootstrapProvision } from './bootstrap-provision';
+import { LiveUpdatesHandler } from './live-updates-handler';
+import { PassthroughHandler, runChannel } from './passthrough-handler';
+import { Publisher } from './publisher';
+import { ShadowHandler, ShadowTopics, ShadowUpdate } from './shadow-handler';
 import { TransactionManager } from './transaction-manager';
-import {
-  buildSignedUrlsRequestMessage,
-  buildStatusResponseMessage
-} from './message-builder';
-import { reboot } from '../device-control/device-control';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+const exec_promise = promisify(exec);
 
 export class DeviceAgentCloudConnection {
   private shadowHandler: ShadowHandler;
   public publisher: Publisher;
-  private cmdStatusMgr: CmdStatusManager;
   private liveUpdatesHandler: LiveUpdatesHandler;
   private txnMgr: TransactionManager;
   private device = awsIot.device;
 
   private clientId = getDeviceUuid();
   private host = getIoTCoreEndpointUrl();
+  private port = 8883;
   private readonly toDeviceTopic = getToDeviceTopic(this.clientId);
   private readonly secureTunnelNotifyTopic = `$aws/things/${this.clientId}/tunnels/notify`;
+  private readonly secureTunnelHandler =
+    SecureTunnelHandlerSingleton.getInstance();
   // FIXME: Add support for multiple simultaneous project updates
   private appCfgUpdateQueue: ShadowUpdate[] = [];
 
@@ -106,7 +107,8 @@ export class DeviceAgentCloudConnection {
             appReleaseHash
           }
         };
-        const message = await buildSignedUrlsRequestMessage(
+        const message = buildSignedUrlsRequestMessage(
+          this.clientId,
           signedUrlsRequestPayload,
           txId
         );
@@ -272,62 +274,6 @@ export class DeviceAgentCloudConnection {
     }
   }
 
-  // eslint-disable-next-line
-  private async atomicCmd<T extends any[]>(props: {
-    func: (...args: T) => Promise<boolean>;
-    args: T;
-    projectId: string;
-    txId: string;
-  }): Promise<boolean> {
-    const { func, args, projectId, txId } = props;
-    try {
-      await this.cmdStatusMgr.start(projectId);
-      await this.liveUpdatesHandler.enableTransactionStatus({
-        txId
-      });
-      const completed = await func(...args);
-      if (completed) {
-        await this.cmdStatusMgr.stop(projectId);
-        await this.liveUpdatesHandler.disableTransactionStatus({
-          txId
-        });
-        const successStatusResponsePayload = await getStatusResponsePayload(
-          keyMirrors.statusResponse.success,
-          ''
-        );
-        // Send final status message
-        const message = await buildStatusResponseMessage(
-          successStatusResponsePayload,
-          txId
-        );
-        this.publisher.publishToClient(message);
-      }
-      return completed;
-    } catch (e) {
-      logger.error(
-        `Failed to execute cmd for ${projectId}:\n${e.message}\n${e.stack}`
-      );
-      const message: string = e.message;
-
-      // uninstall the failed app to put system back in good state
-      await this.cmdStatusMgr.stop(projectId);
-      await this.liveUpdatesHandler.disableTransactionStatus({
-        txId
-      });
-      const failureStatusResponsePayload = await getStatusResponsePayload(
-        keyMirrors.statusResponse.failure,
-        message
-      );
-      // Send final status message
-      const failureStatusResponseMessage = await buildStatusResponseMessage(
-        failureStatusResponsePayload,
-        txId
-      );
-      this.publisher.publishToClient(failureStatusResponseMessage);
-      return true;
-    }
-  }
-
   private handleAppConfigUpdate = async (
     update: ShadowUpdate,
     txId: string
@@ -353,7 +299,8 @@ export class DeviceAgentCloudConnection {
           models: updatedModels
         }
       };
-      const message = await buildSignedUrlsRequestMessage(
+      const message = buildSignedUrlsRequestMessage(
+        this.clientId,
         modelsOnlyUrlsRequestPayload,
         txId
       );
@@ -397,17 +344,19 @@ export class DeviceAgentCloudConnection {
       caPath: AWS_ROOT_CERTIFICATE_FILE_PATH,
       clientId: this.clientId,
       host: this.host,
-      port: 8883,
+      port: this.port,
       keepalive: 1 // time before re-connect attempt on dropped connection, default is 400 seconds
     });
     this.publisher = new Publisher(this.device, this.clientId);
     this.shadowHandler = new ShadowHandler(this.clientId, this.publisher);
-    this.cmdStatusMgr = new CmdStatusManager();
     this.liveUpdatesHandler = new LiveUpdatesHandler(
       this.publisher,
       this.clientId
     );
-    this.txnMgr = new TransactionManager();
+    this.txnMgr = new TransactionManager(
+      this.publisher,
+      this.liveUpdatesHandler
+    );
 
     this.subscribe(this.toDeviceTopic);
     this.subscribe(this.secureTunnelNotifyTopic);
@@ -416,6 +365,8 @@ export class DeviceAgentCloudConnection {
     this.subscribe(this.shadowHandler.shadowTopics.projects.updateDelta);
     this.subscribe(this.shadowHandler.shadowTopics.projects.updateAccepted);
     this.subscribe(this.shadowHandler.shadowTopics.projects.updateRejected);
+    this.subscribe(this.shadowHandler.shadowTopics.secureTunnel.updateDelta);
+    this.subscribe(this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted);
   }
 
   public getClientId(): string {
@@ -431,7 +382,7 @@ export class DeviceAgentCloudConnection {
   }
 
   public isCmdInProgress(projectId: string): boolean {
-    return this.cmdStatusMgr.isCmdInProgress(projectId);
+    return this.txnMgr.isOngoingTransactionForProjectID(projectId);
   }
 
   public async updateProjectShadow(projectId: string) {
@@ -458,23 +409,29 @@ export class DeviceAgentCloudConnection {
       return;
     }
     const txId = message.txId;
+    const {
+      app_state_control,
+      app_version_control,
+      live_state_updates,
+      app_install_response,
+      models_install_response,
+      status_response,
+      device_action
+    } = keyMirrors.toDeviceAgentMessageType;
     switch (message.messageType) {
-      case keyMirrors.toDeviceAgentMessageType.app_state_control: {
+      case app_state_control: {
         // txId sent from cloud, just need to continue it
         const payload = message.payload;
         const projectId = payload.projectId;
 
         try {
-          this.txnMgr.addTransaction(txId, projectId, payload.baseCommand);
-          const completed = await this.atomicCmd({
-            func: this.handleAppStateControl,
-            args: [message.payload],
+          await this.txnMgr.runTransactionStep({
+            func: () => this.handleAppStateControl(message.payload),
             projectId,
-            txId
+            txId,
+            start: true,
+            stepName: payload.baseCommand
           });
-          if (completed) {
-            this.txnMgr.completeTransaction(txId);
-          }
         } catch (e) {
           logger.error(
             `Error processing application state control request: ${e}!`
@@ -483,40 +440,33 @@ export class DeviceAgentCloudConnection {
 
         break;
       }
-      case keyMirrors.toDeviceAgentMessageType.app_version_control: {
+      case app_version_control: {
         // txId sent from cloud, just need to continue it
         const payload = message.payload;
         const projectId = payload.projectId;
-        const appReleaseHash =
-          payload.baseCommand === keyMirrors.appVersionControl.install
-            ? payload.appReleaseHash
-            : undefined;
         try {
-          this.txnMgr.addTransaction(txId, projectId, payload.baseCommand);
-          const completed = await this.atomicCmd({
-            func: this.handleAppVersionControl,
-            args: [payload, txId],
+          await this.txnMgr.runTransactionStep({
+            func: () => this.handleAppVersionControl(payload, txId),
             projectId,
-            txId
+            txId,
+            start: true,
+            stepName: payload.baseCommand
           });
-          if (completed) {
-            this.txnMgr.completeTransaction(txId);
-          }
         } catch (e) {
           logger.error(`Error processing application install request: ${e}!`);
         }
 
         break;
       }
-      case keyMirrors.toDeviceAgentMessageType.live_state_updates: {
+      case live_state_updates: {
         const payload = message.payload;
         // TODO: Send response?
-        await this.liveUpdatesHandler.handleToggles(payload, txId);
+        void this.liveUpdatesHandler.handleToggles(payload, txId);
         break;
       }
-      case keyMirrors.toDeviceAgentMessageType.app_install_response: {
+      case app_install_response: {
         const payload = message.payload;
-        const { projectId, appReleaseHash } = payload.appInstallResponse;
+        const { projectId } = payload.appInstallResponse;
         if (txId !== this.txnMgr.getTransactionFromProject(projectId)) {
           throw new Error(
             `App install response received a message for a transaction ID ${txId} that is not currently underway (${this.txnMgr.getTransactionFromProject(
@@ -524,19 +474,16 @@ export class DeviceAgentCloudConnection {
             )})!`
           );
         }
-        const completed = await this.atomicCmd({
-          func: this.handleAppInstallCloudResponsePayload,
-          args: [payload],
+        await this.txnMgr.runTransactionStep({
+          func: () => this.handleAppInstallCloudResponsePayload(payload),
           projectId,
-          txId
+          txId,
+          start: false,
+          stepName: message.messageType
         });
-        if (completed) {
-          this.txnMgr.completeTransaction(txId);
-        }
-
         break;
       }
-      case keyMirrors.toDeviceAgentMessageType.models_install_response: {
+      case models_install_response: {
         // This message doesn't have appReleaseHash in it's payload, but
         // atomicCmd should be able to read it from the installed app
         const payload = message.payload;
@@ -548,43 +495,73 @@ export class DeviceAgentCloudConnection {
             )})!`
           );
         }
-        const completed = await this.atomicCmd({
-          func: this.handleModelsInstallCloudResponsePayload,
-          args: [payload],
+        await this.txnMgr.runTransactionStep({
+          func: () => this.handleModelsInstallCloudResponsePayload(payload),
           projectId,
-          txId
+          txId,
+          start: false,
+          stepName: message.messageType
         });
-        if (completed) {
+        break;
+      }
+      case status_response: {
+        const { failure } = keyMirrors.statusResponse;
+        if (message.payload.status === failure) {
           this.txnMgr.completeTransaction(txId);
-        }
 
+          const failureStatusResponsePayload: StatusResponsePayload = {
+            status: keyMirrors.statusResponse.failure,
+            message: message.payload.message
+          };
+          // Send final status message
+          const failureStatusResponseMessage =
+            buildToClientStatusResponseMessage(
+              this.clientId,
+              failureStatusResponsePayload,
+              txId
+            );
+          this.publisher.publishToClient(failureStatusResponseMessage);
+        }
         break;
       }
-      case keyMirrors.toDeviceAgentMessageType.device_action: {
+      case device_action: {
         try {
-          const successStatusResponsePayload = await getStatusResponsePayload(
-            keyMirrors.statusResponse.success,
-            ''
-          );
-          const successStatusResponseMessage = await buildStatusResponseMessage(
-            successStatusResponsePayload,
+          const statusResponsePayload: StatusResponsePayload = {
+            status: keyMirrors.statusResponse.in_progress
+          };
+          const statusResponseMessage = buildToClientStatusResponseMessage(
+            this.clientId,
+            statusResponsePayload,
             txId
           );
-          this.publisher.publishToClient(successStatusResponseMessage);
+          this.publisher.publishToClient(statusResponseMessage);
 
           await this.handleDeviceAction(message.payload);
+
+          const successStatusResponsePayload: StatusResponsePayload = {
+            status: keyMirrors.statusResponse.success
+          };
+          const successStatusResponseMessage =
+            buildToClientStatusResponseMessage(
+              this.clientId,
+              successStatusResponsePayload,
+              txId
+            );
+          this.publisher.publishToClient(successStatusResponseMessage);
         } catch (e) {
           logger.error(
             `There was a problem performing device action '${message.payload.action}': ${e.message}`
           );
-          const failureStatusResponsePayload = await getStatusResponsePayload(
-            keyMirrors.statusResponse.failure,
-            e.message
-          );
-          const failureStatusResponseMessage = await buildStatusResponseMessage(
-            failureStatusResponsePayload,
-            txId
-          );
+          const failureStatusResponsePayload: StatusResponsePayload = {
+            status: keyMirrors.statusResponse.failure,
+            message: e.message
+          };
+          const failureStatusResponseMessage =
+            buildToClientStatusResponseMessage(
+              this.clientId,
+              failureStatusResponsePayload,
+              txId
+            );
           this.publisher.publishToClient(failureStatusResponseMessage);
         }
         break;
@@ -609,7 +586,7 @@ export class DeviceAgentCloudConnection {
     );
     switch (topic) {
       case this.shadowHandler.shadowTopics.projects.getAccepted:
-      case this.shadowHandler.shadowTopics.projects.updateDelta: {
+      case this.shadowHandler.shadowTopics.projects.updateAccepted: {
         const shadowUpdates = await this.shadowHandler.handleShadowTopic({
           topic,
           payload: message.state,
@@ -620,16 +597,13 @@ export class DeviceAgentCloudConnection {
             const projectId = shadowUpdate.projectId;
             const txId = shadowUpdate.txId;
             try {
-              this.txnMgr.addTransaction(txId, projectId, topic);
-              const completed = await this.atomicCmd({
-                func: this.handleAppConfigUpdate,
-                args: [shadowUpdate, txId],
+              await this.txnMgr.runTransactionStep({
+                func: () => this.handleAppConfigUpdate(shadowUpdate, txId),
                 projectId,
-                txId
+                txId,
+                start: true,
+                stepName: topic
               });
-              if (completed) {
-                this.txnMgr.completeTransaction(txId);
-              }
             } catch (e) {
               logger.error(`Error handling shadow message: ${e.message}`);
             }
@@ -638,7 +612,7 @@ export class DeviceAgentCloudConnection {
         break;
       }
       case this.shadowHandler.shadowTopics.projects.getRejected:
-      case this.shadowHandler.shadowTopics.projects.updateAccepted:
+      case this.shadowHandler.shadowTopics.projects.updateDelta:
       case this.shadowHandler.shadowTopics.projects.updateRejected:
         // Not handling these for now
         break;
@@ -650,8 +624,24 @@ export class DeviceAgentCloudConnection {
         break;
 
       case this.secureTunnelNotifyTopic:
-        await secureTunnelNotifyHandler(message);
+        await this.secureTunnelHandler.secureTunnelNotifyHandler(message);
+        break;
+      case this.shadowHandler.shadowTopics.secureTunnel.updateDelta: {
+        logger.info(`Received secure tunnel update: ${message}`);
+        const reported = await this.secureTunnelHandler.syncShadowToDeviceState(
+          message
+        );
+        this.publisher.publish(
+          this.shadowHandler.shadowTopics.secureTunnel.update,
+          JSON.stringify({ state: { reported } })
+        );
+        break;
+      }
+      case this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted: {
+        logger.info(`Received secure tunnel deleteAccepted: ${message}`);
+        await this.secureTunnelHandler.destroy();
         break;
+      }
       default:
         logger.error(`Unexpected topic, ignoring! ${topic}`);
     }
@@ -692,9 +682,63 @@ export class DeviceAgentCloudConnection {
 
     this.device.on('offline', () => {
       logger.warn(`Device Agent is offline ${new Date().toLocaleString()}`);
+      void this.logConnectionInfo();
     });
   }
 
+  public async logConnectionInfo() {
+    try {
+      /**
+       * We're using the 'netcat' or 'nc' command to test the connection to the IoT Core endpoint.
+       * This command doesn't always exit (see below), so
+       * we use timeout to break out of the prompt
+       * and catch the resulting error/parse the resulting stderr
+       *
+       * Sample command for current host and port:
+       * nc -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 8883
+       *
+       * Sample output when port is not blocked and host is reachable:
+       * $ nc  -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 443
+       * Connection to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 443 port [tcp/https] succeeded!
+       *
+       *
+       * Sample output when port is blocked (will repeatedly try until ctrl-C out):
+       * $ nc -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 8883
+       * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
+       * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
+       * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
+       * ^C
+       *
+       *
+       * Sample command/output when the port isn't enable on that host:
+       * $ nc -zv -w 1 localhost 8883
+       * nc: connect to localhost port 8883 (tcp) failed: Connection refused
+       */
+      await exec_promise(`nc -zv -w 1 ${this.host} ${this.port}`, {
+        timeout: 2000
+      });
+    } catch (err) {
+      const output = JSON.stringify(err['stderr']);
+      if (output.indexOf('not known') !== -1) {
+        logger.warn(
+          'Iot Core endpoint appears to be unreachable, internet connection may be unstable or the host may be down.'
+        );
+      } else if (output.indexOf('timed out') !== -1) {
+        logger.warn(
+          `Internet connection appears fine, however the endpoint was not reachable on the current connection port: ${this.port}\nPlease check if a firewall is in place.`
+        );
+      } else if (output.indexOf('refused') !== -1) {
+        logger.warn(
+          `The connection was refused, likely ${this.host} is not running a service on ${this.port}.`
+        );
+      } else {
+        logger.warn(
+          `Output from checking connection to ${this.host} on ${this.port}: ${output}`
+        );
+      }
+    }
+  }
+
   public async stop() {
     // FIXME: This method is currently only used by the CLI, and shadow messages
     // can be lost since we aren't waiting for responses so sleep for a short

package/src/cloud-connection/live-updates-handler.test.ts

@@ -3,6 +3,11 @@ import { Publisher } from './publisher';
 
 global.setTimeout = jest.fn() as unknown as typeof setTimeout;
 
+// https://github.com/facebook/react-native/issues/35701
+Object.defineProperty(global, 'performance', {
+  writable: true
+});
+
 const testTrueToggles = {
   deviceStats: true,
   appState: true
@@ -66,4 +71,19 @@ describe('Test Live Updates Handler', () => {
     expect(liveUpdatesHandler.getAppStateLiveUpdates()).toBe(false);
     expect(liveUpdatesHandler.getAppLogsLiveUpdates()).toBe(false);
   });
+
+  test('timeout turns off live updates', async () => {
+    jest.useFakeTimers({ legacyFakeTimers: true });
+
+    void liveUpdatesHandler.handleToggles(testTrueToggles, emptyTxId);
+    expect(liveUpdatesHandler.getDeviceStatsLiveUpdates()).toBe(true);
+    expect(liveUpdatesHandler.getAppStateLiveUpdates()).toBe(true);
+    expect(liveUpdatesHandler.getAppLogsLiveUpdates()).toBe(false);
+
+    jest.runAllTimers();
+
+    expect(liveUpdatesHandler.getDeviceStatsLiveUpdates()).toBe(false);
+    expect(liveUpdatesHandler.getAppStateLiveUpdates()).toBe(false);
+    expect(liveUpdatesHandler.getAppLogsLiveUpdates()).toBe(false);
+  });
 });