@alwaysai/device-agent 1.5.0 → 2.0.0

package/src/cloud-connection/device-agent-cloud-connection.ts

@@ -10,16 +10,27 @@ import {
   SignedUrlsRequestPayload,
   ToCloudMessage,
   ToDeviceAgentMessage,
-  getToDeviceTopic,
+  buildAppLogsMessage,
+  buildAppStateMessage,
+  buildDeviceStatsMessage,
   buildSignedUrlsRequestMessage,
   buildToClientStatusResponseMessage,
-  StatusResponsePayload,
+  getToDeviceTopic,
+  getUpdateDeltaStateFromMessage,
   keyMirrors,
-  validateToDeviceAgentMessage,
-  validateSecureTunnelShadowUpdate
+  validateSecureTunnelShadowUpdate,
+  validateToDeviceAgentMessage
 } from '@alwaysai/device-agent-schemas';
+import {
+  DEVICE_CERTIFICATE_FILE_PATH,
+  DEVICE_PRIVATE_KEY_FILE_PATH
+} from 'alwaysai/lib/infrastructure';
+import { stringifyError } from 'alwaysai/lib/util';
+import { exec } from 'child_process';
 import { existsSync } from 'fs';
+import { promisify } from 'util';
 import {
+  getAppLogs,
   installApp,
   restartApp,
   startApp,
@@ -29,37 +40,33 @@ import {
   updateModelsWithPresignedUrls
 } from '../application-control';
 import { createAppBackup, rollbackApp } from '../application-control/backup';
+import { pruneModels } from '../application-control/models';
 import { reboot } from '../device-control/device-control';
 import { ALWAYSAI_ANALYTICS_PASSTHROUGH } from '../environment';
 import { AgentConfigFile } from '../infrastructure/agent-config';
+import { getBootstrapPrivateKeyFilePath } from '../infrastructure/device-certificate';
+import { migrateFromLegacyCertsAndTokens } from '../infrastructure/legacy-migration/legacy-migration';
+import { requiredConfigFilesPresentAndValid } from '../infrastructure/required-config-checks';
 import { getIoTCoreEndpointUrl } from '../infrastructure/urls';
 import { SecureTunnelHandlerSingleton } from '../secure-tunneling/secure-tunneling';
-import { cloudModeReady } from '../util/cloud-mode-ready';
-import {
-  AWS_ROOT_CERTIFICATE_FILE_PATH,
-  BOOTSTRAP_CERTIFICATES_DIR_PATH,
-  BOOTSTRAP_PRIVATE_KEY_FILE_PATH,
-  DEVICE_CERTIFICATE_FILE_PATH,
-  DEVICE_PRIVATE_KEY_FILE_PATH
-} from '../util/directories';
+import AaiError from '../util/aai-error';
+import { getDeviceAgentVersion } from '../util/check-for-updates';
+import { AWS_ROOT_CERTIFICATE_FILE_PATH } from '../util/directories';
 import { getDeviceUuid } from '../util/get-device-id';
 import { logger } from '../util/logger';
 import sleep from '../util/sleep';
 import { bootstrapProvision } from './bootstrap-provision';
 import { LiveUpdatesHandler } from './live-updates-handler';
+import { getAppStatePayload, getDeviceStatsPayload } from './messages';
 import { PassthroughHandler } from './passthrough-handler';
 import { Publisher } from './publisher';
 import { ShadowHandler, ShadowUpdate } from './shadow-handler';
 import { TransactionManager } from './transaction-manager';
-import { exec } from 'child_process';
-import { promisify } from 'util';
-import { pruneModels } from '../application-control/models';
-import { getDeviceAgentVersion } from '../util/check-for-updates';
 
 const exec_promise = promisify(exec);
 
 export class DeviceAgentCloudConnection {
-  private shadowHandler: ShadowHandler;
+  public shadowHandler: ShadowHandler;
   public publisher: Publisher;
   private liveUpdatesHandler: LiveUpdatesHandler;
   private txnMgr: TransactionManager;
@@ -73,384 +80,182 @@ export class DeviceAgentCloudConnection {
   private readonly secureTunnelHandler =
     SecureTunnelHandlerSingleton.getInstance();
 
-  private handleAppStateControl = async (
-    payload: AppStateControlPayload
-  ): Promise<boolean> => {
-    const { baseCommand, projectId } = payload;
-    switch (baseCommand) {
-      case keyMirrors.appStateControl.start:
-        await startApp({ projectId });
-        break;
-      case keyMirrors.appStateControl.stop:
-        await stopApp({ projectId });
-        break;
-      case keyMirrors.appStateControl.restart:
-        await restartApp({ projectId });
-        break;
-    }
-    return true;
-  };
+  constructor() {
+    this.device = awsIot.device({
+      keyPath: DEVICE_PRIVATE_KEY_FILE_PATH,
+      certPath: DEVICE_CERTIFICATE_FILE_PATH,
+      caPath: AWS_ROOT_CERTIFICATE_FILE_PATH,
+      clientId: this.clientId,
+      host: this.host,
+      port: this.port,
+      keepalive: 10 // time before re-connect attempt on dropped connection, default is 400 seconds
+    });
 
-  private handleAppVersionControl = async (
-    payload:
-      | AppVersionControlInstallPayload
-      | AppVersionControlUninstallPayload,
-    txId: string
-  ): Promise<boolean> => {
-    switch (payload.baseCommand) {
-      case keyMirrors.appVersionControl.install: {
-        const { projectId, appReleaseHash } = payload;
+    this.publisher = new Publisher(this.device, this.clientId);
+    this.shadowHandler = new ShadowHandler(this.clientId, this.publisher);
+    this.liveUpdatesHandler = new LiveUpdatesHandler();
+    this.txnMgr = new TransactionManager(
+      this.publisher,
+      this.liveUpdatesHandler
+    );
 
-        const signedUrlsRequestPayload: SignedUrlsRequestPayload = {
-          signedUrlsRequest: {
-            projectId,
-            appReleaseHash
-          }
-        };
-        const message = buildSignedUrlsRequestMessage(
-          this.clientId,
-          signedUrlsRequestPayload,
-          txId
-        );
-        await this.publishCloudRequest(message);
-        return false;
-      }
-      case keyMirrors.appVersionControl.uninstall: {
-        const { projectId } = payload;
-        await this.atomicApplicationUninstall(projectId);
-        return true;
-      }
-      default:
-        logger.warn(
-          `Ignore App Version Control packet: ${JSON.stringify(
-            payload,
-            null,
-            2
-          )}`
-        );
-        return true;
+    this.subscribe(this.toDeviceTopic);
+    this.subscribe(this.secureTunnelNotifyTopic);
+    for (const topic of this.shadowHandler.projectShadowTopics) {
+      this.subscribe(topic);
     }
-  };
-
-  private handleAppInstallCloudResponsePayload = async (
-    payload: AppInstallResponsePayload
-  ): Promise<boolean> => {
-    const {
-      projectId,
-      appReleaseHash,
-      appInstallPayload,
-      modelsInstallPayload
-    } = payload.appInstallResponse;
-    const signedUrlsPayload = {
-      appInstallPayload,
-      modelsInstallPayload
-    };
-    await this.atomicApplicationUpdate(async () => {
-      this.shadowHandler.clearProjectShadow(projectId);
-      await installApp({ projectId, appReleaseHash, signedUrlsPayload });
-    }, projectId);
-    return true;
-  };
+    this.subscribe(this.shadowHandler.shadowTopics.secureTunnel.updateDelta);
+    this.subscribe(this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted);
 
-  private handleModelsInstallCloudResponsePayload = async (
-    payload: ModelsInstallResponsePayload,
-    txId: string
-  ): Promise<boolean> => {
-    const projectId = payload.modelsInstallResponse.projectId;
+    this.setupHandlers();
+  }
 
-    const update = this.txnMgr.getAppCfgUpdateFromTxID(txId);
-    if (update === undefined) {
-      throw new Error(
-        'Unknown error while updating models via application config! No config present for model update.'
-      );
-    }
-    const { appCfgUpdate, envVarUpdate } = update;
-    if (appCfgUpdate) {
-      await this.atomicApplicationUpdate(
-        async () =>
-          await updateModelsWithPresignedUrls({
-            projectId,
-            modelInstallPayloads: payload.modelsInstallResponse.newModels,
-            newAppCfg: appCfgUpdate.newAppCfg
-          }),
-        projectId
-      );
-    }
+  /*=================================================================
+                            Public interface
+  =================================================================*/
 
-    if (envVarUpdate) {
-      await this.atomicApplicationUpdate(
-        async () =>
-          await this.shadowHandler.updateProjectEnvVars({
-            projectId,
-            envVars: envVarUpdate.envVars
-          }),
-        projectId,
-        true
-      );
-    }
+  public getClientId(): string {
+    return this.clientId;
+  }
 
-    return true;
-  };
+  public isCmdInProgress(projectId: string): boolean {
+    return this.txnMgr.isOngoingTransactionForProjectID(projectId);
+  }
 
-  private async handleDeviceAction(payload: DeviceActionPayload) {
-    const { system_restart } = keyMirrors.deviceAction;
-    switch (payload.action) {
-      case system_restart: {
-        await reboot();
-        break;
-      }
-      default: {
-        logger.info(
-          `Unrecognized device action requested: '${payload.action}'.`
-        );
-      }
+  public async handleMessage(topic: string, message: any) {
+    logger.debug(
+      `Received message: ${JSON.stringify({ topic, message }, null, 2)}`
+    );
+    // ProjectShadow messages
+    if (this.shadowHandler.projectShadowTopics.includes(topic)) {
+      await this.handleProjectShadowMessage(topic, message);
+    } else if (topic === this.toDeviceTopic) {
+      await this.handleDeviceAgentMessage({
+        topic,
+        message
+      });
+      // SecureTunnelNotify messages
+    } else if (topic === this.secureTunnelNotifyTopic) {
+      await this.secureTunnelHandler.secureTunnelNotifyHandler(message);
+      // SecureTunnel messages
+    } else if (
+      topic === this.shadowHandler.shadowTopics.secureTunnel.updateDelta
+    ) {
+      await this.handleSecureTunnelMessage(message);
+    } else if (
+      topic === this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted
+    ) {
+      logger.info(`Received secure tunnel deleteAccepted: ${message}`);
+      await this.secureTunnelHandler.destroy();
+    } else {
+      logger.error(`Unexpected topic, ignoring! ${topic}`);
     }
   }
 
-  private async publishCloudRequest(message: ToCloudMessage) {
-    this.publisher.publishToCloud(message);
+  public async stop() {
+    // This method is currently only used by the CLI, and shadow messages can be
+    // lost since we aren't waiting for responses so sleep for a short time to
+    // receive them
+    await sleep(1000);
+    this.device.end();
   }
 
-  private subscribe(topic: string) {
-    logger.info(`Subscribing to ${topic}`);
-    this.device.subscribe(topic);
-  }
+  /*=================================================================
+                            Private interface
+  =================================================================*/
 
-  private async atomicApplicationUninstall(projectId: string) {
-    try {
-      await uninstallApp({ projectId });
-      this.shadowHandler.clearProjectShadow(projectId);
-    } catch (e) {
-      logger.error(`Failed to uninstall ${projectId}: ${e.message}`);
-      throw e;
-    }
-  }
+  private setupHandlers() {
+    this.device.on('connect', (connack: any) => {
+      logger.info('Device Agent has connected to the cloud');
+      // FIXME: EI-709 Skip this request for now to prevent kicking off another
+      // shadow update process if IoT Core disconnect occurs during app config update
+      //this.shadowHandler.getShadowUpdates();
+      void this.shadowHandler.updateSystemInfoShadow();
+    });
 
-  // eslint-disable-next-line
-  private async atomicApplicationUpdate <F extends () => any>(
-    func: F,
-    projectId: string,
-    skipUpdateShadow?: boolean
-  ): Promise<ReturnType<F>> {
-    // First try to create a backup, so that there is one available if something goes wrong in the next try:catch.
-    if (await AgentConfigFile().isAppPresent({ projectId })) {
+    this.device.on('disconnect', () => {
+      logger.warn('Device Agent has been disconnected from the cloud');
+    });
+
+    this.device.on('reconnect', () => {
+      logger.info(
+        `Device Agent attempting to re-connect ${new Date().toLocaleString()}`
+      );
+    });
+
+    this.device.on('error', function (e) {
+      logger.error(`Error connecting to cloud!\n${stringifyError(e)}`);
+    });
+
+    this.device.on('message', async (topic: string, payload: string) => {
       try {
-        await createAppBackup({ projectId });
+        const jsonPacket = JSON.parse(payload);
+        await this.handleMessage(topic, jsonPacket);
       } catch (e) {
-        logger.error(
-          `Could not create a backup for the project: ${projectId}:\n${e.message}\n${e.stack}`
-        );
+        logger.error(`Error parsing message!\n${stringifyError(e)}`);
       }
-    }
+    });
 
-    try {
-      const out: ReturnType<F> = await func();
-      if (!skipUpdateShadow)
-        await this.shadowHandler.updateProjectShadow(projectId);
-      return out;
-    } catch (errorAppUpdate) {
-      logger.error(
-        `Failed to update ${projectId}:\n${JSON.stringify(errorAppUpdate)}}`
-      );
-      // If something goes wrong, first try to rollback
-      try {
-        await rollbackApp({ projectId });
-        logger.error(
-          `Application update failed, rolled back to previous version: ${errorAppUpdate}`
+    this.device.on('offline', () => {
+      logger.warn(`Device Agent is offline ${new Date().toLocaleString()}`);
+      void this.logConnectionInfo();
+    });
+  }
+
+  private async logConnectionInfo() {
+    try {
+      /**
+       * We're using the 'netcat' or 'nc' command to test the connection to the IoT Core endpoint.
+       * This command doesn't always exit (see below), so
+       * we use timeout to break out of the prompt
+       * and catch the resulting error/parse the resulting stderr
+       *
+       * Sample command for current host and port:
+       * nc -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 8883
+       *
+       * Sample output when port is not blocked and host is reachable:
+       * $ nc  -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 443
+       * Connection to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 443 port [tcp/https] succeeded!
+       *
+       *
+       * Sample output when port is blocked (will repeatedly try until ctrl-C out):
+       * $ nc -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 8883
+       * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
+       * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
+       * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
+       * ^C
+       *
+       *
+       * Sample command/output when the port isn't enable on that host:
+       * $ nc -zv -w 1 localhost 8883
+       * nc: connect to localhost port 8883 (tcp) failed: Connection refused
+       */
+      await exec_promise(`nc -zv -w 1 ${this.host} ${this.port}`, {
+        timeout: 2000
+      });
+    } catch (err) {
+      const output = JSON.stringify(err['stderr']);
+      if (output.indexOf('not known') !== -1) {
+        logger.warn(
+          'Iot Core endpoint appears to be unreachable, internet connection may be unstable or the host may be down.'
         );
-      } catch (errorRollbackApp) {
-        // and if that fails, uninstall the app as a last resort.
-        try {
-          await this.atomicApplicationUninstall(projectId);
-        } catch {
-          // atomicApplicationUninstall handles failing, so there's nothing to handle here.
-        }
-        logger.error(
-          `Application update failed, rolled back to previous version: ${errorAppUpdate}`
+      } else if (output.indexOf('timed out') !== -1) {
+        logger.warn(
+          `Internet connection appears fine, however the endpoint was not reachable on the current connection port: ${this.port}\nPlease check if a firewall is in place.`
         );
-        throw new Error(
-          `Application update and rollback failed, uninstalled the application: ${errorAppUpdate}`
+      } else if (output.indexOf('refused') !== -1) {
+        logger.warn(
+          `The connection was refused, likely ${this.host} is not running a service on ${this.port}.`
         );
-      }
-      throw new Error(
-        `Application update failed, rolled the application back: ${errorAppUpdate}`
-      );
-    }
-  }
-
-  private handleProjectShadowConfigUpdate = async (
-    update: ShadowUpdate,
-    txId: string
-  ): Promise<boolean> => {
-    const { projectId, appCfgUpdate, envVarUpdate } = update;
-
-    if (
-      appCfgUpdate?.updatedModels &&
-      Object.keys(appCfgUpdate.updatedModels).length
-    ) {
-      // When there are model updates request signed URLs and wait to apply config changes
-      const { updatedModels } = appCfgUpdate;
-
-      logger.debug(
-        `Requesting presigned urls from cloud for model versions: ${JSON.stringify(
-          updatedModels
-        )}`
-      );
-      const modelsOnlyUrlsRequestPayload: SignedUrlsRequestPayload = {
-        modelsOnlyUrlsRequest: {
-          projectId,
-          models: updatedModels
-        }
-      };
-      const message = buildSignedUrlsRequestMessage(
-        this.clientId,
-        modelsOnlyUrlsRequestPayload,
-        txId
-      );
-      this.publisher.publishToCloud(message);
-
-      this.txnMgr.setAppCfgUpdateToTx(txId, update);
-
-      return false;
-    }
-
-    if (appCfgUpdate) {
-      await this.atomicApplicationUpdate(async () => {
-        await pruneModels({
-          projectId,
-          appCfg: appCfgUpdate.newAppCfg
-        });
-        await updateAppCfg({
-          projectId,
-          newAppCfg: appCfgUpdate.newAppCfg
-        });
-      }, projectId);
-    }
-
-    if (envVarUpdate) {
-      await this.atomicApplicationUpdate(
-        async () =>
-          await this.shadowHandler.updateProjectEnvVars({
-            projectId,
-            envVars: envVarUpdate.envVars
-          }),
-        projectId,
-        true
-      );
-    }
-    return true;
-  };
-
-  private async handleProjectShadowMessage(topic: string, message: any) {
-    const shadowUpdates = await this.shadowHandler.handleProjectShadow({
-      topic,
-      payload: message,
-      clientToken: message.clientToken
-    });
-    if (shadowUpdates.length) {
-      const shadowUpdatePromises: Promise<void>[] = [];
-      for (const shadowUpdate of shadowUpdates) {
-        const projectId = shadowUpdate.projectId;
-        const txId = shadowUpdate.txId;
-        shadowUpdatePromises.push(
-          this.txnMgr
-            .runTransactionStep({
-              func: () =>
-                this.handleProjectShadowConfigUpdate(shadowUpdate, txId),
-              projectId,
-              txId,
-              start: true,
-              stepName: topic
-            })
-            .catch((e: Error) => {
-              logger.error(
-                `There was an issue updating project shadow config: ${JSON.stringify(
-                  e
-                )}`
-              );
-            })
+      } else {
+        logger.warn(
+          `Output from checking connection to ${this.host} on ${this.port}: ${output}`
         );
       }
-
-      await Promise.all(shadowUpdatePromises);
-    }
-  }
-
-  public async handleSecureTunnelMessage(payload: any): Promise<void> {
-    logger.info(`Received secure tunnel update: ${JSON.stringify(payload)}`);
-    const state = payload.state;
-    if (!state) {
-      logger.debug(`No state found in message: ${JSON.stringify(payload)}`);
-      return;
-    }
-    const valid = validateSecureTunnelShadowUpdate(state);
-    if (!valid) {
-      logger.error(
-        `Error validating message: ${JSON.stringify(
-          { payload, errors: validateSecureTunnelShadowUpdate.errors },
-          null,
-          2
-        )}`
-      );
-      return;
-    }
-    const secureTunnelUpdate =
-      await this.secureTunnelHandler.syncShadowToDeviceState(payload);
-    await this.shadowHandler.updateSecureTunnelShadow(secureTunnelUpdate);
-    return;
-  }
-
-  /*=================================================================
-                            Public interface
-  =================================================================*/
-
-  constructor() {
-    this.device = awsIot.device({
-      keyPath: DEVICE_PRIVATE_KEY_FILE_PATH,
-      certPath: DEVICE_CERTIFICATE_FILE_PATH,
-      caPath: AWS_ROOT_CERTIFICATE_FILE_PATH,
-      clientId: this.clientId,
-      host: this.host,
-      port: this.port,
-      keepalive: 1 // time before re-connect attempt on dropped connection, default is 400 seconds
-    });
-    this.publisher = new Publisher(this.device, this.clientId);
-    this.shadowHandler = new ShadowHandler(this.clientId, this.publisher);
-    this.liveUpdatesHandler = new LiveUpdatesHandler(
-      this.publisher,
-      this.clientId
-    );
-    this.txnMgr = new TransactionManager(
-      this.publisher,
-      this.liveUpdatesHandler
-    );
-
-    this.subscribe(this.toDeviceTopic);
-    this.subscribe(this.secureTunnelNotifyTopic);
-    for (const topic of this.shadowHandler.projectShadowTopics) {
-      this.subscribe(topic);
     }
-    this.subscribe(this.shadowHandler.shadowTopics.secureTunnel.updateDelta);
-    this.subscribe(this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted);
-  }
-
-  public getClientId(): string {
-    return this.clientId;
-  }
-
-  public getToDeviceTopic() {
-    return this.toDeviceTopic;
-  }
-
-  public isCmdInProgress(projectId: string): boolean {
-    return this.txnMgr.isOngoingTransactionForProjectID(projectId);
-  }
-
-  public async updateProjectShadow(projectId: string) {
-    await this.shadowHandler.updateProjectShadow(projectId);
   }
 
-  public async handleDeviceAgentMessage({
+  private async handleDeviceAgentMessage({
     topic,
     message
   }: {
@@ -491,11 +296,22 @@ export class DeviceAgentCloudConnection {
             projectId,
             txId,
             start: true,
+            liveUpdatesPublishFn: async () =>
+              this.publisher.publishToClient(
+                buildToClientStatusResponseMessage(
+                  this.clientId,
+                  { status: keyMirrors.statusResponse.in_progress },
+                  txId
+                ),
+                logger.silly
+              ),
             stepName: payload.baseCommand
           });
         } catch (e) {
           logger.error(
-            `Error processing application state control request: ${e}!`
+            `Error processing application state control request for ${projectId}!\n${stringifyError(
+              e
+            )}`
           );
         }
 
@@ -511,18 +327,98 @@ export class DeviceAgentCloudConnection {
             projectId,
             txId,
             start: true,
+            liveUpdatesPublishFn: async () =>
+              this.publisher.publishToClient(
+                buildToClientStatusResponseMessage(
+                  this.clientId,
+                  { status: keyMirrors.statusResponse.in_progress },
+                  txId
+                ),
+                logger.silly
+              ),
             stepName: payload.baseCommand
           });
         } catch (e) {
-          logger.error(`Error processing application install request: ${e}!`);
+          logger.error(
+            `Error processing application install request for ${projectId}!\n${stringifyError(
+              e
+            )}`
+          );
         }
 
         break;
       }
       case live_state_updates: {
-        const payload = message.payload;
-        // TODO: Send response?
-        void this.liveUpdatesHandler.handleToggles(payload, txId);
+        const { deviceStats, appState, appLogs } = message.payload;
+
+        if (deviceStats !== undefined) {
+          if (deviceStats) {
+            await this.liveUpdatesHandler.enable(
+              keyMirrors.toClientMessageType.device_stats,
+              async () =>
+                this.publisher.publishToClient(
+                  buildDeviceStatsMessage(
+                    this.clientId,
+                    await getDeviceStatsPayload(),
+                    txId
+                  ),
+                  logger.silly
+                )
+            );
+          } else {
+            this.liveUpdatesHandler.disable(
+              keyMirrors.toClientMessageType.device_stats
+            );
+          }
+        }
+
+        if (appState !== undefined) {
+          if (appState) {
+            await this.liveUpdatesHandler.enable(
+              keyMirrors.toClientMessageType.app_state,
+              async () =>
+                this.publisher.publishToClient(
+                  buildAppStateMessage(
+                    this.clientId,
+                    await getAppStatePayload(),
+                    txId
+                  ),
+                  logger.silly
+                )
+            );
+          } else {
+            this.liveUpdatesHandler.disable(
+              keyMirrors.toClientMessageType.app_state
+            );
+          }
+        }
+
+        if (appLogs !== undefined) {
+          if (appLogs.toggle) {
+            await this.liveUpdatesHandler.startStream(
+              appLogs.projectId,
+              async () =>
+                await getAppLogs({
+                  projectId: appLogs.projectId,
+                  args: ['--tail', '100', '--no-log-prefix']
+                }),
+              async (logChunk: string) =>
+                this.publisher.publishToClient(
+                  buildAppLogsMessage(
+                    this.clientId,
+                    {
+                      projectId: appLogs.projectId,
+                      logChunk
+                    },
+                    txId
+                  ),
+                  logger.silly
+                )
+            );
+          } else {
+            this.liveUpdatesHandler.stopStream(appLogs.projectId);
+          }
+        }
         break;
       }
       case app_install_response: {
@@ -569,62 +465,59 @@ export class DeviceAgentCloudConnection {
       case status_response: {
         const { failure } = keyMirrors.statusResponse;
         if (message.payload.status === failure) {
-          this.txnMgr.completeTransaction(txId);
-
-          const failureStatusResponsePayload: StatusResponsePayload = {
-            status: keyMirrors.statusResponse.failure,
-            message: message.payload.message
-          };
-          // Send final status message
-          const failureStatusResponseMessage =
+          this.txnMgr.completeTransaction(
+            txId,
             buildToClientStatusResponseMessage(
               this.clientId,
-              failureStatusResponsePayload,
+              {
+                status: keyMirrors.statusResponse.failure,
+                message: message.payload.message
+              },
               txId
-            );
-          this.publisher.publishToClient(failureStatusResponseMessage);
+            )
+          );
         }
         break;
       }
       case device_action: {
         try {
-          const statusResponsePayload: StatusResponsePayload = {
-            status: keyMirrors.statusResponse.in_progress
-          };
-          const statusResponseMessage = buildToClientStatusResponseMessage(
-            this.clientId,
-            statusResponsePayload,
-            txId
+          this.publisher.publishToClient(
+            buildToClientStatusResponseMessage(
+              this.clientId,
+              {
+                status: keyMirrors.statusResponse.in_progress
+              },
+              txId
+            )
           );
-          this.publisher.publishToClient(statusResponseMessage);
 
           await this.handleDeviceAction(message.payload);
 
-          const successStatusResponsePayload: StatusResponsePayload = {
-            status: keyMirrors.statusResponse.success
-          };
-          const successStatusResponseMessage =
+          this.publisher.publishToClient(
             buildToClientStatusResponseMessage(
               this.clientId,
-              successStatusResponsePayload,
+              {
+                status: keyMirrors.statusResponse.success
+              },
               txId
-            );
-          this.publisher.publishToClient(successStatusResponseMessage);
+            )
+          );
         } catch (e) {
           logger.error(
-            `There was a problem performing device action '${message.payload.action}': ${e.message}`
+            `There was a problem performing device action '${
+              message.payload.action
+            }'!\n${stringifyError(e)}`
           );
-          const failureStatusResponsePayload: StatusResponsePayload = {
-            status: keyMirrors.statusResponse.failure,
-            message: e.message
-          };
-          const failureStatusResponseMessage =
+          this.publisher.publishToClient(
             buildToClientStatusResponseMessage(
               this.clientId,
-              failureStatusResponsePayload,
+              {
+                status: keyMirrors.statusResponse.failure,
+                message: e.message
+              },
               txId
-            );
-          this.publisher.publishToClient(failureStatusResponseMessage);
+            )
+          );
         }
         break;
       }
@@ -639,134 +532,347 @@ export class DeviceAgentCloudConnection {
     }
   }
 
-  public async handleMessage(topic: string, message: any) {
-    logger.debug(
-      `Received message: ${JSON.stringify({ topic, message }, null, 2)}`
-    );
-    // ProjectShadow messages
-    if (this.shadowHandler.projectShadowTopics.includes(topic)) {
-      await this.handleProjectShadowMessage(topic, message);
-    } else if (topic === this.toDeviceTopic) {
-      await this.handleDeviceAgentMessage({
-        topic,
-        message
-      });
-      // SecureTunnelNotify messages
-    } else if (topic === this.secureTunnelNotifyTopic) {
-      await this.secureTunnelHandler.secureTunnelNotifyHandler(message);
-      // SecureTunnel messages
-    } else if (
-      topic === this.shadowHandler.shadowTopics.secureTunnel.updateDelta
-    ) {
-      await this.handleSecureTunnelMessage(message);
-    } else if (
-      topic === this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted
-    ) {
-      logger.info(`Received secure tunnel deleteAccepted: ${message}`);
-      await this.secureTunnelHandler.destroy();
-    } else {
-      logger.error(`Unexpected topic, ignoring! ${topic}`);
+  private handleAppStateControl = async (
+    payload: AppStateControlPayload
+  ): Promise<boolean> => {
+    const { baseCommand, projectId } = payload;
+    switch (baseCommand) {
+      case keyMirrors.appStateControl.start:
+        await startApp({ projectId });
+        break;
+      case keyMirrors.appStateControl.stop:
+        await stopApp({ projectId });
+        break;
+      case keyMirrors.appStateControl.restart:
+        await restartApp({ projectId });
+        break;
     }
-  }
+    return true;
+  };
 
-  public async setupHandlers() {
-    this.device.on('connect', (connack: any) => {
-      logger.info('Device Agent has connected to the cloud');
-      // FIXME: EI-709 Skip this request for now to prevent kicking off another
-      // shadow update process if IoT Core disconnect occurs during app config update
-      //this.shadowHandler.getShadowUpdates();
-      void this.shadowHandler.updateSystemInfoShadow();
-    });
+  private handleAppVersionControl = async (
+    payload:
+      | AppVersionControlInstallPayload
+      | AppVersionControlUninstallPayload,
+    txId: string
+  ): Promise<boolean> => {
+    switch (payload.baseCommand) {
+      case keyMirrors.appVersionControl.install: {
+        const { projectId, appReleaseHash } = payload;
 
-    this.device.on('disconnect', () => {
-      logger.warn('Device Agent has been disconnected from the cloud');
-    });
+        const signedUrlsRequestPayload: SignedUrlsRequestPayload = {
+          signedUrlsRequest: {
+            projectId,
+            appReleaseHash
+          }
+        };
+        const message = buildSignedUrlsRequestMessage(
+          this.clientId,
+          signedUrlsRequestPayload,
+          txId
+        );
+        await this.publishCloudRequest(message);
+        return false;
+      }
+      case keyMirrors.appVersionControl.uninstall: {
+        const { projectId } = payload;
+        await this.atomicApplicationUninstall(projectId);
+        return true;
+      }
+      default:
+        logger.warn(
+          `Ignore App Version Control packet: ${JSON.stringify(
+            payload,
+            null,
+            2
+          )}`
+        );
+        return true;
+    }
+  };
 
-    this.device.on('reconnect', () => {
-      logger.info(
-        `Device Agent attempting to re-connect ${new Date().toLocaleString()}`
+  private handleAppInstallCloudResponsePayload = async (
+    payload: AppInstallResponsePayload
+  ): Promise<boolean> => {
+    const {
+      projectId,
+      appReleaseHash,
+      appInstallPayload,
+      modelsInstallPayload
+    } = payload.appInstallResponse;
+    const signedUrlsPayload = {
+      appInstallPayload,
+      modelsInstallPayload
+    };
+    await this.atomicApplicationUpdate(async () => {
+      this.shadowHandler.clearProjectShadow(projectId);
+      await installApp({ projectId, appReleaseHash, signedUrlsPayload });
+    }, projectId);
+    return true;
+  };
+
+  private handleModelsInstallCloudResponsePayload = async (
+    payload: ModelsInstallResponsePayload,
+    txId: string
+  ): Promise<boolean> => {
+    const projectId = payload.modelsInstallResponse.projectId;
+
+    const update = this.txnMgr.getAppCfgUpdateFromTxID(txId);
+    if (update === undefined) {
+      throw new Error(
+        'Unknown error while updating models via application config! No config present for model update.'
       );
-    });
+    }
+    const { appCfgUpdate, envVarUpdate } = update;
+    if (appCfgUpdate) {
+      await this.atomicApplicationUpdate(
+        async () =>
+          await updateModelsWithPresignedUrls({
+            projectId,
+            modelInstallPayloads: payload.modelsInstallResponse.newModels,
+            newAppCfg: appCfgUpdate.newAppCfg
+          }),
+        projectId
+      );
+    }
 
-    this.device.on('error', function (error) {
-      const errorString = error.message.toString();
-      logger.error(`${errorString}`);
-    });
+    if (envVarUpdate) {
+      await this.atomicApplicationUpdate(
+        async () =>
+          await this.shadowHandler.updateProjectEnvVars({
+            projectId,
+            envVars: envVarUpdate.envVars
+          }),
+        projectId,
+        true
+      );
+    }
 
-    this.device.on('message', async (topic: string, payload: string) => {
-      try {
-        const jsonPacket = JSON.parse(payload);
-        await this.handleMessage(topic, jsonPacket);
-      } catch (e) {
-        logger.error(`Error parsing message: ${e.message}`);
+    return true;
+  };
+
+  private async handleDeviceAction(payload: DeviceActionPayload) {
+    const { system_restart } = keyMirrors.deviceAction;
+    switch (payload.action) {
+      case system_restart: {
+        await reboot();
+        break;
       }
-    });
+      default: {
+        logger.info(
+          `Unrecognized device action requested: '${payload.action}'.`
+        );
+      }
+    }
+  }
 
-    this.device.on('offline', () => {
-      logger.warn(`Device Agent is offline ${new Date().toLocaleString()}`);
-      void this.logConnectionInfo();
-    });
+  private async publishCloudRequest(message: ToCloudMessage) {
+    this.publisher.publishToCloud(message);
+  }
+
+  private subscribe(topic: string) {
+    logger.info(`Subscribing to ${topic}`);
+    this.device.subscribe(topic);
   }
 
-  public async logConnectionInfo() {
+  private async atomicApplicationUninstall(projectId: string) {
     try {
-      /**
-       * We're using the 'netcat' or 'nc' command to test the connection to the IoT Core endpoint.
-       * This command doesn't always exit (see below), so
-       * we use timeout to break out of the prompt
-       * and catch the resulting error/parse the resulting stderr
-       *
-       * Sample command for current host and port:
-       * nc -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 8883
-       *
-       * Sample output when port is not blocked and host is reachable:
-       * $ nc  -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 443
-       * Connection to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 443 port [tcp/https] succeeded!
-       *
-       *
-       * Sample output when port is blocked (will repeatedly try until ctrl-C out):
-       * $ nc -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 8883
-       * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
-       * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
-       * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
-       * ^C
-       *
-       *
-       * Sample command/output when the port isn't enable on that host:
-       * $ nc -zv -w 1 localhost 8883
-       * nc: connect to localhost port 8883 (tcp) failed: Connection refused
-       */
-      await exec_promise(`nc -zv -w 1 ${this.host} ${this.port}`, {
-        timeout: 2000
-      });
-    } catch (err) {
-      const output = JSON.stringify(err['stderr']);
-      if (output.indexOf('not known') !== -1) {
-        logger.warn(
-          'Iot Core endpoint appears to be unreachable, internet connection may be unstable or the host may be down.'
+      await uninstallApp({ projectId });
+      this.shadowHandler.clearProjectShadow(projectId);
+    } catch (e) {
+      logger.error(`Failed to uninstall ${projectId}!\n${stringifyError(e)}`);
+      throw e;
+    }
+  }
+
+  // eslint-disable-next-line
+  private async atomicApplicationUpdate<F extends () => any>(
+    func: F,
+    projectId: string,
+    skipUpdateShadow?: boolean
+  ): Promise<ReturnType<F>> {
+    if (await AgentConfigFile().isAppPresent({ projectId })) {
+      // Reject the application update if app is present but not ready
+      if (!(await AgentConfigFile().isAppReady({ projectId }))) {
+        throw new Error('Application already has installation in progress!');
+      }
+
+      // Try to create a backup, so that there is one available if something goes wrong in the next try:catch.
+      try {
+        await createAppBackup({ projectId });
+      } catch (e) {
+        logger.error(
+          `Could not create a backup for the project: ${projectId}!\n${stringifyError(
+            e
+          )}`
         );
-      } else if (output.indexOf('timed out') !== -1) {
-        logger.warn(
-          `Internet connection appears fine, however the endpoint was not reachable on the current connection port: ${this.port}\nPlease check if a firewall is in place.`
+      }
+    }
+
+    try {
+      const out: ReturnType<F> = await func();
+      if (!skipUpdateShadow)
+        await this.shadowHandler.updateProjectShadow(projectId);
+      return out;
+    } catch (errorAppUpdate) {
+      logger.error(
+        `Failed to update ${projectId}!\n${stringifyError(errorAppUpdate)}`
+      );
+      // If something goes wrong, first try to rollback
+      try {
+        await rollbackApp({ projectId });
+      } catch (errorRollbackApp) {
+        logger.error(
+          `Application rollback failed for ${projectId}!\n${stringifyError(
+            errorRollbackApp
+          )}`
         );
-      } else if (output.indexOf('refused') !== -1) {
-        logger.warn(
-          `The connection was refused, likely ${this.host} is not running a service on ${this.port}.`
+        // and if that fails, uninstall the app as a last resort.
+        try {
+          await this.atomicApplicationUninstall(projectId);
+        } catch {
+          // atomicApplicationUninstall logs failure, so there's nothing to do here.
+        }
+        throw new AaiError(
+          'Application update and rollback failed, uninstalled the application!',
+          { cause: errorAppUpdate }
         );
-      } else {
-        logger.warn(
-          `Output from checking connection to ${this.host} on ${this.port}: ${output}`
+      }
+      throw new Error(
+        'Application update failed, rolled the application back!',
+        { cause: errorAppUpdate }
+      );
+    }
+  }
+
+  private handleProjectShadowConfigUpdate = async (
+    update: ShadowUpdate,
+    txId: string
+  ): Promise<boolean> => {
+    const { projectId, appCfgUpdate, envVarUpdate } = update;
+
+    if (
+      appCfgUpdate?.updatedModels &&
+      Object.keys(appCfgUpdate.updatedModels).length
+    ) {
+      // When there are model updates request signed URLs and wait to apply config changes
+      const { updatedModels } = appCfgUpdate;
+
+      logger.debug(
+        `Requesting presigned urls from cloud for model versions: ${JSON.stringify(
+          updatedModels
+        )}`
+      );
+      const modelsOnlyUrlsRequestPayload: SignedUrlsRequestPayload = {
+        modelsOnlyUrlsRequest: {
+          projectId,
+          models: updatedModels
+        }
+      };
+      const message = buildSignedUrlsRequestMessage(
+        this.clientId,
+        modelsOnlyUrlsRequestPayload,
+        txId
+      );
+      this.publisher.publishToCloud(message);
+
+      this.txnMgr.setAppCfgUpdateToTx(txId, update);
+
+      return false;
+    }
+
+    if (appCfgUpdate) {
+      await this.atomicApplicationUpdate(async () => {
+        await pruneModels({
+          projectId,
+          appCfg: appCfgUpdate.newAppCfg
+        });
+        await updateAppCfg({
+          projectId,
+          newAppCfg: appCfgUpdate.newAppCfg
+        });
+      }, projectId);
+    }
+
+    if (envVarUpdate) {
+      await this.atomicApplicationUpdate(
+        async () =>
+          await this.shadowHandler.updateProjectEnvVars({
+            projectId,
+            envVars: envVarUpdate.envVars
+          }),
+        projectId,
+        true
+      );
+    }
+    return true;
+  };
+
+  private async handleProjectShadowMessage(topic: string, message: any) {
+    const shadowUpdates = await this.shadowHandler.handleProjectShadow({
+      topic,
+      payload: message,
+      clientToken: message.clientToken
+    });
+    if (shadowUpdates.length) {
+      const shadowUpdatePromises: Promise<void>[] = [];
+      for (const shadowUpdate of shadowUpdates) {
+        const projectId = shadowUpdate.projectId;
+        const txId = shadowUpdate.txId;
+        shadowUpdatePromises.push(
+          this.txnMgr
+            .runTransactionStep({
+              func: () =>
+                this.handleProjectShadowConfigUpdate(shadowUpdate, txId),
+              projectId,
+              txId,
+              start: true,
+              liveUpdatesPublishFn: async () =>
+                this.publisher.publishToClient(
+                  buildToClientStatusResponseMessage(
+                    this.clientId,
+                    { status: keyMirrors.statusResponse.in_progress },
+                    txId
+                  ),
+                  logger.silly
+                ),
+              stepName: topic
+            })
+            .catch((e) => {
+              logger.error(
+                `There was an issue updating project shadow config for ${projectId}!\n${stringifyError(
+                  e
+                )}`
+              );
+            })
         );
       }
+
+      await Promise.all(shadowUpdatePromises);
     }
   }
 
-  public async stop() {
-    // This method is currently only used by the CLI, and shadow messages can be
-    // lost since we aren't waiting for responses so sleep for a short time to
-    // receive them
-    await sleep(1000);
-    this.device.end();
+  public async handleSecureTunnelMessage(payload: any): Promise<void> {
+    logger.info(`Received secure tunnel update: ${JSON.stringify(payload)}`);
+    const state = getUpdateDeltaStateFromMessage(payload);
+    if (!state) {
+      logger.debug(`No state found in message: ${JSON.stringify(payload)}`);
+      return;
+    }
+    const valid = validateSecureTunnelShadowUpdate(state);
+    if (!valid) {
+      logger.error(
+        `Error validating message: ${JSON.stringify(
+          { payload, errors: validateSecureTunnelShadowUpdate.errors },
+          null,
+          2
+        )}`
+      );
+      return;
+    }
+    const secureTunnelUpdate =
+      await this.secureTunnelHandler.syncShadowToDeviceState(payload);
+    await this.shadowHandler.updateSecureTunnelShadow(secureTunnelUpdate);
   }
 }
 
@@ -774,20 +880,28 @@ export async function runDeviceAgentCloudInterface() {
   logger.info(
     `Starting alwaysAI Device Agent v${await getDeviceAgentVersion()}`
   );
-  if (cloudModeReady()) {
+  if (existsSync(getBootstrapPrivateKeyFilePath())) {
+    await bootstrapProvision();
+    return;
+  }
+
+  const filesAlreadyMigrated = await requiredConfigFilesPresentAndValid();
+  if (!filesAlreadyMigrated) {
+    logger.debug('Attempting configuration file migration.');
+    await migrateFromLegacyCertsAndTokens();
+  }
+
+  if (await requiredConfigFilesPresentAndValid()) {
     const deviceAgent = new DeviceAgentCloudConnection();
-    await deviceAgent.setupHandlers();
     if (ALWAYSAI_ANALYTICS_PASSTHROUGH === true) {
+      const shadowHandler = deviceAgent.shadowHandler;
       const publisher = deviceAgent.publisher;
-      const passthroughHandler = new PassthroughHandler(publisher);
+      const passthroughHandler = new PassthroughHandler(
+        publisher,
+        shadowHandler
+      );
       await passthroughHandler.setup();
     }
-  } else if (existsSync(BOOTSTRAP_PRIVATE_KEY_FILE_PATH())) {
-    await bootstrapProvision();
-  } else if (existsSync(BOOTSTRAP_CERTIFICATES_DIR_PATH())) {
-    throw new Error(
-      "Device has not been created using 'aai-agent device init' or there has been an issue with device initialization"
-    );
   } else {
     throw new Error(
       "Set device agent to local mode and retry the 'aai-agent device init' command"