@alwaysai/device-agent 1.5.0 → 2.0.0

package/lib/cloud-connection/device-agent-cloud-connection.js

@@ -4,34 +4,37 @@ exports.runDeviceAgentCloudInterface = exports.DeviceAgentCloudConnection = void
 // eslint-disable-next-line
 const awsIot = require('aws-iot-device-sdk');
 const device_agent_schemas_1 = require("@alwaysai/device-agent-schemas");
+const infrastructure_1 = require("alwaysai/lib/infrastructure");
+const util_1 = require("alwaysai/lib/util");
+const child_process_1 = require("child_process");
 const fs_1 = require("fs");
+const util_2 = require("util");
 const application_control_1 = require("../application-control");
 const backup_1 = require("../application-control/backup");
+const models_1 = require("../application-control/models");
 const device_control_1 = require("../device-control/device-control");
 const environment_1 = require("../environment");
 const agent_config_1 = require("../infrastructure/agent-config");
+const device_certificate_1 = require("../infrastructure/device-certificate");
+const legacy_migration_1 = require("../infrastructure/legacy-migration/legacy-migration");
+const required_config_checks_1 = require("../infrastructure/required-config-checks");
 const urls_1 = require("../infrastructure/urls");
 const secure_tunneling_1 = require("../secure-tunneling/secure-tunneling");
-const cloud_mode_ready_1 = require("../util/cloud-mode-ready");
+const aai_error_1 = require("../util/aai-error");
+const check_for_updates_1 = require("../util/check-for-updates");
 const directories_1 = require("../util/directories");
 const get_device_id_1 = require("../util/get-device-id");
 const logger_1 = require("../util/logger");
 const sleep_1 = require("../util/sleep");
 const bootstrap_provision_1 = require("./bootstrap-provision");
 const live_updates_handler_1 = require("./live-updates-handler");
+const messages_1 = require("./messages");
 const passthrough_handler_1 = require("./passthrough-handler");
 const publisher_1 = require("./publisher");
 const shadow_handler_1 = require("./shadow-handler");
 const transaction_manager_1 = require("./transaction-manager");
-const child_process_1 = require("child_process");
-const util_1 = require("util");
-const models_1 = require("../application-control/models");
-const check_for_updates_1 = require("../util/check-for-updates");
-const exec_promise = (0, util_1.promisify)(child_process_1.exec);
+const exec_promise = (0, util_2.promisify)(child_process_1.exec);
 class DeviceAgentCloudConnection {
-    /*=================================================================
-                              Public interface
-    =================================================================*/
     constructor() {
         this.device = awsIot.device;
         this.clientId = (0, get_device_id_1.getDeviceUuid)();
@@ -152,17 +155,17 @@ class DeviceAgentCloudConnection {
             return true;
         };
         this.device = awsIot.device({
-            keyPath: directories_1.DEVICE_PRIVATE_KEY_FILE_PATH,
-            certPath: directories_1.DEVICE_CERTIFICATE_FILE_PATH,
+            keyPath: infrastructure_1.DEVICE_PRIVATE_KEY_FILE_PATH,
+            certPath: infrastructure_1.DEVICE_CERTIFICATE_FILE_PATH,
             caPath: directories_1.AWS_ROOT_CERTIFICATE_FILE_PATH,
             clientId: this.clientId,
             host: this.host,
             port: this.port,
-            keepalive: 1 // time before re-connect attempt on dropped connection, default is 400 seconds
+            keepalive: 10 // time before re-connect attempt on dropped connection, default is 400 seconds
         });
         this.publisher = new publisher_1.Publisher(this.device, this.clientId);
         this.shadowHandler = new shadow_handler_1.ShadowHandler(this.clientId, this.publisher);
-        this.liveUpdatesHandler = new live_updates_handler_1.LiveUpdatesHandler(this.publisher, this.clientId);
+        this.liveUpdatesHandler = new live_updates_handler_1.LiveUpdatesHandler();
         this.txnMgr = new transaction_manager_1.TransactionManager(this.publisher, this.liveUpdatesHandler);
         this.subscribe(this.toDeviceTopic);
         this.subscribe(this.secureTunnelNotifyTopic);
@@ -171,127 +174,133 @@ class DeviceAgentCloudConnection {
         }
         this.subscribe(this.shadowHandler.shadowTopics.secureTunnel.updateDelta);
         this.subscribe(this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted);
+        this.setupHandlers();
     }
-    async handleDeviceAction(payload) {
-        const { system_restart } = device_agent_schemas_1.keyMirrors.deviceAction;
-        switch (payload.action) {
-            case system_restart: {
-                await (0, device_control_1.reboot)();
-                break;
-            }
-            default: {
-                logger_1.logger.info(`Unrecognized device action requested: '${payload.action}'.`);
-            }
-        }
-    }
-    async publishCloudRequest(message) {
-        this.publisher.publishToCloud(message);
+    /*=================================================================
+                              Public interface
+    =================================================================*/
+    getClientId() {
+        return this.clientId;
     }
-    subscribe(topic) {
-        logger_1.logger.info(`Subscribing to ${topic}`);
-        this.device.subscribe(topic);
+    isCmdInProgress(projectId) {
+        return this.txnMgr.isOngoingTransactionForProjectID(projectId);
     }
-    async atomicApplicationUninstall(projectId) {
-        try {
-            await (0, application_control_1.uninstallApp)({ projectId });
-            this.shadowHandler.clearProjectShadow(projectId);
+    async handleMessage(topic, message) {
+        logger_1.logger.debug(`Received message: ${JSON.stringify({ topic, message }, null, 2)}`);
+        // ProjectShadow messages
+        if (this.shadowHandler.projectShadowTopics.includes(topic)) {
+            await this.handleProjectShadowMessage(topic, message);
         }
-        catch (e) {
-            logger_1.logger.error(`Failed to uninstall ${projectId}: ${e.message}`);
-            throw e;
+        else if (topic === this.toDeviceTopic) {
+            await this.handleDeviceAgentMessage({
+                topic,
+                message
+            });
+            // SecureTunnelNotify messages
+        }
+        else if (topic === this.secureTunnelNotifyTopic) {
+            await this.secureTunnelHandler.secureTunnelNotifyHandler(message);
+            // SecureTunnel messages
+        }
+        else if (topic === this.shadowHandler.shadowTopics.secureTunnel.updateDelta) {
+            await this.handleSecureTunnelMessage(message);
+        }
+        else if (topic === this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted) {
+            logger_1.logger.info(`Received secure tunnel deleteAccepted: ${message}`);
+            await this.secureTunnelHandler.destroy();
+        }
+        else {
+            logger_1.logger.error(`Unexpected topic, ignoring! ${topic}`);
         }
     }
-    // eslint-disable-next-line
-    async atomicApplicationUpdate(func, projectId, skipUpdateShadow) {
-        // First try to create a backup, so that there is one available if something goes wrong in the next try:catch.
-        if (await (0, agent_config_1.AgentConfigFile)().isAppPresent({ projectId })) {
+    async stop() {
+        // This method is currently only used by the CLI, and shadow messages can be
+        // lost since we aren't waiting for responses so sleep for a short time to
+        // receive them
+        await (0, sleep_1.default)(1000);
+        this.device.end();
+    }
+    /*=================================================================
+                              Private interface
+    =================================================================*/
+    setupHandlers() {
+        this.device.on('connect', (connack) => {
+            logger_1.logger.info('Device Agent has connected to the cloud');
+            // FIXME: EI-709 Skip this request for now to prevent kicking off another
+            // shadow update process if IoT Core disconnect occurs during app config update
+            //this.shadowHandler.getShadowUpdates();
+            void this.shadowHandler.updateSystemInfoShadow();
+        });
+        this.device.on('disconnect', () => {
+            logger_1.logger.warn('Device Agent has been disconnected from the cloud');
+        });
+        this.device.on('reconnect', () => {
+            logger_1.logger.info(`Device Agent attempting to re-connect ${new Date().toLocaleString()}`);
+        });
+        this.device.on('error', function (e) {
+            logger_1.logger.error(`Error connecting to cloud!\n${(0, util_1.stringifyError)(e)}`);
+        });
+        this.device.on('message', async (topic, payload) => {
             try {
-                await (0, backup_1.createAppBackup)({ projectId });
+                const jsonPacket = JSON.parse(payload);
+                await this.handleMessage(topic, jsonPacket);
             }
             catch (e) {
-                logger_1.logger.error(`Could not create a backup for the project: ${projectId}:\n${e.message}\n${e.stack}`);
+                logger_1.logger.error(`Error parsing message!\n${(0, util_1.stringifyError)(e)}`);
             }
-        }
+        });
+        this.device.on('offline', () => {
+            logger_1.logger.warn(`Device Agent is offline ${new Date().toLocaleString()}`);
+            void this.logConnectionInfo();
+        });
+    }
+    async logConnectionInfo() {
         try {
-            const out = await func();
-            if (!skipUpdateShadow)
-                await this.shadowHandler.updateProjectShadow(projectId);
-            return out;
+            /**
+             * We're using the 'netcat' or 'nc' command to test the connection to the IoT Core endpoint.
+             * This command doesn't always exit (see below), so
+             * we use timeout to break out of the prompt
+             * and catch the resulting error/parse the resulting stderr
+             *
+             * Sample command for current host and port:
+             * nc -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 8883
+             *
+             * Sample output when port is not blocked and host is reachable:
+             * $ nc  -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 443
+             * Connection to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 443 port [tcp/https] succeeded!
+             *
+             *
+             * Sample output when port is blocked (will repeatedly try until ctrl-C out):
+             * $ nc -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 8883
+             * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
+             * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
+             * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
+             * ^C
+             *
+             *
+             * Sample command/output when the port isn't enable on that host:
+             * $ nc -zv -w 1 localhost 8883
+             * nc: connect to localhost port 8883 (tcp) failed: Connection refused
+             */
+            await exec_promise(`nc -zv -w 1 ${this.host} ${this.port}`, {
+                timeout: 2000
+            });
         }
-        catch (errorAppUpdate) {
-            logger_1.logger.error(`Failed to update ${projectId}:\n${JSON.stringify(errorAppUpdate)}}`);
-            // If something goes wrong, first try to rollback
-            try {
-                await (0, backup_1.rollbackApp)({ projectId });
-                logger_1.logger.error(`Application update failed, rolled back to previous version: ${errorAppUpdate}`);
+        catch (err) {
+            const output = JSON.stringify(err['stderr']);
+            if (output.indexOf('not known') !== -1) {
+                logger_1.logger.warn('Iot Core endpoint appears to be unreachable, internet connection may be unstable or the host may be down.');
             }
-            catch (errorRollbackApp) {
-                // and if that fails, uninstall the app as a last resort.
-                try {
-                    await this.atomicApplicationUninstall(projectId);
-                }
-                catch (_a) {
-                    // atomicApplicationUninstall handles failing, so there's nothing to handle here.
-                }
-                logger_1.logger.error(`Application update failed, rolled back to previous version: ${errorAppUpdate}`);
-                throw new Error(`Application update and rollback failed, uninstalled the application: ${errorAppUpdate}`);
+            else if (output.indexOf('timed out') !== -1) {
+                logger_1.logger.warn(`Internet connection appears fine, however the endpoint was not reachable on the current connection port: ${this.port}\nPlease check if a firewall is in place.`);
             }
-            throw new Error(`Application update failed, rolled the application back: ${errorAppUpdate}`);
-        }
-    }
-    async handleProjectShadowMessage(topic, message) {
-        const shadowUpdates = await this.shadowHandler.handleProjectShadow({
-            topic,
-            payload: message,
-            clientToken: message.clientToken
-        });
-        if (shadowUpdates.length) {
-            const shadowUpdatePromises = [];
-            for (const shadowUpdate of shadowUpdates) {
-                const projectId = shadowUpdate.projectId;
-                const txId = shadowUpdate.txId;
-                shadowUpdatePromises.push(this.txnMgr
-                    .runTransactionStep({
-                    func: () => this.handleProjectShadowConfigUpdate(shadowUpdate, txId),
-                    projectId,
-                    txId,
-                    start: true,
-                    stepName: topic
-                })
-                    .catch((e) => {
-                    logger_1.logger.error(`There was an issue updating project shadow config: ${JSON.stringify(e)}`);
-                }));
+            else if (output.indexOf('refused') !== -1) {
+                logger_1.logger.warn(`The connection was refused, likely ${this.host} is not running a service on ${this.port}.`);
+            }
+            else {
+                logger_1.logger.warn(`Output from checking connection to ${this.host} on ${this.port}: ${output}`);
             }
-            await Promise.all(shadowUpdatePromises);
-        }
-    }
-    async handleSecureTunnelMessage(payload) {
-        logger_1.logger.info(`Received secure tunnel update: ${JSON.stringify(payload)}`);
-        const state = payload.state;
-        if (!state) {
-            logger_1.logger.debug(`No state found in message: ${JSON.stringify(payload)}`);
-            return;
-        }
-        const valid = (0, device_agent_schemas_1.validateSecureTunnelShadowUpdate)(state);
-        if (!valid) {
-            logger_1.logger.error(`Error validating message: ${JSON.stringify({ payload, errors: device_agent_schemas_1.validateSecureTunnelShadowUpdate.errors }, null, 2)}`);
-            return;
         }
-        const secureTunnelUpdate = await this.secureTunnelHandler.syncShadowToDeviceState(payload);
-        await this.shadowHandler.updateSecureTunnelShadow(secureTunnelUpdate);
-        return;
-    }
-    getClientId() {
-        return this.clientId;
-    }
-    getToDeviceTopic() {
-        return this.toDeviceTopic;
-    }
-    isCmdInProgress(projectId) {
-        return this.txnMgr.isOngoingTransactionForProjectID(projectId);
-    }
-    async updateProjectShadow(projectId) {
-        await this.shadowHandler.updateProjectShadow(projectId);
     }
     async handleDeviceAgentMessage({ topic, message }) {
         const valid = (0, device_agent_schemas_1.validateToDeviceAgentMessage)(message);
@@ -313,11 +322,12 @@ class DeviceAgentCloudConnection {
                         projectId,
                         txId,
                         start: true,
+                        liveUpdatesPublishFn: async () => this.publisher.publishToClient((0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, { status: device_agent_schemas_1.keyMirrors.statusResponse.in_progress }, txId), logger_1.logger.silly),
                         stepName: payload.baseCommand
                     });
                 }
                 catch (e) {
-                    logger_1.logger.error(`Error processing application state control request: ${e}!`);
+                    logger_1.logger.error(`Error processing application state control request for ${projectId}!\n${(0, util_1.stringifyError)(e)}`);
                 }
                 break;
             }
@@ -331,18 +341,47 @@ class DeviceAgentCloudConnection {
                         projectId,
                         txId,
                         start: true,
+                        liveUpdatesPublishFn: async () => this.publisher.publishToClient((0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, { status: device_agent_schemas_1.keyMirrors.statusResponse.in_progress }, txId), logger_1.logger.silly),
                         stepName: payload.baseCommand
                     });
                 }
                 catch (e) {
-                    logger_1.logger.error(`Error processing application install request: ${e}!`);
+                    logger_1.logger.error(`Error processing application install request for ${projectId}!\n${(0, util_1.stringifyError)(e)}`);
                 }
                 break;
             }
             case live_state_updates: {
-                const payload = message.payload;
-                // TODO: Send response?
-                void this.liveUpdatesHandler.handleToggles(payload, txId);
+                const { deviceStats, appState, appLogs } = message.payload;
+                if (deviceStats !== undefined) {
+                    if (deviceStats) {
+                        await this.liveUpdatesHandler.enable(device_agent_schemas_1.keyMirrors.toClientMessageType.device_stats, async () => this.publisher.publishToClient((0, device_agent_schemas_1.buildDeviceStatsMessage)(this.clientId, await (0, messages_1.getDeviceStatsPayload)(), txId), logger_1.logger.silly));
+                    }
+                    else {
+                        this.liveUpdatesHandler.disable(device_agent_schemas_1.keyMirrors.toClientMessageType.device_stats);
+                    }
+                }
+                if (appState !== undefined) {
+                    if (appState) {
+                        await this.liveUpdatesHandler.enable(device_agent_schemas_1.keyMirrors.toClientMessageType.app_state, async () => this.publisher.publishToClient((0, device_agent_schemas_1.buildAppStateMessage)(this.clientId, await (0, messages_1.getAppStatePayload)(), txId), logger_1.logger.silly));
+                    }
+                    else {
+                        this.liveUpdatesHandler.disable(device_agent_schemas_1.keyMirrors.toClientMessageType.app_state);
+                    }
+                }
+                if (appLogs !== undefined) {
+                    if (appLogs.toggle) {
+                        await this.liveUpdatesHandler.startStream(appLogs.projectId, async () => await (0, application_control_1.getAppLogs)({
+                            projectId: appLogs.projectId,
+                            args: ['--tail', '100', '--no-log-prefix']
+                        }), async (logChunk) => this.publisher.publishToClient((0, device_agent_schemas_1.buildAppLogsMessage)(this.clientId, {
+                            projectId: appLogs.projectId,
+                            logChunk
+                        }, txId), logger_1.logger.silly));
+                    }
+                    else {
+                        this.liveUpdatesHandler.stopStream(appLogs.projectId);
+                    }
+                }
                 break;
             }
             case app_install_response: {
@@ -380,39 +419,29 @@ class DeviceAgentCloudConnection {
             case status_response: {
                 const { failure } = device_agent_schemas_1.keyMirrors.statusResponse;
                 if (message.payload.status === failure) {
-                    this.txnMgr.completeTransaction(txId);
-                    const failureStatusResponsePayload = {
+                    this.txnMgr.completeTransaction(txId, (0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, {
                         status: device_agent_schemas_1.keyMirrors.statusResponse.failure,
                         message: message.payload.message
-                    };
-                    // Send final status message
-                    const failureStatusResponseMessage = (0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, failureStatusResponsePayload, txId);
-                    this.publisher.publishToClient(failureStatusResponseMessage);
+                    }, txId));
                 }
                 break;
             }
             case device_action: {
                 try {
-                    const statusResponsePayload = {
+                    this.publisher.publishToClient((0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, {
                         status: device_agent_schemas_1.keyMirrors.statusResponse.in_progress
-                    };
-                    const statusResponseMessage = (0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, statusResponsePayload, txId);
-                    this.publisher.publishToClient(statusResponseMessage);
+                    }, txId));
                     await this.handleDeviceAction(message.payload);
-                    const successStatusResponsePayload = {
+                    this.publisher.publishToClient((0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, {
                         status: device_agent_schemas_1.keyMirrors.statusResponse.success
-                    };
-                    const successStatusResponseMessage = (0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, successStatusResponsePayload, txId);
-                    this.publisher.publishToClient(successStatusResponseMessage);
+                    }, txId));
                 }
                 catch (e) {
-                    logger_1.logger.error(`There was a problem performing device action '${message.payload.action}': ${e.message}`);
-                    const failureStatusResponsePayload = {
+                    logger_1.logger.error(`There was a problem performing device action '${message.payload.action}'!\n${(0, util_1.stringifyError)(e)}`);
+                    this.publisher.publishToClient((0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, {
                         status: device_agent_schemas_1.keyMirrors.statusResponse.failure,
                         message: e.message
-                    };
-                    const failureStatusResponseMessage = (0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, failureStatusResponsePayload, txId);
-                    this.publisher.publishToClient(failureStatusResponseMessage);
+                    }, txId));
                 }
                 break;
             }
@@ -420,140 +449,140 @@ class DeviceAgentCloudConnection {
                 logger_1.logger.error(`Invalid client message: '${JSON.stringify({ topic, message, txId }, null, 2)}'`);
         }
     }
-    async handleMessage(topic, message) {
-        logger_1.logger.debug(`Received message: ${JSON.stringify({ topic, message }, null, 2)}`);
-        // ProjectShadow messages
-        if (this.shadowHandler.projectShadowTopics.includes(topic)) {
-            await this.handleProjectShadowMessage(topic, message);
-        }
-        else if (topic === this.toDeviceTopic) {
-            await this.handleDeviceAgentMessage({
-                topic,
-                message
-            });
-            // SecureTunnelNotify messages
-        }
-        else if (topic === this.secureTunnelNotifyTopic) {
-            await this.secureTunnelHandler.secureTunnelNotifyHandler(message);
-            // SecureTunnel messages
-        }
-        else if (topic === this.shadowHandler.shadowTopics.secureTunnel.updateDelta) {
-            await this.handleSecureTunnelMessage(message);
+    async handleDeviceAction(payload) {
+        const { system_restart } = device_agent_schemas_1.keyMirrors.deviceAction;
+        switch (payload.action) {
+            case system_restart: {
+                await (0, device_control_1.reboot)();
+                break;
+            }
+            default: {
+                logger_1.logger.info(`Unrecognized device action requested: '${payload.action}'.`);
+            }
         }
-        else if (topic === this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted) {
-            logger_1.logger.info(`Received secure tunnel deleteAccepted: ${message}`);
-            await this.secureTunnelHandler.destroy();
+    }
+    async publishCloudRequest(message) {
+        this.publisher.publishToCloud(message);
+    }
+    subscribe(topic) {
+        logger_1.logger.info(`Subscribing to ${topic}`);
+        this.device.subscribe(topic);
+    }
+    async atomicApplicationUninstall(projectId) {
+        try {
+            await (0, application_control_1.uninstallApp)({ projectId });
+            this.shadowHandler.clearProjectShadow(projectId);
         }
-        else {
-            logger_1.logger.error(`Unexpected topic, ignoring! ${topic}`);
+        catch (e) {
+            logger_1.logger.error(`Failed to uninstall ${projectId}!\n${(0, util_1.stringifyError)(e)}`);
+            throw e;
         }
     }
-    async setupHandlers() {
-        this.device.on('connect', (connack) => {
-            logger_1.logger.info('Device Agent has connected to the cloud');
-            // FIXME: EI-709 Skip this request for now to prevent kicking off another
-            // shadow update process if IoT Core disconnect occurs during app config update
-            //this.shadowHandler.getShadowUpdates();
-            void this.shadowHandler.updateSystemInfoShadow();
-        });
-        this.device.on('disconnect', () => {
-            logger_1.logger.warn('Device Agent has been disconnected from the cloud');
-        });
-        this.device.on('reconnect', () => {
-            logger_1.logger.info(`Device Agent attempting to re-connect ${new Date().toLocaleString()}`);
-        });
-        this.device.on('error', function (error) {
-            const errorString = error.message.toString();
-            logger_1.logger.error(`${errorString}`);
-        });
-        this.device.on('message', async (topic, payload) => {
+    // eslint-disable-next-line
+    async atomicApplicationUpdate(func, projectId, skipUpdateShadow) {
+        if (await (0, agent_config_1.AgentConfigFile)().isAppPresent({ projectId })) {
+            // Reject the application update if app is present but not ready
+            if (!(await (0, agent_config_1.AgentConfigFile)().isAppReady({ projectId }))) {
+                throw new Error('Application already has installation in progress!');
+            }
+            // Try to create a backup, so that there is one available if something goes wrong in the next try:catch.
             try {
-                const jsonPacket = JSON.parse(payload);
-                await this.handleMessage(topic, jsonPacket);
+                await (0, backup_1.createAppBackup)({ projectId });
             }
             catch (e) {
-                logger_1.logger.error(`Error parsing message: ${e.message}`);
+                logger_1.logger.error(`Could not create a backup for the project: ${projectId}!\n${(0, util_1.stringifyError)(e)}`);
             }
-        });
-        this.device.on('offline', () => {
-            logger_1.logger.warn(`Device Agent is offline ${new Date().toLocaleString()}`);
-            void this.logConnectionInfo();
-        });
-    }
-    async logConnectionInfo() {
+        }
         try {
-            /**
-             * We're using the 'netcat' or 'nc' command to test the connection to the IoT Core endpoint.
-             * This command doesn't always exit (see below), so
-             * we use timeout to break out of the prompt
-             * and catch the resulting error/parse the resulting stderr
-             *
-             * Sample command for current host and port:
-             * nc -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 8883
-             *
-             * Sample output when port is not blocked and host is reachable:
-             * $ nc  -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 443
-             * Connection to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 443 port [tcp/https] succeeded!
-             *
-             *
-             * Sample output when port is blocked (will repeatedly try until ctrl-C out):
-             * $ nc -zv -w 1 a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com 8883
-             * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
-             * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
-             * nc: connect to a3tzi5g7sq5zsj-ats.iot.us-west-2.amazonaws.com port 8883 (tcp) timed out: Operation now in progress
-             * ^C
-             *
-             *
-             * Sample command/output when the port isn't enable on that host:
-             * $ nc -zv -w 1 localhost 8883
-             * nc: connect to localhost port 8883 (tcp) failed: Connection refused
-             */
-            await exec_promise(`nc -zv -w 1 ${this.host} ${this.port}`, {
-                timeout: 2000
-            });
+            const out = await func();
+            if (!skipUpdateShadow)
+                await this.shadowHandler.updateProjectShadow(projectId);
+            return out;
         }
-        catch (err) {
-            const output = JSON.stringify(err['stderr']);
-            if (output.indexOf('not known') !== -1) {
-                logger_1.logger.warn('Iot Core endpoint appears to be unreachable, internet connection may be unstable or the host may be down.');
-            }
-            else if (output.indexOf('timed out') !== -1) {
-                logger_1.logger.warn(`Internet connection appears fine, however the endpoint was not reachable on the current connection port: ${this.port}\nPlease check if a firewall is in place.`);
+        catch (errorAppUpdate) {
+            logger_1.logger.error(`Failed to update ${projectId}!\n${(0, util_1.stringifyError)(errorAppUpdate)}`);
+            // If something goes wrong, first try to rollback
+            try {
+                await (0, backup_1.rollbackApp)({ projectId });
             }
-            else if (output.indexOf('refused') !== -1) {
-                logger_1.logger.warn(`The connection was refused, likely ${this.host} is not running a service on ${this.port}.`);
+            catch (errorRollbackApp) {
+                logger_1.logger.error(`Application rollback failed for ${projectId}!\n${(0, util_1.stringifyError)(errorRollbackApp)}`);
+                // and if that fails, uninstall the app as a last resort.
+                try {
+                    await this.atomicApplicationUninstall(projectId);
+                }
+                catch (_a) {
+                    // atomicApplicationUninstall logs failure, so there's nothing to do here.
+                }
+                throw new aai_error_1.default('Application update and rollback failed, uninstalled the application!', { cause: errorAppUpdate });
             }
-            else {
-                logger_1.logger.warn(`Output from checking connection to ${this.host} on ${this.port}: ${output}`);
+            throw new Error('Application update failed, rolled the application back!', { cause: errorAppUpdate });
+        }
+    }
+    async handleProjectShadowMessage(topic, message) {
+        const shadowUpdates = await this.shadowHandler.handleProjectShadow({
+            topic,
+            payload: message,
+            clientToken: message.clientToken
+        });
+        if (shadowUpdates.length) {
+            const shadowUpdatePromises = [];
+            for (const shadowUpdate of shadowUpdates) {
+                const projectId = shadowUpdate.projectId;
+                const txId = shadowUpdate.txId;
+                shadowUpdatePromises.push(this.txnMgr
+                    .runTransactionStep({
+                    func: () => this.handleProjectShadowConfigUpdate(shadowUpdate, txId),
+                    projectId,
+                    txId,
+                    start: true,
+                    liveUpdatesPublishFn: async () => this.publisher.publishToClient((0, device_agent_schemas_1.buildToClientStatusResponseMessage)(this.clientId, { status: device_agent_schemas_1.keyMirrors.statusResponse.in_progress }, txId), logger_1.logger.silly),
+                    stepName: topic
+                })
+                    .catch((e) => {
+                    logger_1.logger.error(`There was an issue updating project shadow config for ${projectId}!\n${(0, util_1.stringifyError)(e)}`);
+                }));
             }
+            await Promise.all(shadowUpdatePromises);
         }
     }
-    async stop() {
-        // This method is currently only used by the CLI, and shadow messages can be
-        // lost since we aren't waiting for responses so sleep for a short time to
-        // receive them
-        await (0, sleep_1.default)(1000);
-        this.device.end();
+    async handleSecureTunnelMessage(payload) {
+        logger_1.logger.info(`Received secure tunnel update: ${JSON.stringify(payload)}`);
+        const state = (0, device_agent_schemas_1.getUpdateDeltaStateFromMessage)(payload);
+        if (!state) {
+            logger_1.logger.debug(`No state found in message: ${JSON.stringify(payload)}`);
+            return;
+        }
+        const valid = (0, device_agent_schemas_1.validateSecureTunnelShadowUpdate)(state);
+        if (!valid) {
+            logger_1.logger.error(`Error validating message: ${JSON.stringify({ payload, errors: device_agent_schemas_1.validateSecureTunnelShadowUpdate.errors }, null, 2)}`);
+            return;
+        }
+        const secureTunnelUpdate = await this.secureTunnelHandler.syncShadowToDeviceState(payload);
+        await this.shadowHandler.updateSecureTunnelShadow(secureTunnelUpdate);
     }
 }
 exports.DeviceAgentCloudConnection = DeviceAgentCloudConnection;
 async function runDeviceAgentCloudInterface() {
     logger_1.logger.info(`Starting alwaysAI Device Agent v${await (0, check_for_updates_1.getDeviceAgentVersion)()}`);
-    if ((0, cloud_mode_ready_1.cloudModeReady)()) {
+    if ((0, fs_1.existsSync)((0, device_certificate_1.getBootstrapPrivateKeyFilePath)())) {
+        await (0, bootstrap_provision_1.bootstrapProvision)();
+        return;
+    }
+    const filesAlreadyMigrated = await (0, required_config_checks_1.requiredConfigFilesPresentAndValid)();
+    if (!filesAlreadyMigrated) {
+        logger_1.logger.debug('Attempting configuration file migration.');
+        await (0, legacy_migration_1.migrateFromLegacyCertsAndTokens)();
+    }
+    if (await (0, required_config_checks_1.requiredConfigFilesPresentAndValid)()) {
         const deviceAgent = new DeviceAgentCloudConnection();
-        await deviceAgent.setupHandlers();
         if (environment_1.ALWAYSAI_ANALYTICS_PASSTHROUGH === true) {
+            const shadowHandler = deviceAgent.shadowHandler;
             const publisher = deviceAgent.publisher;
-            const passthroughHandler = new passthrough_handler_1.PassthroughHandler(publisher);
+            const passthroughHandler = new passthrough_handler_1.PassthroughHandler(publisher, shadowHandler);
             await passthroughHandler.setup();
         }
     }
-    else if ((0, fs_1.existsSync)((0, directories_1.BOOTSTRAP_PRIVATE_KEY_FILE_PATH)())) {
-        await (0, bootstrap_provision_1.bootstrapProvision)();
-    }
-    else if ((0, fs_1.existsSync)((0, directories_1.BOOTSTRAP_CERTIFICATES_DIR_PATH)())) {
-        throw new Error("Device has not been created using 'aai-agent device init' or there has been an issue with device initialization");
-    }
     else {
         throw new Error("Set device agent to local mode and retry the 'aai-agent device init' command");
     }