@alwaysai/device-agent 2.0.2-0 → 2.0.3

package/src/application-control/install.ts

@@ -16,7 +16,8 @@ import {
   appCleanDocker,
   getPythonVenvPaths,
   createPythonVenv,
-  installPythonReqs
+  installPythonReqs,
+  TargetHardware
 } from 'alwaysai/lib/core/app';
 import { DOCKER_IMAGE_ID_INITIAL_VALUE } from 'alwaysai/lib/constants';
 import { runInDir } from '../util/run-in-dir';
@@ -30,8 +31,10 @@ import {
 } from '../local-connection/constants';
 import { DOCKERFILE, PYTHON_REQUIREMENTS_FILE_NAME } from 'alwaysai/lib/paths';
 import { downloadToFile } from '../util/file';
+import { ALWAYSAI_TARGET_HW_OVERRIDE, parseTargetHW } from '../environment';
 import { setEnvInternal } from './environment-variables';
 import { AppConfig } from '@alwaysai/app-configuration-schemas';
+import { CliTerseError } from '@alwaysai/alwayscli';
 
 type SignedUrlPayloadType = {
   appInstallPayload: {
@@ -80,7 +83,7 @@ export async function installApp(props: {
         await stopApp({ projectId });
       } catch (e) {
         logger.error(
-          `Could not stop the application. Old container might still be present after installation.\n${stringifyError(
+          `Could not stop the application. Old container might still be present after installation. Error:\n${stringifyError(
             e
           )}`
         );
@@ -234,15 +237,18 @@ async function checkValidProjectFiles({ appDir }) {
   if (!AppJsonFile(appDir).readIfExists()) {
     throw new Error('App JSON file does not exist!');
   }
-
   // write target json
   if (!fs.existsSync(path.join(appDir, DOCKERFILE))) {
     throw new Error('No Dockerfile found for application!');
   }
+  const targetHw: TargetHardware =
+    parseTargetHW(ALWAYSAI_TARGET_HW_OVERRIDE) ??
+    (await getTargetHardwareType({}));
+  logger.debug(`Target Hardware selected to: ${targetHw}`);
   TargetJsonFile(appDir).write({
     targetProtocol: 'docker:',
     dockerImageId: DOCKER_IMAGE_ID_INITIAL_VALUE,
-    targetHardware: await getTargetHardwareType({})
+    targetHardware: targetHw
   });
 }
 
@@ -260,7 +266,7 @@ export async function uninstallApp(props: {
     await stopApp({ projectId });
   } catch (e) {
     logger.warn(
-      `Failed to stop ${projectId}, may be left running...\n${stringifyError(
+      `Failed to stop ${projectId}, may be left running... Error:\n${stringifyError(
         e
       )}`
     );

package/src/cloud-connection/base-message-handler.ts

@@ -1,4 +1,4 @@
-import { logger, stringifyError } from 'alwaysai/lib/util';
+import { stringifyError } from 'alwaysai/lib/util';
 import { uninstallApp, rollbackApp } from '../application-control';
 import { createAppBackup } from '../application-control/backup';
 import { AgentConfigFile } from '../infrastructure/agent-config';
@@ -12,6 +12,7 @@ import {
   ErrorFunction,
   SuccessFunction
 } from './transaction-manager';
+import { logger } from '../util/logger';
 
 export interface HandlerContext {
   clientId: string;
@@ -52,7 +53,9 @@ export abstract class BaseHandler {
       await uninstallApp({ projectId });
       this.shadowHandler.clearProjectShadow(projectId);
     } catch (e) {
-      logger.error(`Failed to uninstall ${projectId}!\n${stringifyError(e)}`);
+      logger.error(
+        `Failed to uninstall ${projectId}! Error:\n${stringifyError(e)}`
+      );
       throw e;
     }
   }
@@ -73,7 +76,7 @@ export abstract class BaseHandler {
         await createAppBackup({ projectId });
       } catch (e) {
         logger.error(
-          `Could not create a backup for the project: ${projectId}!\n${stringifyError(
+          `Could not create a backup for the project: ${projectId}! Error:\n${stringifyError(
             e
           )}`
         );
@@ -87,14 +90,16 @@ export abstract class BaseHandler {
       return out;
     } catch (errorAppUpdate) {
       logger.error(
-        `Failed to update ${projectId}!\n${stringifyError(errorAppUpdate)}`
+        `Failed to update ${projectId}! Error:\n${stringifyError(
+          errorAppUpdate
+        )}`
       );
       // If something goes wrong, first try to rollback
       try {
         await rollbackApp({ projectId });
       } catch (errorRollbackApp) {
         logger.error(
-          `Application rollback failed for ${projectId}!\n${stringifyError(
+          `Application rollback failed for ${projectId}! Error:\n${stringifyError(
             errorRollbackApp
           )}`
         );

package/src/cloud-connection/connection-manager.ts

@@ -16,40 +16,32 @@ export class ConnectionManager extends MessageDispatcher<any> {
   private clientId: string;
   private host: string;
   private port: number;
-  private device: awsIot.device | null = null;
+  private device: awsIot.device;
   private subscribedTopics: Set<string> = new Set();
+  private connected = false;
 
   constructor(clientId: string, host: string, port: number) {
     super();
     this.clientId = clientId;
     this.host = host;
     this.port = port;
+    this.device = awsIot.device({
+      keyPath: DEVICE_PRIVATE_KEY_FILE_PATH,
+      certPath: DEVICE_CERTIFICATE_FILE_PATH,
+      caPath: AWS_ROOT_CERTIFICATE_FILE_PATH,
+      clientId: this.clientId,
+      host: this.host,
+      port: this.port,
+      keepalive: 10 // time before re-connect attempt on dropped connection, default is 400 seconds
+    });
   }
 
   public getIoTDevice() {
     return this.device;
   }
 
-  public connect(cb): void {
-    try {
-      this.device = awsIot.device({
-        keyPath: DEVICE_PRIVATE_KEY_FILE_PATH,
-        certPath: DEVICE_CERTIFICATE_FILE_PATH,
-        caPath: AWS_ROOT_CERTIFICATE_FILE_PATH,
-        clientId: this.clientId,
-        host: this.host,
-        port: this.port,
-        keepalive: 10 // time before re-connect attempt on dropped connection, default is 400 seconds
-      });
-      this.setupHandlers(cb);
-    } catch (error) {
-      logger.error(`Error connecting to cloud!\n${stringifyError(error)}`);
-      this.handleReconnection();
-    }
-  }
-
   public isConnected(): boolean {
-    return this.device ? this.device.connected : false;
+    return this.connected;
   }
 
   public registerHandler(topic: string, handler: MessageHandler) {
@@ -58,16 +50,11 @@ export class ConnectionManager extends MessageDispatcher<any> {
   }
 
   public disconnect(): void {
-    if (this.device) {
-      this.device.end();
-      logger.debug(`Device Agent has been disconnected from the AWS IoT Core.`);
-    }
+    this.device.end();
+    logger.debug(`Device Agent has been disconnected from the AWS IoT Core.`);
   }
 
   public subscribe(topic: string): void {
-    if (!this.device) {
-      throw new Error('Must call connect() before subscribe()!');
-    }
     if (!this.subscribedTopics.has(topic)) {
       this.device.subscribe(topic);
       this.subscribedTopics.add(topic);
@@ -76,9 +63,6 @@ export class ConnectionManager extends MessageDispatcher<any> {
   }
 
   public unsubscribe(topic: string): void {
-    if (!this.device) {
-      throw new Error('Must call connect() before unsubscribe()!');
-    }
     if (this.subscribedTopics.has(topic)) {
       this.device.unsubscribe(topic);
       this.subscribedTopics.delete(topic);
@@ -86,25 +70,16 @@ export class ConnectionManager extends MessageDispatcher<any> {
     }
   }
 
-  private handleReconnection() {
-    logger.debug(`Attempting to reconnect to AWS IoT Core...`);
-    setTimeout(
-      () =>
-        this.connect(() => {
-          // Do nothing
-        }),
-      5000
-    ); // try in 5 seconds.
-  }
-
-  private setupHandlers(cb): void {
+  public initConnectionHandlers(connectCallback: () => void): void {
     this.device.on('connect', (connack: any) => {
       logger.info('Device Agent has connected to the cloud.');
-      cb();
+      this.connected = true;
+      connectCallback();
     });
 
     this.device.on('disconnect', () => {
       logger.warn('Device Agent has been disconnected from the cloud');
+      this.connected = false;
     });
 
     this.device.on('reconnect', () => {
@@ -115,17 +90,20 @@ export class ConnectionManager extends MessageDispatcher<any> {
 
     this.device.on('error', function (e) {
       logger.error(
-        `Error connecting to the AWS IoT Core!\n${stringifyError(e)}`
+        `Error connecting to the AWS IoT Core! Error:\n${stringifyError(e)}`
       );
+      this.connected = false;
     });
 
     this.device.on('close', () => {
       logger.warn('Device Agent AWS IoT Core connection closed.');
+      this.connected = false;
     });
 
     this.device.on('offline', () => {
       logger.warn(`Device Agent is offline ${new Date().toLocaleString()}`);
       void this.logConnectionInfo();
+      this.connected = false;
     });
 
     this.device.on('message', async (topic: string, payload: string) => {
@@ -136,7 +114,7 @@ export class ConnectionManager extends MessageDispatcher<any> {
         const jsonPacket = JSON.parse(payload);
         this.dispatch(topic, jsonPacket);
       } catch (e) {
-        logger.error(`Error parsing message!\n${stringifyError(e)}`);
+        logger.error(`Error parsing message! Error:\n${stringifyError(e)}`);
       }
     });
   }

package/src/cloud-connection/device-agent-cloud-connection.ts

@@ -41,8 +41,6 @@ export class DeviceAgentCloudConnection {
     SecureTunnelHandlerSingleton.getInstance();
 
   constructor() {
-    this.liveUpdatesHandler = new LiveUpdatesHandler();
-
     // Initialize & setup the connection
     this.connectionManager = new ConnectionManager(
       this.clientId,
@@ -52,92 +50,91 @@ export class DeviceAgentCloudConnection {
 
     this.publisher = new Publisher(this.connectionManager, this.clientId);
     this.shadowHandler = new ShadowHandler(this.clientId, this.publisher);
-
-    this.connectionManager.connect(() => {
-      void this.shadowHandler.initShadows();
-      this.publisher.publish(JOB_HANDLER_TOPICS.START_NEXT, JSON.stringify({}));
-    });
-
+    this.liveUpdatesHandler = new LiveUpdatesHandler();
     this.transactionManager = new TransactionManager(this.liveUpdatesHandler);
 
-    // Construct a HandlerContext used by all the message handlers
-    const handlerContext: HandlerContext = {
-      clientId: this.clientId,
-      txnMgr: this.transactionManager,
-      publisher: this.publisher,
-      shadowHandler: this.shadowHandler,
-      liveUpdatesHandler: this.liveUpdatesHandler,
-      secureTunnelHandler: this.secureTunnelHandler
-    };
-
-    // Instantiate & register message handlers for Project Shadow topics
-    const projectShadowMessageHandler = new ProjectShadowMessageHandler(
-      handlerContext
-    );
-    this.shadowHandler.projectShadowTopics.forEach((topic) => {
+    this.connectionManager.initConnectionHandlers(() => {
+      // Construct a HandlerContext used by all the message handlers
+      const handlerContext: HandlerContext = {
+        clientId: this.clientId,
+        txnMgr: this.transactionManager,
+        publisher: this.publisher,
+        shadowHandler: this.shadowHandler,
+        liveUpdatesHandler: this.liveUpdatesHandler,
+        secureTunnelHandler: this.secureTunnelHandler
+      };
+
+      // Instantiate & register message handlers for Project Shadow topics
+      const projectShadowMessageHandler = new ProjectShadowMessageHandler(
+        handlerContext
+      );
+      this.shadowHandler.projectShadowTopics.forEach((topic) => {
+        this.connectionManager.registerHandler(
+          topic,
+          projectShadowMessageHandler
+        );
+      });
+
+      // Instantiate & register message handlers for to-device and secureTunnel topics
       this.connectionManager.registerHandler(
-        topic,
-        projectShadowMessageHandler
+        this.toDeviceTopic,
+        new DeviceAgentMessageHandler(
+          handlerContext,
+          (txId: string, errorMsg: string) => {
+            const msg = buildToClientStatusResponseMessage(
+              this.publisher.getClientId(),
+              {
+                status: keyMirrors.statusResponse.failure,
+                message: errorMsg
+              },
+              txId
+            );
+            this.publisher.publishToClient(msg);
+          },
+          (txId: string) => {
+            const msg = buildToClientStatusResponseMessage(
+              this.publisher.getClientId(),
+              { status: keyMirrors.statusResponse.success },
+              txId
+            );
+            this.publisher.publishToClient(msg);
+          }
+        )
       );
-    });
 
-    // Instantiate & register message handlers for to-device and secureTunnel topics
-    this.connectionManager.registerHandler(
-      this.toDeviceTopic,
-      new DeviceAgentMessageHandler(
-        handlerContext,
-        (txId: string, errorMsg: string) => {
-          const msg = buildToClientStatusResponseMessage(
-            this.publisher.getClientId(),
-            {
-              status: keyMirrors.statusResponse.failure,
-              message: errorMsg
-            },
-            txId
-          );
-          this.publisher.publishToClient(msg);
-        },
-        (txId: string) => {
-          const msg = buildToClientStatusResponseMessage(
-            this.publisher.getClientId(),
-            { status: keyMirrors.statusResponse.success },
-            txId
-          );
-          this.publisher.publishToClient(msg);
-        }
-      )
-    );
-
-    const secureTunnelMessageHandler = new SecureTunnelMessageHandler(
-      handlerContext
-    );
-    this.connectionManager.registerHandler(
-      secureTunnelMessageHandler.getNotifyTopic(),
-      secureTunnelMessageHandler
-    );
+      const secureTunnelMessageHandler = new SecureTunnelMessageHandler(
+        handlerContext
+      );
+      this.connectionManager.registerHandler(
+        secureTunnelMessageHandler.getNotifyTopic(),
+        secureTunnelMessageHandler
+      );
 
-    this.connectionManager.registerHandler(
-      this.shadowHandler.shadowTopics.secureTunnel.updateDelta,
-      secureTunnelMessageHandler
-    );
-    this.connectionManager.registerHandler(
-      this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted,
-      secureTunnelMessageHandler
-    );
+      this.connectionManager.registerHandler(
+        this.shadowHandler.shadowTopics.secureTunnel.updateDelta,
+        secureTunnelMessageHandler
+      );
+      this.connectionManager.registerHandler(
+        this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted,
+        secureTunnelMessageHandler
+      );
 
-    const jobHandler = new JobHandler(handlerContext);
+      const jobHandler = new JobHandler(handlerContext);
+      const JOB_HANDLER_TOPICS = jobHandler.getJobTopics();
 
-    const JOB_HANDLER_TOPICS = jobHandler.getJobTopic();
+      this.connectionManager.registerHandler(
+        JOB_HANDLER_TOPICS.NOTIFY_NEXT,
+        jobHandler
+      );
 
-    this.connectionManager.registerHandler(
-      JOB_HANDLER_TOPICS.NOTIFY_NEXT,
-      jobHandler
-    );
+      this.connectionManager.registerHandler(
+        JOB_HANDLER_TOPICS.START_NEXT_ACCEPTED,
+        jobHandler
+      );
+      this.publisher.publish(JOB_HANDLER_TOPICS.START_NEXT, JSON.stringify({}));
 
-    this.connectionManager.registerHandler(
-      JOB_HANDLER_TOPICS.START_NEXT_ACCEPTED,
-      jobHandler
-    );
+      void this.shadowHandler.initShadows();
+    });
   }
 
   /*=================================================================
@@ -158,7 +155,22 @@ export class DeviceAgentCloudConnection {
     await sleep(1000);
     this.connectionManager.disconnect();
   }
-  public async handleMessage(topic: string, message: any) {
+
+  // CLI methods
+
+  public async waitForConnection() {
+    while (!this.connectionManager.isConnected()) {
+      await sleep(1000);
+    }
+  }
+
+  public async waitForCmd(projectId: string) {
+    while (this.isCmdInProgress(projectId)) {
+      await sleep(1000);
+    }
+  }
+
+  public async sendCliCmd(topic: string, message: any) {
     this.connectionManager.dispatch(topic, message);
   }
 }
@@ -179,16 +191,12 @@ export async function runDeviceAgentCloudInterface() {
   }
 
   if (await requiredConfigFilesPresentAndValid()) {
-    const deviceAgent = new DeviceAgentCloudConnection();
-    if (ALWAYSAI_ANALYTICS_PASSTHROUGH === true) {
-      const shadowHandler = deviceAgent.shadowHandler;
-      const publisher = deviceAgent.publisher;
-      const passthroughHandler = new PassthroughHandler(
-        publisher,
-        shadowHandler
-      );
-      await passthroughHandler.setup();
-    }
+    const cloudConnection = new DeviceAgentCloudConnection();
+    const passthroughHandler = new PassthroughHandler(
+      cloudConnection.publisher,
+      cloudConnection.shadowHandler
+    );
+    await passthroughHandler.run();
   } else {
     throw new Error(
       "Set device agent to local mode and retry the 'aai-agent device init' command"

package/src/cloud-connection/device-agent-message-handler.ts

@@ -27,7 +27,7 @@ import {
   ModelsInstallResponseMessage
 } from '@alwaysai/device-agent-schemas/lib/app-action-schema';
 import { DeviceActionMessage } from '@alwaysai/device-agent-schemas/lib/device-action-schema';
-import { logger, stringifyError } from 'alwaysai/lib/util';
+import { stringifyError } from 'alwaysai/lib/util';
 import {
   startApp,
   stopApp,
@@ -45,6 +45,7 @@ import {
   AppConfig,
   validateAppConfig
 } from '@alwaysai/app-configuration-schemas';
+import { logger } from '../util/logger';
 
 export type AppContent = {
   projectId: string;
@@ -160,7 +161,7 @@ class AppStateMessageHandler
       });
     } catch (e) {
       logger.error(
-        `Error processing application state control request for ${projectId}!\n${stringifyError(
+        `Error processing application state control request for ${projectId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -215,7 +216,7 @@ class AppVersionControlMessageHandler
       });
     } catch (e) {
       logger.error(
-        `Error processing application install request for ${projectId}!\n${stringifyError(
+        `Error processing application install request for ${projectId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -310,7 +311,9 @@ class AppVersionControlMessageHandler
         }
       } catch (e) {
         logger.error(
-          `Could not parse the appConfig for transaction!\n${stringifyError(e)}`
+          `Could not parse the appConfig for transaction! Error:\n${stringifyError(
+            e
+          )}`
         );
       }
     }
@@ -340,9 +343,9 @@ class AppVersionControlMessageHandler
           appContent.envVars = envvarsUpdate;
         }
       } catch (e) {
-        // throw here
+        // TODO: throw here
         logger.error(
-          `Could not parse the environment variables for transaction!\n${stringifyError(
+          `Could not parse the environment variables for transaction! Error:\n${stringifyError(
             e
           )}`
         );
@@ -610,7 +613,7 @@ class DeviceActionMessageHandler
       logger.error(
         `There was a problem performing device action '${
           message.payload.action
-        }'!\n${stringifyError(e)}`
+        }'! Error: \n${stringifyError(e)}`
       );
       this.publisher.publishToClient(
         buildToClientStatusResponseMessage(

package/src/cloud-connection/live-updates-handler.ts

@@ -14,6 +14,9 @@ interface IntervalOptions {
   ms?: number;
 }
 
+/*
+Responsible for managing the lifecycle of periodic updates and streams.
+*/
 export class LiveUpdatesHandler {
   private killAllTimeout: ReturnType<typeof setTimeout>;
   private livingIntervals: Record<string, ReturnType<typeof setInterval>> = {};
@@ -31,12 +34,14 @@ export class LiveUpdatesHandler {
     transactionId?: string,
     options?: IntervalOptions
   ) {
+    logger.silly(`LiveUpdatesHandler: Enabling ${intervalType}`);
     this.restartKillAllTimeout();
 
     const key = this.generateIntervalKey(intervalType, transactionId);
 
     this.safeSetInterval(key, publishingFn, options);
 
+    // Call publishing function right away for immediate results
     await publishingFn();
   }
 
@@ -44,6 +49,7 @@ export class LiveUpdatesHandler {
     intervalType: ToClientMessageTypeValue,
     transactionId?: string
   ) {
+    logger.silly(`LiveUpdatesHandler: Disabling ${intervalType}`);
     const key = this.generateIntervalKey(intervalType, transactionId);
     clearInterval(this.livingIntervals[key]);
     delete this.livingIntervals[key];
@@ -54,7 +60,7 @@ export class LiveUpdatesHandler {
     streamGetter: () => Promise<NodeJS.ReadableStream | null>,
     publishingFn: (logChunk: string) => void
   ) {
-    logger.info(`Starting log stream for ${projectId}`);
+    logger.info(`LiveUpdatesHandler: Starting log stream for ${projectId}`);
 
     this.livingStreams.add(projectId);
 
@@ -84,7 +90,7 @@ export class LiveUpdatesHandler {
     });
 
     readable.on('finished', () => {
-      logger.info(`Strean complete. ProjectId: ${projectId}`);
+      logger.info(`Stream complete. ProjectId: ${projectId}`);
     });
   }
 
@@ -105,8 +111,8 @@ export class LiveUpdatesHandler {
       try {
         return await streamGetter();
       } catch (e) {
-        logger.info(
-          `Failed to start app logs, retrying in 1 second.\n${stringifyError(
+        logger.error(
+          `Failed to start app logs, retrying in 1 second. Error:\n${stringifyError(
             e
           )}`
         );
@@ -117,7 +123,8 @@ export class LiveUpdatesHandler {
     return null;
   }
 
-  // do not await any functions in the setSafeInterval other than inside setInterval()  as per EI-1694.
+  // Do not await any functions in setSafeInterval other than inside
+  // setInterval() as per EI-1694.
   private safeSetInterval(
     key: string,
     publishingFn: () => Promise<void>,