@alwaysai/device-agent 2.0.2 → 2.1.0-1

package/src/cloud-connection/base-message-handler.ts

@@ -1,4 +1,4 @@
-import { logger, stringifyError } from 'alwaysai/lib/util';
+import { stringifyError } from 'alwaysai/lib/util';
 import { uninstallApp, rollbackApp } from '../application-control';
 import { createAppBackup } from '../application-control/backup';
 import { AgentConfigFile } from '../infrastructure/agent-config';
@@ -12,6 +12,7 @@ import {
   ErrorFunction,
   SuccessFunction
 } from './transaction-manager';
+import { logger } from '../util/logger';
 
 export interface HandlerContext {
   clientId: string;
@@ -52,7 +53,9 @@ export abstract class BaseHandler {
       await uninstallApp({ projectId });
       this.shadowHandler.clearProjectShadow(projectId);
     } catch (e) {
-      logger.error(`Failed to uninstall ${projectId}!\n${stringifyError(e)}`);
+      logger.error(
+        `Failed to uninstall ${projectId}! Error:\n${stringifyError(e)}`
+      );
       throw e;
     }
   }
@@ -73,7 +76,7 @@ export abstract class BaseHandler {
         await createAppBackup({ projectId });
       } catch (e) {
         logger.error(
-          `Could not create a backup for the project: ${projectId}!\n${stringifyError(
+          `Could not create a backup for the project: ${projectId}! Error:\n${stringifyError(
             e
           )}`
         );
@@ -87,14 +90,16 @@ export abstract class BaseHandler {
       return out;
     } catch (errorAppUpdate) {
       logger.error(
-        `Failed to update ${projectId}!\n${stringifyError(errorAppUpdate)}`
+        `Failed to update ${projectId}! Error:\n${stringifyError(
+          errorAppUpdate
+        )}`
       );
       // If something goes wrong, first try to rollback
       try {
         await rollbackApp({ projectId });
       } catch (errorRollbackApp) {
         logger.error(
-          `Application rollback failed for ${projectId}!\n${stringifyError(
+          `Application rollback failed for ${projectId}! Error:\n${stringifyError(
             errorRollbackApp
           )}`
         );

package/src/cloud-connection/connection-manager.ts

@@ -90,7 +90,7 @@ export class ConnectionManager extends MessageDispatcher<any> {
 
     this.device.on('error', function (e) {
       logger.error(
-        `Error connecting to the AWS IoT Core!\n${stringifyError(e)}`
+        `Error connecting to the AWS IoT Core! Error:\n${stringifyError(e)}`
       );
       this.connected = false;
     });
@@ -114,7 +114,7 @@ export class ConnectionManager extends MessageDispatcher<any> {
         const jsonPacket = JSON.parse(payload);
         this.dispatch(topic, jsonPacket);
       } catch (e) {
-        logger.error(`Error parsing message!\n${stringifyError(e)}`);
+        logger.error(`Error parsing message! Error:\n${stringifyError(e)}`);
       }
     });
   }

package/src/cloud-connection/device-agent-cloud-connection.ts

@@ -191,16 +191,12 @@ export async function runDeviceAgentCloudInterface() {
   }
 
   if (await requiredConfigFilesPresentAndValid()) {
-    const deviceAgent = new DeviceAgentCloudConnection();
-    if (ALWAYSAI_ANALYTICS_PASSTHROUGH === true) {
-      const shadowHandler = deviceAgent.shadowHandler;
-      const publisher = deviceAgent.publisher;
-      const passthroughHandler = new PassthroughHandler(
-        publisher,
-        shadowHandler
-      );
-      await passthroughHandler.setup();
-    }
+    const cloudConnection = new DeviceAgentCloudConnection();
+    const passthroughHandler = new PassthroughHandler(
+      cloudConnection.publisher,
+      cloudConnection.shadowHandler
+    );
+    await passthroughHandler.run();
   } else {
     throw new Error(
       "Set device agent to local mode and retry the 'aai-agent device init' command"

package/src/cloud-connection/device-agent-message-handler.ts

@@ -27,7 +27,7 @@ import {
   ModelsInstallResponseMessage
 } from '@alwaysai/device-agent-schemas/lib/app-action-schema';
 import { DeviceActionMessage } from '@alwaysai/device-agent-schemas/lib/device-action-schema';
-import { logger, stringifyError } from 'alwaysai/lib/util';
+import { stringifyError } from 'alwaysai/lib/util';
 import {
   startApp,
   stopApp,
@@ -45,6 +45,7 @@ import {
   AppConfig,
   validateAppConfig
 } from '@alwaysai/app-configuration-schemas';
+import { logger } from '../util/logger';
 
 export type AppContent = {
   projectId: string;
@@ -160,7 +161,7 @@ class AppStateMessageHandler
       });
     } catch (e) {
       logger.error(
-        `Error processing application state control request for ${projectId}!\n${stringifyError(
+        `Error processing application state control request for ${projectId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -215,7 +216,7 @@ class AppVersionControlMessageHandler
       });
     } catch (e) {
       logger.error(
-        `Error processing application install request for ${projectId}!\n${stringifyError(
+        `Error processing application install request for ${projectId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -310,7 +311,9 @@ class AppVersionControlMessageHandler
         }
       } catch (e) {
         logger.error(
-          `Could not parse the appConfig for transaction!\n${stringifyError(e)}`
+          `Could not parse the appConfig for transaction! Error:\n${stringifyError(
+            e
+          )}`
         );
       }
     }
@@ -340,9 +343,9 @@ class AppVersionControlMessageHandler
           appContent.envVars = envvarsUpdate;
         }
       } catch (e) {
-        // throw here
+        // TODO: throw here
         logger.error(
-          `Could not parse the environment variables for transaction!\n${stringifyError(
+          `Could not parse the environment variables for transaction! Error:\n${stringifyError(
             e
           )}`
         );
@@ -610,7 +613,7 @@ class DeviceActionMessageHandler
       logger.error(
         `There was a problem performing device action '${
           message.payload.action
-        }'!\n${stringifyError(e)}`
+        }'! Error: \n${stringifyError(e)}`
       );
       this.publisher.publishToClient(
         buildToClientStatusResponseMessage(

package/src/cloud-connection/live-updates-handler.ts

@@ -14,6 +14,9 @@ interface IntervalOptions {
   ms?: number;
 }
 
+/*
+Responsible for managing the lifecycle of periodic updates and streams.
+*/
 export class LiveUpdatesHandler {
   private killAllTimeout: ReturnType<typeof setTimeout>;
   private livingIntervals: Record<string, ReturnType<typeof setInterval>> = {};
@@ -31,12 +34,14 @@ export class LiveUpdatesHandler {
     transactionId?: string,
     options?: IntervalOptions
   ) {
+    logger.silly(`LiveUpdatesHandler: Enabling ${intervalType}`);
     this.restartKillAllTimeout();
 
     const key = this.generateIntervalKey(intervalType, transactionId);
 
     this.safeSetInterval(key, publishingFn, options);
 
+    // Call publishing function right away for immediate results
     await publishingFn();
   }
 
@@ -44,6 +49,7 @@ export class LiveUpdatesHandler {
     intervalType: ToClientMessageTypeValue,
     transactionId?: string
   ) {
+    logger.silly(`LiveUpdatesHandler: Disabling ${intervalType}`);
     const key = this.generateIntervalKey(intervalType, transactionId);
     clearInterval(this.livingIntervals[key]);
     delete this.livingIntervals[key];
@@ -54,7 +60,7 @@ export class LiveUpdatesHandler {
     streamGetter: () => Promise<NodeJS.ReadableStream | null>,
     publishingFn: (logChunk: string) => void
   ) {
-    logger.info(`Starting log stream for ${projectId}`);
+    logger.info(`LiveUpdatesHandler: Starting log stream for ${projectId}`);
 
     this.livingStreams.add(projectId);
 
@@ -84,7 +90,7 @@ export class LiveUpdatesHandler {
     });
 
     readable.on('finished', () => {
-      logger.info(`Strean complete. ProjectId: ${projectId}`);
+      logger.info(`Stream complete. ProjectId: ${projectId}`);
     });
   }
 
@@ -105,8 +111,8 @@ export class LiveUpdatesHandler {
       try {
         return await streamGetter();
       } catch (e) {
-        logger.info(
-          `Failed to start app logs, retrying in 1 second.\n${stringifyError(
+        logger.error(
+          `Failed to start app logs, retrying in 1 second. Error:\n${stringifyError(
             e
           )}`
         );
@@ -117,7 +123,8 @@ export class LiveUpdatesHandler {
     return null;
   }
 
-  // do not await any functions in the setSafeInterval other than inside setInterval()  as per EI-1694.
+  // Do not await any functions in setSafeInterval other than inside
+  // setInterval() as per EI-1694.
   private safeSetInterval(
     key: string,
     publishingFn: () => Promise<void>,

package/src/cloud-connection/passthrough-handler.ts

@@ -11,9 +11,9 @@ import {
   LOCAL_CONNECTION_ROUTING_KEY
 } from '../local-connection/constants';
 import {
-  setupRabbitMQContainer,
+  runRabbitMQContainer,
   stopRabbitMQContainer
-} from '../local-connection/rabbitmq-connection';
+} from '../local-connection/rabbitmq-container';
 import { logger } from '../util/logger';
 import sleep from '../util/sleep';
 import { Publisher } from './publisher';
@@ -23,6 +23,11 @@ const messageQueue: any[] = [];
 const ackQueue: any[] = [];
 const MAX_LOCAL_CONNECTION_ATTEMPTS = 10;
 
+/*
+Responsible for managing the lifecycle of the Local Connection container
+(RabbitMQ), the connection to it, and the passthrough of messages received
+from the local connection to endpoints of provided `Publisher`.
+*/
 export class PassthroughHandler {
   public publisher: Publisher;
   public shadowHandler: ShadowHandler;
@@ -35,8 +40,8 @@ export class PassthroughHandler {
     this.shadowHandler = shadowHandler;
   }
 
-  runChannel = async () => {
-    logger.debug('Beginning to consume packets');
+  private async runLocalConnectionChannel() {
+    logger.debug('Beginning to consume packets from Local Connection');
     await this.channel.consume(
       this.packetQueue,
       (msg) => {
@@ -84,12 +89,12 @@ export class PassthroughHandler {
               } else {
                 this.channel.ack(msg);
                 logger.debug(
-                  `Received & acknowledged a RabbitMQ Package of unknown structure: ${parsedPacket}`
+                  `Received & acknowledged a RabbitMQ packet of unknown structure: ${parsedPacket}`
                 );
               }
             } catch (e) {
               logger.error(
-                `There was a problem parsing RabbitMQ packet!\n${stringifyError(
+                `There was a problem parsing RabbitMQ packet! Error:\n${stringifyError(
                   e
                 )}`
               );
@@ -103,12 +108,12 @@ export class PassthroughHandler {
         noAck: false // When true, RabbitMQ deletes message as soon as it is consumed
       }
     );
-  };
+  }
 
-  async establishLocalConnection(): Promise<void> {
+  private async establishLocalConnection(): Promise<void> {
     let connectAttempts = 0;
     let connected = false;
-    logger.debug(`Establishing local connection...`);
+    logger.debug(`Establishing Local Connection...`);
     while (
       connectAttempts <= MAX_LOCAL_CONNECTION_ATTEMPTS &&
       this.connection === undefined
@@ -117,14 +122,40 @@ export class PassthroughHandler {
         this.connection = await amqp.connect(
           `amqp://${LOCAL_CONNECTION_HOST}:${LOCAL_CONNECTION_PORT}`
         );
-        this.channel = await this.connection.createChannel();
-        this.connection.on('error', async () => {
-          logger.error(`Local connection failed. Attempting to reconnect...`);
-          await stopRabbitMQContainer();
+
+        this.connection.on('error', async (e) => {
+          logger.error(
+            `Local Connection failed due to ${stringifyError(
+              e
+            )}. Reconnect will be handled by "close"`
+          );
+        });
+
+        this.connection.on('close', async () => {
+          // Close should be called for every connection loss, error or not.
+          logger.warn(
+            `Local Connection closed by server. Attempting to reconnect...`
+          );
+          const stopped = await stopRabbitMQContainer();
+          if (stopped === false) {
+            logger.warn(
+              'Failed to stop Local Connection container. Restart may fail...'
+            );
+          }
           this.connection = undefined;
-          await this.setup();
+          await this.run();
         });
 
+        this.connection.on('blocked', async () => {
+          logger.warn(`Local Connection blocked.`);
+        });
+
+        this.connection.on('unblocked', async () => {
+          logger.warn(`Local Connection unblocked.`);
+        });
+
+        this.channel = await this.connection.createChannel();
+
         connected = true;
       } catch (e) {
         const timeTillNextAttemptMs = 1000 + 1000 * connectAttempts;
@@ -143,7 +174,7 @@ export class PassthroughHandler {
       await this.channel.assertQueue(this.packetQueue, {
         durable: true
       });
-      logger.info(`Local connection established.`);
+      logger.info(`Local Connection established.`);
     } else {
       throw new Error(
         'Unable to establish connection to alwaysAI Local Connection, please try restarting Device Agent.'
@@ -166,30 +197,40 @@ export class PassthroughHandler {
     );
   }
 
-  async setup() {
-    if (ALWAYSAI_ANALYTICS_PASSTHROUGH === true) {
-      logger.debug(
-        `Setting up alwaysAI Local Connection on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
-      );
-      await this.publishPassthroughStatusUpdate('starting');
-      await setupRabbitMQContainer();
-      try {
-        await this.establishLocalConnection();
-        await this.runChannel();
-        await this.publishPassthroughStatusUpdate(
-          'running',
-          `Passthrough running on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
-        );
-      } catch (e) {
-        logger.error(
-          `There was a problem maintaining RabbitMQ connection!\n${stringifyError(
-            e
-          )}`
-        );
-        await this.publishPassthroughStatusUpdate('error', stringifyError(e));
-      }
-    } else {
+  async run() {
+    if (ALWAYSAI_ANALYTICS_PASSTHROUGH === false) {
+      logger.debug(`alwaysAI Local Connection disabled`);
       await this.publishPassthroughStatusUpdate('disabled');
+      return;
     }
+
+    logger.debug(
+      `Setting up alwaysAI Local Connection on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
+    );
+    await this.publishPassthroughStatusUpdate('starting');
+    const containerUp = await runRabbitMQContainer();
+    if (!containerUp) {
+      await this.publishPassthroughStatusUpdate(
+        'error',
+        'Local Connection container failed to start!'
+      );
+      return;
+    }
+    try {
+      await this.establishLocalConnection();
+      await this.runLocalConnectionChannel();
+    } catch (e) {
+      logger.error(
+        `There was a problem maintaining RabbitMQ connection! Error:\n${stringifyError(
+          e
+        )}`
+      );
+      await this.publishPassthroughStatusUpdate('error', stringifyError(e));
+      return;
+    }
+    await this.publishPassthroughStatusUpdate(
+      'running',
+      `Passthrough running on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
+    );
   }
 }

package/src/cloud-connection/shadow-handler.ts

@@ -125,7 +125,7 @@ export class ShadowHandler {
       newAppCfg = JSON.parse(appConfig);
     } catch (e) {
       logger.error(
-        `Could not parse the appConfig for transaction ${txId}!\n${stringifyError(
+        `Could not parse the appConfig for transaction ${txId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -172,7 +172,7 @@ export class ShadowHandler {
       newEnvVars = JSON.parse(envVars);
     } catch (e) {
       logger.error(
-        `Could not parse the environment variables for transaction ${txId}!\n${stringifyError(
+        `Could not parse the environment variables for transaction ${txId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -493,7 +493,7 @@ export class ProjectShadowMessageHandler
             })
             .catch((e) => {
               logger.error(
-                `There was an issue updating project shadow config for ${projectId}!\n${stringifyError(
+                `There was an issue updating project shadow config for ${projectId}! Error:\n${stringifyError(
                   e
                 )}`
               );

package/src/cloud-connection/shadow.ts

@@ -45,7 +45,9 @@ export const getAppCfgModelsDiff = async ({
       });
   } catch (e) {
     logger.error(
-      `Error parsing app config update for ${projectId}!\n${stringifyError(e)}`
+      `Error parsing app config update for ${projectId}! Error:\n${stringifyError(
+        e
+      )}`
     );
   }
 

package/src/cloud-connection/transaction-manager.test.ts

@@ -115,78 +115,97 @@ describe('Test Transaction Manager', () => {
   test('Attempt to start an ongoing transaction, results in failure', async () => {
     const txId = generateTxId();
     const projectId = generateRandomProjectId();
+    let error;
+    const errorFn = jest.fn().mockImplementation((txid, message) => {
+      error = message;
+    });
     await txnMgr.runTransactionStep({
       func: func_incomplete,
       projectId,
       txId,
       start: true,
-      stepName: 'step1'
+      stepName: 'step1',
+      errorFn
+    });
+    expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
+    expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(errorFn).toBeCalledTimes(0);
+
+    await txnMgr.runTransactionStep({
+      func: func_incomplete,
+      projectId,
+      txId,
+      start: true,
+      stepName: 'step2',
+      errorFn
     });
-    try {
-      await txnMgr.runTransactionStep({
-        func: func_incomplete,
-        projectId,
-        txId,
-        start: true,
-        stepName: 'step2'
-      });
-      throw new Error('Expected starting transaction to fail!');
-    } catch (e) {
-      console.log(e);
-      expect(e.code).toBe(txnMgr.Errors.TRANSACTION_ONGOING);
-    }
     expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
     expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(errorFn).toBeCalledTimes(1);
+    expect(error.includes(`Transaction ${txId} already ongoing`)).toBeTruthy();
   });
 
   test('Attempt to continue a transaction that is not ongoing', async () => {
     const txId = generateTxId();
     const projectId = generateRandomProjectId();
-    try {
-      await txnMgr.runTransactionStep({
-        func: func_incomplete,
-        projectId,
-        txId,
-        start: false,
-        stepName: 'step1'
-      });
-      throw new Error('Expected continue transaction to fail!');
-    } catch (e) {
-      console.log(e);
-      expect(e.code).toBe(txnMgr.Errors.TRANSACTION_NOT_ONGOING);
-    }
+    let error;
+    const errorFn = jest.fn().mockImplementation((txid, message) => {
+      error = message;
+    });
+    await txnMgr.runTransactionStep({
+      func: func_incomplete,
+      projectId,
+      txId,
+      start: false,
+      stepName: 'step1',
+      errorFn
+    });
     expect(txnMgr.isOngoingTransaction(txId)).toBe(false);
     expect(txnMgr.isOngoingTransactionForProjectID(projectId)).toBe(false);
     expect(txnMgr.isAnyOngoingTransaction()).toBe(false);
+    expect(errorFn).toBeCalledTimes(1);
+    expect(
+      error.includes(
+        `Cannot update transaction ${txId} since it doesn't exist!`
+      )
+    ).toBeTruthy();
   });
 
   test('Attempt to start a transaction for a project with an ongoing transaction, results in failure', async () => {
     const txId = generateTxId();
     const txId2 = generateTxId();
     const projectId = generateRandomProjectId();
+    let error;
+    const errorFn = jest.fn().mockImplementation((txid, message) => {
+      error = message;
+    });
     await txnMgr.runTransactionStep({
       func: func_incomplete,
       projectId,
       txId,
       start: true,
-      stepName: 'step1'
+      stepName: 'step1',
+      errorFn
+    });
+    expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
+    expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(errorFn).toBeCalledTimes(0);
+
+    await txnMgr.runTransactionStep({
+      func: func_incomplete,
+      projectId,
+      txId: txId2,
+      start: true,
+      stepName: 'step2',
+      errorFn
     });
-    try {
-      await txnMgr.runTransactionStep({
-        func: func_incomplete,
-        projectId,
-        txId: txId2,
-        start: true,
-        stepName: 'step2'
-      });
-      throw new Error('Expected start transaction to fail!');
-    } catch (e) {
-      console.log(e);
-      expect(e.code).toBe(txnMgr.Errors.PROJECT_TRANSACTION_ONGOING);
-    }
     expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
     expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
     expect(txnMgr.getProjectFromTransaction(txId2)).toBeUndefined();
+    expect(errorFn).toBeCalledTimes(1);
+    expect(
+      error.includes(`Project ${projectId} already has an ongoing transaction`)
+    ).toBeTruthy();
   });
 
   test('Handle error in step function', async () => {

package/src/cloud-connection/transaction-manager.ts

@@ -1,6 +1,5 @@
 import {
   ToClientStatusResponseMessage,
-  buildToClientStatusResponseMessage,
   keyMirrors
 } from '@alwaysai/device-agent-schemas';
 import { LiveUpdatesHandler } from './live-updates-handler';
@@ -39,7 +38,7 @@ export class TransactionManager {
     if (this.detailsByTx[txId]) {
       const txnDetails = this.detailsByTx[txId];
       throw new CodedError(
-        `Transaction ${txId} already ongoing!\n${JSON.stringify(
+        `Transaction ${txId} already ongoing:\n${JSON.stringify(
           txnDetails,
           null,
           2
@@ -52,7 +51,7 @@ export class TransactionManager {
     if (this.detailsByProject[projectId]) {
       const txnDetails = this.detailsByProject[projectId];
       throw new CodedError(
-        `Project ${projectId} already has an ongoing transaction!\n${JSON.stringify(
+        `Project ${projectId} already has an ongoing transaction:\n${JSON.stringify(
           txnDetails,
           null,
           2
@@ -126,16 +125,30 @@ export class TransactionManager {
       errorFn,
       successFn
     } = props;
-    if (start) {
-      await this.startTransaction(
-        txId,
-        projectId,
-        liveUpdatesPublishFn,
-        stepName
+    try {
+      if (start) {
+        await this.startTransaction(
+          txId,
+          projectId,
+          liveUpdatesPublishFn,
+          stepName
+        );
+      } else {
+        this.updateTransaction(txId, stepName);
+      }
+    } catch (e) {
+      logger.error(
+        `Failed to start or update cmd for ${projectId}! Error:\n${stringifyError(
+          e
+        )}`
       );
-    } else {
-      this.updateTransaction(txId, stepName);
+
+      if (errorFn) {
+        errorFn(txId, e.message);
+      }
+      return;
     }
+
     try {
       const completed = await func();
       if (completed) {
@@ -146,13 +159,14 @@ export class TransactionManager {
       }
     } catch (e) {
       logger.error(
-        `Failed to execute cmd for ${projectId}!\n${stringifyError(e)}`
+        `Failed to execute cmd for ${projectId}! Error:\n${stringifyError(e)}`
       );
 
       this.completeTransaction(txId);
       if (errorFn) {
         errorFn(txId, e.message);
       }
+      return;
     }
   }