@alwaysai/device-agent 2.0.2 → 2.1.0-0

package/src/cloud-connection/device-agent-message-handler.ts

@@ -27,7 +27,7 @@ import {
   ModelsInstallResponseMessage
 } from '@alwaysai/device-agent-schemas/lib/app-action-schema';
 import { DeviceActionMessage } from '@alwaysai/device-agent-schemas/lib/device-action-schema';
-import { logger, stringifyError } from 'alwaysai/lib/util';
+import { stringifyError } from 'alwaysai/lib/util';
 import {
   startApp,
   stopApp,
@@ -45,6 +45,7 @@ import {
   AppConfig,
   validateAppConfig
 } from '@alwaysai/app-configuration-schemas';
+import { logger } from '../util/logger';
 
 export type AppContent = {
   projectId: string;
@@ -160,7 +161,7 @@ class AppStateMessageHandler
       });
     } catch (e) {
       logger.error(
-        `Error processing application state control request for ${projectId}!\n${stringifyError(
+        `Error processing application state control request for ${projectId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -215,7 +216,7 @@ class AppVersionControlMessageHandler
       });
     } catch (e) {
       logger.error(
-        `Error processing application install request for ${projectId}!\n${stringifyError(
+        `Error processing application install request for ${projectId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -310,7 +311,9 @@ class AppVersionControlMessageHandler
         }
       } catch (e) {
         logger.error(
-          `Could not parse the appConfig for transaction!\n${stringifyError(e)}`
+          `Could not parse the appConfig for transaction! Error:\n${stringifyError(
+            e
+          )}`
         );
       }
     }
@@ -340,9 +343,9 @@ class AppVersionControlMessageHandler
           appContent.envVars = envvarsUpdate;
         }
       } catch (e) {
-        // throw here
+        // TODO: throw here
         logger.error(
-          `Could not parse the environment variables for transaction!\n${stringifyError(
+          `Could not parse the environment variables for transaction! Error:\n${stringifyError(
             e
           )}`
         );
@@ -610,7 +613,7 @@ class DeviceActionMessageHandler
       logger.error(
         `There was a problem performing device action '${
           message.payload.action
-        }'!\n${stringifyError(e)}`
+        }'! Error: \n${stringifyError(e)}`
       );
       this.publisher.publishToClient(
         buildToClientStatusResponseMessage(

package/src/cloud-connection/live-updates-handler.ts

@@ -14,6 +14,9 @@ interface IntervalOptions {
   ms?: number;
 }
 
+/*
+Responsible for managing the lifecycle of periodic updates and streams.
+*/
 export class LiveUpdatesHandler {
   private killAllTimeout: ReturnType<typeof setTimeout>;
   private livingIntervals: Record<string, ReturnType<typeof setInterval>> = {};
@@ -31,12 +34,14 @@ export class LiveUpdatesHandler {
     transactionId?: string,
     options?: IntervalOptions
   ) {
+    logger.silly(`LiveUpdatesHandler: Enabling ${intervalType}`);
     this.restartKillAllTimeout();
 
     const key = this.generateIntervalKey(intervalType, transactionId);
 
     this.safeSetInterval(key, publishingFn, options);
 
+    // Call publishing function right away for immediate results
     await publishingFn();
   }
 
@@ -44,6 +49,7 @@ export class LiveUpdatesHandler {
     intervalType: ToClientMessageTypeValue,
     transactionId?: string
   ) {
+    logger.silly(`LiveUpdatesHandler: Disabling ${intervalType}`);
     const key = this.generateIntervalKey(intervalType, transactionId);
     clearInterval(this.livingIntervals[key]);
     delete this.livingIntervals[key];
@@ -54,7 +60,7 @@ export class LiveUpdatesHandler {
     streamGetter: () => Promise<NodeJS.ReadableStream | null>,
     publishingFn: (logChunk: string) => void
   ) {
-    logger.info(`Starting log stream for ${projectId}`);
+    logger.info(`LiveUpdatesHandler: Starting log stream for ${projectId}`);
 
     this.livingStreams.add(projectId);
 
@@ -84,7 +90,7 @@ export class LiveUpdatesHandler {
     });
 
     readable.on('finished', () => {
-      logger.info(`Strean complete. ProjectId: ${projectId}`);
+      logger.info(`Stream complete. ProjectId: ${projectId}`);
     });
   }
 
@@ -105,8 +111,8 @@ export class LiveUpdatesHandler {
       try {
         return await streamGetter();
       } catch (e) {
-        logger.info(
-          `Failed to start app logs, retrying in 1 second.\n${stringifyError(
+        logger.error(
+          `Failed to start app logs, retrying in 1 second. Error:\n${stringifyError(
             e
           )}`
         );
@@ -117,7 +123,8 @@ export class LiveUpdatesHandler {
     return null;
   }
 
-  // do not await any functions in the setSafeInterval other than inside setInterval()  as per EI-1694.
+  // Do not await any functions in setSafeInterval other than inside
+  // setInterval() as per EI-1694.
   private safeSetInterval(
     key: string,
     publishingFn: () => Promise<void>,

package/src/cloud-connection/passthrough-handler.ts

@@ -11,18 +11,21 @@ import {
   LOCAL_CONNECTION_ROUTING_KEY
 } from '../local-connection/constants';
 import {
-  setupRabbitMQContainer,
+  runRabbitMQContainer,
   stopRabbitMQContainer
-} from '../local-connection/rabbitmq-connection';
+} from '../local-connection/rabbitmq-container';
 import { logger } from '../util/logger';
 import sleep from '../util/sleep';
 import { Publisher } from './publisher';
 import { ShadowHandler } from './shadow-handler';
 
-const messageQueue: any[] = [];
-const ackQueue: any[] = [];
 const MAX_LOCAL_CONNECTION_ATTEMPTS = 10;
 
+/*
+Responsible for managing the lifecycle of the Local Connection container
+(RabbitMQ), the connection to it, and the passthrough of messages received
+from the local connection to endpoints of provided `Publisher`.
+*/
 export class PassthroughHandler {
   public publisher: Publisher;
   public shadowHandler: ShadowHandler;
@@ -35,80 +38,86 @@ export class PassthroughHandler {
     this.shadowHandler = shadowHandler;
   }
 
-  runChannel = async () => {
-    logger.debug('Beginning to consume packets');
+  private async runLocalConnectionChannel() {
+    logger.debug('Beginning to consume packets from Local Connection');
+
     await this.channel.consume(
       this.packetQueue,
-      (msg) => {
-        // NOTE: this needs to be an arrow function and then the whole contents of processPublish are below
-        if (msg?.content !== undefined) {
-          const packet = JSON.parse(msg.content.toString());
-          messageQueue.push({ packet, msg });
-          while (messageQueue.length > 0) {
-            const entry = messageQueue.shift();
-            const { packet, msg } = entry;
+      async (msg) => {
+        if (!msg?.content) return;
+
+        let packet: string;
+
+        try {
+          const raw = msg.content.toString();
+          // Currently, edgeIQ is calling json.dumps twice, so a single JSON.parse
+          // results in a string return value, which is needed by publishToCloudWithAck.
+          // We will update the edgeIQ implementation in the future.
+          packet = JSON.parse(raw);
+        } catch (err) {
+          logger.error(`Error parsing RabbitMQ packet: ${stringifyError(err)}`);
+          this.channel.ack(msg);
+          logger.debug('Problematic packet was acknowledged');
+          return;
+        }
+
+        // We do a second call to JSON.parse here only to retrieve the 'action' field
+        // NOTE: this second call to JSON.parse seems to alter the packet contents
+        // (at least when a rounded floating point value is included)
+        const action = JSON.parse(packet)?.['action'];
+
+        switch (action) {
+          case 'analytics': {
             try {
-              const parsedPacket = JSON.parse(packet);
-              if (parsedPacket?.['action']) {
-                switch (parsedPacket['action']) {
-                  case 'analytics':
-                    ackQueue.push(msg);
-                    // FIXME: put real topic here
-                    this.publisher.publishToCloudWithAck(
-                      packet,
-                      (errOrResp) => {
-                        while (ackQueue.length > 0) {
-                          const msg = ackQueue.shift();
-                          if (errOrResp === true) {
-                            this.channel.ack(msg); // acknowledge, allow queue to discard
-                          } else if (errOrResp === false) {
-                            this.channel.reject(msg, true); // reject and requeue
-                          }
-                        }
-                      }
-                    );
-                    break;
-                  case 'heartbeat':
-                    this.channel.ack(msg);
-                    logger.silly(
-                      `Heartbeat package received & acknowledged: ${packet}`
-                    );
-                    break;
-                  default:
-                    this.channel.ack(msg);
-                    logger.debug(
-                      `Unknown 'action' package received & acknowledged: ${packet}`
-                    );
-                    break;
-                }
-              } else {
+              const success = await this.publisher.publishToCloudWithAck(
+                packet
+              );
+
+              if (success) {
                 this.channel.ack(msg);
-                logger.debug(
-                  `Received & acknowledged a RabbitMQ Package of unknown structure: ${parsedPacket}`
-                );
+              } else {
+                this.channel.reject(msg, true); // requeue on failure
               }
-            } catch (e) {
+            } catch (err) {
               logger.error(
-                `There was a problem parsing RabbitMQ packet!\n${stringifyError(
-                  e
-                )}`
+                `Error publishing analytics packet: ${stringifyError(err)}`
+              );
+              this.channel.reject(msg, true);
+            }
+            break;
+          }
+
+          case 'heartbeat': {
+            this.channel.ack(msg);
+            logger.silly(
+              `Heartbeat packet received & acknowledged:\n${packet}`
+            );
+            break;
+          }
+
+          default: {
+            this.channel.ack(msg);
+            if (action) {
+              logger.debug(
+                `Unknown 'action' packet received & acknowledged:\n${packet}`
+              );
+            } else {
+              logger.debug(
+                `Received & acknowledged a RabbitMQ packet of unknown structure:\n${packet}`
               );
-              this.channel.ack(msg);
-              logger.debug('Problematic packet was acknowledged');
             }
+            break;
           }
         }
       },
-      {
-        noAck: false // When true, RabbitMQ deletes message as soon as it is consumed
-      }
+      { noAck: false } // message must be explicitly acked/rejected
     );
-  };
+  }
 
-  async establishLocalConnection(): Promise<void> {
+  private async establishLocalConnection(): Promise<void> {
     let connectAttempts = 0;
     let connected = false;
-    logger.debug(`Establishing local connection...`);
+    logger.debug(`Establishing Local Connection...`);
     while (
       connectAttempts <= MAX_LOCAL_CONNECTION_ATTEMPTS &&
       this.connection === undefined
@@ -117,14 +126,40 @@ export class PassthroughHandler {
         this.connection = await amqp.connect(
           `amqp://${LOCAL_CONNECTION_HOST}:${LOCAL_CONNECTION_PORT}`
         );
-        this.channel = await this.connection.createChannel();
-        this.connection.on('error', async () => {
-          logger.error(`Local connection failed. Attempting to reconnect...`);
-          await stopRabbitMQContainer();
+
+        this.connection.on('error', async (e) => {
+          logger.error(
+            `Local Connection failed due to ${stringifyError(
+              e
+            )}. Reconnect will be handled by "close"`
+          );
+        });
+
+        this.connection.on('close', async () => {
+          // Close should be called for every connection loss, error or not.
+          logger.warn(
+            `Local Connection closed by server. Attempting to reconnect...`
+          );
+          const stopped = await stopRabbitMQContainer();
+          if (stopped === false) {
+            logger.warn(
+              'Failed to stop Local Connection container. Restart may fail...'
+            );
+          }
           this.connection = undefined;
-          await this.setup();
+          await this.run();
+        });
+
+        this.connection.on('blocked', async () => {
+          logger.warn(`Local Connection blocked.`);
+        });
+
+        this.connection.on('unblocked', async () => {
+          logger.warn(`Local Connection unblocked.`);
         });
 
+        this.channel = await this.connection.createChannel();
+
         connected = true;
       } catch (e) {
         const timeTillNextAttemptMs = 1000 + 1000 * connectAttempts;
@@ -143,7 +178,7 @@ export class PassthroughHandler {
       await this.channel.assertQueue(this.packetQueue, {
         durable: true
       });
-      logger.info(`Local connection established.`);
+      logger.info(`Local Connection established.`);
     } else {
       throw new Error(
         'Unable to establish connection to alwaysAI Local Connection, please try restarting Device Agent.'
@@ -166,30 +201,40 @@ export class PassthroughHandler {
     );
   }
 
-  async setup() {
-    if (ALWAYSAI_ANALYTICS_PASSTHROUGH === true) {
-      logger.debug(
-        `Setting up alwaysAI Local Connection on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
-      );
-      await this.publishPassthroughStatusUpdate('starting');
-      await setupRabbitMQContainer();
-      try {
-        await this.establishLocalConnection();
-        await this.runChannel();
-        await this.publishPassthroughStatusUpdate(
-          'running',
-          `Passthrough running on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
-        );
-      } catch (e) {
-        logger.error(
-          `There was a problem maintaining RabbitMQ connection!\n${stringifyError(
-            e
-          )}`
-        );
-        await this.publishPassthroughStatusUpdate('error', stringifyError(e));
-      }
-    } else {
+  async run() {
+    if (ALWAYSAI_ANALYTICS_PASSTHROUGH === false) {
+      logger.debug(`alwaysAI Local Connection disabled`);
       await this.publishPassthroughStatusUpdate('disabled');
+      return;
     }
+
+    logger.debug(
+      `Setting up alwaysAI Local Connection on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
+    );
+    await this.publishPassthroughStatusUpdate('starting');
+    const containerUp = await runRabbitMQContainer();
+    if (!containerUp) {
+      await this.publishPassthroughStatusUpdate(
+        'error',
+        'Local Connection container failed to start!'
+      );
+      return;
+    }
+    try {
+      await this.establishLocalConnection();
+      await this.runLocalConnectionChannel();
+    } catch (e) {
+      logger.error(
+        `There was a problem maintaining RabbitMQ connection! Error:\n${stringifyError(
+          e
+        )}`
+      );
+      await this.publishPassthroughStatusUpdate('error', stringifyError(e));
+      return;
+    }
+    await this.publishPassthroughStatusUpdate(
+      'running',
+      `Passthrough running on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
+    );
   }
 }

package/src/cloud-connection/publisher.ts

@@ -7,6 +7,7 @@ import {
 } from '@alwaysai/device-agent-schemas';
 import * as winston from 'winston';
 import { ConnectionManager } from './connection-manager';
+import { mqtt5 } from 'aws-iot-device-sdk-v2';
 
 export class Publisher {
   private connectionManager: ConnectionManager;
@@ -28,6 +29,12 @@ export class Publisher {
   ) {
     // TODO: topic validation
     // By default, log the published message at debug level, unless otherwise specified
+    const publishPacket: mqtt5.PublishPacket = {
+      topicName: topic,
+      qos: this.connectionManager.qos,
+      payload: Buffer.from(payload)
+    };
+
     customLogger(
       `Publishing message:\nTopic: ${topic}\nMessage: ${JSON.stringify(
         JSON.parse(payload),
@@ -35,37 +42,32 @@ export class Publisher {
         2
       )}`
     );
-    this.connectionManager
-      .getIoTDevice()
-      .publish(topic, payload, (err: any) => {
-        if (err) {
-          logger.error(
-            `Error publishing message: \nTopic: ${topic}\nMessage: ${payload}\nError: ${err}`
-          );
-        }
-      });
+
+    // TODO: Change publish() to async and resolve issues where it's used synchronously.
+    void this.connectionManager.getIoTDevice().publish(publishPacket);
   }
 
-  public publishToCloudWithAck(
-    payload: string,
-    ackNackCallback: CallableFunction
-  ) {
+  public async publishToCloudWithAck(payload: string): Promise<boolean> {
     const topic = this.toCloudTopic;
-    this.connectionManager
-      .getIoTDevice()
-      .publish(topic, payload, { qos: 1 }, (err: any, resp: any) => {
-        if (err) {
-          logger.error(
-            `Error publishing message: \nTopic: ${topic}\nMessage: ${payload}\nError: ${err}`
-          );
-          ackNackCallback(false);
-        } else if (resp) {
-          logger.debug(
-            `Successfully published message: \nTopic: ${topic}\nMessage: ${payload}`
-          );
-          ackNackCallback(true);
-        }
-      });
+
+    const publishPacket: mqtt5.PublishPacket = {
+      topicName: topic,
+      qos: this.connectionManager.qos,
+      payload: Buffer.from(payload)
+    };
+
+    try {
+      await this.connectionManager.getIoTDevice().publish(publishPacket);
+      logger.debug(
+        `Successfully published message:\nTopic: ${topic}\nMessage: ${payload}`
+      );
+      return true;
+    } catch (err) {
+      logger.error(
+        `Error publishing message:\nTopic: ${topic}\nMessage: ${payload}\nError: ${err}`
+      );
+      return false;
+    }
   }
 
   public publishDeviceAgentMessage(

package/src/cloud-connection/shadow-handler.ts

@@ -125,7 +125,7 @@ export class ShadowHandler {
       newAppCfg = JSON.parse(appConfig);
     } catch (e) {
       logger.error(
-        `Could not parse the appConfig for transaction ${txId}!\n${stringifyError(
+        `Could not parse the appConfig for transaction ${txId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -172,7 +172,7 @@ export class ShadowHandler {
       newEnvVars = JSON.parse(envVars);
     } catch (e) {
       logger.error(
-        `Could not parse the environment variables for transaction ${txId}!\n${stringifyError(
+        `Could not parse the environment variables for transaction ${txId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -493,7 +493,7 @@ export class ProjectShadowMessageHandler
             })
             .catch((e) => {
               logger.error(
-                `There was an issue updating project shadow config for ${projectId}!\n${stringifyError(
+                `There was an issue updating project shadow config for ${projectId}! Error:\n${stringifyError(
                   e
                 )}`
               );

package/src/cloud-connection/shadow.ts

@@ -45,7 +45,9 @@ export const getAppCfgModelsDiff = async ({
       });
   } catch (e) {
     logger.error(
-      `Error parsing app config update for ${projectId}!\n${stringifyError(e)}`
+      `Error parsing app config update for ${projectId}! Error:\n${stringifyError(
+        e
+      )}`
     );
   }
 

package/src/cloud-connection/transaction-manager.test.ts

@@ -115,78 +115,97 @@ describe('Test Transaction Manager', () => {
   test('Attempt to start an ongoing transaction, results in failure', async () => {
     const txId = generateTxId();
     const projectId = generateRandomProjectId();
+    let error;
+    const errorFn = jest.fn().mockImplementation((txid, message) => {
+      error = message;
+    });
     await txnMgr.runTransactionStep({
       func: func_incomplete,
       projectId,
       txId,
       start: true,
-      stepName: 'step1'
+      stepName: 'step1',
+      errorFn
+    });
+    expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
+    expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(errorFn).toBeCalledTimes(0);
+
+    await txnMgr.runTransactionStep({
+      func: func_incomplete,
+      projectId,
+      txId,
+      start: true,
+      stepName: 'step2',
+      errorFn
     });
-    try {
-      await txnMgr.runTransactionStep({
-        func: func_incomplete,
-        projectId,
-        txId,
-        start: true,
-        stepName: 'step2'
-      });
-      throw new Error('Expected starting transaction to fail!');
-    } catch (e) {
-      console.log(e);
-      expect(e.code).toBe(txnMgr.Errors.TRANSACTION_ONGOING);
-    }
     expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
     expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(errorFn).toBeCalledTimes(1);
+    expect(error.includes(`Transaction ${txId} already ongoing`)).toBeTruthy();
   });
 
   test('Attempt to continue a transaction that is not ongoing', async () => {
     const txId = generateTxId();
     const projectId = generateRandomProjectId();
-    try {
-      await txnMgr.runTransactionStep({
-        func: func_incomplete,
-        projectId,
-        txId,
-        start: false,
-        stepName: 'step1'
-      });
-      throw new Error('Expected continue transaction to fail!');
-    } catch (e) {
-      console.log(e);
-      expect(e.code).toBe(txnMgr.Errors.TRANSACTION_NOT_ONGOING);
-    }
+    let error;
+    const errorFn = jest.fn().mockImplementation((txid, message) => {
+      error = message;
+    });
+    await txnMgr.runTransactionStep({
+      func: func_incomplete,
+      projectId,
+      txId,
+      start: false,
+      stepName: 'step1',
+      errorFn
+    });
     expect(txnMgr.isOngoingTransaction(txId)).toBe(false);
     expect(txnMgr.isOngoingTransactionForProjectID(projectId)).toBe(false);
     expect(txnMgr.isAnyOngoingTransaction()).toBe(false);
+    expect(errorFn).toBeCalledTimes(1);
+    expect(
+      error.includes(
+        `Cannot update transaction ${txId} since it doesn't exist!`
+      )
+    ).toBeTruthy();
   });
 
   test('Attempt to start a transaction for a project with an ongoing transaction, results in failure', async () => {
     const txId = generateTxId();
     const txId2 = generateTxId();
     const projectId = generateRandomProjectId();
+    let error;
+    const errorFn = jest.fn().mockImplementation((txid, message) => {
+      error = message;
+    });
     await txnMgr.runTransactionStep({
       func: func_incomplete,
       projectId,
       txId,
       start: true,
-      stepName: 'step1'
+      stepName: 'step1',
+      errorFn
+    });
+    expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
+    expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(errorFn).toBeCalledTimes(0);
+
+    await txnMgr.runTransactionStep({
+      func: func_incomplete,
+      projectId,
+      txId: txId2,
+      start: true,
+      stepName: 'step2',
+      errorFn
     });
-    try {
-      await txnMgr.runTransactionStep({
-        func: func_incomplete,
-        projectId,
-        txId: txId2,
-        start: true,
-        stepName: 'step2'
-      });
-      throw new Error('Expected start transaction to fail!');
-    } catch (e) {
-      console.log(e);
-      expect(e.code).toBe(txnMgr.Errors.PROJECT_TRANSACTION_ONGOING);
-    }
     expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
     expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
     expect(txnMgr.getProjectFromTransaction(txId2)).toBeUndefined();
+    expect(errorFn).toBeCalledTimes(1);
+    expect(
+      error.includes(`Project ${projectId} already has an ongoing transaction`)
+    ).toBeTruthy();
   });
 
   test('Handle error in step function', async () => {