@alwaysai/device-agent 2.0.2-0 → 2.0.3

package/src/cloud-connection/passthrough-handler.ts

@@ -11,9 +11,9 @@ import {
   LOCAL_CONNECTION_ROUTING_KEY
 } from '../local-connection/constants';
 import {
-  setupRabbitMQContainer,
+  runRabbitMQContainer,
   stopRabbitMQContainer
-} from '../local-connection/rabbitmq-connection';
+} from '../local-connection/rabbitmq-container';
 import { logger } from '../util/logger';
 import sleep from '../util/sleep';
 import { Publisher } from './publisher';
@@ -23,6 +23,11 @@ const messageQueue: any[] = [];
 const ackQueue: any[] = [];
 const MAX_LOCAL_CONNECTION_ATTEMPTS = 10;
 
+/*
+Responsible for managing the lifecycle of the Local Connection container
+(RabbitMQ), the connection to it, and the passthrough of messages received
+from the local connection to endpoints of provided `Publisher`.
+*/
 export class PassthroughHandler {
   public publisher: Publisher;
   public shadowHandler: ShadowHandler;
@@ -35,8 +40,8 @@ export class PassthroughHandler {
     this.shadowHandler = shadowHandler;
   }
 
-  runChannel = async () => {
-    logger.debug('Beginning to consume packets');
+  private async runLocalConnectionChannel() {
+    logger.debug('Beginning to consume packets from Local Connection');
     await this.channel.consume(
       this.packetQueue,
       (msg) => {
@@ -84,12 +89,12 @@ export class PassthroughHandler {
               } else {
                 this.channel.ack(msg);
                 logger.debug(
-                  `Received & acknowledged a RabbitMQ Package of unknown structure: ${parsedPacket}`
+                  `Received & acknowledged a RabbitMQ packet of unknown structure: ${parsedPacket}`
                 );
               }
             } catch (e) {
               logger.error(
-                `There was a problem parsing RabbitMQ packet!\n${stringifyError(
+                `There was a problem parsing RabbitMQ packet! Error:\n${stringifyError(
                   e
                 )}`
               );
@@ -103,12 +108,12 @@ export class PassthroughHandler {
         noAck: false // When true, RabbitMQ deletes message as soon as it is consumed
       }
     );
-  };
+  }
 
-  async establishLocalConnection(): Promise<void> {
+  private async establishLocalConnection(): Promise<void> {
     let connectAttempts = 0;
     let connected = false;
-    logger.debug(`Establishing local connection...`);
+    logger.debug(`Establishing Local Connection...`);
     while (
       connectAttempts <= MAX_LOCAL_CONNECTION_ATTEMPTS &&
       this.connection === undefined
@@ -117,14 +122,40 @@ export class PassthroughHandler {
         this.connection = await amqp.connect(
           `amqp://${LOCAL_CONNECTION_HOST}:${LOCAL_CONNECTION_PORT}`
         );
-        this.channel = await this.connection.createChannel();
-        this.connection.on('error', async () => {
-          logger.error(`Local connection failed. Attempting to reconnect...`);
-          await stopRabbitMQContainer();
+
+        this.connection.on('error', async (e) => {
+          logger.error(
+            `Local Connection failed due to ${stringifyError(
+              e
+            )}. Reconnect will be handled by "close"`
+          );
+        });
+
+        this.connection.on('close', async () => {
+          // Close should be called for every connection loss, error or not.
+          logger.warn(
+            `Local Connection closed by server. Attempting to reconnect...`
+          );
+          const stopped = await stopRabbitMQContainer();
+          if (stopped === false) {
+            logger.warn(
+              'Failed to stop Local Connection container. Restart may fail...'
+            );
+          }
           this.connection = undefined;
-          await this.setup();
+          await this.run();
         });
 
+        this.connection.on('blocked', async () => {
+          logger.warn(`Local Connection blocked.`);
+        });
+
+        this.connection.on('unblocked', async () => {
+          logger.warn(`Local Connection unblocked.`);
+        });
+
+        this.channel = await this.connection.createChannel();
+
         connected = true;
       } catch (e) {
         const timeTillNextAttemptMs = 1000 + 1000 * connectAttempts;
@@ -143,7 +174,7 @@ export class PassthroughHandler {
       await this.channel.assertQueue(this.packetQueue, {
         durable: true
       });
-      logger.info(`Local connection established.`);
+      logger.info(`Local Connection established.`);
     } else {
       throw new Error(
         'Unable to establish connection to alwaysAI Local Connection, please try restarting Device Agent.'
@@ -166,30 +197,40 @@ export class PassthroughHandler {
     );
   }
 
-  async setup() {
-    if (ALWAYSAI_ANALYTICS_PASSTHROUGH === true) {
-      logger.debug(
-        `Setting up alwaysAI Local Connection on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
-      );
-      await this.publishPassthroughStatusUpdate('starting');
-      await setupRabbitMQContainer();
-      try {
-        await this.establishLocalConnection();
-        await this.runChannel();
-        await this.publishPassthroughStatusUpdate(
-          'running',
-          `Passthrough running on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
-        );
-      } catch (e) {
-        logger.error(
-          `There was a problem maintaining RabbitMQ connection!\n${stringifyError(
-            e
-          )}`
-        );
-        await this.publishPassthroughStatusUpdate('error', stringifyError(e));
-      }
-    } else {
+  async run() {
+    if (ALWAYSAI_ANALYTICS_PASSTHROUGH === false) {
+      logger.debug(`alwaysAI Local Connection disabled`);
       await this.publishPassthroughStatusUpdate('disabled');
+      return;
     }
+
+    logger.debug(
+      `Setting up alwaysAI Local Connection on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
+    );
+    await this.publishPassthroughStatusUpdate('starting');
+    const containerUp = await runRabbitMQContainer();
+    if (!containerUp) {
+      await this.publishPassthroughStatusUpdate(
+        'error',
+        'Local Connection container failed to start!'
+      );
+      return;
+    }
+    try {
+      await this.establishLocalConnection();
+      await this.runLocalConnectionChannel();
+    } catch (e) {
+      logger.error(
+        `There was a problem maintaining RabbitMQ connection! Error:\n${stringifyError(
+          e
+        )}`
+      );
+      await this.publishPassthroughStatusUpdate('error', stringifyError(e));
+      return;
+    }
+    await this.publishPassthroughStatusUpdate(
+      'running',
+      `Passthrough running on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
+    );
   }
 }

package/src/cloud-connection/shadow-handler.ts

@@ -125,7 +125,7 @@ export class ShadowHandler {
       newAppCfg = JSON.parse(appConfig);
     } catch (e) {
       logger.error(
-        `Could not parse the appConfig for transaction ${txId}!\n${stringifyError(
+        `Could not parse the appConfig for transaction ${txId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -172,7 +172,7 @@ export class ShadowHandler {
       newEnvVars = JSON.parse(envVars);
     } catch (e) {
       logger.error(
-        `Could not parse the environment variables for transaction ${txId}!\n${stringifyError(
+        `Could not parse the environment variables for transaction ${txId}! Error:\n${stringifyError(
           e
         )}`
       );
@@ -493,7 +493,7 @@ export class ProjectShadowMessageHandler
             })
             .catch((e) => {
               logger.error(
-                `There was an issue updating project shadow config for ${projectId}!\n${stringifyError(
+                `There was an issue updating project shadow config for ${projectId}! Error:\n${stringifyError(
                   e
                 )}`
               );

package/src/cloud-connection/shadow.ts

@@ -45,7 +45,9 @@ export const getAppCfgModelsDiff = async ({
       });
   } catch (e) {
     logger.error(
-      `Error parsing app config update for ${projectId}!\n${stringifyError(e)}`
+      `Error parsing app config update for ${projectId}! Error:\n${stringifyError(
+        e
+      )}`
     );
   }
 

package/src/cloud-connection/transaction-manager.test.ts

@@ -115,78 +115,97 @@ describe('Test Transaction Manager', () => {
   test('Attempt to start an ongoing transaction, results in failure', async () => {
     const txId = generateTxId();
     const projectId = generateRandomProjectId();
+    let error;
+    const errorFn = jest.fn().mockImplementation((txid, message) => {
+      error = message;
+    });
     await txnMgr.runTransactionStep({
       func: func_incomplete,
       projectId,
       txId,
       start: true,
-      stepName: 'step1'
+      stepName: 'step1',
+      errorFn
+    });
+    expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
+    expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(errorFn).toBeCalledTimes(0);
+
+    await txnMgr.runTransactionStep({
+      func: func_incomplete,
+      projectId,
+      txId,
+      start: true,
+      stepName: 'step2',
+      errorFn
     });
-    try {
-      await txnMgr.runTransactionStep({
-        func: func_incomplete,
-        projectId,
-        txId,
-        start: true,
-        stepName: 'step2'
-      });
-      throw new Error('Expected starting transaction to fail!');
-    } catch (e) {
-      console.log(e);
-      expect(e.code).toBe(txnMgr.Errors.TRANSACTION_ONGOING);
-    }
     expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
     expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(errorFn).toBeCalledTimes(1);
+    expect(error.includes(`Transaction ${txId} already ongoing`)).toBeTruthy();
   });
 
   test('Attempt to continue a transaction that is not ongoing', async () => {
     const txId = generateTxId();
     const projectId = generateRandomProjectId();
-    try {
-      await txnMgr.runTransactionStep({
-        func: func_incomplete,
-        projectId,
-        txId,
-        start: false,
-        stepName: 'step1'
-      });
-      throw new Error('Expected continue transaction to fail!');
-    } catch (e) {
-      console.log(e);
-      expect(e.code).toBe(txnMgr.Errors.TRANSACTION_NOT_ONGOING);
-    }
+    let error;
+    const errorFn = jest.fn().mockImplementation((txid, message) => {
+      error = message;
+    });
+    await txnMgr.runTransactionStep({
+      func: func_incomplete,
+      projectId,
+      txId,
+      start: false,
+      stepName: 'step1',
+      errorFn
+    });
     expect(txnMgr.isOngoingTransaction(txId)).toBe(false);
     expect(txnMgr.isOngoingTransactionForProjectID(projectId)).toBe(false);
     expect(txnMgr.isAnyOngoingTransaction()).toBe(false);
+    expect(errorFn).toBeCalledTimes(1);
+    expect(
+      error.includes(
+        `Cannot update transaction ${txId} since it doesn't exist!`
+      )
+    ).toBeTruthy();
   });
 
   test('Attempt to start a transaction for a project with an ongoing transaction, results in failure', async () => {
     const txId = generateTxId();
     const txId2 = generateTxId();
     const projectId = generateRandomProjectId();
+    let error;
+    const errorFn = jest.fn().mockImplementation((txid, message) => {
+      error = message;
+    });
     await txnMgr.runTransactionStep({
       func: func_incomplete,
       projectId,
       txId,
       start: true,
-      stepName: 'step1'
+      stepName: 'step1',
+      errorFn
+    });
+    expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
+    expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(errorFn).toBeCalledTimes(0);
+
+    await txnMgr.runTransactionStep({
+      func: func_incomplete,
+      projectId,
+      txId: txId2,
+      start: true,
+      stepName: 'step2',
+      errorFn
     });
-    try {
-      await txnMgr.runTransactionStep({
-        func: func_incomplete,
-        projectId,
-        txId: txId2,
-        start: true,
-        stepName: 'step2'
-      });
-      throw new Error('Expected start transaction to fail!');
-    } catch (e) {
-      console.log(e);
-      expect(e.code).toBe(txnMgr.Errors.PROJECT_TRANSACTION_ONGOING);
-    }
     expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
     expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
     expect(txnMgr.getProjectFromTransaction(txId2)).toBeUndefined();
+    expect(errorFn).toBeCalledTimes(1);
+    expect(
+      error.includes(`Project ${projectId} already has an ongoing transaction`)
+    ).toBeTruthy();
   });
 
   test('Handle error in step function', async () => {

package/src/cloud-connection/transaction-manager.ts

@@ -1,6 +1,5 @@
 import {
   ToClientStatusResponseMessage,
-  buildToClientStatusResponseMessage,
   keyMirrors
 } from '@alwaysai/device-agent-schemas';
 import { LiveUpdatesHandler } from './live-updates-handler';
@@ -39,7 +38,7 @@ export class TransactionManager {
     if (this.detailsByTx[txId]) {
       const txnDetails = this.detailsByTx[txId];
       throw new CodedError(
-        `Transaction ${txId} already ongoing!\n${JSON.stringify(
+        `Transaction ${txId} already ongoing:\n${JSON.stringify(
           txnDetails,
           null,
           2
@@ -52,7 +51,7 @@ export class TransactionManager {
     if (this.detailsByProject[projectId]) {
       const txnDetails = this.detailsByProject[projectId];
       throw new CodedError(
-        `Project ${projectId} already has an ongoing transaction!\n${JSON.stringify(
+        `Project ${projectId} already has an ongoing transaction:\n${JSON.stringify(
           txnDetails,
           null,
           2
@@ -126,16 +125,30 @@ export class TransactionManager {
       errorFn,
       successFn
     } = props;
-    if (start) {
-      await this.startTransaction(
-        txId,
-        projectId,
-        liveUpdatesPublishFn,
-        stepName
+    try {
+      if (start) {
+        await this.startTransaction(
+          txId,
+          projectId,
+          liveUpdatesPublishFn,
+          stepName
+        );
+      } else {
+        this.updateTransaction(txId, stepName);
+      }
+    } catch (e) {
+      logger.error(
+        `Failed to start or update cmd for ${projectId}! Error:\n${stringifyError(
+          e
+        )}`
       );
-    } else {
-      this.updateTransaction(txId, stepName);
+
+      if (errorFn) {
+        errorFn(txId, e.message);
+      }
+      return;
     }
+
     try {
       const completed = await func();
       if (completed) {
@@ -146,13 +159,14 @@ export class TransactionManager {
       }
     } catch (e) {
       logger.error(
-        `Failed to execute cmd for ${projectId}!\n${stringifyError(e)}`
+        `Failed to execute cmd for ${projectId}! Error:\n${stringifyError(e)}`
       );
 
       this.completeTransaction(txId);
       if (errorFn) {
         errorFn(txId, e.message);
       }
+      return;
     }
   }
 

package/src/device-control/device-control.ts

@@ -137,30 +137,40 @@ export async function getNetworkInfo() {
 }
 
 export async function getDockerVersion() {
+  let result: string | undefined;
   try {
     const spawner = JsSpawner();
-    const result = JSON.parse(
-      await spawner.run({
-        exe: 'docker',
-        args: ['info', '--format', 'json']
-      })
-    );
-    return String(result.ClientInfo.Version);
+    result = await spawner.run({
+      exe: 'docker',
+      args: ['info', '--format', 'json']
+    });
+    const parsedResult = JSON.parse(result);
+    return String(parsedResult.ClientInfo.Version);
   } catch (e) {
-    logger.warn(`Cannot get Docker version!\n${stringifyError(e)}`);
+    logger.warn(
+      `Cannot get Docker version! Result: ${result} Error:\n${stringifyError(
+        e
+      )}`
+    );
     return 'Not found';
   }
 }
 
 export async function getDockerComposeVersion() {
+  let result: string | undefined;
   try {
     const spawner = JsSpawner();
-    return await spawner.run({
+    result = await spawner.run({
       exe: 'docker',
       args: ['compose', 'version']
     });
+    return result;
   } catch (e) {
-    logger.warn(`Cannot get Docker Compose version!\n${stringifyError(e)}`);
+    logger.warn(
+      `Cannot get Docker Compose version! Results: ${result} Error:\n${stringifyError(
+        e
+      )}`
+    );
     return 'Not found';
   }
 }
@@ -173,7 +183,7 @@ export async function getNpmVersion() {
       args: ['--version']
     });
   } catch (e) {
-    logger.warn(`Cannot get npm version!\n${stringifyError(e)}`);
+    logger.warn(`Cannot get npm version! Error:\n${stringifyError(e)}`);
     return 'Not found';
   }
 }
@@ -186,7 +196,7 @@ export async function getNodeVersion() {
       args: ['-v']
     });
   } catch (e) {
-    logger.warn(`Cannot get Node version!\n${stringifyError(e)}`);
+    logger.warn(`Cannot get Node version! Error:\n${stringifyError(e)}`);
     return 'Not found';
   }
 }
@@ -230,7 +240,7 @@ export async function getLastBootTime() {
 
     return String(new Date(`${tokens[2]} ${tokens[3]} ${tokens[4]}`));
   } catch (e) {
-    logger.error(`Issue getting last boot time!\n${stringifyError(e)}`);
+    logger.error(`Issue getting last boot time! Error:\n${stringifyError(e)}`);
     return undefined;
   }
 }

package/src/docker/docker-compose.ts

@@ -19,7 +19,9 @@ export function importDockerCompose():
       execSync('docker-compose -v').toString();
       logger.warn('Using docker-compose V1. Please consider updating to V2.');
     } catch (e) {
-      logger.warn(`Could not determine compose!\n${stringifyError(e)}`);
+      logger.warn(
+        `Could not determine compose version! Error:\n${stringifyError(e)}`
+      );
     }
   }
   dockerCmd = 'docker-compose';

package/src/environment.ts

@@ -1,4 +1,6 @@
+import { TargetHardware } from 'alwaysai/lib/core/app';
 import { platform } from 'os';
+import { logger } from './util/logger';
 
 export const ALWAYSAI_OS_PLATFORM = parseOsPlatform(
   process.env.ALWAYSAI_OS_PLATFORM
@@ -22,6 +24,21 @@ export const ALWAYSAI_LOCAL_CONNECTION_PORT =
   process.env.ALWAYSAI_LOCAL_CONNECTION_PORT;
 export const ALWAYSAI_LOCAL_CONNECTION_ANALYTICS_ROUTING_KEY =
   process.env.ALWAYSAI_LOCAL_CONNECTION_ANALYTICS_ROUTING_KEY;
+export const ALWAYSAI_TARGET_HW_OVERRIDE = process.env.ALWAYSAI_TARGET_HW;
+
+export function parseTargetHW(
+  hardwareType?: string
+): TargetHardware | undefined {
+  const target = hardwareType ? TargetHardware[hardwareType] : undefined;
+  if (!target && hardwareType) {
+    logger.error(
+      `Incorrect TargetHardware: ${hardwareType}. Ignoring the environment variable. Allowed TargetHardware types: ${Object.keys(
+        TargetHardware
+      )}`
+    );
+  }
+  return target;
+}
 
 function parseOsPlatform(str: string | undefined): NodeJS.Platform {
   switch (str) {

package/src/index.ts

@@ -11,11 +11,30 @@ import { runDeviceAgentCloudInterface } from './cloud-connection/device-agent-cl
 import { AgentConfigFile } from './infrastructure/agent-config';
 import { ALWAYSAI_DEVICE_AGENT_MODE } from './environment';
 import { checkForUpdatesAndPrompt } from './util/check-for-updates';
+import { logger } from './util/logger';
+import { getSystemId } from './infrastructure/system-id';
+import { stringifyError } from 'alwaysai/lib/util';
 
 if (module === require.main) {
   if (!AgentConfigFile().exists()) {
     AgentConfigFile().initialize();
   }
+  logger.debug(`Starting Device Agent with system ID: ${getSystemId()}`);
+
+  // Catch any unhandled errors to prevent Device Agent crash
+  process.on('unhandledRejection', (reason: Error, p: Promise<any>) => {
+    logger.error(
+      `Unhandled Rejection at: ${JSON.stringify(p)} reason: ${stringifyError(
+        reason
+      )}`
+    );
+  });
+  process.on('uncaughtException', (error: Error) => {
+    logger.error(
+      `Caught exception: ${stringifyError(error)}\n` +
+        `Exception origin: ${error.stack}`
+    );
+  });
 
   if (ALWAYSAI_DEVICE_AGENT_MODE === 'cloud') {
     void runDeviceAgentCloudInterface();

package/src/infrastructure/config-check-utility.ts

@@ -14,7 +14,7 @@ export const validConfigMessage = 'Device config is present and valid.';
 export async function checkDeviceConfigPresent(
   baseDir?: string
 ): Promise<boolean> {
-  logger.debug('Checking for required device configuraton file.');
+  logger.debug('Checking for required device configuration file.');
   const deviceConfig = DeviceConfigFile(
     baseDir
       ? join(baseDir, getDeviceConfigPath(), DEVICE_CONFIG_FILE_NAME)
@@ -48,7 +48,7 @@ export async function checkCertificatesPresent(baseDir?: string) {
 export async function checkAaiConfigPresent(
   baseDir?: string
 ): Promise<boolean> {
-  logger.debug('Checking for required alwaysai configuraton file.');
+  logger.debug('Checking for required alwaysAI configuration file.');
   const localAaiCfg = new LocalAaiCfg(baseDir);
 
   try {

package/src/infrastructure/legacy-migration/legacy-migration.ts

@@ -17,7 +17,7 @@ import rimraf from 'rimraf';
 import {
   checkRabbitMQContainerRunning,
   stopRabbitMQContainer
-} from '../../local-connection/rabbitmq-connection';
+} from '../../local-connection/rabbitmq-container';
 import { copyDir } from '../../util/copy-dir';
 import {
   getDeviceAgentConfigPath,
@@ -54,7 +54,7 @@ export async function migrateAgentConfigFile(baseDir?: string) {
   try {
     parsedAgentConfig = legacyAgentConfigFile.read();
   } catch (e) {
-    logger.error(`Error reading agent config for:\n${stringifyError(e)}`);
+    logger.error(`Error reading agent config! Error:\n${stringifyError(e)}`);
     logger.error(
       `Agent Config Errors: ${JSON.stringify(
         legacyAgentConfigFile.getErrors(),
@@ -73,7 +73,7 @@ export async function migrateAgentConfigFile(baseDir?: string) {
     // we can remove it after migrating it
     await rimraf(legacyAgentConfigFile.path);
   } catch (e) {
-    logger.error(`Error writing agent config for:\n${stringifyError(e)}`);
+    logger.error(`Error writing agent config! Error:\n${stringifyError(e)}`);
     logger.error(
       `Agent Config Errors: ${JSON.stringify(
         newAgentConfig.getErrors(),
@@ -199,17 +199,12 @@ export async function migrateFromLegacyCertsAndTokens(
     await migrateAgentConfigFile(baseDir);
 
     // if rabbitmq container is present, stop container and migrate files
-    try {
-      if (await checkRabbitMQContainerRunning()) {
-        await stopRabbitMQContainer();
+    if (await checkRabbitMQContainerRunning()) {
+      if ((await stopRabbitMQContainer()) === false) {
+        logger.error(
+          `You may need to manually stop the container by running docker-compose down in the following directory: ${getDeviceAgentDockerComposePath()}`
+        );
       }
-    } catch (e) {
-      logger.error(
-        `You may need to manually stop the container by running docker-compose down in the following directory: ${getDeviceAgentDockerComposePath()}`
-      );
-      logger.debug(
-        `Error in checking / stopping RabbitMQ container!\n${stringifyError(e)}`
-      );
     }
     if (await exists(getDeviceAgentConfigPath(baseDir))) {
       await copyDir({