@alwaysai/device-agent 1.2.0 → 1.3.0

package/src/cloud-connection/live-updates-handler.test.ts

@@ -1,6 +1,8 @@
 import { LiveUpdatesHandler } from './live-updates-handler';
 import { Publisher } from './publisher';
 
+global.setTimeout = jest.fn() as unknown as typeof setTimeout;
+
 const testTrueToggles = {
   deviceStats: true,
   appState: true
@@ -15,8 +17,6 @@ const mockClient = jest.fn();
 const clientId = 'test-client';
 const emptyTxId = '';
 
-jest.spyOn(global, 'setTimeout');
-
 // NOTE: this was the way I found to mock private class functions
 const mockStartPublishingLiveUpdates = jest.spyOn(
   LiveUpdatesHandler.prototype as any,

package/src/cloud-connection/live-updates-handler.ts

@@ -69,7 +69,7 @@ export class LiveUpdatesHandler {
         logChunk: logStr
       };
       const message = await buildAppLogsMessage(payload, this.clientId);
-      this.publisher.publishToClient(message);
+      this.publisher.publishToClient(message, logger.silly);
     });
 
     readable.on('error', (error) => {
@@ -153,14 +153,8 @@ export class LiveUpdatesHandler {
         const payload: ToClientMessagePayload = await payloadBuilderFunction(
           ...args
         );
-        logger.debug(
-          `payload returned from builder: ${JSON.stringify(payload)}`
-        );
         const message = await messageBuilderFunction(payload, txId);
-        logger.debug(
-          `message returned from builder: ${JSON.stringify(message)}`
-        );
-        this.publisher.publishToClient(message);
+        this.publisher.publishToClient(message, logger.silly);
       } catch (e) {
         logger.error(
           `Error publishing live updates for ${messageType}: ${e.message}`

package/src/cloud-connection/passthrough-handler.ts

@@ -92,22 +92,23 @@ function processPublish(passthroughHandler: PassthroughHandler) {
             );
             break;
           case 'heartbeat':
-            logger.debug(`Recieved heartbeat packet: ${packet}`);
             passthroughHandler.channel.ack(msg);
-            logger.debug(`Heartbeat package acknowledged`);
+            logger.debug(
+              `Heartbeat package received & acknowledged: ${packet}`
+            );
             break;
           default:
-            logger.debug(`Recieved unknown 'action' packet: ${packet}`);
             passthroughHandler.channel.ack(msg);
-            logger.debug(`Unknown 'action' package acknowledged`);
+            logger.debug(
+              `Unknown 'action' package received & acknowledged: ${packet}`
+            );
             break;
         }
       } else {
+        passthroughHandler.channel.ack(msg);
         logger.debug(
-          `Received a RabbitMQ packet that did not have an 'action' field ${parsedPacket}`
+          `Received & acknowledged a RabbitMQ Package of unknown structure: ${parsedPacket}`
         );
-        passthroughHandler.channel.ack(msg);
-        logger.debug(`Package of unknown structure acknowledged`);
       }
     } catch (e) {
       logger.error(`There was a problem parsing RabbitMQ packet ${e}`);

package/src/cloud-connection/publisher.ts

@@ -5,6 +5,7 @@ import {
   getToClientTopic,
   getToCloudTopic
 } from '@alwaysai/device-agent-schemas';
+import * as winston from 'winston';
 
 export class Publisher {
   private client: any;
@@ -19,8 +20,20 @@ export class Publisher {
     this.toCloudTopic = getToCloudTopic(this.clientId);
   }
 
-  public publish(topic: string, payload: string) {
+  public publish(
+    topic: string,
+    payload: string,
+    customLogger: winston.LeveledLogMethod = logger.debug
+  ) {
     // TODO: topic validation
+    // By default, log the published message at debug level, unless otherwise specified
+    customLogger(
+      `Publishing message:\nTopic: ${topic}\nMessage: ${JSON.stringify(
+        JSON.parse(payload),
+        null,
+        2
+      )}`
+    );
     this.client.publish(topic, payload, (err: any) => {
       if (err) {
         logger.error(
@@ -52,21 +65,25 @@ export class Publisher {
 
   public publishDeviceAgentMessage(
     topic: string,
-    message: ToClientMessage | ToCloudMessage
+    message: ToClientMessage | ToCloudMessage,
+    logger?: winston.LeveledLogMethod
   ) {
     const messageStr = JSON.stringify(message);
-    logger.debug(
-      `Publishing message:\n${JSON.stringify({ topic, message }, null, 2)}`
-    );
-    this.publish(topic, messageStr);
+    this.publish(topic, messageStr, logger);
   }
 
-  public publishToClient(message: ToClientMessage) {
-    this.publishDeviceAgentMessage(this.toClientTopic, message);
+  public publishToClient(
+    message: ToClientMessage,
+    logger?: winston.LeveledLogMethod
+  ) {
+    this.publishDeviceAgentMessage(this.toClientTopic, message, logger);
   }
 
-  public publishToCloud(message: ToCloudMessage) {
+  public publishToCloud(
+    message: ToCloudMessage,
+    logger?: winston.LeveledLogMethod
+  ) {
     // Can edit topic field in message here if we want
-    this.publishDeviceAgentMessage(this.toCloudTopic, message);
+    this.publishDeviceAgentMessage(this.toCloudTopic, message, logger);
   }
 }

package/src/cloud-connection/shadow-handler.test.ts

@@ -113,6 +113,7 @@ describe('Test Shadow Handler', () => {
       expect(updates.length).toBe(1);
       expect(updates[0]).toEqual({
         projectId: projectId1,
+        txId: expect.any(String),
         appCfgUpdate: {
           newAppCfg: appCfg1,
           updatedModels: {
@@ -157,6 +158,7 @@ describe('Test Shadow Handler', () => {
       expect(updates.length).toBe(1);
       expect(updates[0]).toEqual({
         projectId: projectId1,
+        txId: expect.any(String),
         appCfgUpdate: {
           newAppCfg: appCfg1,
           updatedModels: {
@@ -222,6 +224,7 @@ describe('Test Shadow Handler', () => {
       expect(updates.length).toBe(2);
       expect(updates[0]).toEqual({
         projectId: projectId1,
+        txId: expect.any(String),
         appCfgUpdate: {
           newAppCfg: appCfg1,
           updatedModels: {
@@ -232,6 +235,7 @@ describe('Test Shadow Handler', () => {
       });
       expect(updates[1]).toEqual({
         projectId: projectId2,
+        txId: expect.any(String),
         appCfgUpdate: {
           newAppCfg: appCfg2,
           updatedModels: {
@@ -274,6 +278,7 @@ describe('Test Shadow Handler', () => {
       expect(updates.length).toBe(1);
       expect(updates[0]).toEqual({
         projectId: projectId1,
+        txId: expect.any(String),
         appCfgUpdate: {
           newAppCfg: appCfg1
         }
@@ -337,6 +342,7 @@ describe('Test Shadow Handler', () => {
       expect(updates.length).toBe(1);
       expect(updates[0]).toEqual({
         projectId: projectId1,
+        txId: expect.any(String),
         envVarUpdate: {
           envVars: envVars1
         }
@@ -361,6 +367,7 @@ describe('Test Shadow Handler', () => {
       expect(updates.length).toBe(1);
       expect(updates[0]).toEqual({
         projectId: projectId1,
+        txId: expect.any(String),
         envVarUpdate: {
           envVars: envVars1
         }
@@ -391,12 +398,14 @@ describe('Test Shadow Handler', () => {
       expect(updates.length).toBe(2);
       expect(updates[0]).toEqual({
         projectId: projectId1,
+        txId: expect.any(String),
         envVarUpdate: {
           envVars: envVars1
         }
       });
       expect(updates[1]).toEqual({
         projectId: projectId2,
+        txId: expect.any(String),
         envVarUpdate: {
           envVars: envVars2
         }

package/src/cloud-connection/shadow-handler.ts

@@ -7,6 +7,7 @@ import { logger } from '../util/logger';
 import { Publisher } from './publisher';
 import { AppConfigModels, getAppCfgModelsDiff } from './shadow';
 import { getSystemInformation } from '../device-control/device-control';
+import { generateTxId } from '@alwaysai/device-agent-schemas';
 
 export interface ShadowTopics {
   projects: {
@@ -35,6 +36,7 @@ export type EnvVarUpdate = {
 
 export type ShadowUpdate = {
   projectId: string;
+  txId: string;
   appCfgUpdate?: AppConfigUpdate;
   envVarUpdate?: EnvVarUpdate;
 };
@@ -77,7 +79,9 @@ export class ShadowHandler {
 
     for (const projectId of deltaKeys) {
       const projectShadow = delta[projectId];
-      const shadowUpdate: ShadowUpdate = { projectId };
+      // For incoming shadow updates, there will be no TxID, so it needs to be generated here.
+      const txId = generateTxId();
+      const shadowUpdate: ShadowUpdate = { projectId, txId };
 
       if (projectShadow.appConfig) {
         const newAppCfg = JSON.parse(projectShadow.appConfig);
@@ -140,7 +144,7 @@ export class ShadowHandler {
       case this.shadowTopics.projects.updateDelta:
         if (clientToken === this.clientId) {
           logger.debug(
-            `Ignoring message sent from self: ${JSON.stringify(
+            `Ignoring delta caused by Device Agent: ${JSON.stringify(
               { topic, payload },
               null,
               2
@@ -179,9 +183,6 @@ export class ShadowHandler {
       clientToken: this.clientId
     };
     const topic = this.shadowTopics.systemInfo.update;
-    logger.debug(
-      `Publishing message:\n${JSON.stringify({ topic, packet }, null, 2)}`
-    );
     this.publisher.publish(topic, JSON.stringify(packet));
   }
 
@@ -200,9 +201,6 @@ export class ShadowHandler {
       clientToken: this.clientId
     };
     const topic = this.shadowTopics.projects.update;
-    logger.debug(
-      `Publishing message:\n${JSON.stringify({ topic, packet }, null, 2)}`
-    );
     this.publisher.publish(topic, JSON.stringify(packet));
   }
 
@@ -211,9 +209,6 @@ export class ShadowHandler {
     const packet = {
       clientToken: this.clientId
     };
-    logger.debug(
-      `Publishing message:\n${JSON.stringify({ topic, packet }, null, 2)}`
-    );
     this.publisher.publish(topic, JSON.stringify(packet));
   }
 
@@ -232,9 +227,6 @@ export class ShadowHandler {
       },
       clientToken: this.clientId
     };
-    logger.debug(
-      `Publishing message:\n${JSON.stringify({ topic, packet }, null, 2)}`
-    );
     this.publisher.publish(topic, JSON.stringify(packet));
   }
 }

package/src/cloud-connection/transaction-manager.test.ts

@@ -0,0 +1,73 @@
+import { generateTxId } from '@alwaysai/device-agent-schemas';
+import { TransactionManager } from './transaction-manager';
+import { v4 as uuidv4 } from 'uuid';
+
+describe('Test Transaction Manager', () => {
+  let txnMgr: TransactionManager;
+
+  beforeEach(() => {
+    txnMgr = new TransactionManager();
+    jest.clearAllMocks();
+  });
+
+  function generateRandomProjectId() {
+    return uuidv4();
+  }
+
+  test('Add a new transaction', async () => {
+    const txId = generateTxId();
+    const projectId = generateRandomProjectId();
+    txnMgr.addTransaction(txId, projectId);
+    expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
+  });
+
+  test('Add multiple transactions to different projects', async () => {
+    const numTransactions = 3;
+    const projectTxIdList: any = [];
+    for (let i = 0; i < numTransactions; i++) {
+      const txId = generateTxId();
+      const projectId = generateRandomProjectId();
+      txnMgr.addTransaction(txId, projectId);
+      projectTxIdList.push({ projectId, txId });
+    }
+    projectTxIdList.forEach(({ txId, projectId }) => {
+      expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+      expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
+    });
+  });
+
+  test('Attempt to add existing transaction to a project, results in failure', async () => {
+    const txId = generateTxId();
+    const projectId = generateRandomProjectId();
+    txnMgr.addTransaction(txId, projectId);
+    expect(() => txnMgr.addTransaction(txId, projectId)).toThrow(
+      `Transaction ID ${txId} already exists for this projectId ${projectId}!`
+    );
+    expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
+    expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+  });
+
+  test('Attempt to add a transaction to a project with an ongoing transaction, results in failure', async () => {
+    const txId = generateTxId();
+    const txId2 = generateTxId();
+    const projectId = generateRandomProjectId();
+    txnMgr.addTransaction(txId, projectId);
+    expect(() => txnMgr.addTransaction(txId2, projectId)).toThrow(
+      `This Project ID ${projectId} already has an ongoing transaction!`
+    );
+    expect(txnMgr.getTransactionFromProject(projectId)).toEqual(txId);
+    expect(txnMgr.getProjectFromTransaction(txId)).toEqual(projectId);
+    expect(txnMgr.getProjectFromTransaction(txId2)).toBeUndefined();
+  });
+
+  test('Test remove transaction from queue', async () => {
+    const txId = generateTxId();
+    const projectId = generateRandomProjectId();
+    txnMgr.addTransaction(txId, projectId);
+    txnMgr.completeTransaction(txId);
+
+    expect(txnMgr.getTransactionFromProject(projectId)).toBeUndefined();
+    expect(txnMgr.getProjectFromTransaction(txId)).toBeUndefined();
+  });
+});

package/src/cloud-connection/transaction-manager.ts

@@ -0,0 +1,43 @@
+export class TransactionManager {
+  private txToProject: Record<string, string> = {};
+  private projectToTx: Record<string, string> = {};
+
+  constructor() {
+    this.txToProject = {};
+    this.projectToTx = {};
+  }
+
+  public addTransaction(txId: string, projectId: string) {
+    // Check if the Transaction already exists
+    if (this.txToProject[txId]) {
+      throw new Error(
+        `Transaction ID ${txId} already exists for this projectId ${projectId}!`
+      );
+    } else {
+      // Check if there is any ongoing Transactions for Project ID
+      if (this.projectToTx[projectId]) {
+        throw new Error(
+          `This Project ID ${projectId} already has an ongoing transaction!`
+        );
+      } else {
+        // Map the Transaction ID with Project ID
+        this.txToProject[txId] = projectId;
+        this.projectToTx[projectId] = txId;
+      }
+    }
+  }
+
+  public getTransactionFromProject(projectId: string) {
+    return this.projectToTx[projectId];
+  }
+
+  public getProjectFromTransaction(txId: string) {
+    return this.txToProject[txId];
+  }
+
+  public completeTransaction(txId: string) {
+    const projectId = this.txToProject[txId];
+    delete this.txToProject[txId];
+    delete this.projectToTx[projectId];
+  }
+}

package/src/device-control/device-control.ts

@@ -1,9 +1,11 @@
 import { SystemInformationPayload } from '@alwaysai/device-agent-schemas';
 import { logger } from '../util/logger';
-// eslint-disable-next-line
-const osu = require('node-os-utils');
-// eslint-disable-next-line
-const si = require('systeminformation');
+import * as osu from 'node-os-utils';
+import * as si from 'systeminformation';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+const exec_promise = promisify(exec);
 
 // Device Stats
 export async function getCpuDetails() {
@@ -16,6 +18,8 @@ export async function getCpuDetails() {
 }
 
 export async function getDiskDetails() {
+  // Types incorrectly specify diskname as required instead of optional
+  // @ts-expect-error
   const driveInfo = await osu.drive.info();
   return {
     usedGb: parseFloat(driveInfo.usedGb),
@@ -26,8 +30,8 @@ export async function getDiskDetails() {
 export async function getMemDetails() {
   const memInfo = await osu.mem.info();
   return {
-    usedMb: parseFloat(memInfo.usedMemMb),
-    freeMb: parseFloat(memInfo.freeMemMb)
+    usedMb: memInfo.usedMemMb,
+    freeMb: memInfo.freeMemMb
   };
 }
 
@@ -52,7 +56,7 @@ export async function getCpuInfo() {
     vendor: cpuInfo.vendor,
     model: cpuInfo.model,
     cores: cpuInfo.cores,
-    physicalCores: cpuInfo.phsyicalCores,
+    physicalCores: cpuInfo.physicalCores,
     efficiencyCores: cpuInfo.efficiencyCores,
     processors: cpuInfo.processors
   };
@@ -78,16 +82,19 @@ export async function getDeviceInfo() {
     model: deviceInfo.model,
     version: deviceInfo.version,
     serial:
-      deviceInfo.serial && deviceInfo.serial !== '-' ? deviceInfo.serial : null,
+      deviceInfo.serial && deviceInfo.serial !== '-'
+        ? deviceInfo.serial
+        : undefined,
     virtual: deviceInfo.virtual
   };
 }
 
 export async function getNetworkInfo() {
   const networkInterfaces = await si.networkInterfaces();
-  const defaultNetworkInterface = networkInterfaces.filter(
-    (iface: any) => iface.ip4 !== '127.0.0.1'
-  )[0];
+
+  const defaultNetworkInterface = Array.isArray(networkInterfaces)
+    ? networkInterfaces.filter((iface: any) => iface.ip4 !== '127.0.0.1')[0]
+    : networkInterfaces;
   return {
     ipv4Address: defaultNetworkInterface.ip4,
     ipv6Address: defaultNetworkInterface.ip6,
@@ -95,6 +102,17 @@ export async function getNetworkInfo() {
   };
 }
 
+export async function getPackageVersions() {
+  // eslint-disable-next-line
+  const agentJson = require('../../package.json');
+  // eslint-disable-next-line
+  const deviceAgentSchemasJson = require('../../node_modules/@alwaysai/device-agent-schemas/package.json');
+  return {
+    agent: agentJson.version,
+    deviceAgentSchemas: deviceAgentSchemasJson.version
+  };
+}
+
 export async function getSystemInformation(): Promise<SystemInformationPayload> {
   try {
     const systemInfo: SystemInformationPayload = {
@@ -102,7 +120,8 @@ export async function getSystemInformation(): Promise<SystemInformationPayload>
       cpu: await getCpuInfo(),
       disk: await getDiskInfo(),
       device: await getDeviceInfo(),
-      network: await getNetworkInfo()
+      network: await getNetworkInfo(),
+      versions: await getPackageVersions()
     };
     return systemInfo;
   } catch (e) {
@@ -110,3 +129,22 @@ export async function getSystemInformation(): Promise<SystemInformationPayload>
   }
   return {};
 }
+
+export async function reboot() {
+  try {
+    // FIXME: This command must always be run with sudo. The user must enable access
+    // in the sudoers file for it to work.
+    // If passwordless access is not set, will prompt user for password. Use
+    // timeout to break out of the prompt
+    const result = await exec_promise('sudo /sbin/shutdown -r now', {
+      timeout: 5000
+    });
+    logger.info(result.stdout.trim());
+  } catch (err) {
+    throw new Error(
+      `Could not reboot device. You may need to add passwordless access to '/sbin/shutdown'. ${JSON.stringify(
+        err
+      )}`
+    );
+  }
+}

package/src/subcommands/app/env-vars.ts

@@ -7,6 +7,7 @@ import {
 import { EnvVars, getAllEnvs } from '../../application-control';
 import { DeviceAgentCloudConnection } from '../../cloud-connection/device-agent-cloud-connection';
 import sleep from '../../util/sleep';
+import { logger } from '../../util/logger';
 
 export const getAllEnvsCliLeaf = CliLeaf({
   name: 'get-all-envs',
@@ -49,25 +50,31 @@ export const setEnvCliLeaf = CliLeaf({
       if (nameVal.length !== 2) {
         throw new CliTerseError(`Invalid argument: ${arg}`);
       }
-      envVars[service][nameVal[0]] = nameVal[1];
+      const value = nameVal[1] === '' ? null : nameVal[1];
+      envVars[service][nameVal[0]] = value;
     });
     const deviceAgent = new DeviceAgentCloudConnection();
     await deviceAgent.setupHandlers();
 
-    const topic = deviceAgent.getShadowTopics().projects.updateDelta;
-
-    const message = {
-      version: 3,
-      timestamp: 0,
+    // Update the shadow as a client
+    const topic = deviceAgent.getShadowTopics().projects.update;
+    const packet = {
       state: {
-        [project]: {
-          envVars
+        desired: {
+          [project]: {
+            envVars
+          }
         }
       },
-      clientToken: 'not-self'
+      clientToken: 'client'
     };
+    logger.debug(
+      `Publishing message:\n${JSON.stringify({ topic, packet }, null, 2)}`
+    );
+    deviceAgent.publisher.publish(topic, JSON.stringify(packet));
+    // Sleep for extra time to ensure time for shadow response
+    await sleep(10000);
 
-    await deviceAgent.handleMessage(topic, message);
     while (deviceAgent.isCmdInProgress(project)) {
       await sleep(1000);
     }

package/src/subcommands/device/clean.ts

@@ -1,12 +1,12 @@
 import { CliLeaf } from '@alwaysai/alwayscli';
-import { rimraf } from 'rimraf';
 import { logger } from '../../util/logger';
 import { AgentConfigFile } from '../../infrastructure/agent-config';
 import { DeviceConfigFile } from 'alwaysai/lib/core/device';
 import {
   APP_ROOT,
   CREDENTIALS_FILE_PATH,
-  DEVICE_AGENT_CFG_PATH
+  DEVICE_AGENT_CFG_PATH,
+  DEVICE_AGENT_DOCKER_COMPOSE_PATH
 } from '../../util/directories';
 import {
   checkRabbitMQContainerRunning,
@@ -14,33 +14,42 @@ import {
 } from '../../local-connection/rabbitmq-connection';
 import { stopApp } from '../../application-control';
 import { LOCAL_CERT_AND_KEY_DIR } from 'alwaysai/lib/paths';
+import safeRimraf from '../../util/safe-rimraf';
 
 export const cleanCliLeaf = CliLeaf({
   name: 'clean',
   description: 'Remove all provisioning files',
   async action(_, opts) {
-    if (await checkRabbitMQContainerRunning()) {
-      await stopRabbitMQContainer();
+    logger.info('Removing provisioning files.');
+    try {
+      if (await checkRabbitMQContainerRunning()) {
+        await stopRabbitMQContainer();
+      }
+    } catch (e) {
+      logger.error(
+        `You may need to manually stop the container by running docker-compose down in the following directory: ${DEVICE_AGENT_DOCKER_COMPOSE_PATH}`
+      );
+      logger.debug(`Error in checking / stopping RabbitMQ container: ${e}`);
     }
-    logger.debug(`Removing ${DEVICE_AGENT_CFG_PATH}`);
-    await rimraf(DEVICE_AGENT_CFG_PATH);
+
+    await safeRimraf(DEVICE_AGENT_CFG_PATH);
+
     logger.debug('Checking for alwaysAI applications still running');
     const apps = await AgentConfigFile().getApps();
     for (const app of apps) {
       const { projectId } = app;
       await stopApp({ projectId });
     }
-    logger.info('Cleaning device configuration');
-    logger.debug(`Removing ${LOCAL_CERT_AND_KEY_DIR}`);
-    await rimraf(LOCAL_CERT_AND_KEY_DIR);
+    await safeRimraf(LOCAL_CERT_AND_KEY_DIR);
+
     logger.debug(`Removing ${AgentConfigFile().path}`);
     AgentConfigFile().remove();
     logger.debug(`Removing ${DeviceConfigFile().path}`);
     DeviceConfigFile().remove();
-    logger.debug(`Removing ${CREDENTIALS_FILE_PATH}`);
-    await rimraf(CREDENTIALS_FILE_PATH);
-    logger.debug(`Removing ${APP_ROOT}`);
-    await rimraf(APP_ROOT);
+
+    await safeRimraf(CREDENTIALS_FILE_PATH);
+    await safeRimraf(APP_ROOT);
+
     logger.info('Device configuration cleaned');
   }
 });