@alwaysai/device-agent 1.3.0 → 1.3.1-1

package/src/cloud-connection/live-updates-handler.ts

@@ -1,27 +1,21 @@
 import {
   AppLogsPayload,
-  ToClientMessagePayload,
   keyMirrors,
   LiveStateUpdatesTogglePayload,
-  ToClientMessage
+  ToClientMessage,
+  buildAppLogsMessage,
+  buildAppStateMessage,
+  buildDeviceStatsMessage,
+  StatusResponsePayload,
+  buildToClientStatusResponseMessage
 } from '@alwaysai/device-agent-schemas';
 import { getAppLogs } from '../application-control';
 import { logger } from '../util/logger';
 import sleep from '../util/sleep';
 import { Publisher } from './publisher';
-import {
-  getStatusResponsePayload,
-  getAppStatePayload,
-  getDeviceStatsPayload
-} from './messages';
+import { getAppStatePayload, getDeviceStatsPayload } from './messages';
 import { ToClientMessageTypeValue } from '@alwaysai/device-agent-schemas';
 import { ALWAYSAI_LIVE_UPDATES_TIMEOUT_MS } from '../environment';
-import {
-  buildAppLogsMessage,
-  buildAppStateMessage,
-  buildDeviceStatsMessage,
-  buildStatusResponseMessage
-} from './message-builder';
 
 const LIVE_UPDATES_TIMEOUT = ALWAYSAI_LIVE_UPDATES_TIMEOUT_MS
   ? parseInt(ALWAYSAI_LIVE_UPDATES_TIMEOUT_MS)
@@ -48,8 +42,9 @@ export class LiveUpdatesHandler {
   private appLogStreams = new Set<string>();
   private transactionStatuses = new Set<string>();
 
-  private async startAppLogStream(projectId: string) {
+  private async startAppLogStream(projectId: string, txId: string) {
     logger.info(`Starting log stream for ${projectId}`);
+
     this.appLogStreams.add(projectId);
     const readable = await getAppLogs({
       projectId,
@@ -68,7 +63,7 @@ export class LiveUpdatesHandler {
         projectId,
         logChunk: logStr
       };
-      const message = await buildAppLogsMessage(payload, this.clientId);
+      const message = buildAppLogsMessage(this.clientId, payload, txId);
       this.publisher.publishToClient(message, logger.silly);
     });
 
@@ -117,7 +112,7 @@ export class LiveUpdatesHandler {
       this.liveUpdatesAlive.device_stats = toggles.deviceStats;
     }
     if (toggles.appState !== undefined) {
-      this.liveUpdatesAlive.app_logs = toggles.appState;
+      this.liveUpdatesAlive.app_state = toggles.appState;
     }
   }
 
@@ -132,36 +127,25 @@ export class LiveUpdatesHandler {
     }, LIVE_UPDATES_TIMEOUT);
   }
 
-  private async startPublishingLiveUpdates<T extends any[]>(
+  private async startPublishingLiveUpdates(
     messageType: ToClientMessageTypeValue,
-    payloadBuilderFunction: (...args: T) => Promise<ToClientMessagePayload>,
-    messageBuilderFunction: (
-      payload: ToClientMessagePayload,
-      txId: string
-    ) => Promise<ToClientMessage>,
-    args: T,
+    getMessage: () => Promise<ToClientMessage>,
     txId: string
   ) {
     logger.info(`Turned on live updates for ${messageType}`);
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-      try {
-        if (!this.continuePublishing(messageType, txId)) {
-          logger.info(`Turned off live updates for ${messageType}`);
-          break;
-        }
-        const payload: ToClientMessagePayload = await payloadBuilderFunction(
-          ...args
-        );
-        const message = await messageBuilderFunction(payload, txId);
+    try {
+      while (this.continuePublishing(messageType, txId)) {
+        const message = await getMessage();
         this.publisher.publishToClient(message, logger.silly);
-      } catch (e) {
-        logger.error(
-          `Error publishing live updates for ${messageType}: ${e.message}`
-        );
+
+        await sleep(this.getLiveUpdatesInterval(messageType));
       }
-      await sleep(this.getLiveUpdatesInterval(messageType));
+    } catch (e) {
+      logger.error(
+        `Error publishing live updates for ${messageType}: ${e.message}`
+      );
     }
+    logger.info(`Turned off live updates for ${messageType}`);
   }
 
   /*=================================================================
@@ -196,9 +180,16 @@ export class LiveUpdatesHandler {
       // Don't wait for this call to finish since it loops until disabled
       void this.startPublishingLiveUpdates(
         keyMirrors.toClientMessageType.status_response,
-        getStatusResponsePayload,
-        buildStatusResponseMessage,
-        [keyMirrors.statusResponse.in_progress, ''],
+        async () => {
+          const payload: StatusResponsePayload = {
+            status: keyMirrors.statusResponse.in_progress
+          };
+          return buildToClientStatusResponseMessage(
+            this.clientId,
+            payload,
+            txId
+          );
+        },
         txId
       );
     }
@@ -220,41 +211,43 @@ export class LiveUpdatesHandler {
     const { deviceStats, appState, appLogs } = toggles;
     this.restartLiveUpdatesTimeout();
 
-    const currentDeviceStats = this.getDeviceStatsLiveUpdates();
     if (deviceStats !== undefined) {
+      const currentDeviceStats = this.getDeviceStatsLiveUpdates();
       this.liveUpdatesAlive.device_stats = deviceStats;
       if (deviceStats && currentDeviceStats !== true) {
         // Don't wait for this call to finish since it loops until disabled
         void this.startPublishingLiveUpdates(
           keyMirrors.toClientMessageType.device_stats,
-          getDeviceStatsPayload,
-          buildDeviceStatsMessage,
-          [],
+          async () => {
+            const payload = await getDeviceStatsPayload();
+            return buildDeviceStatsMessage(this.clientId, payload, txId);
+          },
           txId
         );
       }
     }
 
-    const currentAppState = this.getAppStateLiveUpdates();
     if (appState !== undefined) {
+      const currentAppState = this.getAppStateLiveUpdates();
       this.liveUpdatesAlive.app_state = appState;
       if (appState && currentAppState !== true) {
         // Don't wait for this call to finish since it loops until disabled
         void this.startPublishingLiveUpdates(
           keyMirrors.toClientMessageType.app_state,
-          getAppStatePayload,
-          buildAppStateMessage,
-          [],
+          async () => {
+            const payload = await getAppStatePayload();
+            return buildAppStateMessage(this.clientId, payload, txId);
+          },
           txId
         );
       }
     }
 
-    const currentAppLogs = this.getAppLogsLiveUpdates();
     if (appLogs !== undefined) {
+      const currentAppLogs = this.getAppLogsLiveUpdates();
       if (appLogs.toggle && currentAppLogs !== true) {
         // Don't wait for this call to finish since it loops until disabled
-        void this.startAppLogStream(appLogs.projectId);
+        void this.startAppLogStream(appLogs.projectId, txId);
       } else {
         this.appLogStreams.delete(appLogs.projectId);
       }

package/src/cloud-connection/messages.ts

@@ -1,10 +1,8 @@
 import {
   AppState,
   AppStatePayload,
-  DeviceStatsPayload,
-  StatusResponsePayload
+  DeviceStatsPayload
 } from '@alwaysai/device-agent-schemas';
-import { StatusResponseValue } from '@alwaysai/device-agent-schemas/lib/constants';
 import { getAppState } from '../application-control';
 import {
   getCpuDetails,
@@ -27,17 +25,6 @@ export async function getAppStatePayload(): Promise<AppStatePayload> {
   return appStatePayload;
 }
 
-export async function getStatusResponsePayload(
-  status: StatusResponseValue,
-  message: string
-): Promise<StatusResponsePayload> {
-  const statusResponsePayload: StatusResponsePayload = {
-    status,
-    message
-  };
-  return statusResponsePayload;
-}
-
 export async function getDeviceStatsPayload(): Promise<DeviceStatsPayload> {
   const cpuDetails = await getCpuDetails();
   const diskDetails = await getDiskDetails();

package/src/cloud-connection/publisher.ts

@@ -86,4 +86,8 @@ export class Publisher {
     // Can edit topic field in message here if we want
     this.publishDeviceAgentMessage(this.toCloudTopic, message, logger);
   }
+
+  public getClientId(): string {
+    return this.clientId;
+  }
 }

package/src/cloud-connection/shadow-handler.test.ts

@@ -2,6 +2,8 @@ import { AppConfig } from '@alwaysai/app-configuration-schemas';
 import { readAppCfgFile } from '../application-control';
 import { Publisher } from './publisher';
 import { ShadowHandler } from './shadow-handler';
+import { Logger } from 'winston';
+import { logger } from '../util/logger';
 
 jest.mock('../application-control');
 jest.mock('./publisher');
@@ -23,7 +25,7 @@ describe('Test Shadow Handler', () => {
     //FIXME: Invalid input is silently ignored, need input validation
     expect(async () => {
       await shadowHandler.handleShadowTopic({
-        topic: shadowHandler.shadowTopics.projects.updateDelta,
+        topic: shadowHandler.shadowTopics.projects.updateAccepted,
         payload: Buffer.from('test-payload'),
         clientToken: ''
       });
@@ -56,7 +58,7 @@ describe('Test Shadow Handler', () => {
     };
 
     const updates = await shadowHandler.handleShadowTopic({
-      topic: shadowHandler.shadowTopics.projects.updateDelta,
+      topic: shadowHandler.shadowTopics.projects.updateAccepted,
       payload,
       clientToken: clientId
     });
@@ -65,11 +67,13 @@ describe('Test Shadow Handler', () => {
 
   test('handle project shadow empty delta', async () => {
     const payload = {
-      [projectId1]: {}
+      desired: {
+        [projectId1]: {}
+      }
     };
 
     const updates = await shadowHandler.handleShadowTopic({
-      topic: shadowHandler.shadowTopics.projects.updateDelta,
+      topic: shadowHandler.shadowTopics.projects.updateAccepted,
       payload,
       clientToken: ''
     });
@@ -98,7 +102,7 @@ describe('Test Shadow Handler', () => {
       };
 
       const payload = {
-        delta: {
+        desired: {
           [projectId1]: {
             appConfig: JSON.stringify(appCfg1)
           }
@@ -106,7 +110,7 @@ describe('Test Shadow Handler', () => {
       };
 
       const updates = await shadowHandler.handleShadowTopic({
-        topic: shadowHandler.shadowTopics.projects.getAccepted,
+        topic: shadowHandler.shadowTopics.projects.updateAccepted,
         payload,
         clientToken: ''
       });
@@ -145,13 +149,15 @@ describe('Test Shadow Handler', () => {
       };
 
       const payload = {
-        [projectId1]: {
-          appConfig: JSON.stringify(appCfg1)
+        desired: {
+          [projectId1]: {
+            appConfig: JSON.stringify(appCfg1)
+          }
         }
       };
 
       const updates = await shadowHandler.handleShadowTopic({
-        topic: shadowHandler.shadowTopics.projects.updateDelta,
+        topic: shadowHandler.shadowTopics.projects.updateAccepted,
         payload,
         clientToken: ''
       });
@@ -208,16 +214,18 @@ describe('Test Shadow Handler', () => {
         }
       };
       const payload = {
-        [projectId1]: {
-          appConfig: JSON.stringify(appCfg1)
-        },
-        [projectId2]: {
-          appConfig: JSON.stringify(appCfg2)
+        desired: {
+          [projectId1]: {
+            appConfig: JSON.stringify(appCfg1)
+          },
+          [projectId2]: {
+            appConfig: JSON.stringify(appCfg2)
+          }
         }
       };
 
       const updates = await shadowHandler.handleShadowTopic({
-        topic: shadowHandler.shadowTopics.projects.updateDelta,
+        topic: shadowHandler.shadowTopics.projects.updateAccepted,
         payload,
         clientToken: ''
       });
@@ -265,13 +273,15 @@ describe('Test Shadow Handler', () => {
       };
 
       const payload = {
-        [projectId1]: {
-          appConfig: JSON.stringify(appCfg1)
+        desired: {
+          [projectId1]: {
+            appConfig: JSON.stringify(appCfg1)
+          }
         }
       };
 
       const updates = await shadowHandler.handleShadowTopic({
-        topic: shadowHandler.shadowTopics.projects.updateDelta,
+        topic: shadowHandler.shadowTopics.projects.updateAccepted,
         payload,
         clientToken: ''
       });
@@ -307,27 +317,73 @@ describe('Test Shadow Handler', () => {
       };
 
       const payload = {
-        [projectId1]: {
-          appConfig: JSON.stringify(appCfg1)
+        desired: {
+          [projectId1]: {
+            appConfig: JSON.stringify(appCfg1)
+          }
         }
       };
 
       const updates = await shadowHandler.handleShadowTopic({
-        topic: shadowHandler.shadowTopics.projects.updateDelta,
+        topic: shadowHandler.shadowTopics.projects.updateAccepted,
         payload,
         clientToken: ''
       });
       expect(updates.length).toBe(0);
     });
+
+    test('handles an unparsable object in a project shadow appCfg delta', async () => {
+      const ogAppCfg1: AppConfig = {
+        scripts: {
+          start: 'python app.py'
+        },
+        models: {}
+      };
+      jest.mocked(readAppCfgFile).mockResolvedValue(ogAppCfg1);
+
+      // This appCfg is invalid on it's own (see values below)
+      const appCfg1 = {
+        scripts: {
+          start: 'python app.py'
+        },
+        models: {
+          'alwaysai/mobilenet_ssd': '3', // string instead of int
+          'alwaysai/yolo_v4': '5' // string instead of int
+        }
+      };
+
+      const payload = {
+        desired: {
+          [projectId1]: {
+            appConfig: appCfg1 // This is missing JSON.stringify() making this an unparsable object.
+          }
+        }
+      };
+
+      const loggerSpy = jest
+        .spyOn(logger, 'error')
+        .mockReturnValue({} as unknown as Logger);
+
+      const updates = await shadowHandler.handleShadowTopic({
+        topic: shadowHandler.shadowTopics.projects.updateAccepted,
+        payload,
+        clientToken: ''
+      });
+
+      expect(updates.length).toBe(0);
+      expect(loggerSpy).toHaveBeenCalledWith(
+        expect.stringContaining('Could not parse the appConfig')
+      );
+    });
   });
 
   describe('handle project shadow env vars', () => {
-    test('handle project shadow env vars get response with delta', async () => {
+    test('handle a response from the getAccepted from the cloud', async () => {
       const envVars1 = {
         VAR0: 'value0'
       };
       const payload = {
-        delta: {
+        reported: {
           [projectId1]: {
             envVars: envVars1
           }
@@ -339,28 +395,22 @@ describe('Test Shadow Handler', () => {
         payload,
         clientToken: ''
       });
-      expect(updates.length).toBe(1);
-      expect(updates[0]).toEqual({
-        projectId: projectId1,
-        txId: expect.any(String),
-        envVarUpdate: {
-          envVars: envVars1
-        }
-      });
+      expect(updates.length).toBe(0);
     });
-
     test('handle project shadow env vars update delta', async () => {
       const envVars1 = {
         VAR1: 'value1'
       };
       const payload = {
-        [projectId1]: {
-          envVars: envVars1
+        desired: {
+          [projectId1]: {
+            envVars: envVars1
+          }
         }
       };
 
       const updates = await shadowHandler.handleShadowTopic({
-        topic: shadowHandler.shadowTopics.projects.updateDelta,
+        topic: shadowHandler.shadowTopics.projects.updateAccepted,
         payload,
         clientToken: ''
       });
@@ -382,16 +432,18 @@ describe('Test Shadow Handler', () => {
         VAR2: null
       };
       const payload = {
-        [projectId1]: {
-          envVars: envVars1
-        },
-        [projectId2]: {
-          envVars: envVars2
+        desired: {
+          [projectId1]: {
+            envVars: envVars1
+          },
+          [projectId2]: {
+            envVars: envVars2
+          }
         }
       };
 
       const updates = await shadowHandler.handleShadowTopic({
-        topic: shadowHandler.shadowTopics.projects.updateDelta,
+        topic: shadowHandler.shadowTopics.projects.updateAccepted,
         payload,
         clientToken: ''
       });

package/src/cloud-connection/shadow-handler.ts

@@ -2,12 +2,12 @@ import {
   AppConfig,
   validateAppConfig
 } from '@alwaysai/app-configuration-schemas';
+import { generateTxId } from '@alwaysai/device-agent-schemas';
 import { EnvVars, getAllEnvs, readAppCfgFile } from '../application-control';
+import { getSystemInformation } from '../device-control/device-control';
 import { logger } from '../util/logger';
 import { Publisher } from './publisher';
 import { AppConfigModels, getAppCfgModelsDiff } from './shadow';
-import { getSystemInformation } from '../device-control/device-control';
-import { generateTxId } from '@alwaysai/device-agent-schemas';
 
 export interface ShadowTopics {
   projects: {
@@ -23,6 +23,18 @@ export interface ShadowTopics {
   systemInfo: {
     update: string;
   };
+  secureTunnel: {
+    get: string;
+    getAccepted: string;
+    getRejected: string;
+    update: string;
+    updateDelta: string;
+    updateAccepted: string;
+    updateRejected: string;
+    delete: string;
+    deleteAccepted: string;
+    deleteRejected: string;
+  };
 }
 
 export type AppConfigUpdate = {
@@ -50,20 +62,32 @@ export class ShadowHandler {
   constructor(clientId: string, publisher: Publisher) {
     this.clientId = clientId;
     this.publisher = publisher;
-    this.shadowPrefix = `$aws/things/${this.clientId}/shadow/name/`;
+    this.shadowPrefix = `$aws/things/${this.clientId}/shadow/name`;
     this.shadowTopics = {
       projects: {
-        get: `${this.shadowPrefix}projects/get`,
-        getAccepted: `${this.shadowPrefix}projects/get/accepted`,
-        getRejected: `${this.shadowPrefix}projects/get/rejected`,
-        update: `${this.shadowPrefix}projects/update`,
-        updateDelta: `${this.shadowPrefix}projects/update/delta`,
-        updateAccepted: `${this.shadowPrefix}projects/update/accepted`,
-        updateRejected: `${this.shadowPrefix}projects/update/rejected`,
-        delete: `${this.shadowPrefix}projects/delete`
+        get: `${this.shadowPrefix}/projects/get`,
+        getAccepted: `${this.shadowPrefix}/projects/get/accepted`,
+        getRejected: `${this.shadowPrefix}/projects/get/rejected`,
+        update: `${this.shadowPrefix}/projects/update`,
+        updateDelta: `${this.shadowPrefix}/projects/update/delta`,
+        updateAccepted: `${this.shadowPrefix}/projects/update/accepted`,
+        updateRejected: `${this.shadowPrefix}/projects/update/rejected`,
+        delete: `${this.shadowPrefix}/projects/delete`
       },
       systemInfo: {
-        update: `${this.shadowPrefix}system-info/update`
+        update: `${this.shadowPrefix}/system-info/update`
+      },
+      secureTunnel: {
+        get: `${this.shadowPrefix}/secure-tunnel/get`,
+        getAccepted: `${this.shadowPrefix}/secure-tunnel/get/accepted`,
+        getRejected: `${this.shadowPrefix}/secure-tunnel/get/rejected`,
+        update: `${this.shadowPrefix}/secure-tunnel/update`,
+        updateDelta: `${this.shadowPrefix}/secure-tunnel/update/delta`,
+        updateAccepted: `${this.shadowPrefix}/secure-tunnel/update/accepted`,
+        updateRejected: `${this.shadowPrefix}/secure-tunnel/update/rejected`,
+        delete: `${this.shadowPrefix}/secure-tunnel/delete`,
+        deleteAccepted: `${this.shadowPrefix}/secure-tunnel/delete/accepted`,
+        deleteRejected: `${this.shadowPrefix}/secure-tunnel/delete/rejected`
       }
     };
   }
@@ -84,7 +108,15 @@ export class ShadowHandler {
       const shadowUpdate: ShadowUpdate = { projectId, txId };
 
       if (projectShadow.appConfig) {
-        const newAppCfg = JSON.parse(projectShadow.appConfig);
+        let newAppCfg: any;
+        try {
+          newAppCfg = JSON.parse(projectShadow.appConfig);
+        } catch (error) {
+          logger.error(
+            `Could not parse the appConfig for transaction ${txId}!\n${error}`
+          );
+          continue;
+        }
         if (!validateAppConfig(newAppCfg)) {
           // FIXME: Raise an exception to be handled at higher layer
           logger.error(
@@ -124,6 +156,7 @@ export class ShadowHandler {
         updates.push(shadowUpdate);
       }
     }
+
     return updates;
   }
 
@@ -141,7 +174,7 @@ export class ShadowHandler {
     // TODO: make use a function like the other topic getters
     const shadowName = topic.split('/')[5];
     switch (topic) {
-      case this.shadowTopics.projects.updateDelta:
+      case this.shadowTopics.projects.updateAccepted:
         if (clientToken === this.clientId) {
           logger.debug(
             `Ignoring delta caused by Device Agent: ${JSON.stringify(
@@ -152,14 +185,16 @@ export class ShadowHandler {
           );
           break;
         }
-        return await this.handleNamedShadowUpdate({ delta: payload });
+        return await this.handleNamedShadowUpdate({ delta: payload.desired });
       case this.shadowTopics.projects.getAccepted:
-        if (payload['delta']) {
-          return await this.handleNamedShadowUpdate({
-            delta: payload['delta']
-          });
-        } else {
-          logger.info(`No delta updates in named shadow '${shadowName}'`);
+        if (clientToken !== this.clientId) {
+          logger.debug(
+            `Ignoring get message initiated by the cloud/console: ${JSON.stringify(
+              { topic, payload },
+              null,
+              2
+            )}`
+          );
         }
         break;
       default:
@@ -189,6 +224,7 @@ export class ShadowHandler {
   public async updateProjectShadow(projectId: string) {
     const appCfg = await readAppCfgFile({ projectId });
     const envVars = await getAllEnvs({ projectId });
+
     const packet = {
       state: {
         reported: {