@alwaysai/device-agent 0.0.12 → 0.0.14

package/src/cloud-connection/shadow-handler.test.ts

@@ -0,0 +1,361 @@
+import { AppConfig } from '@alwaysai/app-configuration-schemas';
+import { readAppCfgFile } from '../application-control';
+import { Publisher } from './publisher';
+import { ShadowHandler } from './shadow-handler';
+
+jest.mock('../application-control');
+jest.mock('./publisher');
+const mockClient = jest.fn();
+const clientId = 'test-client';
+const projectId1 = 'test-project';
+const projectId2 = 'test-project-2';
+
+describe('Test Shadow Handler', () => {
+  let publisher: Publisher;
+  let shadowHandler: ShadowHandler;
+
+  beforeEach(() => {
+    publisher = new Publisher(mockClient, clientId);
+    shadowHandler = new ShadowHandler(clientId, publisher);
+  });
+
+  test.skip('reject buffer payload', async () => {
+    //FIXME: Invalid input is silently ignored, need input validation
+    expect(() => {
+      shadowHandler.handleShadowTopic({
+        topic: shadowHandler.shadowTopics.projects.updateDelta,
+        payload: Buffer.from('test-payload')
+      });
+    }).toThrow(Error);
+  });
+
+  test('ignore message from self', async () => {
+    const ogAppCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/mobilenet_ssd': 2
+      }
+    };
+    jest.mocked(readAppCfgFile).mockResolvedValue(ogAppCfg1);
+
+    const appCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/yolo_v3': 4
+      }
+    };
+    const payload = {
+      [projectId1]: {
+        appConfig: JSON.stringify(appCfg1)
+      },
+      clientToken: clientId
+    };
+
+    const appCfgUpdates = await shadowHandler.handleShadowTopic({
+      topic: shadowHandler.shadowTopics.projects.updateDelta,
+      payload
+    });
+    expect(appCfgUpdates.length).toBe(0);
+  });
+
+  test('handle shadow get response with updated models', async () => {
+    const ogAppCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/mobilenet_ssd': 3
+      }
+    };
+
+    jest.mocked(readAppCfgFile).mockResolvedValue(ogAppCfg1);
+    const appCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/yolo_v3': 4
+      }
+    };
+
+    const payload = {
+      delta: {
+        [projectId1]: {
+          appConfig: JSON.stringify(appCfg1)
+        }
+      }
+    };
+
+    const appCfgUpdates = await shadowHandler.handleShadowTopic({
+      topic: shadowHandler.shadowTopics.projects.getAccepted,
+      payload
+    });
+    expect(appCfgUpdates.length).toBe(1);
+    expect(appCfgUpdates[0]).toEqual({
+      projectId: projectId1,
+      newAppCfg: appCfg1,
+      updatedModels: {
+        'alwaysai/yolo_v3': 4
+      }
+    });
+  });
+
+  test('handle shadow delta without app config', async () => {
+    const ogAppCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/mobilenet_ssd': 2
+      }
+    };
+    jest.mocked(readAppCfgFile).mockResolvedValue(ogAppCfg1);
+
+    const payload = {
+      [projectId1]: {}
+    };
+
+    const appCfgUpdates = await shadowHandler.handleShadowTopic({
+      topic: shadowHandler.shadowTopics.projects.updateDelta,
+      payload
+    });
+    expect(appCfgUpdates.length).toBe(0);
+  });
+
+  test('handle shadow delta with updated models', async () => {
+    const ogAppCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/mobilenet_ssd': 2
+      }
+    };
+
+    jest.mocked(readAppCfgFile).mockResolvedValue(ogAppCfg1);
+    const appCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/mobilenet_ssd': 3,
+        'alwaysai/yolo_v4': 5
+      }
+    };
+
+    const payload = {
+      [projectId1]: {
+        appConfig: JSON.stringify(appCfg1)
+      }
+    };
+
+    const appCfgUpdates = await shadowHandler.handleShadowTopic({
+      topic: shadowHandler.shadowTopics.projects.updateDelta,
+      payload
+    });
+    expect(appCfgUpdates.length).toBe(1);
+    expect(appCfgUpdates[0]).toEqual({
+      projectId: projectId1,
+      newAppCfg: appCfg1,
+      updatedModels: {
+        'alwaysai/mobilenet_ssd': 3,
+        'alwaysai/yolo_v4': 5
+      }
+    });
+  });
+
+  test('handle shadow delta with updated models for two apps', async () => {
+    const ogAppCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/mobilenet_ssd': 2
+      }
+    };
+    jest.mocked(readAppCfgFile).mockResolvedValueOnce(ogAppCfg1);
+    const ogAppCfg2: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/yolo_v4': 5
+      }
+    };
+    jest.mocked(readAppCfgFile).mockResolvedValueOnce(ogAppCfg2);
+
+    const appCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/mobilenet_ssd': 3,
+        'alwaysai/yolo_v4': 5
+      }
+    };
+    const appCfg2: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/yolo_v4': 5,
+        'alwaysai/human_pose': 7
+      }
+    };
+    const payload = {
+      [projectId1]: {
+        appConfig: JSON.stringify(appCfg1)
+      },
+      [projectId2]: {
+        appConfig: JSON.stringify(appCfg2)
+      }
+    };
+
+    const appCfgUpdates = await shadowHandler.handleShadowTopic({
+      topic: shadowHandler.shadowTopics.projects.updateDelta,
+      payload
+    });
+    expect(appCfgUpdates.length).toBe(2);
+    expect(appCfgUpdates[0]).toEqual({
+      projectId: projectId1,
+      newAppCfg: appCfg1,
+      updatedModels: {
+        'alwaysai/mobilenet_ssd': 3,
+        'alwaysai/yolo_v4': 5
+      }
+    });
+    expect(appCfgUpdates[1]).toEqual({
+      projectId: projectId2,
+      newAppCfg: appCfg2,
+      updatedModels: {
+        'alwaysai/human_pose': 7
+      }
+    });
+  });
+
+  test('handle shadow delta with updated analytics', async () => {
+    const ogAppCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {}
+    };
+    jest.mocked(readAppCfgFile).mockResolvedValue(ogAppCfg1);
+
+    const appCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {},
+      analytics: {
+        enable_cloud_publish: true
+      }
+    };
+
+    const payload = {
+      [projectId1]: {
+        appConfig: JSON.stringify(appCfg1)
+      }
+    };
+
+    const appCfgUpdates = await shadowHandler.handleShadowTopic({
+      topic: shadowHandler.shadowTopics.projects.updateDelta,
+      payload
+    });
+    expect(appCfgUpdates.length).toBe(1);
+    expect(appCfgUpdates[0]).toEqual({
+      projectId: projectId1,
+      newAppCfg: appCfg1
+    });
+  });
+
+  test('handle shadow delta with invalid app config', async () => {
+    const ogAppCfg1: AppConfig = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/mobilenet_ssd': 2
+      }
+    };
+
+    jest.mocked(readAppCfgFile).mockResolvedValue(ogAppCfg1);
+    const appCfg1 = {
+      scripts: {
+        start: 'python app.py'
+      },
+      models: {
+        'alwaysai/mobilenet_ssd': '3',
+        'alwaysai/yolo_v4': '5'
+      }
+    };
+
+    const payload = {
+      [projectId1]: {
+        appConfig: JSON.stringify(appCfg1)
+      }
+    };
+
+    const appCfgUpdates = await shadowHandler.handleShadowTopic({
+      topic: shadowHandler.shadowTopics.projects.updateDelta,
+      payload
+    });
+    expect(appCfgUpdates.length).toBe(0);
+  });
+
+  test.skip('publish app state', async () => {
+    // FIXME: For some reason publisher is not being called...
+    const testAppCfg: AppConfig = {
+      scripts: {
+        start: ''
+      },
+      models: {}
+    };
+    jest.mocked(readAppCfgFile).mockResolvedValue(testAppCfg);
+
+    shadowHandler.publishAppState(projectId1);
+    expect(jest.mocked(readAppCfgFile)).toBeCalledWith({ projectId1 });
+    const packet = {
+      state: {
+        reported: {
+          [projectId1]: { appConfig: JSON.stringify(testAppCfg) }
+        }
+      },
+      clientToken: clientId
+    };
+    expect(jest.mocked(publisher.publish)).toBeCalledWith(
+      shadowHandler.shadowTopics.projects.update,
+      JSON.stringify(packet)
+    );
+  });
+
+  test('get shadow updates', async () => {
+    shadowHandler.getShadowUpdates();
+    expect(jest.mocked(publisher.publish)).toBeCalledWith(
+      shadowHandler.shadowTopics.projects.get,
+      JSON.stringify({
+        clientToken: clientId
+      })
+    );
+  });
+
+  test('delete project shadow', async () => {
+    shadowHandler.deleteProjectShadow(projectId1);
+    const packet = {
+      state: {
+        reported: {
+          [projectId1]: null
+        }
+      },
+      clientToken: clientId
+    };
+    expect(jest.mocked(publisher.publish)).toBeCalledWith(
+      shadowHandler.shadowTopics.projects.update,
+      JSON.stringify(packet)
+    );
+  });
+});

package/src/cloud-connection/shadow-handler.ts

@@ -0,0 +1,175 @@
+import {
+  AppConfig,
+  validateAppConfig
+} from '@alwaysai/app-configuration-schemas';
+import { readAppCfgFile } from '../application-control';
+import { logger } from '../util/logger';
+import { Publisher } from './publisher';
+import { AppConfigModels, getAppCfgModelsDiff } from './shadow';
+
+export interface ShadowTopics {
+  projects: {
+    update: string;
+    get: string;
+    updateDelta: string;
+    getAccepted: string;
+    delete: string;
+  };
+}
+
+export type AppConfigUpdate = {
+  projectId: string;
+  newAppCfg: AppConfig;
+  updatedModels?: AppConfigModels;
+};
+
+export class ShadowHandler {
+  private clientId: string;
+  private publisher: Publisher;
+  public readonly shadowPrefix: string;
+  public readonly shadowTopics: ShadowTopics;
+
+  constructor(clientId: string, publisher: Publisher) {
+    this.clientId = clientId;
+    this.publisher = publisher;
+    this.shadowPrefix = `$aws/things/${this.clientId}/shadow/name/`;
+    this.shadowTopics = {
+      projects: {
+        update: `${this.shadowPrefix}projects/update`,
+        get: `${this.shadowPrefix}projects/get`,
+        updateDelta: `${this.shadowPrefix}projects/update/delta`,
+        getAccepted: `${this.shadowPrefix}projects/get/accepted`,
+        delete: `${this.shadowPrefix}projects/delete`
+      }
+    };
+  }
+
+  private async handleNamedShadowUpdate({
+    delta
+  }: {
+    delta: any;
+  }): Promise<AppConfigUpdate[]> {
+    const appConfigUpdates: AppConfigUpdate[] = [];
+
+    const deltaKeys = Object.keys(delta);
+
+    for (const projectId of deltaKeys) {
+      const projectShadow = delta[projectId];
+
+      if (projectShadow.appConfig) {
+        const newAppCfg = JSON.parse(projectShadow.appConfig);
+        if (!validateAppConfig(newAppCfg)) {
+          // FIXME: Raise an exception to be handled at higher layer
+          logger.error(
+            `Received invalid app config for ${projectId}!\n${JSON.stringify(
+              validateAppConfig.errors,
+              null,
+              2
+            )}`
+          );
+          continue;
+        }
+        const { updatedModels } = await getAppCfgModelsDiff({
+          newAppCfg,
+          projectId
+        });
+
+        if (updatedModels && Object.keys(updatedModels).length) {
+          appConfigUpdates.push({ projectId, newAppCfg, updatedModels });
+        } else {
+          appConfigUpdates.push({ projectId, newAppCfg });
+        }
+      } else {
+        logger.warn(
+          `Ignoring shadow update for ${projectId} due to no app config`
+        );
+      }
+    }
+    return appConfigUpdates;
+  }
+
+  // Public interface
+
+  public async handleShadowTopic({
+    topic,
+    payload
+  }: {
+    topic: string;
+    payload: any;
+  }): Promise<AppConfigUpdate[]> {
+    // TODO: make use a function like the other topic getters
+    const shadowName = topic.split('/')[5];
+    switch (topic) {
+      case this.shadowTopics.projects.updateDelta:
+        if (payload.clientToken === this.clientId) {
+          logger.debug(
+            `Ignoring message sent from self: ${JSON.stringify(
+              { topic, payload },
+              null,
+              2
+            )}`
+          );
+          break;
+        }
+        return await this.handleNamedShadowUpdate({ delta: payload });
+      case this.shadowTopics.projects.getAccepted:
+        if (payload['delta']) {
+          return await this.handleNamedShadowUpdate({
+            delta: payload['delta']
+          });
+        } else {
+          logger.info(`No delta updates in named shadow '${shadowName}'`);
+        }
+        break;
+      default:
+        logger.info(
+          `Ignoring shadow message: ${JSON.stringify(
+            { topic, payload },
+            null,
+            2
+          )}`
+        );
+    }
+    return [];
+  }
+
+  public async publishAppState(projectId: string) {
+    const appCfg = await readAppCfgFile({ projectId });
+    const packet = {
+      state: {
+        reported: {
+          [projectId]: { appConfig: JSON.stringify(appCfg) }
+        }
+      },
+      clientToken: this.clientId
+    };
+    this.publisher.publish(
+      this.shadowTopics.projects.update,
+      JSON.stringify(packet)
+    );
+  }
+
+  public getShadowUpdates() {
+    this.publisher.publish(
+      this.shadowTopics.projects.get,
+      JSON.stringify({
+        clientToken: this.clientId
+      })
+    );
+  }
+
+  public deleteProjectShadow(projectId: string) {
+    const packet = {
+      state: {
+        reported: {
+          [projectId]: null
+        }
+      },
+      clientToken: this.clientId
+    };
+    this.publisher.publish(
+      this.shadowTopics.projects.update,
+      JSON.stringify(packet)
+    );
+  }
+}

package/src/cloud-connection/shadow.ts

@@ -0,0 +1,50 @@
+import { logger } from '../util/logger';
+import { readAppCfgFile } from '../application-control';
+import { AppConfig } from '@alwaysai/app-configuration-schemas';
+
+export type AppConfigModels = {
+  [modelId: string]: number;
+};
+
+export type AppConfigScripts = {
+  start: string;
+};
+
+export const getAppCfgModelsDiff = async ({
+  newAppCfg,
+  projectId
+}: {
+  newAppCfg: AppConfig;
+  projectId: string;
+}) => {
+  const updatedModels: AppConfigModels = {};
+  const untouchedModels: AppConfigModels = {};
+  const newScripts: AppConfigScripts = { start: '' };
+
+  try {
+    const shadowScripts = newAppCfg.scripts;
+    const shadowModels = newAppCfg.models;
+
+    const localAppCfg = await readAppCfgFile({ projectId });
+    const localModels = localAppCfg.models;
+
+    Object.keys(shadowModels).forEach((modelId: string) => {
+      const localVersion = localModels[modelId];
+      const shadowVersion = shadowModels[modelId];
+      if (!localVersion || localVersion !== shadowVersion) {
+        updatedModels[modelId] = shadowVersion;
+      } else {
+        untouchedModels[modelId] = localVersion;
+      }
+    });
+
+    shadowScripts &&
+      Object.keys(shadowScripts).forEach((scriptName: string) => {
+        newScripts[scriptName] = shadowScripts[scriptName];
+      });
+  } catch (e) {
+    logger.error('Error parsing app config update: ', e);
+  }
+
+  return { scripts: newScripts, updatedModels, untouchedModels };
+};

package/src/device-control/device-control.ts

@@ -1,3 +1,4 @@
+// eslint-disable-next-line
 const osu = require('node-os-utils');
 
 export async function getCpuUtil(): Promise<number> {

package/src/docker/docker-cmd.ts

@@ -6,7 +6,7 @@ export async function runDockerLogin(props: { token: string }) {
   const server = '994534263224.dkr.ecr.us-west-2.amazonaws.com';
   const output = await spawner.run({
     exe: 'docker',
-    args: ['login', '--username', 'AWS', '--password', token, server],
+    args: ['login', '--username', 'AWS', '--password', token, server]
   });
   return output;
 }

package/src/docker/docker-compose-cmd.ts

@@ -1,12 +1,15 @@
 import { JsSpawner } from 'alwaysai/lib/util';
 
-export async function runDockerComposeCmd(props: { args: string[]; dir: string }) {
+export async function runDockerComposeCmd(props: {
+  args: string[];
+  dir: string;
+}) {
   const { args, dir } = props;
   const spawner = JsSpawner();
   const output = await spawner.run({
     exe: 'docker-compose',
     args,
-    cwd: dir,
+    cwd: dir
   });
   return output;
 }

package/src/endpoints.ts

@@ -1,19 +1,19 @@
-import { getSystemId } from "alwaysai/lib/infrastructure";
+import { getSystemId } from './infrastructure/system-id';
 
 export const getSecondLevelDomain = () => {
-  let domain = "";
+  let domain = '';
   switch (getSystemId()) {
-    case "development":
-      domain = "a6i0.net";
+    case 'development':
+      domain = 'a6i0.net';
       break;
-    case "qa":
-      domain = "a6i1.net";
+    case 'qa':
+      domain = 'a6i1.net';
       break;
-    case "production":
-      domain = "alwaysai.co";
+    case 'production':
+      domain = 'alwaysai.co';
       break;
     default:
-      domain = "alwaysai.co";
+      domain = 'alwaysai.co';
       break;
   }
   return domain;

package/src/environment.ts

@@ -1,10 +1,18 @@
 import { platform } from 'os';
 
-export const ALWAYSAI_OS_PLATFORM = parseOsPlatform(process.env.ALWAYSAI_OS_PLATFORM);
-export const ALWAYSAI_SHOW_HIDDEN = parseBoolean(process.env.ALWAYSAI_SHOW_HIDDEN);
-export const ALWAYSAI_DEVICE_AGENT_MODE = process.env.ALWAYSAI_DEVICE_AGENT_MODE;
+export const ALWAYSAI_OS_PLATFORM = parseOsPlatform(
+  process.env.ALWAYSAI_OS_PLATFORM
+);
+export const ALWAYSAI_SHOW_HIDDEN = parseBoolean(
+  process.env.ALWAYSAI_SHOW_HIDDEN
+);
+export const ALWAYSAI_DEVICE_AGENT_MODE =
+  process.env.ALWAYSAI_DEVICE_AGENT_MODE;
 export const ALWAYSAI_LOG_LEVEL = process.env.AAI_LOG_LEVEL;
 export const ALWAYSAI_LOG_TO_CONSOLE = process.env.ALWAYSAI_LOG_TO_CONSOLE;
+export const ALWAYSAI_ANALYTICS_PASSTHROUGH = parseBoolean(
+  process.env.ALWAYSAI_ANALYTICS_PASSTHROUGH
+);
 
 function parseOsPlatform(str: string | undefined): NodeJS.Platform {
   switch (str) {