@alwaysai/device-agent 0.0.11 → 0.0.13

package/src/cloud-connection/device-agent.ts

@@ -1,19 +1,18 @@
-const awsIot = require("aws-iot-device-sdk");
+// eslint-disable-next-line
+const awsIot = require('aws-iot-device-sdk');
 import {
-  BOOTSTRAP_CLAIM_ID_FILE_NAME,
-  BOOTSTRAP_CLAIM_ID_FILE_PATH,
   DEVICE_PRIVATE_KEY_FILE_NAME,
   CERTIFICATE_OWNERSHIP_TOKEN_FILE_NAME,
   DEVICE_CLAIM_ID_FILE_NAME,
-  DEVICE_CERTIFICATE_FILE_NAME,
-} from "../util/directories";
-import { getTargetHardwareUuid } from "../infrastructure/certificates-and-tokens";
-import { getDeviceId } from "../util/get-device-id";
-import { LOCAL_CERT_AND_KEY_DIR } from "alwaysai/lib/constants";
-import { JsSpawner } from "alwaysai/lib/util";
-import { logger } from "alwaysai/lib/util/logger";
-import { DeviceAgentCloudConnection } from "./device-agent-cloud-connection";
-const process = require("process");
+  DEVICE_CERTIFICATE_FILE_NAME
+} from '../util/directories';
+import { getDeviceUuid } from '../util/get-device-id';
+import { LOCAL_CERT_AND_KEY_DIR } from 'alwaysai/lib/constants';
+import { JsSpawner } from 'alwaysai/lib/util';
+import { logger } from '../util/logger';
+import { getTargetHardwareUuid } from 'alwaysai/lib/core/app';
+// eslint-disable-next-line
+const process = require('process');
 
 interface DeviceAgentConfigType {
   keyPath: string;
@@ -24,12 +23,12 @@ interface DeviceAgentConfigType {
 }
 
 interface FleetProvisionTemplateMessageType {
-  certificateOwnershipToken: String;
+  certificateOwnershipToken: string;
   parameters: {
-    hardwareId: String;
-    deviceUuid: String;
-    certificateId: String;
-    deviceType: String;
+    hardwareId: string;
+    deviceUuid: string;
+    certificateId: string;
+    deviceType: string;
   };
 }
 
@@ -38,10 +37,10 @@ export class DeviceAgent {
     this.device = awsIot.device(config);
   }
 
-  public deviceType = "aai-device";
+  public deviceType = 'aai-device';
   public device = awsIot.device;
   public hardwareId = async () => await getTargetHardwareUuid(JsSpawner());
-  public deviceId = getDeviceId();
+  public deviceId = getDeviceUuid();
 
   public publishMessage(topic: string, message: string) {
     this.device.publish(topic, message);
@@ -51,40 +50,39 @@ export class DeviceAgent {
 export class BootstrapAgent extends DeviceAgent {
   public async subscribeToAllTopics() {
     const AWS_CERTIFICATE_ACCEPT_TOPIC =
-      "$aws/certificates/create/json/accepted";
+      '$aws/certificates/create/json/accepted';
     const PROVISIONING_ACCEPTED_TOPIC =
-      "$aws/provisioning-templates/FleetProvisionTemplate/provision/json/accepted";
+      '$aws/provisioning-templates/FleetProvisionTemplate/provision/json/accepted';
 
     const topics = [AWS_CERTIFICATE_ACCEPT_TOPIC, PROVISIONING_ACCEPTED_TOPIC];
     const resp = this.device.subscribe(
       topics,
       function (err: any, granted: { topic: string; qos: number }[]) {
-        logger.debug(granted);
+        logger.debug(`Bootstrap Agent: ${JSON.stringify(granted, null, 2)}`);
       }
     );
   }
 
   public async handleAwsCertificateTopics(topic: string, payload: any) {
     switch (topic) {
-      case "$aws/certificates/create/json/accepted": {
+      case '$aws/certificates/create/json/accepted': {
         const {
           certificateId,
           certificatePem,
           privateKey,
-          certificateOwnershipToken,
+          certificateOwnershipToken
         } = JSON.parse(payload);
-        console.log(certificateId);
 
         const certSpawner = JsSpawner({ path: LOCAL_CERT_AND_KEY_DIR });
 
         await certSpawner.writeFile(
-          DEVICE_CERTIFICATE_FILE_NAME,
+          DEVICE_CERTIFICATE_FILE_NAME(),
           certificatePem
         );
 
-        await certSpawner.writeFile(DEVICE_PRIVATE_KEY_FILE_NAME, privateKey);
+        await certSpawner.writeFile(DEVICE_PRIVATE_KEY_FILE_NAME(), privateKey);
 
-        await certSpawner.writeFile(DEVICE_CLAIM_ID_FILE_NAME, certificateId);
+        await certSpawner.writeFile(DEVICE_CLAIM_ID_FILE_NAME(), certificateId);
 
         await certSpawner.writeFile(
           CERTIFICATE_OWNERSHIP_TOKEN_FILE_NAME,
@@ -97,24 +95,19 @@ export class BootstrapAgent extends DeviceAgent {
             hardwareId: await this.hardwareId(),
             deviceUuid: this.deviceId,
             certificateId,
-            deviceType: this.deviceType,
-          },
+            deviceType: this.deviceType
+          }
         };
-        console.log(content);
 
         this.publishMessage(
-          "$aws/provisioning-templates/FleetProvisionTemplate/provision/json",
+          '$aws/provisioning-templates/FleetProvisionTemplate/provision/json',
           JSON.stringify(content)
         );
 
         break;
       }
-      case "$aws/provisioning-templates/FleetProvisionTemplate/provision/json/accepted": {
-        console.log("success");
-        //  Resolve the `child_process` module, and `spawn`
-        //  a new process.
-        //  The `child_process` module lets us
-        //  access OS functionalities by running any bash command.`.
+      case '$aws/provisioning-templates/FleetProvisionTemplate/provision/json/accepted': {
+        logger.info('Device agent provisioning: success');
         process.exit();
       }
     }

package/src/cloud-connection/live-updates-handler.ts

@@ -0,0 +1,226 @@
+import {
+  AppLogsMessage,
+  DeviceAgentMessagePayload,
+  keyMirrors,
+  LiveUpdatesToggles
+} from '@alwaysai/device-agent-schemas';
+import { getAppLogs } from '../application-control';
+import { logger } from '../util/logger';
+import sleep from '../util/sleep';
+import { Publisher } from './publisher';
+import { getAppStateMessage, getDeviceStatsMessage } from './messages';
+import { AgentMessageTypeValue } from '@alwaysai/device-agent-schemas/lib/constants';
+import { AppInstallStatusManager } from './app-install-status';
+
+export class LiveUpdatesHandler {
+  private publisher: Publisher;
+  private appInstallStatusMgr: AppInstallStatusManager;
+
+  private liveUpdatesTimeout: ReturnType<typeof setTimeout>;
+  private liveUpdatesAlive = {
+    [keyMirrors.agentMessageType.device_stats]: false,
+    [keyMirrors.agentMessageType.app_state]: false,
+    [keyMirrors.agentMessageType.app_logs]: false,
+    [keyMirrors.agentMessageType.app_install_status]: false
+  };
+  private liveUpdatesSleepIntervals = {
+    [keyMirrors.agentMessageType.device_stats]: 5000,
+    [keyMirrors.agentMessageType.app_state]: 5000,
+    [keyMirrors.agentMessageType.app_logs]: 5000,
+    [keyMirrors.agentMessageType.app_install_status]: 5000
+  };
+  private appLogStreams = new Set<string>();
+  private appInstallStatuses = new Set<string>();
+
+  constructor(
+    publisher: Publisher,
+    appInstallStatusMgr: AppInstallStatusManager
+  ) {
+    this.publisher = publisher;
+    this.appInstallStatusMgr = appInstallStatusMgr;
+  }
+
+  private async startAppLogStream(projectId: string) {
+    logger.info(`Starting log stream for ${projectId}`);
+    this.appLogStreams.add(projectId);
+    const readable = await getAppLogs({
+      projectId,
+      args: ['--tail', '100', '--no-log-prefix']
+    });
+    readable.on('data', (chunk: Buffer) => {
+      if (!this.appLogStreams.has(projectId)) {
+        // why doesn't typescript know about this function?
+        // @ts-ignore
+        readable.destroy();
+        logger.info(`App log stream terminated for project ${projectId}`);
+        return;
+      }
+      const logStr = chunk.toString();
+      const message: AppLogsMessage = {
+        messageType: keyMirrors.agentMessageType.app_logs,
+        appLogs: {
+          projectId,
+          logChunk: logStr
+        }
+      };
+      this.publisher.publishToClient(message);
+    });
+
+    readable.on('error', (error) => {
+      logger.error(
+        `App log stream terminated for project ${projectId}: ${error}`
+      );
+    });
+
+    readable.on('finished', () => {
+      logger.info(`App logs finished piping for project ${projectId}`);
+    });
+  }
+
+  // Use arrow function to allow `this` to be accessed
+  private getAppInstallStatusMessage = async (appReleaseHash: string) => {
+    const appInstallStatus =
+      this.appInstallStatusMgr.getStatusPacket(appReleaseHash);
+    const appInstallStatusMessage = {
+      messageType: keyMirrors.agentMessageType.app_install_status,
+      appInstallStatus
+    };
+    return appInstallStatusMessage;
+  };
+
+  public async startPublishingLiveUpdates<T extends any[]>(
+    messageType: AgentMessageTypeValue,
+    getMessageData: (...args: T) => Promise<DeviceAgentMessagePayload>,
+    args: T
+  ) {
+    logger.info(`Turned on live updates for ${messageType}`);
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      try {
+        const message = await getMessageData(...args);
+        this.publisher.publishToClient(message);
+      } catch (e) {
+        logger.error(
+          `Error publishing live updates for ${messageType}: ${e.message}`
+        );
+        break;
+      }
+      if (!this.continuePublishing(messageType, ...args)) {
+        logger.info(`Turned off live updates for ${messageType}`);
+        break;
+      }
+      await sleep(this.getLiveUpdatesInterval(messageType));
+    }
+  }
+
+  private continuePublishing(flag: AgentMessageTypeValue, data?: any): boolean {
+    switch (flag) {
+      case keyMirrors.agentMessageType.device_stats:
+      case keyMirrors.agentMessageType.app_state:
+        return this.liveUpdatesAlive[flag];
+      case keyMirrors.agentMessageType.app_install_status: {
+        const appReleaseHash: string = data;
+        return this.appInstallStatuses.has(appReleaseHash);
+      }
+      default:
+        logger.error(`Unrecognized publishable flag ${flag}`);
+        return false;
+    }
+  }
+
+  private getLiveUpdatesInterval(flag: AgentMessageTypeValue): number {
+    const exists = this.liveUpdatesSleepIntervals[flag];
+    if (exists) {
+      return exists;
+    }
+    logger.error(`Unrecognized live updates flag ${flag}`);
+    return -1;
+  }
+
+  private setLiveUpdates(toggles: LiveUpdatesToggles) {
+    if (toggles.deviceStats) {
+      this.liveUpdatesAlive.device_stats = toggles.deviceStats;
+    }
+    if (toggles.appState) {
+      this.liveUpdatesAlive.app_state = toggles.appState;
+    }
+  }
+
+  private restartLiveUpdatesTimeout() {
+    clearTimeout(this.liveUpdatesTimeout);
+    this.liveUpdatesTimeout = setTimeout(() => {
+      this.setLiveUpdates({
+        deviceStats: false,
+        appState: false
+      });
+      this.appLogStreams.clear();
+      // TODO: Make constant, not hard coded
+    }, 600000); // 10 min
+  }
+
+  public async update({
+    deviceStats,
+    appState,
+    appLogs,
+    appInstallStatus
+  }: {
+    deviceStats?: boolean;
+    appState?: boolean;
+    appLogs?: {
+      projectId: string;
+      toggle: boolean;
+    };
+    appInstallStatus?: {
+      appReleaseHash: string;
+      toggle: boolean;
+    };
+  }) {
+    this.restartLiveUpdatesTimeout();
+    if (deviceStats !== undefined) {
+      this.liveUpdatesAlive.device_stats = deviceStats;
+      if (deviceStats) {
+        this.startPublishingLiveUpdates(
+          keyMirrors.agentMessageType.device_stats,
+          getDeviceStatsMessage,
+          []
+        );
+      }
+    }
+
+    if (appState !== undefined) {
+      this.liveUpdatesAlive.app_state = appState;
+      if (appState) {
+        this.startPublishingLiveUpdates(
+          keyMirrors.agentMessageType.app_state,
+          getAppStateMessage,
+          []
+        );
+      }
+    }
+
+    if (appLogs !== undefined) {
+      if (appLogs.toggle) {
+        this.startAppLogStream(appLogs.projectId);
+      } else {
+        this.appLogStreams.delete(appLogs.projectId);
+      }
+    }
+
+    if (appInstallStatus !== undefined) {
+      if (appInstallStatus.toggle) {
+        this.liveUpdatesAlive.app_install_status = appInstallStatus.toggle;
+        this.appInstallStatuses.add(appInstallStatus.appReleaseHash);
+        this.startPublishingLiveUpdates(
+          keyMirrors.agentMessageType.app_install_status,
+          this.getAppInstallStatusMessage,
+          [appInstallStatus.appReleaseHash]
+        );
+      } else {
+        this.appInstallStatuses.delete(appInstallStatus.appReleaseHash);
+      }
+      if (this.appInstallStatuses.size === 0) {
+        this.liveUpdatesAlive.app_install_status = false;
+      }
+    }
+  }
+}

package/src/cloud-connection/messages.ts

@@ -0,0 +1,39 @@
+import { AppStatePacket, keyMirrors } from '@alwaysai/device-agent-schemas';
+import { getAppStatus } from '../application-control';
+import {
+  getCpuUtil,
+  getDiskUtil,
+  getMemUtil
+} from '../device-control/device-control';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+
+export async function getAppStateMessage() {
+  const appStateMessage: AppStatePacket[] = [];
+  const apps = await AgentConfigFile().getReadyApps();
+  for (const app of apps) {
+    const projectId = app.projectId;
+    const status = await getAppStatus({ projectId });
+    appStateMessage.push(status);
+  }
+  const appStatePackage = {
+    messageType: keyMirrors.agentMessageType.app_state,
+    appState: appStateMessage
+  };
+  return appStatePackage;
+}
+
+export async function getDeviceStatsMessage() {
+  const cpuUsage = await getCpuUtil();
+  const diskUtil = await getDiskUtil();
+  const memUtil = await getMemUtil();
+
+  const deviceStatsMessage = {
+    messageType: keyMirrors.agentMessageType.device_stats,
+    deviceStats: {
+      cpuUsage,
+      diskUtil,
+      usedMemoryPercentage: memUtil
+    }
+  };
+  return deviceStatsMessage;
+}

package/src/cloud-connection/publisher.ts

@@ -0,0 +1,65 @@
+import {
+  DeviceAgentMessage,
+  DeviceAgentMessagePayload,
+  getClientTopic,
+  getCloudTopic
+} from '@alwaysai/device-agent-schemas';
+import { logger } from '../util/logger';
+
+export class Publisher {
+  private client: any;
+  private clientId: string;
+  private readonly toClientTopic: string;
+  private readonly toCloudTopic: string;
+
+  constructor(client: any, clientId: string) {
+    this.client = client;
+    this.clientId = clientId;
+    this.toClientTopic = getClientTopic(this.clientId);
+    this.toCloudTopic = getCloudTopic(this.clientId);
+  }
+
+  private buildMessagePacket(
+    topic: string,
+    payload: DeviceAgentMessagePayload
+  ): DeviceAgentMessage {
+    const packet = {
+      timestamp: new Date().toUTCString(),
+      deviceId: this.clientId,
+      topic,
+      payload
+    };
+    return packet;
+  }
+
+  public publish(topic: string, payload: string) {
+    // TODO: topic validation
+    this.client.publish(topic, payload, (err: any) => {
+      if (err) {
+        logger.error(
+          `Error publishing message: \nTopic: ${topic}\nMessage: ${payload}\nError: ${err}`
+        );
+      }
+    });
+  }
+
+  public publishDeviceAgentPayload(
+    topic: string,
+    payload: DeviceAgentMessagePayload
+  ) {
+    const packet = this.buildMessagePacket(topic, payload);
+    const packetStr = JSON.stringify(packet);
+    logger.debug(
+      `Publishing message:\n${JSON.stringify({ topic, packet }, null, 2)}`
+    );
+    this.publish(topic, packetStr);
+  }
+
+  public publishToClient(payload: DeviceAgentMessagePayload) {
+    this.publishDeviceAgentPayload(this.toClientTopic, payload);
+  }
+
+  public publishToCloud(payload: DeviceAgentMessagePayload) {
+    this.publishDeviceAgentPayload(this.toCloudTopic, payload);
+  }
+}

package/src/cloud-connection/shadow-handler.ts

@@ -0,0 +1,154 @@
+import {
+  AppConfig,
+  validateAppConfig
+} from '@alwaysai/app-configuration-schemas';
+import { readAppCfgFile } from '../application-control';
+import { logger } from '../util/logger';
+import { Publisher } from './publisher';
+import { AppConfigModels, getAppCfgModelsDiff } from './shadow';
+
+interface ShadowTopics {
+  projects: {
+    update: string;
+    get: string;
+    updateDelta: string;
+    getAccepted: string;
+    delete: string;
+  };
+}
+
+export type AppConfigUpdate = {
+  projectId: string;
+  newAppCfg: AppConfig;
+  updatedModels?: AppConfigModels;
+};
+
+export class ShadowHandler {
+  private clientId: string;
+  private publisher: Publisher;
+  public readonly shadowPrefix: string;
+  public readonly shadowTopics: ShadowTopics;
+
+  constructor(clientId: string, publisher: Publisher) {
+    this.clientId = clientId;
+    this.publisher = publisher;
+    this.shadowPrefix = `$aws/things/${this.clientId}/shadow/name/`;
+    this.shadowTopics = {
+      projects: {
+        update: `${this.shadowPrefix}projects/update`,
+        get: `${this.shadowPrefix}projects/get`,
+        updateDelta: `${this.shadowPrefix}projects/update/delta`,
+        getAccepted: `${this.shadowPrefix}projects/get/accepted`,
+        delete: `${this.shadowPrefix}projects/delete`
+      }
+    };
+  }
+
+  private async handleNamedShadowUpdate({
+    delta
+  }: {
+    delta: any;
+  }): Promise<AppConfigUpdate[]> {
+    const appConfigUpdates: AppConfigUpdate[] = [];
+
+    const deltaKeys = Object.keys(delta);
+
+    for (const projectId of deltaKeys) {
+      const projectShadow = delta[projectId];
+
+      if (projectShadow.appConfig) {
+        const newAppCfg = JSON.parse(projectShadow.appConfig);
+        if (!validateAppConfig(newAppCfg)) {
+          logger.error(
+            `Received invalid app config for ${projectId}!/n${JSON.stringify(
+              validateAppConfig.errors,
+              null,
+              2
+            )}`
+          );
+          continue;
+        }
+        const { updatedModels } = await getAppCfgModelsDiff({
+          newAppCfg,
+          projectId
+        });
+
+        if (updatedModels && Object.keys(updatedModels).length) {
+          appConfigUpdates.push({ projectId, newAppCfg, updatedModels });
+        } else {
+          appConfigUpdates.push({ projectId, newAppCfg });
+        }
+      }
+    }
+    return appConfigUpdates;
+  }
+
+  // Public interface
+
+  public async handleShadowTopic({
+    topic,
+    payload
+  }: {
+    topic: string;
+    payload: any;
+  }): Promise<AppConfigUpdate[]> {
+    // TODO: make use a function like the other topic getters
+    const shadowName = topic.split('/')[5];
+    switch (topic) {
+      case this.shadowTopics.projects.updateDelta:
+        return await this.handleNamedShadowUpdate({ delta: payload });
+      case this.shadowTopics.projects.getAccepted:
+        if (payload['delta']) {
+          return await this.handleNamedShadowUpdate({
+            delta: payload['delta']
+          });
+        } else {
+          logger.info(`No delta updates in named shadow '${shadowName}'`);
+        }
+        break;
+      default:
+        logger.info(
+          `Ignoring shadow message: ${JSON.stringify(
+            { topic, payload },
+            null,
+            2
+          )}`
+        );
+    }
+    return [];
+  }
+
+  public async publishAppState(projectId: string) {
+    const appCfg = await readAppCfgFile({ projectId });
+    const packet = {
+      state: {
+        reported: {
+          [projectId]: { appConfig: JSON.stringify(appCfg) }
+        }
+      },
+      clientToken: this.clientId
+    };
+    this.publisher.publish(
+      this.shadowTopics.projects.update,
+      JSON.stringify(packet)
+    );
+  }
+
+  public getShadowUpdates() {
+    this.publisher.publish(
+      this.shadowTopics.projects.get,
+      JSON.stringify({
+        clientToken: this.clientId
+      })
+    );
+  }
+
+  public deleteProjectShadow() {
+    this.publisher.publish(
+      `${this.shadowTopics.projects.delete}`,
+      JSON.stringify({
+        clientToken: this.clientId
+      })
+    );
+  }
+}

package/src/cloud-connection/shadow.ts

@@ -0,0 +1,50 @@
+import { logger } from '../util/logger';
+import { readAppCfgFile } from '../application-control';
+import { AppConfig } from '@alwaysai/app-configuration-schemas';
+
+export type AppConfigModels = {
+  [modelId: string]: number;
+};
+
+export type AppConfigScripts = {
+  start: string;
+};
+
+export const getAppCfgModelsDiff = async ({
+  newAppCfg,
+  projectId
+}: {
+  newAppCfg: AppConfig;
+  projectId: string;
+}) => {
+  const updatedModels: AppConfigModels = {};
+  const untouchedModels: AppConfigModels = {};
+  const newScripts: AppConfigScripts = { start: '' };
+
+  try {
+    const shadowScripts = newAppCfg.scripts;
+    const shadowModels = newAppCfg.models;
+
+    const localAppCfg = await readAppCfgFile({ projectId });
+    const localModels = localAppCfg.models;
+
+    Object.keys(shadowModels).forEach((modelId: string) => {
+      const localVersion = localModels[modelId];
+      const shadowVersion = shadowModels[modelId];
+      if (!localVersion || localVersion !== shadowVersion) {
+        updatedModels[modelId] = shadowVersion;
+      } else {
+        untouchedModels[modelId] = localVersion;
+      }
+    });
+
+    shadowScripts &&
+      Object.keys(shadowScripts).forEach((scriptName: string) => {
+        newScripts[scriptName] = shadowScripts[scriptName];
+      });
+  } catch (e) {
+    logger.error('Error parsing app config update: ', e);
+  }
+
+  return { scripts: newScripts, updatedModels, untouchedModels };
+};

package/src/device-control/device-control.ts

@@ -1,3 +1,4 @@
+// eslint-disable-next-line
 const osu = require('node-os-utils');
 
 export async function getCpuUtil(): Promise<number> {

package/src/docker/docker-cmd.ts

@@ -6,7 +6,7 @@ export async function runDockerLogin(props: { token: string }) {
   const server = '994534263224.dkr.ecr.us-west-2.amazonaws.com';
   const output = await spawner.run({
     exe: 'docker',
-    args: ['login', '--username', 'AWS', '--password', token, server],
+    args: ['login', '--username', 'AWS', '--password', token, server]
   });
   return output;
 }