@alwaysai/device-agent 0.0.1-2.1-beta-provision

package/src/cloud-connection/device-agent-cloud-connection.ts

@@ -0,0 +1,673 @@
+const awsIot = require("aws-iot-device-sdk");
+import { getIoTCoreEndpointUrl } from "../infrastructure/urls";
+import { existsSync, createReadStream } from "fs";
+import { rmBootstrapCertsAndClose } from "../util/clean-certs";
+import {
+  getPrivateKeyFilePath,
+  getCertificateFilePath,
+  BOOTSTRAP_DEVICE_CERTIFICATE_FILE_PATH,
+  BOOTSTRAP_DEVICE_PRIVATE_KEY_FILE_PATH,
+  AWS_ROOT_CERTIFICATE_FILE_PATH,
+  BOOTSTRAP_CERTIFICATES_DIR_PATH,
+} from "../util/directories";
+
+import { BootstrapAgent } from "./device-agent";
+
+import sleep from "../util/sleep";
+import {
+  startApp,
+  stopApp,
+  restartApp,
+  getAppLogs,
+  getAppStatus,
+} from "../application-control/status";
+import { installApp, uninstallApp } from "../application-control/install";
+import {
+  keyMirrors,
+  validateClientMessage,
+  DeviceAgentMessagePayload,
+  AppInstallStatusPacket,
+  AppStateControlPacket,
+  AppVersionControlPacket,
+  LiveUpdatesToggles,
+  AppStatePacket,
+  LiveUpdatesToggleMessage,
+  DeviceStatsMessage,
+  AppStateMessage,
+  AppInstallStatusMessage,
+  SignedUrlsRequestMessage,
+  DeviceAgentMessage,
+  ClientMessage,
+  getClientTopic,
+  getCloudTopic,
+  getDeviceTopic,
+} from "@alwaysai/device-agent-schemas";
+import { getDeviceId } from "../util/get-device-id";
+import { logger } from "../util/logger";
+import {
+  getCpuUtil,
+  getDiskUtil,
+  getMemUtil,
+} from "../device-control/device-control";
+import { AgentConfigFile } from "../infrastructure/agent-config";
+import {
+  buildApp,
+  getAppConfig,
+  getAppDir,
+} from "../application-control/utils";
+import { updateModelsWithPresignedUrls } from "../application-control/models";
+import { updateAppConfig } from "../application-control/config";
+
+export class DeviceAgentCloudConnection {
+  private clientId = getDeviceId();
+  private host = getIoTCoreEndpointUrl();
+  private appInstallStatus: AppInstallStatusPacket;
+  private liveUpdatesTimeout: ReturnType<typeof setTimeout>;
+  private liveUpdatesAlive = {
+    [keyMirrors.agentMessageType.device_stats]: false,
+    [keyMirrors.agentMessageType.app_state]: false,
+    [keyMirrors.agentMessageType.app_logs]: false,
+  };
+  private liveUpdatesSleepIntervals = {
+    [keyMirrors.agentMessageType.device_stats]: 5000,
+    [keyMirrors.agentMessageType.app_state]: 5000,
+    [keyMirrors.agentMessageType.app_logs]: 5000,
+    [keyMirrors.agentMessageType.app_install_status]: 5000,
+  };
+  private appLogStreams = new Set<string>();
+  private readonly shadowPrefix = `$aws/things/${this.clientId}/shadow/name/`;
+  private readonly shadowTopics = {
+    projects: {
+      updateDelta: `${this.shadowPrefix}projects/update/delta`,
+      getAccepted: `${this.shadowPrefix}projects/get/accepted`,
+    },
+  };
+  private readonly toCloudTopic = getCloudTopic(this.clientId);
+  private readonly toClientTopic = getClientTopic(this.clientId);
+  private readonly toDeviceTopic = getDeviceTopic(this.clientId);
+
+  // device shadow utils
+
+  public getShadowPrefix() {
+    return this.shadowPrefix;
+  }
+
+  private async handleNamedShadowUpdate({ payload }: { payload: string }) {
+    const delta = JSON.parse(payload);
+    const deltaKeys = Object.keys(delta);
+
+    for (const projectId of deltaKeys) {
+      const projectShadow = delta[projectId];
+      if (projectShadow.appConfig) {
+        const appConfig = projectShadow.appConfig;
+        const appDir = getAppDir(projectId);
+        await updateAppConfig(projectId, appConfig);
+
+        if (appConfig.models) {
+          this.publishCloudRequest({
+            messageType: keyMirrors.agentMessageType.signed_urls_request,
+            modelsOnlyUrlsRequest: {
+              projectId,
+              models: appConfig.models,
+            },
+          });
+        }
+
+        if (appConfig.scripts && !appConfig.models) {
+          const appState = await getAppStatus({ projectId });
+
+          await buildApp({ appDir });
+
+          if (
+            appState.services.length &&
+            appState.services[0].state !== keyMirrors.appState.stopped
+          ) {
+            restartApp({ projectId });
+          }
+          await this.publishReportedState(projectId);
+        }
+      }
+    }
+  }
+
+  private async startAppLogStream(projectId: string) {
+    this.appLogStreams.add(projectId);
+    const readable = await getAppLogs({
+      projectId,
+      args: ["--tail", "100", "--no-log-prefix"],
+    });
+    readable.on("data", (chunk: Buffer) => {
+      if (!this.appLogStreams.has(projectId)) {
+        // why doesn't typescript know about this function?
+        // @ts-ignore
+        readable.destroy();
+        logger.info(`App log stream terminated for project ${projectId}`);
+        return;
+      }
+      const logStr = chunk.toString();
+      const message = {
+        messageType: keyMirrors.agentMessageType.app_logs,
+        appLogs: {
+          projectId,
+          logChunk: logStr,
+        },
+      };
+      const packet = this.buildMessagePacket(
+        this.getClientId(),
+        this.toClientTopic,
+        message
+      );
+      this.publishMessage(this.toClientTopic, JSON.stringify(packet));
+    });
+
+    readable.on("error", (error) => {
+      logger.error(
+        `App log stream terminated for project ${projectId}: ${error}`
+      );
+    });
+
+    readable.on("finished", () => {
+      logger.info(`App logs finished piping for project ${projectId}`);
+    });
+  }
+
+  // must contain app release hash
+  private initAppInstallStatus(installationStatus: AppInstallStatusPacket) {
+    this.appInstallStatus = installationStatus;
+  }
+
+  private updateAppInstallStatus(
+    installationStatus: Omit<AppInstallStatusPacket, "appReleaseHash">
+  ) {
+    this.appInstallStatus.status = installationStatus.status;
+    this.appInstallStatus.message = installationStatus.message;
+  }
+
+  private getAppInstallStatus(): AppInstallStatusPacket {
+    return this.appInstallStatus;
+  }
+
+  // Message Builders
+  private buildMessagePacket(
+    deviceId: string,
+    topic: string,
+    payload: DeviceAgentMessagePayload
+  ): DeviceAgentMessage {
+    const packet = {
+      timestamp: new Date().toUTCString(),
+      deviceId,
+      topic,
+      payload,
+    };
+    return packet;
+  }
+
+  private async getAppStateMessage(): Promise<AppStateMessage> {
+    const appStateMessage: AppStatePacket[] = [];
+    const apps = await AgentConfigFile().getReadyApps();
+    for (const app of apps) {
+      const projectId = app.projectId;
+      const status = await getAppStatus({ projectId });
+      appStateMessage.push(status);
+    }
+    const appStatePackage = {
+      messageType: keyMirrors.agentMessageType.app_state,
+      appState: appStateMessage,
+    };
+    return appStatePackage;
+  }
+
+  private async getDeviceStatsMessage(): Promise<DeviceStatsMessage> {
+    const cpuUsage = await getCpuUtil();
+    const diskUtil = await getDiskUtil();
+    const memUtil = await getMemUtil();
+
+    const deviceStatsMessage = {
+      messageType: keyMirrors.agentMessageType.device_stats,
+      deviceStats: {
+        cpuUsage,
+        diskUtil,
+        usedMemoryPercentage: memUtil,
+      },
+    };
+    return deviceStatsMessage;
+  }
+
+  // must be arrow function due to this context when function is passed as param
+  private getAppInstallStatusMessage =
+    async (): Promise<AppInstallStatusMessage> => {
+      const appInstallStatus = this.getAppInstallStatus();
+      const appInstallStatusMessage = {
+        messageType: keyMirrors.agentMessageType.app_install_status,
+        appInstallStatus,
+      };
+      return appInstallStatusMessage;
+    };
+
+  private async startPublishingLiveUpdates(
+    topic: string,
+    messageType: string,
+    getMessageData: () => Promise<DeviceAgentMessagePayload>
+  ) {
+    while (true) {
+      try {
+        const message = await getMessageData();
+        const packet = this.buildMessagePacket(
+          this.getClientId(),
+          topic,
+          message
+        );
+        this.publishMessage(topic, JSON.stringify(packet));
+      } catch (e) {
+        logger.error(
+          `Error publishing live updates for ${messageType}: ${e.message}`
+        );
+        break;
+      }
+      if (!this.continuePublishing(messageType)) {
+        logger.info(`Turned off live updates for ${messageType}`);
+        break;
+      }
+      await sleep(this.getLiveUpdatesInterval(messageType));
+    }
+  }
+
+  private continuePublishing(flag: string): boolean {
+    switch (flag) {
+      case keyMirrors.agentMessageType.device_stats:
+      case keyMirrors.agentMessageType.app_state:
+        return this.liveUpdatesAlive[flag];
+      case keyMirrors.agentMessageType.app_install_status:
+        return (
+          this.appInstallStatus.status ===
+          keyMirrors.appInstallStatus.in_progress
+        );
+      default:
+        logger.error(`Unrecognized publishable flag ${flag}`);
+        return false;
+    }
+  }
+
+  private getLiveUpdatesInterval(flag: string): number {
+    const exists = this.liveUpdatesSleepIntervals[flag];
+    if (exists) {
+      return exists;
+    }
+    logger.error(`Unrecognized live updates flag ${flag}`);
+    return -1;
+  }
+
+  private setLiveUpdates(toggles: LiveUpdatesToggles) {
+    if (toggles.deviceStats) {
+      this.liveUpdatesAlive.device_stats = toggles.deviceStats;
+    }
+    if (toggles.appState) {
+      this.liveUpdatesAlive.app_state = toggles.appState;
+    }
+  }
+
+  private handleAppStateControl(payload: AppStateControlPacket) {
+    const { baseCommand, projectId } = payload;
+    switch (baseCommand) {
+      case keyMirrors.appStateControl.start:
+        startApp({ projectId });
+        break;
+      case keyMirrors.appStateControl.stop:
+        stopApp({ projectId });
+        break;
+      case keyMirrors.appStateControl.restart:
+        restartApp({ projectId });
+        break;
+    }
+  }
+
+  private handleAppVersionControl(payload: AppVersionControlPacket) {
+    const { projectId, appReleaseHash } = payload;
+    const signedUrlsRequest = { projectId, appReleaseHash };
+    this.publishCloudRequest({
+      messageType: keyMirrors.agentMessageType.signed_urls_request,
+      signedUrlsRequest,
+    });
+  }
+
+  private handleDeviceCommand = async (packet: any) => {
+    // TODO
+  };
+
+  private handleAgentCommand(message: LiveUpdatesToggleMessage) {
+    switch (message.messageType) {
+      case keyMirrors.clientMessageType.live_state_updates:
+        this.liveUpdatesBroker(message.liveUpdatesToggles);
+        break;
+      default:
+        logger.error(
+          `Invalid agent action message type from message '${message}'`
+        );
+    }
+  }
+
+  private restartLiveUpdatesTimeout() {
+    clearTimeout(this.liveUpdatesTimeout);
+    this.liveUpdatesTimeout = setTimeout(() => {
+      this.setLiveUpdates({
+        deviceStats: false,
+        appState: false,
+      });
+      this.appLogStreams.clear();
+      // TODO: Make constant, not hard coded
+    }, 600000); // 10 min
+  }
+
+  private async liveUpdatesBroker({
+    deviceStats,
+    appState,
+    appLogs,
+  }: {
+    deviceStats?: boolean;
+    appState?: boolean;
+    appLogs?: {
+      projectId: string;
+      toggle: boolean;
+    };
+  }) {
+    this.restartLiveUpdatesTimeout();
+    if (deviceStats !== undefined) {
+      this.liveUpdatesAlive.device_stats = deviceStats;
+      if (deviceStats) {
+        this.startPublishingLiveUpdates(
+          this.toClientTopic,
+          keyMirrors.agentMessageType.device_stats,
+          this.getDeviceStatsMessage
+        );
+      }
+    }
+
+    if (appState !== undefined) {
+      this.liveUpdatesAlive.app_state = appState;
+      if (appState) {
+        this.startPublishingLiveUpdates(
+          this.toClientTopic,
+          keyMirrors.agentMessageType.app_state,
+          this.getAppStateMessage
+        );
+      }
+    }
+
+    if (appLogs !== undefined) {
+      if (appLogs.toggle) {
+        this.startAppLogStream(appLogs.projectId);
+      } else {
+        this.appLogStreams.delete(appLogs.projectId);
+      }
+    }
+  }
+
+  private async publishReportedState(projectId) {
+    const newAppCfg = await getAppConfig(projectId);
+    const packet = {
+      state: {
+        reported: {
+          [projectId]: { appConfig: newAppCfg },
+        },
+      },
+    };
+    this.publishMessage(
+      `${this.shadowPrefix}projects/update`,
+      JSON.stringify(packet)
+    );
+  }
+
+  private async publishCloudRequest(payload: SignedUrlsRequestMessage) {
+    const topic = this.toCloudTopic;
+    const deviceRequestPacket = this.buildMessagePacket(
+      this.getClientId(),
+      topic,
+      payload
+    );
+    this.publishMessage(topic, JSON.stringify({ deviceRequestPacket }));
+  }
+
+  // Public Methods
+
+  public device = awsIot.device;
+
+  constructor() {
+    this.device = awsIot.device({
+      keyPath: getPrivateKeyFilePath(),
+      certPath: getCertificateFilePath(),
+      caPath: AWS_ROOT_CERTIFICATE_FILE_PATH,
+      clientId: this.clientId,
+      host: this.host,
+      port: 8883,
+    });
+
+    this.device.subscribe(this.toDeviceTopic);
+    this.device.subscribe(this.shadowTopics.projects.getAccepted);
+    this.device.subscribe(this.shadowTopics.projects.updateDelta);
+  }
+
+  public getClientId(): string {
+    return this.clientId;
+  }
+
+  public publishMessage(topic: string, message: string) {
+    // TODO: topic validation
+    this.device.publish(topic, message, (err: any) => {});
+  }
+
+  public async handleClientMessage({
+    topic,
+    message,
+  }: {
+    topic: string;
+    message: ClientMessage;
+  }) {
+    const payload = message.payload;
+    switch (payload.messageType) {
+      case keyMirrors.clientMessageType.app_state_control: {
+        this.handleAppStateControl(payload.appStateControl);
+        break;
+      }
+      case keyMirrors.clientMessageType.app_version_control: {
+        this.handleAppVersionControl(payload.appVersionControl);
+        break;
+      }
+      case keyMirrors.clientMessageType.live_state_updates: {
+        this.handleAgentCommand(payload);
+        break;
+      }
+      case keyMirrors.clientMessageType.app_install_cloud_response: {
+        const {
+          projectId,
+          appReleaseHash,
+          appInstallPayload,
+          modelsInstallPayload,
+        } = payload.appInstallCloudResponse;
+
+        this.initAppInstallStatus({
+          status: keyMirrors.appInstallStatus.in_progress,
+          appReleaseHash,
+        });
+
+        this.startPublishingLiveUpdates(
+          this.toClientTopic,
+          keyMirrors.agentMessageType.app_install_status,
+          this.getAppInstallStatusMessage
+        );
+
+        // Install the app and models
+        try {
+          const signedUrlsPayload = {
+            appInstallPayload,
+            modelsInstallPayload,
+          };
+          await installApp({
+            projectId,
+            appReleaseHash,
+            signedUrlsPayload,
+          });
+          this.updateAppInstallStatus({
+            status: keyMirrors.appInstallStatus.success,
+          });
+
+          // update app config shadow for project
+          await this.publishReportedState(projectId);
+        } catch (e) {
+          logger.error(e);
+          const message: string = e.message;
+
+          // uninstall the failed app to put system back in good state
+          await uninstallApp({ projectId });
+          this.updateAppInstallStatus({
+            status: keyMirrors.appInstallStatus.failure,
+            message,
+          });
+
+          // delete shadow for project
+          this.publishMessage(`${this.shadowPrefix}${projectId}/delete`, "");
+        }
+        break;
+      }
+      case keyMirrors.clientMessageType.models_install_cloud_response: {
+        const { projectId, newModels } = payload.modelsInstallCloudResponse;
+
+        try {
+          await updateModelsWithPresignedUrls(projectId, newModels);
+
+          await this.publishReportedState(projectId);
+        } catch (e) {
+          logger.error(e);
+        }
+        break;
+      }
+      default:
+        logger.error(`Invalid Client Message '${JSON.stringify(payload)}'`);
+    }
+  }
+
+  public async handleShadowTopic({
+    topic,
+    payload,
+  }: {
+    topic: string;
+    payload: string;
+  }) {
+    const shadowName = topic.split("/")[5];
+    const message = JSON.parse(payload);
+    if (topic === this.shadowTopics.projects.updateDelta) {
+      this.handleNamedShadowUpdate({ payload });
+    } else if (topic === this.shadowTopics.projects.getAccepted) {
+      if (message.delta) {
+        this.handleNamedShadowUpdate({
+          payload: JSON.stringify(message.delta),
+        });
+      } else {
+        logger.info(`No delta updates in shadow ${shadowName}`);
+      }
+    }
+  }
+}
+
+export async function runDeviceAgentCloudInterface() {
+  switch (true) {
+    case existsSync(getCertificateFilePath()):
+      {
+        const deviceAgent = new DeviceAgentCloudConnection();
+
+        deviceAgent.device.on("connect", function (connack: any) {
+          logger.info("Device Agent has connected to the cloud");
+
+          // Get shadow updates
+          deviceAgent.publishMessage(
+            `${deviceAgent.getShadowPrefix()}projects/get`,
+            ""
+          );
+        });
+
+        deviceAgent.device.on("disconnect", function () {
+          logger.info("Device Agent has been disconnected from the cloud");
+        });
+
+        deviceAgent.device.on(
+          "message",
+          function (topic: string, payload: string) {
+            logger.debug(JSON.stringify({ topic }));
+            try {
+              const jsonPacket = JSON.parse(payload);
+              if (jsonPacket.hasOwnProperty("state")) {
+                deviceAgent.handleShadowTopic({
+                  topic,
+                  payload: JSON.stringify(jsonPacket.state),
+                });
+              } else {
+                const valid = validateClientMessage(jsonPacket);
+                if (!valid) {
+                  console.error(JSON.stringify(validateClientMessage.errors));
+                } else {
+                  deviceAgent.handleClientMessage({
+                    topic,
+                    message: jsonPacket,
+                  });
+                }
+              }
+            } catch (error) {
+              logger.error(error);
+            }
+          }
+        );
+
+        deviceAgent.device.on("packetsend", (packet: any) => {
+          logger.debug(JSON.stringify({ packet: packet }, null, 2));
+        });
+      }
+
+      break;
+    case existsSync(BOOTSTRAP_DEVICE_PRIVATE_KEY_FILE_PATH):
+      {
+        setTimeout(rmBootstrapCertsAndClose, 60000);
+
+        const clientId = getDeviceId();
+        const bootstrapConfig = {
+          keyPath: BOOTSTRAP_DEVICE_PRIVATE_KEY_FILE_PATH,
+          certPath: BOOTSTRAP_DEVICE_CERTIFICATE_FILE_PATH,
+          caPath: AWS_ROOT_CERTIFICATE_FILE_PATH,
+          clientId,
+          host: getIoTCoreEndpointUrl(),
+        };
+
+        const bootstrapAgent = new BootstrapAgent(bootstrapConfig);
+        bootstrapAgent.subscribeToAllTopics();
+
+        bootstrapAgent.publishMessage("$aws/certificates/create/json", "");
+
+        bootstrapAgent.device.on("connect", () => {
+          logger.info("Your device is being provisioned");
+        });
+
+        bootstrapAgent.device.on(
+          "message",
+          (topic: string, payload: string) => {
+            bootstrapAgent.handleAwsCertificateTopics(topic, payload);
+          }
+        );
+
+        bootstrapAgent.device.on("packetsend", (packet: any) => {
+          logger.debug(
+            JSON.stringify({ packet: packet.subscriptions }, null, 2)
+          );
+        });
+      }
+      break;
+    case existsSync(BOOTSTRAP_CERTIFICATES_DIR_PATH): {
+      logger.info(
+        "Device has not been created using 'device-agent device create' or there has been an issue with device creation"
+      );
+      break;
+    }
+    default: {
+      logger.info(
+        "Set device agent to local mode and retry the 'aai-agent device init' command"
+      );
+    }
+  }
+}