@alwaysai/device-agent 0.0.3 → 0.0.5

package/src/application-control/status.ts

@@ -0,0 +1,156 @@
+import compose from 'docker-compose';
+import { fetchAppReleaseHistory } from 'alwaysai/lib/infrastructure';
+import { JsSpawner } from 'alwaysai/lib/util';
+
+import { runDockerLogin } from '../docker/docker-cmd';
+import { getAppDir, requireAppInstalled } from './utils';
+import { AppStateValue, ServiceStatus, AppState } from '@alwaysai/device-agent-schemas';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+
+export async function listAppReleases(props: { projectId: string }) {
+  const { projectId } = props;
+  const releaseHistory = await fetchAppReleaseHistory(projectId);
+  return releaseHistory;
+}
+
+export async function listAppLatestRelease(props: { projectId: string }) {
+  const { projectId } = props;
+  const releaseHistory = await fetchAppReleaseHistory(projectId);
+  if (releaseHistory.length >= 1) {
+    return releaseHistory[0]['releaseHash'];
+  }
+  return undefined;
+}
+
+export async function getAppStatus(props: { projectId: string }): Promise<AppState> {
+  const { projectId } = props;
+  if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
+    throw new Error('Application is not installed');
+  }
+
+  const appDetails = {
+    projectId,
+    version: await AgentConfigFile().getAppVersion({ projectId }),
+  };
+  if (!(await AgentConfigFile().isAppReady({ projectId }))) {
+    // App is being installed or updated
+    return { appDetails, services: [] };
+  }
+
+  const appDir = getAppDir(projectId);
+
+  // Check if app is running
+  const output = await compose.ps({ cwd: appDir });
+
+  if (output.exitCode !== 0) {
+    throw new Error(
+      `Failed to get application status! stdout=${output.out} stderr=${output.err}`,
+    );
+  }
+
+  const services: ServiceStatus[] = [];
+  if (output.data.services.length === 0) {
+    // If the app has never been started (or has an empty docker-compose.yaml)
+    // the list will be empty.
+    services.push({ name: '', state: 'Stopped' });
+  } else {
+    for (const service of output.data.services) {
+      // Container name has the format:
+      // <projectId>_name_<number>
+      const matchProjectId = new RegExp(`^${projectId}`);
+      const name = service.name
+        .replace(matchProjectId, '')
+        .replace(/\d+$/, '')
+        .replace(/_/g, '');
+
+      let state: AppStateValue;
+      if (service.state.includes('Up')) {
+        state = 'Up';
+      } else if (service.state.includes('Stopped') || service.state.includes('Exit')) {
+        state = 'Stopped';
+      } else if (service.state.includes('Restarting')) {
+        state = 'Restarting';
+      } else {
+        throw new Error(`Unknown state for service! (${service.state})`);
+      }
+      services.push({ name, state });
+    }
+  }
+  return { appDetails, services };
+}
+
+export async function getAppLogs(props: {
+  projectId: string;
+  services?: string[];
+}): Promise<NodeJS.ReadableStream> {
+  const { projectId, services } = props;
+  await requireAppInstalled({ projectId });
+
+  const appDir = getAppDir(projectId);
+
+  const serviceList = services
+    ? services
+    : await (async function () {
+        const appStatus = await getAppStatus({ projectId });
+        const services: string[] = [];
+        for (const service of appStatus.services) {
+          services.push(service.name);
+        }
+        return services;
+      })();
+
+  // Use direct command with spawner in order to get a readable stream
+  return await JsSpawner().runStreaming({
+    exe: 'docker-compose',
+    args: ['logs', '-f', ...serviceList],
+    cwd: appDir,
+  });
+}
+
+export async function startApp(props: {
+  projectId: string;
+  dockerLoginToken?: string;
+}): Promise<void> {
+  const { projectId, dockerLoginToken } = props;
+  await requireAppInstalled({ projectId });
+
+  const appDir = getAppDir(projectId);
+  if (dockerLoginToken !== undefined) {
+    const result = await runDockerLogin({ token: dockerLoginToken });
+    console.log(result);
+  }
+
+  // TODO: Check if app is running
+  // Start app
+  const upOut = await compose.upAll({ cwd: appDir });
+  console.log(upOut);
+  if (upOut.exitCode !== 0) {
+    throw new Error(
+      `Failed to start application! stdout=${upOut.out} stderr=${upOut.err}`,
+    );
+  }
+  console.log('Started', projectId);
+}
+
+export async function stopApp(props: { projectId: string }): Promise<void> {
+  const { projectId } = props;
+  await requireAppInstalled({ projectId });
+
+  const appDir = getAppDir(projectId);
+  // TODO: Check if app is running
+  // Stop app
+  const output = await compose.down({ cwd: appDir });
+  console.log(output);
+  if (output.exitCode !== 0) {
+    throw new Error(
+      `Failed to stop application! stdout=${output.out} stderr=${output.err}`,
+    );
+  }
+  console.log('Stopped', projectId);
+}
+
+export async function restartApp(props: { projectId: string }): Promise<void> {
+  const { projectId } = props;
+  await stopApp({ projectId });
+  await startApp({ projectId });
+}

package/src/application-control/types.ts

@@ -0,0 +1 @@
+export type ModelDetails = { modelId: string; version: number };

package/src/application-control/utils.ts

@@ -1,47 +1,33 @@
 import compose from 'docker-compose';
 import * as path from 'path';
 import * as fs from 'fs';
+import { AAI_DIR } from 'alwaysai/lib/constants';
 
-import { JsSpawner } from '../util/spawner/js-spawner';
-import { DOT_ALWAYSAI_DIR } from '../constants';
 import { runCliCmd } from '../util/run-cli-cmd';
+import { AgentConfigFile } from '../infrastructure/agent-config';
 
-export const APP_ROOT = path.join(DOT_ALWAYSAI_DIR, 'applications');
-const RELEASE_FILENAME = 'alwaysai.release.json';
+export const APP_ROOT = path.join(AAI_DIR, 'applications');
 const DOCKER_COMPOSE_FILENAME = 'docker-compose.yaml';
 
 export function getAppDir(projectId: string): string {
   return path.join(APP_ROOT, projectId);
 }
 
-export async function isAppInstalled(props: { appDir: string }): Promise<boolean> {
-  const { appDir } = props;
-  // TODO: Could do more sophisticated check
-  if (fs.existsSync(appDir)) {
-    return true;
+export async function requireAppInstalled(props: { projectId: string }) {
+  const { projectId } = props;
+  // Ensure an app that is being installed or updated is not modified
+  if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
+    throw new Error('Application is not installed');
+  }
+  if (!(await AgentConfigFile().isAppReady({ projectId }))) {
+    throw new Error('Application is not done installing or updating');
   }
-  return false;
-}
-
-export async function setAppVersion(props: { appDir: string; version: string }) {
-  const { appDir, version } = props;
-  // TODO: Switch to node-config
-  const versionPath = path.join(appDir, RELEASE_FILENAME);
-  const versionData = { version };
-  await JsSpawner().writeFile(versionPath, JSON.stringify(versionData, null, 2));
-}
-
-export async function getAppVersion(props: { appDir: string }) {
-  const { appDir } = props;
-  const versionPath = path.join(appDir, RELEASE_FILENAME);
-  const releaseData = JSON.parse(await JsSpawner().readFile(versionPath));
-  return releaseData.version;
 }
 
 export async function buildApp(props: { appDir: string }) {
   const { appDir } = props;
   await runCliCmd({
-    cmd: ['app', 'configure', '--yes', '--protocol', 'docker:'],
+    cmd: ['app', 'configure', '--yes', '--protocol', 'docker:', '--skip-model-sync'],
     cwd: appDir,
   });
   await runCliCmd({ cmd: ['app', 'install', '--yes', '--clean', '--pull'], cwd: appDir });
@@ -55,7 +41,6 @@ export async function buildApp(props: { appDir: string }) {
   }
 
   const buildOut = await compose.buildAll({ cwd: appDir });
-  console.log(buildOut);
   if (buildOut.exitCode !== 0) {
     throw new Error(
       `Failed to build application! stdout=${buildOut.out} stderr=${buildOut.err}`,

package/src/cloud-connection/device-agent-cloud-connection.ts

@@ -1,15 +1,46 @@
 const awsIot = require('aws-iot-device-sdk');
-import { getIoTCoreEndpointUrl } from '../util/urls';
+import { getIoTCoreEndpointUrl } from '../infrastructure/urls';
 import {
   getPrivateKeyFilePath,
   getCertificateFilePath,
   getRootCertificateFilePath,
 } from '../util/directories';
+import {
+  startApp,
+  stopApp,
+  restartApp,
+  getAppStatus,
+  getAppLogs,
+} from '../application-control/status';
+import { installApp } from '../application-control/install';
+import { getCpuUtil, getDiskUtil, getMemUtil } from '../device-control/device-control';
+import {
+  DeviceAgentMessage,
+  DeviceStatsMessage,
+  AppState,
+  AppStateMessage,
+  ActionMessage,
+  AppLogs,
+  AppLogsMessage,
+  InstallationStatusMessage,
+  InstallationStatus,
+  InstallationStatusEnum,
+} from '@alwaysai/device-agent-schemas';
+import { getDeviceId } from '../util/get-device-id';
+import { AgentConfigFile } from '../infrastructure/agent-config';
 
 export class DeviceAgentCloudConnection {
-  public device = awsIot.device;
-  private clientId = 'DEVICE_AGENT'; // TODO: use CLI to get either the username or the device ID? how do we determine the device ID in this case?
+  private clientId = getDeviceId();
   private host = getIoTCoreEndpointUrl();
+  private publishable = false;
+  private readonly publishInterval = 1000;
+  private readonly installationStatusInterval = 3000;
+  private installationStatus: InstallationStatus;
+  private publishableTimeout: ReturnType<typeof setTimeout>;
+
+  public device = awsIot.device;
+  public agentTopicPrefix = `destination/agent/device/${this.clientId}/topic/`;
+  public cloudTopicPrefix = `destination/cloud/device/${this.clientId}/topic/`;
 
   constructor() {
     this.device = awsIot.device({
@@ -20,34 +51,270 @@ export class DeviceAgentCloudConnection {
       host: this.host,
     });
 
-    // TODO: subscribe to topics here
-    this.device.subscribe('command');
+    this.device.subscribe(`${this.agentTopicPrefix}command`);
+    this.device.subscribe(`${this.agentTopicPrefix}response`);
+  }
+
+  public setInstallationStatus(installationStatus: InstallationStatus) {
+    this.installationStatus = installationStatus;
+  }
+
+  public getInstallationStatus(): InstallationStatus {
+    return this.installationStatus;
+  }
+
+  public getPublishable(): boolean {
+    return this.publishable;
+  }
+
+  public setPublishable(enabled: boolean) {
+    this.publishable = enabled;
+  }
+
+  public restartPublishableTimeout() {
+    clearTimeout(this.publishableTimeout);
+    this.publishableTimeout = setTimeout(() => {
+      this.setPublishable(false);
+    }, 600000); // 10 min
   }
 
-  public getClientId() {
+  public getPublishInterval() {
+    return this.publishInterval;
+  }
+
+  public getInstallationStatusInterval() {
+    return this.installationStatusInterval;
+  }
+
+  public getClientId(): string {
     return this.clientId;
   }
 
   public publishMessage(topic: string, message: string) {
-    console.log('publishing message ', message);
-    this.device.publish(topic, JSON.stringify({ message }));
+    this.device.publish(topic, message);
   }
 }
 
 export function runDeviceAgentCloudInterface() {
   const deviceAgent = new DeviceAgentCloudConnection();
 
-  deviceAgent.device.on('connect', function() {
+  async function buildMessagePacket(
+    topic: string,
+    payload:
+      | DeviceStatsMessage
+      | ActionMessage
+      | AppStateMessage
+      | AppLogsMessage
+      | InstallationStatusMessage,
+  ): Promise<DeviceAgentMessage> {
+    const packet = {
+      timestamp: new Date().toUTCString(),
+      deviceId: deviceAgent.getClientId(),
+      topic,
+      payload,
+    };
+    return packet;
+  }
+
+  async function getAppLogsMessage(): Promise<AppLogsMessage> {
+    const appLogsList: AppLogs[] = [];
+    const apps = await AgentConfigFile().getReadyApps();
+    for (const app of apps) {
+      const projectId = app.projectId;
+      const readable = await getAppLogs({ projectId });
+      const logs: any[] = [];
+      readable.setEncoding('utf8');
+      for await (const chunk of readable) {
+        logs.push(chunk);
+      }
+      const appLogs = {
+        projectId,
+        logs,
+      };
+      appLogsList.push(appLogs);
+    }
+    const applicationStatePackage = {
+      applicationLogs: appLogsList,
+    };
+    return applicationStatePackage;
+  }
+
+  const publishAppLogs = setInterval(async function () {
+    const appLogsMessage = await getAppLogsMessage();
+    const topic = `device/${deviceAgent.getClientId()}/topic/application-management`;
+    const appLogsPacket = await buildMessagePacket(topic, appLogsMessage);
+
+    deviceAgent.publishMessage(topic, JSON.stringify({ appLogsPacket }));
+  }, deviceAgent.getPublishInterval());
+
+  async function getAppStateMessage(): Promise<AppStateMessage> {
+    const appStateMessage: AppState[] = [];
+    const apps = await AgentConfigFile().getApps();
+    for (const app of apps) {
+      const projectId = app.projectId;
+      const status = await getAppStatus({ projectId });
+      appStateMessage.push(status);
+    }
+    const applicationStatePackage = {
+      applicationState: appStateMessage,
+    };
+    return applicationStatePackage;
+  }
+
+  const publishAppState = setInterval(async function () {
+    if (deviceAgent.getPublishable()) {
+      const topic = `${deviceAgent.cloudTopicPrefix}application-management`;
+      const appStateMessage = await getAppStateMessage();
+      const appStatePacket = await buildMessagePacket(topic, appStateMessage);
+      deviceAgent.publishMessage(topic, JSON.stringify({ appStatePacket }));
+    }
+  }, deviceAgent.getPublishInterval());
+
+  async function getDeviceStatsMessage(): Promise<DeviceStatsMessage> {
+    const cpuUsage = await getCpuUtil();
+    const diskUtil = await getDiskUtil();
+    const memUtil = await getMemUtil();
+
+    const deviceStatsMessage = {
+      deviceStats: {
+        cpuUsage,
+        diskUtil,
+        usedMemoryPercentage: memUtil,
+      },
+    };
+    return deviceStatsMessage;
+  }
+
+  const publishDeviceStats = setInterval(async function () {
+    if (deviceAgent.getPublishable()) {
+      const topic = `${deviceAgent.cloudTopicPrefix}device-management`;
+      const deviceStatsMessage = await getDeviceStatsMessage();
+      const deviceStatsPacket = await buildMessagePacket(topic, deviceStatsMessage);
+      deviceAgent.publishMessage(topic, JSON.stringify({ deviceStatsPacket }));
+    }
+  }, deviceAgent.getPublishInterval());
+
+  const publishDeviceRequest = async ({ projectId }) => {
+    const topic = `${deviceAgent.cloudTopicPrefix}request`;
+    const deviceRequestPackage = {
+      timestamp: new Date().toUTCString(),
+      deviceId: deviceAgent.getClientId(),
+      projectId,
+      releaseHash: '',
+      topic,
+    };
+    deviceAgent.publishMessage(
+      topic,
+      JSON.stringify({ device_request: deviceRequestPackage }),
+    );
+  };
+
+  const publishInstallationStatus = async (interval: NodeJS.Timeout) => {
+    const topic = `${deviceAgent.cloudTopicPrefix}installation-status`;
+    const installationStatus = deviceAgent.getInstallationStatus();
+    const installationStatusPacket = await buildMessagePacket(topic, {
+      installationStatus,
+    });
+    deviceAgent.device.publish(topic, JSON.stringify({ installationStatusPacket }));
+    if (installationStatus.status !== InstallationStatusEnum.IN_PROGRESS) {
+      clearInterval(interval);
+    }
+  };
+
+  const handleMessageTopic = async ({ topic, payload }) => {
+    const action = payload['action'];
+    const actionPayload: any = payload[action];
+    const type = topic.split('/').slice(-1)[0];
+
+    if (!type) {
+      return false;
+    }
+
+    switch (action) {
+      /**
+       * Install app package based on the given project ID
+       */
+      case 'install':
+        /**
+          {
+            "action": "install",
+            "install": {
+              "releaseHash": "7fb2a812f9e7aa193208dac353521965da50d755085162066c125592f1ed760b",
+              "projectId": "786e4686-a681-4cff-9e17-1e7d385c0fdb"
+            }
+          }
+         */
+        if (type === 'response') {
+          deviceAgent.setInstallationStatus({
+            status: InstallationStatusEnum.IN_PROGRESS,
+          });
+
+          const installationStatusPing = setInterval(
+            () => publishInstallationStatus(installationStatusPing),
+            deviceAgent.getInstallationStatusInterval(),
+          );
+
+          // Install the app with the given url
+          await (async () => {
+            try {
+              await installApp(actionPayload);
+              deviceAgent.setInstallationStatus({
+                status: InstallationStatusEnum.SUCCESS,
+              });
+            } catch (e) {
+              const reason: string = e.message;
+              deviceAgent.setInstallationStatus({
+                status: InstallationStatusEnum.FAILURE,
+                reason,
+              });
+            }
+          })();
+          break;
+        }
+        publishDeviceRequest(actionPayload);
+        break;
+      case 'start':
+        startApp({ projectId: actionPayload.projectId });
+        break;
+      case 'stop':
+        stopApp({ projectId: actionPayload.projectId });
+        break;
+      case 'restart':
+        restartApp({ projectId: actionPayload.projectId });
+        break;
+      /**
+       * Allow/disallow publishing on this device. By default, (publishable = false)
+       */
+      case 'publishable':
+        /**
+          {
+            "action": "publishable",
+            "publishable": true | false
+          }
+         */
+        deviceAgent.setPublishable(actionPayload);
+        deviceAgent.restartPublishableTimeout();
+        break;
+      default:
+        break;
+    }
+  };
+
+  deviceAgent.device.on('connect', function () {
     deviceAgent.publishMessage('connection', deviceAgent.getClientId());
     console.log('Device Agent has connected to the cloud');
   });
 
-  deviceAgent.device.on('disconnect', function() {
+  deviceAgent.device.on('disconnect', function () {
     console.log('Device Agent has been disconnected from the cloud');
   });
 
-  deviceAgent.device.on('message', function(topic, payload) {
-    // TODO: parse/determine action based on topic/payload here
-    console.log('message', topic, payload.toString());
+  deviceAgent.device.on('message', function (topic, payload) {
+    // ToDo: insert valdiation here for incoming messages.  Maybe we will use a JSON schema for the message structure.
+    try {
+      handleMessageTopic({ topic, payload: JSON.parse(payload) });
+    } catch (error) {
+      console.error(error);
+    }
   });
 }

package/src/docker/docker-cmd.ts

@@ -1,4 +1,4 @@
-import { JsSpawner } from '../util/spawner/js-spawner';
+import { JsSpawner } from 'alwaysai/lib/util';
 
 export async function runDockerLogin(props: { token: string }) {
   const { token } = props;

package/src/docker/docker-compose-cmd.ts

@@ -1,4 +1,4 @@
-import { JsSpawner } from '../util/spawner/js-spawner';
+import { JsSpawner } from 'alwaysai/lib/util';
 
 export async function runDockerComposeCmd(props: { args: string[]; dir: string }) {
   const { args, dir } = props;

package/src/environment.ts

@@ -2,6 +2,8 @@ import { platform } from 'os';
 
 export const ALWAYSAI_OS_PLATFORM = parseOsPlatform(process.env.ALWAYSAI_OS_PLATFORM);
 export const ALWAYSAI_SHOW_HIDDEN = parseBoolean(process.env.ALWAYSAI_SHOW_HIDDEN);
+export const ALWAYSAI_DEVICE_AGENT_MODE = process.env.ALWAYSAI_DEVICE_AGENT_MODE;
+export const ALWAYSAI_LOG_LEVEL = process.env.AAI_LOG_LEVEL;
 
 function parseOsPlatform(str: string | undefined): NodeJS.Platform {
   switch (str) {

package/src/index.ts

@@ -1,21 +1,24 @@
 #!/usr/bin/env node
 
-// This file is the entry point for when this package is run as a CLI. Though
-// this file is not executable, the above "shebang" line is necessary as an
-// indicator to npm that this file is a Node.js script, not a shell script e.g.
+/*
+Though this file is not executable, the above "shebang" line is necessary as an
+indicator to npm that this file is a Node.js script, not a shell script e.g.
+*/
 
 import { runCliAndExit } from '@alwaysai/alwayscli';
 import { root } from './root';
-import { runWebInterface } from './web/web-interface';
 import { runDeviceAgentCloudInterface } from './cloud-connection/device-agent-cloud-connection';
+import { AgentConfigFile } from './infrastructure/agent-config';
+import { ALWAYSAI_DEVICE_AGENT_MODE } from './environment';
 
 if (module === require.main) {
   console.log('Starting alwaysAI Device Agent');
+  if (!AgentConfigFile().exists()) {
+    AgentConfigFile().initialize();
+  }
 
-  if (process.env.ALWAYSAI_DEVICE_AGENT_MODE === 'cloud') {
+  if (ALWAYSAI_DEVICE_AGENT_MODE === 'cloud') {
     runDeviceAgentCloudInterface();
-  } else if (process.env.ALWAYSAI_DEVICE_AGENT_MODE === 'web') {
-    runWebInterface();
   } else {
     runCliAndExit(root, {});
   }