@alwaysai/device-agent 0.0.3 → 0.0.5

package/readme.md

@@ -1,115 +1,219 @@
 # alwaysAI Device Agent
 
+The alwaysAI Device Agent is a tool that runs on an alwaysAI device provisioned
+for production remote deployment. It enables management of one or more alwaysAI
+apps on a particular device via a command line interface.
+
+Note that the Device Agent is still in an experimental phase and these commands
+are likely to change. This guide will be updated with the latest usage as things
+change.
+
 ## System Requirements
 
- 1. Preferably running Linux, but can hack it to work on Mac too
- 2. Docker installed with user access (On Linux: `$ sudo usermod -aG docker $USER`, nothing needed for Mac)
- 3. Docker Compose installed
- 4. npm and node (for build and test)
+* Recent version of Debian or Ubuntu
+* `npm` >= 7.0.0
+* `node` >= 16.0.0
+* Recent version of `docker` with user access
+  * On Linux: `$ sudo usermod -aG docker $USER`
+* Recent version of `docker-compose`
+* `curl` installed.
+
+## Provision Device
 
-## Lint and Build
+### Provision the device
+
+If you're system already meets all the system requirements, you can simply install the device agent via `npm`:
 
 ```
-$ npm run lint:fix
-$ npm run build
+sudo npm install -g @alwaysai/device-agent@latest
 ```
 
-## Usage
+Otherwise, you can run the provision script to install the agent and its dependencies:
 
-There are three modes of operation: CLI, which is great for testing new features, the device-hosted web dashboard, and the IoT Core cloud connection.
+```
+curl -fsSL https://alwaysai-artifacts-prod.s3.us-west-1.amazonaws.com/device-agent/install-device-agent.sh | sudo -E bash -
+```
+
+The Device Agent will be available in the terminal as `aai-agent`.
+
+Next, log in to the Device Agent with the following command:
 
-To enable the cloud IoT Core connection, set the following environment variable:
 ```
-export ALWAYSAI_DEVICE_AGENT_MODE="cloud"
+aai-agent login --email <username> --password <password> [--device <device_id>]
 ```
 
-To enable the device-hosted web dashboard, set the following environment variable:
+Note that if you already have a device registered with the alwaysAI cloud, you
+can optionally provide that device ID in the login command. If you don't yet
+have a device configured, follow the next step to initialize the device.
+
+### Initialize the alwaysAI device
+
+After logging in, run the following command to initialize the current device as
+an alwaysAI device:
+
 ```
-export ALWAYSAI_DEVICE_AGENT_MODE="web"
+$ aai-agent device init --name <name> [--description <description>]
 ```
 
-Any other value will run the CLI.
+For example:
 
-On Mac you need to override the OS to remove some usage restrictions:
 ```
-export ALWAYSAI_OS_PLATFORM="linux"
+$ aai-agent device init --name test-dev-1 --description "Device for testing alwaysAI Device Agent"
 ```
 
-### CLI
+## Run an alwaysAI application on your device
+
+### Publish an application release and install on device
+
+First, you must publish your application to the alwaysAI cloud. From the root
+of your application directory (where the `alwaysai.app.json` file is) on your
+development host:
 
-Here are the available commands:
 ```
-$ node lib --help
-Starting alwaysAI Device Agent
-Usage: aai-agent <subcommand> ...
+aai app configure
+aai app publish
+```
 
-   Manage your alwaysAI production device
+The output of the final command will give you the release hash that was
+published. You can run `aai release list --project <project_id>` to list all
+release versions for the project, where project ID can be found in the
+`alwaysai.project.json` file.
 
-Subcommands:
+Now you can install the application on the device using the device agent. Run
+the following on the device where the Device Agent is installed:
 
-   get-device-info    : Get device info
-   install-test-app   : Install an empty test application
-   get-installed-apps : List all installed apps
-   install-app        : Install an alwaysAI app from a project
-   get-app-status     : Get the status of an installed alwaysAI app
-   start-app          : Start an installed alwaysAI app
-   stop-app           : Stop a running alwaysAI app
-   uninstall-app      : Remove an alwaysAI app
 ```
+aai-agent app install --project <project_id> [--release <release_hash>]
+```
+
+### Control the application
 
-To install a test app, run:
+Run the following commands on the device where the Device Agent is installed:
+
+Get application status:
 ```
-$ node lib install-test-app
+aai-agent app status --project <project_id>
 ```
 
-The test app has the project ID 16ad11bb-8c3a-4a15-923e-49f7a8311dc6.
+Start the application:
+```
+aai-agent app start --project <project_id>
+```
 
-Now you can start, check the status, and stop using the commands:
+Show the application logs:
 ```
-$ node lib start-app --project 16ad11bb-8c3a-4a15-923e-49f7a8311dc6
-Starting alwaysAI Device Agent
-Starting app...
-{
-  exitCode: 0,
-  err: 'Starting 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... \r\n' +
-    'Starting 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... done\r\n',
-  out: ''
-}
-$ node lib get-app-status --project 16ad11bb-8c3a-4a15-923e-49f7a8311dc6
-Starting alwaysAI Device Agent
-[
-  {
-    name: '16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1',
-    command: 'tail -f /dev/null',
-    state: 'Up',
-    ports: []
-  }
-]
-$  node lib stop-app --project 16ad11bb-8c3a-4a15-923e-49f7a8311dc6
-Starting alwaysAI Device Agent
-{
-  exitCode: 0,
-  err: 'Stopping 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... \r\n' +
-    'Stopping 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... done\r\n' +
-    'Removing 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... \r\n' +
-    'Removing 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... done\r\n' +
-    'Removing network 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_default\n',
-  out: ''
-}
+aai-agent app logs --project <project_id>
+```
+
+Stop the application:
+```
+aai-agent app stop --project <project_id>
+```
+
+Uninstall the application:
+```
+aai-agent app uninstall --project <project_id>
+```
+
+### Manage application models
+
+There are several ways to manage the models for your application.
+
+#### Update models to new version of the same model ID
+
+If a new version of a model is published for an existing model ID, and an
+application is already configured to be using that model ID, updating to the
+new model can simply be done with:
+
+```
+aai-agent app stop --project <project_id>
+aai-agent app update-models --project <project_id>
+aai-agent app start --project <project_id>
+```
+
+#### Replace models for an application for new ID
+
+If you'd like to install an entirely new model to an application, replacing the
+model the app was originally configured with, run the following command:
+
+```
+aai-agent app stop --project <project_id>
+aai-agent app replace-models --project <project_id> --models <model_id_1> [<model_id_2> ...]
+aai-agent app start --project <project_id>
+```
+
+If you plan on using this method, you can make your application source model ID
+agnostic by providing the model ID to edgeIQ in the following way:
+
+Select the first model in the config list:
+```
+obj_detect = edgeiq.ObjectDetection(edgeiq._globals.MODEL_ID_LIST[0])
+```
+
+#### Manually download a model package
+
+To download a model package from the alwaysAI cloud and unpack to a specific directory, run:
+
+```
+aai-agent get-model-package <model ID> [--path <destination path>]
 ```
 
-### Web Dashboard
+For example, to download `alwaysai/yolo_v3` to `~/models` run:
+
+```
+aai-agent get-model-package alwaysai/yolo_v3 --path ~/models
+```
 
-You can start the web dashboard by running:
+Once the command completes, you'll see the model package in the
+`~/models/alwaysai/yolo_v3` directory.
+
+### Set and update environment variables
+
+The Device Agent enables you to set and update environment variables for all
+services or a single service of you application.
 
 ```
-node lib
+aai-agent app set-env <key=val> --project <project_id> [--service <service>]
 ```
 
-And clicking the link that comes up.
+For example, to set the following environment variable for the `alwaysai`
+service run the following command on the device:
 
-If you are developing new functionality for the cli in tandem with the device agent, you can run
 ```
-npm i --save /path/to/your/cli/repo
+aai-agent app set-env TEST_ENV=1 --project <project_id> --service alwaysai
 ```
-Remember to not commit the package.json, or revert it when you are done developing a feature
+
+## Device Agent Commands
+
+```
+$ aai-agent --help
+Starting alwaysAI Device Agent
+Usage: aai-agent <subcommand> ...
+
+   Manage your alwaysAI production device
+
+Subcommands:
+
+   login                   : Login to alwaysAI (this is meant for scripted environments)
+   app list                : List all installed apps
+   app list-releases       : List all releases for a given app
+   app list-latest-release : List the latest release hash for a given app
+   app install             : Install an alwaysAI app from a project
+   app status              : Get the status of an installed alwaysAI app
+   app start               : Start an installed alwaysAI app
+   app stop                : Stop a running alwaysAI app
+   app restart             : Restart running alwaysAI app
+   app logs                : Get logs for an application
+   app uninstall           : Remove an alwaysAI app
+   app rollback            : Rollback an alwaysAI app to the previous version
+   app show-models         : Show the application models
+   app add-model           : Add a model to an alwaysAI app
+   app remove-model        : Remove a model from an alwaysAI app
+   app replace-models      : Replace all models of an alwaysAI app with new models
+   app update-models       : Update all models for an alwaysAI app
+   app get-all-envs        : Get environment variables for an application
+   app set-env             : Set environment variables for a service
+   device init             : Initialize device
+   device get-info         : Get device info
+   get-model-package       : Download and unpack a model package
+```

package/src/application-control/backup.ts

@@ -0,0 +1,32 @@
+import * as rimraf from 'rimraf';
+
+import { copyDir } from '../util/copy-dir';
+import { buildApp, getAppDir } from './utils';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+
+export const BACKUP_EXT = '.bak';
+
+export async function createAppBackup(props: { projectId: string }) {
+  const { projectId } = props;
+  const appDir = getAppDir(projectId);
+  const backupAppDir = `${appDir}${BACKUP_EXT}`;
+  rimraf.sync(backupAppDir);
+  await copyDir({ srcPath: appDir, destPath: backupAppDir });
+  await AgentConfigFile().setAppBackup({ projectId });
+  console.log(`Backed up app ${projectId} to ${backupAppDir}`);
+}
+
+export async function rollbackApp(props: { projectId: string }) {
+  const { projectId } = props;
+  const config = await AgentConfigFile().getAppBackup({ projectId });
+  if (!config) {
+    throw new Error(`Backup doesn't exist for ${projectId}`);
+  }
+  const appDir = getAppDir(projectId);
+  const backupAppDir = `${appDir}${BACKUP_EXT}`;
+  rimraf.sync(appDir);
+  await copyDir({ srcPath: backupAppDir, destPath: appDir });
+  await buildApp({ appDir });
+  await AgentConfigFile().setAppInstalled({ projectId, version: config.version });
+  console.log(`Rolled back app ${projectId} to ${config.version}`);
+}

package/src/application-control/environment-variables.ts

@@ -0,0 +1,81 @@
+import { JsSpawner } from 'alwaysai/lib/util';
+import compose from 'docker-compose';
+import { parse, stringify } from 'yaml';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+import { getAppDir } from './utils';
+
+export async function setEnv(props: {
+  projectId: string;
+  vars: string[];
+  service?: string;
+}) {
+  const { projectId, vars } = props;
+  if (!AgentConfigFile().isAppReady({ projectId })) {
+    throw new Error(`App ${projectId} is not ready!`);
+  }
+  const appDir = getAppDir(projectId);
+  const spawner = JsSpawner({ path: appDir });
+  const composeContents = await spawner.readFile('docker-compose.yaml');
+  const composeParsed = parse(composeContents);
+  if ('services' in composeParsed) {
+    let services: string[] = Object.keys(composeParsed['services']);
+    if (props.service) {
+      console.log(services);
+      if (services.includes(props.service)) {
+        services = [props.service];
+      } else {
+        throw new Error(`Service ${props.service} not found in ${services}`);
+      }
+    }
+    for (const s of services) {
+      const service = composeParsed['services'][s];
+      // The environment field overrides the env files, so by appending to
+      // the environment list we can assure the value will be used
+      if ('environment' in service) {
+        const environment: string[] = service['environment'];
+        composeParsed['services'][s]['environment'] = environment.concat(vars);
+      } else {
+        composeParsed['services'][s]['environment'] = vars;
+      }
+    }
+  }
+  const composeOutput = stringify(composeParsed);
+  // Validate new contents
+  await compose.config({ cwd: appDir, configAsString: composeOutput });
+  await spawner.writeFile('docker-compose.yaml', composeOutput);
+}
+
+export async function getAllEnvs(props: { projectId: string }) {
+  const { projectId } = props;
+  if (!AgentConfigFile().isAppReady({ projectId })) {
+    throw new Error(`App ${projectId} is not ready!`);
+  }
+  const appDir = getAppDir(projectId);
+  const spawner = JsSpawner({ path: appDir });
+  const envVars = {};
+  const composeContents = await spawner.readFile('docker-compose.yaml');
+  const composeParsed = parse(composeContents);
+  if ('services' in composeParsed) {
+    const services = Object.keys(composeParsed['services']);
+    for (const s of services) {
+      envVars[s] = [];
+      const service = composeParsed['services'][s];
+      if ('env_file' in service) {
+        const envFiles: string[] = service['env_file'];
+        for (const ef of envFiles) {
+          envVars[s] = envVars[s].concat((await spawner.readFile(ef)).split('\n'));
+        }
+      }
+      if ('environment' in service) {
+        const environment: string[] = service['environment'];
+        envVars[s] = envVars[s].concat(environment);
+      }
+      // Filter out empty lines and comment lines
+      envVars[s] = envVars[s].filter((v) => {
+        return v !== '' && !v.includes('#');
+      });
+    }
+  }
+
+  return envVars;
+}

package/src/application-control/index.ts

@@ -0,0 +1,40 @@
+import { installApp, uninstallApp } from './install';
+import { rollbackApp } from './backup';
+import {
+  listAppReleases,
+  listAppLatestRelease,
+  getAppStatus,
+  startApp,
+  getAppLogs,
+  stopApp,
+  restartApp,
+} from './status';
+import { ModelDetails } from './types';
+import { getAllEnvs, setEnv } from './environment-variables';
+
+export {
+  installApp,
+  uninstallApp,
+  rollbackApp,
+  listAppReleases,
+  listAppLatestRelease,
+  getAppStatus,
+  startApp,
+  getAppLogs,
+  stopApp,
+  restartApp,
+  ModelDetails,
+  getAllEnvs,
+  setEnv,
+};
+
+// CLI-mode only
+import {
+  addModel,
+  getAppModels,
+  removeModel,
+  replaceModels,
+  updateModels,
+} from './models';
+
+export { addModel, getAppModels, removeModel, replaceModels, updateModels };

package/src/application-control/install.ts

@@ -0,0 +1,126 @@
+import * as rimraf from 'rimraf';
+import * as fs from 'fs';
+import * as path from 'path';
+import nodeFetch, { Response } from 'node-fetch';
+import { JsSpawner } from 'alwaysai/lib/util';
+
+import { runCliCmd } from '../util/run-cli-cmd';
+import { buildApp, getAppDir } from './utils';
+import { AppDetails } from '@alwaysai/device-agent-schemas';
+import { BACKUP_EXT, createAppBackup } from './backup';
+import { stopApp } from './status';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+
+export async function getInstalledApps(): Promise<AppDetails[]> {
+  const apps = await AgentConfigFile().getApps();
+  const appDetails: AppDetails[] = [];
+  for (const app of apps) {
+    const { projectId, version } = app;
+    appDetails.push({ projectId, version });
+  }
+  return appDetails;
+}
+
+export type InstallAppProps = {
+  projectId: string;
+  releaseHash: string;
+  presignedAppUrl?: string;
+};
+
+export async function installAppPackage({ localDest, presignedUrl }): Promise<void> {
+  const response = await nodeFetch(presignedUrl);
+  if (response.status !== 200) {
+    // If the URL is invalid; I think we shouldn't get here with the new changes
+    throw new Error(
+      `Status Code: ${response.status}, ${response.statusText}. Response: ${response.body}`,
+    );
+  }
+
+  /**
+   * Write the app package to the hash directory
+   */
+  const stream = response.body.pipe(fs.createWriteStream(localDest));
+  await new Promise((resolve, reject) => {
+    stream.on('finish', resolve);
+    stream.on('error', reject);
+  });
+}
+
+export async function installApp(props: InstallAppProps): Promise<void> {
+  const { projectId, releaseHash, presignedAppUrl } = props;
+  const appDir = getAppDir(projectId);
+  const spawner = JsSpawner();
+  if (await AgentConfigFile().isAppPresent({ projectId })) {
+    if (!(await AgentConfigFile().isAppReady({ projectId }))) {
+      throw new Error('Application already has installation in progress!');
+    }
+    await AgentConfigFile().setAppInstalling({ projectId, version: releaseHash });
+    console.log('Application is already installed, updating');
+    await createAppBackup({ projectId });
+    await spawner.rimraf(appDir);
+  } else {
+    await AgentConfigFile().setAppInstalling({ projectId, version: releaseHash });
+  }
+
+  /**
+   * Prep directory and fetch the app package
+   */
+  await spawner.mkdirp(appDir);
+  const localDest = path.join(appDir, `${path.basename(releaseHash)}.tgz`);
+  if (!presignedAppUrl) {
+    // Download the application using the CLI
+    await runCliCmd({
+      cmd: [
+        'release',
+        'pull',
+        '--yes',
+        '--project',
+        projectId,
+        '--releaseHash',
+        releaseHash,
+      ],
+      cwd: appDir,
+    });
+  } else {
+    await installAppPackage({ localDest, presignedUrl: presignedAppUrl });
+  }
+
+  /**
+   * Unpack app package
+   */
+  await spawner.untar(fs.createReadStream(localDest), appDir);
+  console.log(`Unpacked application to ${appDir}`);
+
+  /**
+   * Build app
+   * ToDo: migrate the model pull to use MQTT workflow
+   */
+  await buildApp({ appDir });
+
+  /**
+   * Remove tar file
+   */
+  await spawner.rimraf(localDest);
+
+  await AgentConfigFile().setAppInstalled({ projectId, version: releaseHash });
+  console.log(`Installed ${projectId} to ${appDir}`);
+}
+
+export async function uninstallApp(props: { projectId: string }): Promise<void> {
+  const { projectId } = props;
+  if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
+    console.log(`Application ${projectId} not installed`);
+    return;
+  }
+  try {
+    await stopApp({ projectId });
+  } catch {
+    console.log('Failed to stop app, may be left running...');
+  }
+  await AgentConfigFile().setAppUninstalled({ projectId });
+  // Delete application directory and backup
+  const appDir = getAppDir(projectId);
+  rimraf.sync(appDir);
+  rimraf.sync(`${appDir}${BACKUP_EXT}`);
+  console.log('Uninstalled', projectId);
+}

package/src/application-control/models.ts

@@ -1,12 +1,36 @@
+import { JsSpawner } from 'alwaysai/lib/util';
+import { existsSync } from 'fs';
+import { join } from 'path';
+
 import { runCliCmd } from '../util/run-cli-cmd';
-import { buildApp, getAppDir, isAppInstalled } from './utils';
+import { ModelDetails } from './types';
+import { buildApp, getAppDir, requireAppInstalled } from './utils';
+
+export async function getAppModels(props: { projectId: string }) {
+  const { projectId } = props;
+  await requireAppInstalled({ projectId });
+
+  const appDir = getAppDir(projectId);
+  const appCfgPath = join(appDir, 'alwaysai.app.json');
+  if (!existsSync(appCfgPath)) {
+    throw new Error('Application config not found!');
+  }
+  const appCfg = JSON.parse(await JsSpawner().readFile(appCfgPath));
+  const modelDetails: ModelDetails[] = [];
+  if (!('models' in appCfg)) {
+    return modelDetails;
+  }
+  for (const model in appCfg['models']) {
+    modelDetails.push({ modelId: model, version: appCfg['models'][model] });
+  }
+  return modelDetails;
+}
 
 export async function addModel(props: { projectId: string; modelId: string }) {
   const { projectId, modelId } = props;
+  await requireAppInstalled({ projectId });
+
   const appDir = getAppDir(projectId);
-  if (!(await isAppInstalled({ appDir }))) {
-    throw new Error('Application is not installed');
-  }
   await runCliCmd({
     cmd: ['app', 'models', 'add', modelId],
     cwd: appDir,
@@ -16,22 +40,38 @@ export async function addModel(props: { projectId: string; modelId: string }) {
 
 export async function removeModel(props: { projectId: string; modelId: string }) {
   const { projectId, modelId } = props;
+  await requireAppInstalled({ projectId });
+
+  const appDir = getAppDir(projectId);
+  await runCliCmd({
+    cmd: ['app', 'models', 'remove', modelId, '--purge'],
+    cwd: appDir,
+  });
+}
+
+export async function replaceModels(props: { projectId: string; modelIds: string[] }) {
+  const { projectId, modelIds } = props;
+  await requireAppInstalled({ projectId });
+
   const appDir = getAppDir(projectId);
-  if (!(await isAppInstalled({ appDir }))) {
-    throw new Error('Application is not installed');
-  }
   await runCliCmd({
-    cmd: ['app', 'models', 'remove', modelId],
+    cmd: ['app', 'models', 'remove-all', '--purge'],
     cwd: appDir,
   });
+  for (const modelId of modelIds) {
+    await runCliCmd({
+      cmd: ['app', 'models', 'add', modelId],
+      cwd: appDir,
+    });
+  }
+  await buildApp({ appDir });
 }
 
 export async function updateModels(props: { projectId: string }) {
   const { projectId } = props;
+  await requireAppInstalled({ projectId });
+
   const appDir = getAppDir(projectId);
-  if (!(await isAppInstalled({ appDir }))) {
-    throw new Error('Application is not installed');
-  }
   await runCliCmd({
     cmd: ['app', 'models', 'update'],
     cwd: appDir,