@alwaysai/device-agent 0.0.3 → 0.0.4-1

package/package.json

@@ -1,9 +1,13 @@
 {
   "name": "@alwaysai/device-agent",
   "description": "The alwaysAI Device Agent",
-  "version": "0.0.3",
-  "main": "lib/exports.js",
-  "types": "lib/exports.d.ts",
+  "version": "0.0.4-1",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "publishConfig": {
+    "access": "public",
+    "tag": "next"
+  },
   "bin": {
     "aai-agent": "lib/index.js"
   },
@@ -12,34 +16,35 @@
     "node": ">=16.0.0"
   },
   "scripts": {
-    "build": "tsc --project tsconfig.build.json && copyfiles -u 1 src/web/*.html src/web/static/* lib/",
+    "build": "tsc --project tsconfig.build.json",
     "clean": "rimraf lib packages",
     "build:clean": "npm run clean && npm run build",
     "build:watch": "tsc --project . --watch",
     "lint": "tslint --project tsconfig.build.json",
     "lint:fix": "npm run lint -- --fix",
-    "test": "npm run lint",
+    "test": "npm run test:unit",
     "test:unit": "jest --coverage src",
     "test:watch": "npm run test:unit -- --watch",
     "preversion": "npm test && npm run build:clean",
     "postversion": "npm publish"
   },
   "dependencies": {
-    "@alwaysai/alwayscli": "0.3.0",
-    "@alwaysai/config-nodejs": "0.0.2",
+    "@alwaysai/alwayscli": "0.3.1",
+    "@alwaysai/config-nodejs": "0.1.0",
+    "@alwaysai/device-agent-schemas": "0.0.1",
     "@carnesen/coded-error": "0.4.0",
     "@types/mkdirp": "1.0.2",
     "@types/pump": "1.1.1",
     "@types/rimraf": "3.0.2",
     "@types/signal-exit": "3.0.1",
     "@types/tar": "6.1.1",
+    "ajv": "8.11.0",
+    "alwaysai": "1.8.0",
     "aws-iot-device-sdk": "2.2.12",
-    "alwaysai": "1.5.1",
     "docker-compose": "0.23.17",
     "express": "4.17.3",
-    "fp-ts": "2.11.5",
-    "io-ts": "2.2.16",
     "mkdirp": "1.0.4",
+    "node-fetch": "2.6.1",
     "node-os-utils": "1.3.6",
     "pump": "3.0.0",
     "rimraf": "3.0.2",
@@ -48,17 +53,17 @@
     "tar": "6.1.11",
     "tree-kill": "1.2.2",
     "winston": "3.3.3",
-    "winston-daily-rotate-file": "4.5.5"
+    "winston-daily-rotate-file": "4.5.5",
+    "yaml": "2.1.1"
   },
   "devDependencies": {
-    "@alwaysai/tsconfig": "0.0.0",
-    "@alwaysai/tslint-config": "0.0.3",
+    "@alwaysai/tsconfig": "0.0.1",
+    "@alwaysai/tslint-config": "0.0.4",
     "@carnesen/run-and-catch": "0.4.3",
     "@types/jest": "27.0.3",
     "@types/node": "16.11.12",
     "@types/sinon": "10.0.6",
     "aws-sdk": "^2.1046.0",
-    "copyfiles": "^2.4.1",
     "cp-cli": "2.0.0",
     "get-stream": "6.0.1",
     "jest": "27.4.3",

package/readme.md

@@ -1,115 +1,192 @@
 # alwaysAI Device Agent
 
+The alwaysAI Device Agent is a tool that runs on an alwaysAI device provisioned
+for production remote deployment. It enables management of one or more alwaysAI
+apps on a particular device via a command line interface.
+
+Note that the Device Agent is still in an experimental phase and these commands
+are likely to change. This guide will be updated with the latest usage as things
+change.
+
 ## System Requirements
 
- 1. Preferably running Linux, but can hack it to work on Mac too
- 2. Docker installed with user access (On Linux: `$ sudo usermod -aG docker $USER`, nothing needed for Mac)
- 3. Docker Compose installed
- 4. npm and node (for build and test)
+* Recent version of Debian or Ubuntu
+* `npm` >= 7.0.0
+* `node` >= 16.0.0
+* Recent version of `docker` with user access
+  * On Linux: `$ sudo usermod -aG docker $USER`
+* Recent version of `docker-compose`
+* `curl` installed.
+
+## Provision Device
+
+### Create an alwaysAI device
 
-## Lint and Build
+On any machine running the alwaysAI CLI you can create a new production device
+in the alwaysAI cloud:
 
 ```
-$ npm run lint:fix
-$ npm run build
+$ aai generate-production-device --name test-dev-1 --description "Device for testing alwaysAI Device Agent"
+✔ Create new production device
+✔ Write device credentials to /home/alwaysai/abc8fbbf-5d7f-4e49-92b7-8feedffeedbf
+Created test-dev-1: device ID=abc8fbbf-5d7f-4e49-92b7-8feedffeedbf
 ```
 
-## Usage
+Note down the device ID for use with Device Agent commands. The device
+credentials are also stored locally for you to keep in a safe place, but you
+won't need those since the Device Agent will update them automatically for you.
 
-There are three modes of operation: CLI, which is great for testing new features, the device-hosted web dashboard, and the IoT Core cloud connection.
+### Provision the device
+
+If you're system already meets all the system requirements, you can simply install the device agent via `npm`:
 
-To enable the cloud IoT Core connection, set the following environment variable:
 ```
-export ALWAYSAI_DEVICE_AGENT_MODE="cloud"
+sudo npm install -g @alwaysai/device-agent@latest
 ```
 
-To enable the device-hosted web dashboard, set the following environment variable:
+Otherwise, you can run the provision script to install the agent and its dependencies:
+
 ```
-export ALWAYSAI_DEVICE_AGENT_MODE="web"
+curl -fsSL https://alwaysai-artifacts-prod.s3.us-west-1.amazonaws.com/device-agent/install-device-agent.sh | sudo -E bash -
 ```
 
-Any other value will run the CLI.
+The Device Agent will be available in the terminal as `aai-agent`.
+
+Next, log in to the Device Agent with the following command:
 
-On Mac you need to override the OS to remove some usage restrictions:
 ```
-export ALWAYSAI_OS_PLATFORM="linux"
+aai-agent login --email <username> --password <password> --device <device_id>
 ```
 
-### CLI
+## Run an alwaysAI application on your device
+
+### Publish an application release and install on device
+
+First, you must publish your application to the alwaysAI cloud. From the root
+of your application directory (where the `alwaysai.app.json` file is) on your
+development host:
 
-Here are the available commands:
 ```
-$ node lib --help
-Starting alwaysAI Device Agent
-Usage: aai-agent <subcommand> ...
+aai app configure
+aai app publish
+```
 
-   Manage your alwaysAI production device
+The output of the final command will give you the release hash that was
+published. You can run `aai release list --project <project_id>` to list all
+release versions for the project, where project ID can be found in the
+`alwaysai.project.json` file.
 
-Subcommands:
+Now you can install the application on the device using the device agent. Run
+the following on the device where the Device Agent is installed:
+
+```
+aai-agent app install --project <project_id> [--release <release_hash>]
+```
+
+### Control the application
+
+Run the following commands on the device where the Device Agent is installed:
 
-   get-device-info    : Get device info
-   install-test-app   : Install an empty test application
-   get-installed-apps : List all installed apps
-   install-app        : Install an alwaysAI app from a project
-   get-app-status     : Get the status of an installed alwaysAI app
-   start-app          : Start an installed alwaysAI app
-   stop-app           : Stop a running alwaysAI app
-   uninstall-app      : Remove an alwaysAI app
+Get application status:
+```
+aai-agent app status --project <project_id>
 ```
 
-To install a test app, run:
+Start the application:
 ```
-$ node lib install-test-app
+aai-agent app start --project <project_id>
 ```
 
-The test app has the project ID 16ad11bb-8c3a-4a15-923e-49f7a8311dc6.
+Show the application logs:
+```
+aai-agent app logs --project <project_id>
+```
 
-Now you can start, check the status, and stop using the commands:
+Stop the application:
 ```
-$ node lib start-app --project 16ad11bb-8c3a-4a15-923e-49f7a8311dc6
-Starting alwaysAI Device Agent
-Starting app...
-{
-  exitCode: 0,
-  err: 'Starting 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... \r\n' +
-    'Starting 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... done\r\n',
-  out: ''
-}
-$ node lib get-app-status --project 16ad11bb-8c3a-4a15-923e-49f7a8311dc6
-Starting alwaysAI Device Agent
-[
-  {
-    name: '16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1',
-    command: 'tail -f /dev/null',
-    state: 'Up',
-    ports: []
-  }
-]
-$  node lib stop-app --project 16ad11bb-8c3a-4a15-923e-49f7a8311dc6
-Starting alwaysAI Device Agent
-{
-  exitCode: 0,
-  err: 'Stopping 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... \r\n' +
-    'Stopping 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... done\r\n' +
-    'Removing 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... \r\n' +
-    'Removing 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... done\r\n' +
-    'Removing network 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_default\n',
-  out: ''
-}
+aai-agent app stop --project <project_id>
 ```
 
-### Web Dashboard
+Uninstall the application:
+```
+aai-agent app uninstall --project <project_id>
+```
+
+### Manage application models
+
+There are several ways to manage the models for your application.
 
-You can start the web dashboard by running:
+#### Update models to new version of the same model ID
+
+If a new version of a model is published for an existing model ID, and an
+application is already configured to be using that model ID, updating to the
+new model can simply be done with:
 
 ```
-node lib
+aai-agent app stop --project <project_id>
+aai-agent app update-models --project <project_id>
+aai-agent app start --project <project_id>
 ```
 
-And clicking the link that comes up.
+#### Replace models for an application for new ID
+
+If you'd like to install an entirely new model to an application, replacing the
+model the app was originally configured with, run the following command:
 
-If you are developing new functionality for the cli in tandem with the device agent, you can run
 ```
-npm i --save /path/to/your/cli/repo
+aai-agent app stop --project <project_id>
+aai-agent app replace-models --project <project_id> --models <model_id_1> [<model_id_2> ...]
+aai-agent app start --project <project_id>
 ```
-Remember to not commit the package.json, or revert it when you are done developing a feature
+
+If you plan on using this method, you can make your application source model ID
+agnostic by providing the model ID to edgeIQ in the following way:
+
+Select the first model in the config list:
+```
+obj_detect = edgeiq.ObjectDetection(edgeiq._globals.MODEL_ID_LIST[0])
+```
+
+### Set and update environment variables
+
+The Device Agent enables you to set and update environment variables for all
+services or a single service of you application. For example, set teh following
+environment variable for the `alwaysai` service run the following command on the
+device:
+
+```
+aai-agent app set-env TEST_ENV=1 --project <project_id> --service alwaysai
+```
+
+## Device Agent Commands
+
+```
+$ aai-agent --help
+Starting alwaysAI Device Agent
+Usage: aai-agent <subcommand> ...
+
+   Manage your alwaysAI production device
+
+Subcommands:
+
+   login                   : Login to alwaysAI (this is meant for scripted environments)
+   app list                : List all installed apps
+   app list-releases       : List all releases for a given app
+   app list-latest-release : List the latest release hash for a given app
+   app install             : Install an alwaysAI app from a project
+   app status              : Get the status of an installed alwaysAI app
+   app start               : Start an installed alwaysAI app
+   app stop                : Stop a running alwaysAI app
+   app restart             : Restart running alwaysAI app
+   app logs                : Get logs for an application
+   app uninstall           : Remove an alwaysAI app
+   app rollback            : Rollback an alwaysAI app to the previous version
+   app show-models         : Show the application models
+   app add-model           : Add a model to an alwaysAI app
+   app remove-model        : Remove a model from an alwaysAI app
+   app replace-models      : Replace all models of an alwaysAI app with new models
+   app update-models       : Update all models for an alwaysAI app
+   app get-all-envs        : Get environment variables for an application
+   app set-env             : Set environment variables for a service
+   get-device-info         : Get device info
+```

package/src/application-control/backup.ts

@@ -0,0 +1,32 @@
+import * as rimraf from 'rimraf';
+
+import { copyDir } from '../util/copy-dir';
+import { buildApp, getAppDir } from './utils';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+
+export const BACKUP_EXT = '.bak';
+
+export async function createAppBackup(props: { projectId: string }) {
+  const { projectId } = props;
+  const appDir = getAppDir(projectId);
+  const backupAppDir = `${appDir}${BACKUP_EXT}`;
+  rimraf.sync(backupAppDir);
+  await copyDir({ srcPath: appDir, destPath: backupAppDir });
+  await AgentConfigFile().setAppBackup({ projectId });
+  console.log(`Backed up app ${projectId} to ${backupAppDir}`);
+}
+
+export async function rollbackApp(props: { projectId: string }) {
+  const { projectId } = props;
+  const config = await AgentConfigFile().getAppBackup({ projectId });
+  if (!config) {
+    throw new Error(`Backup doesn't exist for ${projectId}`);
+  }
+  const appDir = getAppDir(projectId);
+  const backupAppDir = `${appDir}${BACKUP_EXT}`;
+  rimraf.sync(appDir);
+  await copyDir({ srcPath: backupAppDir, destPath: appDir });
+  await buildApp({ appDir });
+  await AgentConfigFile().setAppInstalled({ projectId, version: config.version });
+  console.log(`Rolled back app ${projectId} to ${config.version}`);
+}

package/src/application-control/environment-variables.ts

@@ -0,0 +1,81 @@
+import { JsSpawner } from 'alwaysai/lib/util';
+import compose from 'docker-compose';
+import { parse, stringify } from 'yaml';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+import { getAppDir } from './utils';
+
+export async function setEnv(props: {
+  projectId: string;
+  vars: string[];
+  service?: string;
+}) {
+  const { projectId, vars } = props;
+  if (!AgentConfigFile().isAppReady({ projectId })) {
+    throw new Error(`App ${projectId} is not ready!`);
+  }
+  const appDir = getAppDir(projectId);
+  const spawner = JsSpawner({ path: appDir });
+  const composeContents = await spawner.readFile('docker-compose.yaml');
+  const composeParsed = parse(composeContents);
+  if ('services' in composeParsed) {
+    let services: string[] = Object.keys(composeParsed['services']);
+    if (props.service) {
+      console.log(services);
+      if (services.includes(props.service)) {
+        services = [props.service];
+      } else {
+        throw new Error(`Service ${props.service} not found in ${services}`);
+      }
+    }
+    for (const s of services) {
+      const service = composeParsed['services'][s];
+      // The environment field overrides the env files, so by appending to
+      // the environment list we can assure the value will be used
+      if ('environment' in service) {
+        const environment: string[] = service['environment'];
+        composeParsed['services'][s]['environment'] = environment.concat(vars);
+      } else {
+        composeParsed['services'][s]['environment'] = vars;
+      }
+    }
+  }
+  const composeOutput = stringify(composeParsed);
+  // Validate new contents
+  await compose.config({ cwd: appDir, configAsString: composeOutput });
+  await spawner.writeFile('docker-compose.yaml', composeOutput);
+}
+
+export async function getAllEnvs(props: { projectId: string }) {
+  const { projectId } = props;
+  if (!AgentConfigFile().isAppReady({ projectId })) {
+    throw new Error(`App ${projectId} is not ready!`);
+  }
+  const appDir = getAppDir(projectId);
+  const spawner = JsSpawner({ path: appDir });
+  const envVars = {};
+  const composeContents = await spawner.readFile('docker-compose.yaml');
+  const composeParsed = parse(composeContents);
+  if ('services' in composeParsed) {
+    const services = Object.keys(composeParsed['services']);
+    for (const s of services) {
+      envVars[s] = [];
+      const service = composeParsed['services'][s];
+      if ('env_file' in service) {
+        const envFiles: string[] = service['env_file'];
+        for (const ef of envFiles) {
+          envVars[s] = envVars[s].concat((await spawner.readFile(ef)).split('\n'));
+        }
+      }
+      if ('environment' in service) {
+        const environment: string[] = service['environment'];
+        envVars[s] = envVars[s].concat(environment);
+      }
+      // Filter out empty lines and comment lines
+      envVars[s] = envVars[s].filter((v) => {
+        return v !== '' && !v.includes('#');
+      });
+    }
+  }
+
+  return envVars;
+}

package/src/application-control/index.ts

@@ -0,0 +1,40 @@
+import { installApp, uninstallApp } from './install';
+import { rollbackApp } from './backup';
+import {
+  listAppReleases,
+  listAppLatestRelease,
+  getAppStatus,
+  startApp,
+  getAppLogs,
+  stopApp,
+  restartApp,
+} from './status';
+import { ModelDetails } from './types';
+import { getAllEnvs, setEnv } from './environment-variables';
+
+export {
+  installApp,
+  uninstallApp,
+  rollbackApp,
+  listAppReleases,
+  listAppLatestRelease,
+  getAppStatus,
+  startApp,
+  getAppLogs,
+  stopApp,
+  restartApp,
+  ModelDetails,
+  getAllEnvs,
+  setEnv,
+};
+
+// CLI-mode only
+import {
+  addModel,
+  getAppModels,
+  removeModel,
+  replaceModels,
+  updateModels,
+} from './models';
+
+export { addModel, getAppModels, removeModel, replaceModels, updateModels };

package/src/application-control/install.ts

@@ -0,0 +1,126 @@
+import * as rimraf from 'rimraf';
+import * as fs from 'fs';
+import * as path from 'path';
+import nodeFetch, { Response } from 'node-fetch';
+import { JsSpawner } from 'alwaysai/lib/util';
+
+import { runCliCmd } from '../util/run-cli-cmd';
+import { buildApp, getAppDir } from './utils';
+import { AppDetails } from '@alwaysai/device-agent-schemas';
+import { BACKUP_EXT, createAppBackup } from './backup';
+import { stopApp } from './status';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+
+export async function getInstalledApps(): Promise<AppDetails[]> {
+  const apps = await AgentConfigFile().getApps();
+  const appDetails: AppDetails[] = [];
+  for (const app of apps) {
+    const { projectId, version } = app;
+    appDetails.push({ projectId, version });
+  }
+  return appDetails;
+}
+
+export type InstallAppProps = {
+  projectId: string;
+  releaseHash: string;
+  presignedAppUrl?: string;
+};
+
+export async function installAppPackage({ localDest, presignedUrl }): Promise<void> {
+  const response = await nodeFetch(presignedUrl);
+  if (response.status !== 200) {
+    // If the URL is invalid; I think we shouldn't get here with the new changes
+    throw new Error(
+      `Status Code: ${response.status}, ${response.statusText}. Response: ${response.body}`,
+    );
+  }
+
+  /**
+   * Write the app package to the hash directory
+   */
+  const stream = response.body.pipe(fs.createWriteStream(localDest));
+  await new Promise((resolve, reject) => {
+    stream.on('finish', resolve);
+    stream.on('error', reject);
+  });
+}
+
+export async function installApp(props: InstallAppProps): Promise<void> {
+  const { projectId, releaseHash, presignedAppUrl } = props;
+  const appDir = getAppDir(projectId);
+  const spawner = JsSpawner();
+  if (await AgentConfigFile().isAppPresent({ projectId })) {
+    if (!(await AgentConfigFile().isAppReady({ projectId }))) {
+      throw new Error('Application already has installation in progress!');
+    }
+    await AgentConfigFile().setAppInstalling({ projectId, version: releaseHash });
+    console.log('Application is already installed, updating');
+    await createAppBackup({ projectId });
+    await spawner.rimraf(appDir);
+  } else {
+    await AgentConfigFile().setAppInstalling({ projectId, version: releaseHash });
+  }
+
+  /**
+   * Prep directory and fetch the app package
+   */
+  await spawner.mkdirp(appDir);
+  const localDest = path.join(appDir, `${path.basename(releaseHash)}.tgz`);
+  if (!presignedAppUrl) {
+    // Download the application using the CLI
+    await runCliCmd({
+      cmd: [
+        'release',
+        'pull',
+        '--yes',
+        '--project',
+        projectId,
+        '--releaseHash',
+        releaseHash,
+      ],
+      cwd: appDir,
+    });
+  } else {
+    await installAppPackage({ localDest, presignedUrl: presignedAppUrl });
+  }
+
+  /**
+   * Unpack app package
+   */
+  await spawner.untar(fs.createReadStream(localDest), appDir);
+  console.log(`Unpacked application to ${appDir}`);
+
+  /**
+   * Build app
+   * ToDo: migrate the model pull to use MQTT workflow
+   */
+  await buildApp({ appDir });
+
+  /**
+   * Remove tar file
+   */
+  await spawner.rimraf(localDest);
+
+  await AgentConfigFile().setAppInstalled({ projectId, version: releaseHash });
+  console.log(`Installed ${projectId} to ${appDir}`);
+}
+
+export async function uninstallApp(props: { projectId: string }): Promise<void> {
+  const { projectId } = props;
+  if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
+    console.log(`Application ${projectId} not installed`);
+    return;
+  }
+  try {
+    await stopApp({ projectId });
+  } catch {
+    console.log('Failed to stop app, may be left running...');
+  }
+  await AgentConfigFile().setAppUninstalled({ projectId });
+  // Delete application directory and backup
+  const appDir = getAppDir(projectId);
+  rimraf.sync(appDir);
+  rimraf.sync(`${appDir}${BACKUP_EXT}`);
+  console.log('Uninstalled', projectId);
+}