@alwaysai/device-agent 0.0.4 → 0.0.5

package/lib/util/directories.js

@@ -1,22 +1,22 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getCredentialsFilePath = exports.getRootCertificateFilePath = exports.getCertificateFilePath = exports.getPrivateKeyFilePath = void 0;
+const constants_1 = require("alwaysai/lib/constants");
 const path_1 = require("path");
-const constants_1 = require("../constants");
 function getPrivateKeyFilePath() {
-    return (0, path_1.join)(constants_1.CERT_AND_KEY_DIR, constants_1.DEVICE_PRIVATE_KEY_FILE_NAME);
+    return (0, path_1.join)(constants_1.LOCAL_CERT_AND_KEY_DIR, constants_1.DEVICE_PRIVATE_KEY_FILE_NAME);
 }
 exports.getPrivateKeyFilePath = getPrivateKeyFilePath;
 function getCertificateFilePath() {
-    return (0, path_1.join)(constants_1.CERT_AND_KEY_DIR, constants_1.DEVICE_CERTIFICATE_FILE_NAME);
+    return (0, path_1.join)(constants_1.LOCAL_CERT_AND_KEY_DIR, constants_1.DEVICE_CERTIFICATE_FILE_NAME);
 }
 exports.getCertificateFilePath = getCertificateFilePath;
 function getRootCertificateFilePath() {
-    return (0, path_1.join)(constants_1.CERT_AND_KEY_DIR, constants_1.DEVICE_ROOT_CERT_FILE_NAME);
+    return (0, path_1.join)(constants_1.LOCAL_CERT_AND_KEY_DIR, constants_1.DEVICE_ROOT_CERT_FILE_NAME);
 }
 exports.getRootCertificateFilePath = getRootCertificateFilePath;
 function getCredentialsFilePath() {
-    return (0, path_1.join)(constants_1.TOKEN_DIR, constants_1.DEVICE_TOKEN_FILE_NAME);
+    return (0, path_1.join)(constants_1.LOCAL_AAI_CFG_DIR, constants_1.DEVICE_TOKEN_FILE_NAME);
 }
 exports.getCredentialsFilePath = getCredentialsFilePath;
 //# sourceMappingURL=directories.js.map

package/lib/util/directories.js.map

@@ -1 +1 @@
-{"version":3,"file":"directories.js","sourceRoot":"","sources":["../../src/util/directories.ts"],"names":[],"mappings":";;;AAAA,+BAA4B;AAC5B,4CAOsB;AAEtB,SAAgB,qBAAqB;IACnC,OAAO,IAAA,WAAI,EAAC,4BAAgB,EAAE,wCAA4B,CAAC,CAAC;AAC9D,CAAC;AAFD,sDAEC;AAED,SAAgB,sBAAsB;IACpC,OAAO,IAAA,WAAI,EAAC,4BAAgB,EAAE,wCAA4B,CAAC,CAAC;AAC9D,CAAC;AAFD,wDAEC;AAED,SAAgB,0BAA0B;IACxC,OAAO,IAAA,WAAI,EAAC,4BAAgB,EAAE,sCAA0B,CAAC,CAAC;AAC5D,CAAC;AAFD,gEAEC;AAED,SAAgB,sBAAsB;IACpC,OAAO,IAAA,WAAI,EAAC,qBAAS,EAAE,kCAAsB,CAAC,CAAC;AACjD,CAAC;AAFD,wDAEC"}
+{"version":3,"file":"directories.js","sourceRoot":"","sources":["../../src/util/directories.ts"],"names":[],"mappings":";;;AAAA,sDAOgC;AAChC,+BAA4B;AAE5B,SAAgB,qBAAqB;IACnC,OAAO,IAAA,WAAI,EAAC,kCAAsB,EAAE,wCAA4B,CAAC,CAAC;AACpE,CAAC;AAFD,sDAEC;AAED,SAAgB,sBAAsB;IACpC,OAAO,IAAA,WAAI,EAAC,kCAAsB,EAAE,wCAA4B,CAAC,CAAC;AACpE,CAAC;AAFD,wDAEC;AAED,SAAgB,0BAA0B;IACxC,OAAO,IAAA,WAAI,EAAC,kCAAsB,EAAE,sCAA0B,CAAC,CAAC;AAClE,CAAC;AAFD,gEAEC;AAED,SAAgB,sBAAsB;IACpC,OAAO,IAAA,WAAI,EAAC,6BAAiB,EAAE,kCAAsB,CAAC,CAAC;AACzD,CAAC;AAFD,wDAEC"}

package/package.json

@@ -1,9 +1,13 @@
 {
   "name": "@alwaysai/device-agent",
   "description": "The alwaysAI Device Agent",
-  "version": "0.0.4",
-  "main": "lib/exports.js",
-  "types": "lib/exports.d.ts",
+  "version": "0.0.5",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "publishConfig": {
+    "access": "public",
+    "tag": "next"
+  },
   "bin": {
     "aai-agent": "lib/index.js"
   },
@@ -12,34 +16,35 @@
     "node": ">=16.0.0"
   },
   "scripts": {
-    "build": "tsc --project tsconfig.build.json && copyfiles -u 1 src/web/*.html src/web/static/* lib/",
+    "build": "tsc --project tsconfig.build.json",
     "clean": "rimraf lib packages",
     "build:clean": "npm run clean && npm run build",
     "build:watch": "tsc --project . --watch",
     "lint": "tslint --project tsconfig.build.json",
     "lint:fix": "npm run lint -- --fix",
-    "test": "npm run lint",
+    "test": "npm run test:unit",
     "test:unit": "jest --coverage src",
     "test:watch": "npm run test:unit -- --watch",
     "preversion": "npm test && npm run build:clean",
     "postversion": "npm publish"
   },
   "dependencies": {
-    "@alwaysai/alwayscli": "0.3.0",
-    "@alwaysai/config-nodejs": "0.0.2",
+    "@alwaysai/alwayscli": "0.3.1",
+    "@alwaysai/config-nodejs": "0.1.0",
     "@carnesen/coded-error": "0.4.0",
+    "@alwaysai/device-agent-schemas": "0.0.2",
     "@types/mkdirp": "1.0.2",
     "@types/pump": "1.1.1",
     "@types/rimraf": "3.0.2",
     "@types/signal-exit": "3.0.1",
     "@types/tar": "6.1.1",
+    "ajv": "8.11.0",
+    "alwaysai": "1.8.0",
     "aws-iot-device-sdk": "2.2.12",
-    "alwaysai": "1.6.0",
     "docker-compose": "0.23.17",
     "express": "4.17.3",
-    "fp-ts": "2.11.5",
-    "io-ts": "2.2.16",
     "mkdirp": "1.0.4",
+    "node-fetch": "2.6.1",
     "node-os-utils": "1.3.6",
     "pump": "3.0.0",
     "rimraf": "3.0.2",
@@ -48,17 +53,17 @@
     "tar": "6.1.11",
     "tree-kill": "1.2.2",
     "winston": "3.3.3",
-    "winston-daily-rotate-file": "4.5.5"
+    "winston-daily-rotate-file": "4.5.5",
+    "yaml": "2.1.1"
   },
   "devDependencies": {
-    "@alwaysai/tsconfig": "0.0.0",
-    "@alwaysai/tslint-config": "0.0.3",
+    "@alwaysai/tsconfig": "0.0.1",
+    "@alwaysai/tslint-config": "0.0.4",
     "@carnesen/run-and-catch": "0.4.3",
     "@types/jest": "27.0.3",
     "@types/node": "16.11.12",
     "@types/sinon": "10.0.6",
     "aws-sdk": "^2.1046.0",
-    "copyfiles": "^2.4.1",
     "cp-cli": "2.0.0",
     "get-stream": "6.0.1",
     "jest": "27.4.3",

package/readme.md

@@ -1,122 +1,219 @@
 # alwaysAI Device Agent
 
+The alwaysAI Device Agent is a tool that runs on an alwaysAI device provisioned
+for production remote deployment. It enables management of one or more alwaysAI
+apps on a particular device via a command line interface.
+
+Note that the Device Agent is still in an experimental phase and these commands
+are likely to change. This guide will be updated with the latest usage as things
+change.
+
 ## System Requirements
 
- 1. Preferably running Linux, but can hack it to work on Mac too
- 2. Docker installed with user access (On Linux: `$ sudo usermod -aG docker $USER`, nothing needed for Mac)
- 3. Docker Compose installed
- 4. npm and node (for build and test)
+* Recent version of Debian or Ubuntu
+* `npm` >= 7.0.0
+* `node` >= 16.0.0
+* Recent version of `docker` with user access
+  * On Linux: `$ sudo usermod -aG docker $USER`
+* Recent version of `docker-compose`
+* `curl` installed.
+
+## Provision Device
+
+### Provision the device
+
+If you're system already meets all the system requirements, you can simply install the device agent via `npm`:
+
+```
+sudo npm install -g @alwaysai/device-agent@latest
+```
 
-## Lint and Build
+Otherwise, you can run the provision script to install the agent and its dependencies:
 
 ```
-$ npm run lint:fix
-$ npm run build
+curl -fsSL https://alwaysai-artifacts-prod.s3.us-west-1.amazonaws.com/device-agent/install-device-agent.sh | sudo -E bash -
 ```
 
-## Usage
+The Device Agent will be available in the terminal as `aai-agent`.
 
-There are three modes of operation: CLI, which is great for testing new features, the device-hosted web dashboard, and the IoT Core cloud connection.
+Next, log in to the Device Agent with the following command:
 
-To enable the cloud IoT Core connection, set the following environment variable:
 ```
-export ALWAYSAI_DEVICE_AGENT_MODE="cloud"
+aai-agent login --email <username> --password <password> [--device <device_id>]
 ```
 
-To enable the device-hosted web dashboard, set the following environment variable:
+Note that if you already have a device registered with the alwaysAI cloud, you
+can optionally provide that device ID in the login command. If you don't yet
+have a device configured, follow the next step to initialize the device.
+
+### Initialize the alwaysAI device
+
+After logging in, run the following command to initialize the current device as
+an alwaysAI device:
+
 ```
-export ALWAYSAI_DEVICE_AGENT_MODE="web"
+$ aai-agent device init --name <name> [--description <description>]
 ```
 
-Any other value will run the CLI.
+For example:
 
-On Mac you need to override the OS to remove some usage restrictions:
 ```
-export ALWAYSAI_OS_PLATFORM="linux"
+$ aai-agent device init --name test-dev-1 --description "Device for testing alwaysAI Device Agent"
 ```
 
-### CLI
+## Run an alwaysAI application on your device
+
+### Publish an application release and install on device
+
+First, you must publish your application to the alwaysAI cloud. From the root
+of your application directory (where the `alwaysai.app.json` file is) on your
+development host:
 
-Here are the available commands:
 ```
-$ node lib --help
-Starting alwaysAI Device Agent
-Usage: aai-agent <subcommand> ...
+aai app configure
+aai app publish
+```
 
-   Manage your alwaysAI production device
+The output of the final command will give you the release hash that was
+published. You can run `aai release list --project <project_id>` to list all
+release versions for the project, where project ID can be found in the
+`alwaysai.project.json` file.
 
-Subcommands:
+Now you can install the application on the device using the device agent. Run
+the following on the device where the Device Agent is installed:
 
-   login                   : Login to alwaysAI (this is meant for scripted environments)
-   app list                : List all installed apps
-   app list-releases       : List all releases for a given app
-   app list-latest-release : List the latest release hash for a given app
-   app install             : Install an alwaysAI app from a project
-   app status              : Get the status of an installed alwaysAI app
-   app start               : Start an installed alwaysAI app
-   app logs                : Get logs for an application
-   app stop                : Stop a running alwaysAI app
-   app uninstall           : Remove an alwaysAI app
-   app rollback            : Rollback an alwaysAI app to the previous version
-   app add-model           : Add a model to an alwaysAI app
-   app remove-model        : Remove a model from an alwaysAI app
-   app update-models       : Update all models for an alwaysAI app
-   get-device-info         : Get device info
+```
+aai-agent app install --project <project_id> [--release <release_hash>]
+```
+
+### Control the application
+
+Run the following commands on the device where the Device Agent is installed:
+
+Get application status:
+```
+aai-agent app status --project <project_id>
 ```
 
-To install a test app, run:
+Start the application:
 ```
-$ node lib install-test-app
+aai-agent app start --project <project_id>
 ```
 
-The test app has the project ID 16ad11bb-8c3a-4a15-923e-49f7a8311dc6.
+Show the application logs:
+```
+aai-agent app logs --project <project_id>
+```
 
-Now you can start, check the status, and stop using the commands:
+Stop the application:
 ```
-$ node lib start-app --project 16ad11bb-8c3a-4a15-923e-49f7a8311dc6
-Starting alwaysAI Device Agent
-Starting app...
-{
-  exitCode: 0,
-  err: 'Starting 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... \r\n' +
-    'Starting 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... done\r\n',
-  out: ''
-}
-$ node lib get-app-status --project 16ad11bb-8c3a-4a15-923e-49f7a8311dc6
-Starting alwaysAI Device Agent
-[
-  {
-    name: '16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1',
-    command: 'tail -f /dev/null',
-    state: 'Up',
-    ports: []
-  }
-]
-$  node lib stop-app --project 16ad11bb-8c3a-4a15-923e-49f7a8311dc6
-Starting alwaysAI Device Agent
-{
-  exitCode: 0,
-  err: 'Stopping 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... \r\n' +
-    'Stopping 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... done\r\n' +
-    'Removing 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... \r\n' +
-    'Removing 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_alwaysai_1 ... done\r\n' +
-    'Removing network 16ad11bb-8c3a-4a15-923e-49f7a8311dc6_default\n',
-  out: ''
-}
+aai-agent app stop --project <project_id>
+```
+
+Uninstall the application:
+```
+aai-agent app uninstall --project <project_id>
+```
+
+### Manage application models
+
+There are several ways to manage the models for your application.
+
+#### Update models to new version of the same model ID
+
+If a new version of a model is published for an existing model ID, and an
+application is already configured to be using that model ID, updating to the
+new model can simply be done with:
+
+```
+aai-agent app stop --project <project_id>
+aai-agent app update-models --project <project_id>
+aai-agent app start --project <project_id>
+```
+
+#### Replace models for an application for new ID
+
+If you'd like to install an entirely new model to an application, replacing the
+model the app was originally configured with, run the following command:
+
+```
+aai-agent app stop --project <project_id>
+aai-agent app replace-models --project <project_id> --models <model_id_1> [<model_id_2> ...]
+aai-agent app start --project <project_id>
+```
+
+If you plan on using this method, you can make your application source model ID
+agnostic by providing the model ID to edgeIQ in the following way:
+
+Select the first model in the config list:
+```
+obj_detect = edgeiq.ObjectDetection(edgeiq._globals.MODEL_ID_LIST[0])
+```
+
+#### Manually download a model package
+
+To download a model package from the alwaysAI cloud and unpack to a specific directory, run:
+
 ```
+aai-agent get-model-package <model ID> [--path <destination path>]
+```
+
+For example, to download `alwaysai/yolo_v3` to `~/models` run:
+
+```
+aai-agent get-model-package alwaysai/yolo_v3 --path ~/models
+```
+
+Once the command completes, you'll see the model package in the
+`~/models/alwaysai/yolo_v3` directory.
 
-### Web Dashboard
+### Set and update environment variables
 
-You can start the web dashboard by running:
+The Device Agent enables you to set and update environment variables for all
+services or a single service of you application.
 
 ```
-node lib
+aai-agent app set-env <key=val> --project <project_id> [--service <service>]
 ```
 
-And clicking the link that comes up.
+For example, to set the following environment variable for the `alwaysai`
+service run the following command on the device:
 
-If you are developing new functionality for the cli in tandem with the device agent, you can run
 ```
-npm i --save /path/to/your/cli/repo
+aai-agent app set-env TEST_ENV=1 --project <project_id> --service alwaysai
 ```
-Remember to not commit the package.json, or revert it when you are done developing a feature
+
+## Device Agent Commands
+
+```
+$ aai-agent --help
+Starting alwaysAI Device Agent
+Usage: aai-agent <subcommand> ...
+
+   Manage your alwaysAI production device
+
+Subcommands:
+
+   login                   : Login to alwaysAI (this is meant for scripted environments)
+   app list                : List all installed apps
+   app list-releases       : List all releases for a given app
+   app list-latest-release : List the latest release hash for a given app
+   app install             : Install an alwaysAI app from a project
+   app status              : Get the status of an installed alwaysAI app
+   app start               : Start an installed alwaysAI app
+   app stop                : Stop a running alwaysAI app
+   app restart             : Restart running alwaysAI app
+   app logs                : Get logs for an application
+   app uninstall           : Remove an alwaysAI app
+   app rollback            : Rollback an alwaysAI app to the previous version
+   app show-models         : Show the application models
+   app add-model           : Add a model to an alwaysAI app
+   app remove-model        : Remove a model from an alwaysAI app
+   app replace-models      : Replace all models of an alwaysAI app with new models
+   app update-models       : Update all models for an alwaysAI app
+   app get-all-envs        : Get environment variables for an application
+   app set-env             : Set environment variables for a service
+   device init             : Initialize device
+   device get-info         : Get device info
+   get-model-package       : Download and unpack a model package
+```

package/src/application-control/backup.ts

@@ -1,8 +1,8 @@
 import * as rimraf from 'rimraf';
-import * as fs from 'fs';
 
 import { copyDir } from '../util/copy-dir';
 import { buildApp, getAppDir } from './utils';
+import { AgentConfigFile } from '../infrastructure/agent-config';
 
 export const BACKUP_EXT = '.bak';
 
@@ -10,19 +10,23 @@ export async function createAppBackup(props: { projectId: string }) {
   const { projectId } = props;
   const appDir = getAppDir(projectId);
   const backupAppDir = `${appDir}${BACKUP_EXT}`;
+  rimraf.sync(backupAppDir);
   await copyDir({ srcPath: appDir, destPath: backupAppDir });
+  await AgentConfigFile().setAppBackup({ projectId });
   console.log(`Backed up app ${projectId} to ${backupAppDir}`);
 }
 
 export async function rollbackApp(props: { projectId: string }) {
   const { projectId } = props;
-  const appDir = getAppDir(projectId);
-  const backupAppDir = `${appDir}${BACKUP_EXT}`;
-  if (!fs.existsSync(backupAppDir)) {
+  const config = await AgentConfigFile().getAppBackup({ projectId });
+  if (!config) {
     throw new Error(`Backup doesn't exist for ${projectId}`);
   }
+  const appDir = getAppDir(projectId);
+  const backupAppDir = `${appDir}${BACKUP_EXT}`;
   rimraf.sync(appDir);
-  await copyDir({ srcPath: `${appDir}${BACKUP_EXT}`, destPath: appDir });
+  await copyDir({ srcPath: backupAppDir, destPath: appDir });
   await buildApp({ appDir });
-  console.log(`Rolled back app ${projectId} to previous version`);
+  await AgentConfigFile().setAppInstalled({ projectId, version: config.version });
+  console.log(`Rolled back app ${projectId} to ${config.version}`);
 }

package/src/application-control/environment-variables.ts

@@ -0,0 +1,81 @@
+import { JsSpawner } from 'alwaysai/lib/util';
+import compose from 'docker-compose';
+import { parse, stringify } from 'yaml';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+import { getAppDir } from './utils';
+
+export async function setEnv(props: {
+  projectId: string;
+  vars: string[];
+  service?: string;
+}) {
+  const { projectId, vars } = props;
+  if (!AgentConfigFile().isAppReady({ projectId })) {
+    throw new Error(`App ${projectId} is not ready!`);
+  }
+  const appDir = getAppDir(projectId);
+  const spawner = JsSpawner({ path: appDir });
+  const composeContents = await spawner.readFile('docker-compose.yaml');
+  const composeParsed = parse(composeContents);
+  if ('services' in composeParsed) {
+    let services: string[] = Object.keys(composeParsed['services']);
+    if (props.service) {
+      console.log(services);
+      if (services.includes(props.service)) {
+        services = [props.service];
+      } else {
+        throw new Error(`Service ${props.service} not found in ${services}`);
+      }
+    }
+    for (const s of services) {
+      const service = composeParsed['services'][s];
+      // The environment field overrides the env files, so by appending to
+      // the environment list we can assure the value will be used
+      if ('environment' in service) {
+        const environment: string[] = service['environment'];
+        composeParsed['services'][s]['environment'] = environment.concat(vars);
+      } else {
+        composeParsed['services'][s]['environment'] = vars;
+      }
+    }
+  }
+  const composeOutput = stringify(composeParsed);
+  // Validate new contents
+  await compose.config({ cwd: appDir, configAsString: composeOutput });
+  await spawner.writeFile('docker-compose.yaml', composeOutput);
+}
+
+export async function getAllEnvs(props: { projectId: string }) {
+  const { projectId } = props;
+  if (!AgentConfigFile().isAppReady({ projectId })) {
+    throw new Error(`App ${projectId} is not ready!`);
+  }
+  const appDir = getAppDir(projectId);
+  const spawner = JsSpawner({ path: appDir });
+  const envVars = {};
+  const composeContents = await spawner.readFile('docker-compose.yaml');
+  const composeParsed = parse(composeContents);
+  if ('services' in composeParsed) {
+    const services = Object.keys(composeParsed['services']);
+    for (const s of services) {
+      envVars[s] = [];
+      const service = composeParsed['services'][s];
+      if ('env_file' in service) {
+        const envFiles: string[] = service['env_file'];
+        for (const ef of envFiles) {
+          envVars[s] = envVars[s].concat((await spawner.readFile(ef)).split('\n'));
+        }
+      }
+      if ('environment' in service) {
+        const environment: string[] = service['environment'];
+        envVars[s] = envVars[s].concat(environment);
+      }
+      // Filter out empty lines and comment lines
+      envVars[s] = envVars[s].filter((v) => {
+        return v !== '' && !v.includes('#');
+      });
+    }
+  }
+
+  return envVars;
+}

package/src/application-control/index.ts

@@ -0,0 +1,40 @@
+import { installApp, uninstallApp } from './install';
+import { rollbackApp } from './backup';
+import {
+  listAppReleases,
+  listAppLatestRelease,
+  getAppStatus,
+  startApp,
+  getAppLogs,
+  stopApp,
+  restartApp,
+} from './status';
+import { ModelDetails } from './types';
+import { getAllEnvs, setEnv } from './environment-variables';
+
+export {
+  installApp,
+  uninstallApp,
+  rollbackApp,
+  listAppReleases,
+  listAppLatestRelease,
+  getAppStatus,
+  startApp,
+  getAppLogs,
+  stopApp,
+  restartApp,
+  ModelDetails,
+  getAllEnvs,
+  setEnv,
+};
+
+// CLI-mode only
+import {
+  addModel,
+  getAppModels,
+  removeModel,
+  replaceModels,
+  updateModels,
+} from './models';
+
+export { addModel, getAppModels, removeModel, replaceModels, updateModels };