@alwaysai/device-agent 1.5.0 → 2.0.1

package/src/subcommands/app/env-vars.ts

@@ -4,13 +4,13 @@ import {
   CliStringInput,
   CliTerseError
 } from '@alwaysai/alwayscli';
-import { EnvVars, getAllEnvs } from '../../application-control';
+import { getAllEnvs } from '../../application-control';
 import { DeviceAgentCloudConnection } from '../../cloud-connection/device-agent-cloud-connection';
 import sleep from '../../util/sleep';
-import { logger } from '../../util/logger';
 import {
-  buildUpdateProjectShadowMessage,
-  getShadowTopic
+  buildUpdateShadowMessage,
+  getShadowTopic,
+  EnvVars
 } from '@alwaysai/device-agent-schemas';
 
 export const getAllEnvsCliLeaf = CliLeaf({
@@ -48,27 +48,90 @@ export const setEnvCliLeaf = CliLeaf({
   },
   async action(args, opts) {
     const { project: projectId, service } = opts;
-    const envVars: EnvVars = { [service]: {} };
+    const currentEnvs = await getAllEnvs({ projectId });
+    const envVars: EnvVars = { [service]: { ...currentEnvs[service] } };
+
     args.forEach((arg: string) => {
       const nameVal = arg.split('=');
       if (nameVal.length !== 2) {
         throw new CliTerseError(`Invalid argument: ${arg}`);
       }
-      const value = nameVal[1] === '' ? null : nameVal[1];
+      const value = nameVal[1];
       envVars[service][nameVal[0]] = value;
     });
     const deviceAgent = new DeviceAgentCloudConnection();
-    await deviceAgent.setupHandlers();
 
     const toDesire = {
       [projectId]: {
-        envVars
+        envVars: JSON.stringify(envVars)
+      }
+    };
+    deviceAgent.publisher.publish(
+      getShadowTopic(deviceAgent.getClientId(), 'projects', 'update'),
+      JSON.stringify(
+        buildUpdateShadowMessage({
+          clientId: 'client',
+          desired: toDesire
+        })
+      )
+    );
+    // Sleep for extra time to ensure time for shadow response
+    await sleep(10000);
+
+    while (deviceAgent.isCmdInProgress(projectId)) {
+      await sleep(1000);
+    }
+    await deviceAgent.stop();
+  }
+});
+
+export const rmEnvCliLeaf = CliLeaf({
+  name: 'rm-env',
+  description: 'Remove an environment variable from an application',
+  positionalInput: CliStringInput({
+    description: 'The key of the environment variable to be removed',
+    required: true
+  }),
+  namedInputs: {
+    project: CliStringInput({
+      description: 'Project Id',
+      required: true
+    }),
+    service: CliStringInput({
+      description:
+        'Docker compose service to remove an environment variable from',
+      required: true
+    })
+  },
+  async action(arg, opts) {
+    const { project: projectId, service } = opts;
+    const envVar = arg;
+    const currentEnvs = await getAllEnvs({ projectId: projectId });
+
+    const newEnvs = { ...currentEnvs };
+    if (Object.keys(newEnvs[service]).includes(envVar)) {
+      delete newEnvs[service][envVar];
+    } else {
+      throw new CliTerseError(
+        `Could not find key "${envVar}" in service ${service}. Current keys are:\n ${JSON.stringify(
+          Object.keys(currentEnvs[service]),
+          null,
+          2
+        )}.\nPlease check the key and try again.`
+      );
+    }
+
+    const deviceAgent = new DeviceAgentCloudConnection();
+
+    const toDesire = {
+      [projectId]: {
+        envVars: JSON.stringify(newEnvs)
       }
     };
     deviceAgent.publisher.publish(
       getShadowTopic(deviceAgent.getClientId(), 'projects', 'update'),
       JSON.stringify(
-        buildUpdateProjectShadowMessage({
+        buildUpdateShadowMessage({
           clientId: 'client',
           desired: toDesire
         })

package/src/subcommands/app/index.ts

@@ -1,12 +1,6 @@
 import { CliBranch } from '@alwaysai/alwayscli';
-import { getAllEnvsCliLeaf, setEnvCliLeaf } from './env-vars';
-import {
-  showAppModelsCliLeaf,
-  addModelCliLeaf,
-  removeModelCliLeaf,
-  replaceModelsCliLeaf,
-  updateModelsCliLeaf
-} from './models';
+import { getAllEnvsCliLeaf, setEnvCliLeaf, rmEnvCliLeaf } from './env-vars';
+import { showAppModelsCliLeaf, addModelCliLeaf } from './models';
 import {
   getAppStatusCliLeaf,
   startAppCliLeaf,
@@ -38,11 +32,9 @@ export const appCliBranch = CliBranch({
     rollbackAppCliLeaf,
     showAppModelsCliLeaf,
     addModelCliLeaf,
-    removeModelCliLeaf,
-    replaceModelsCliLeaf,
-    updateModelsCliLeaf,
     getAllEnvsCliLeaf,
     setEnvCliLeaf,
+    rmEnvCliLeaf,
     getAnalyticsCfgCliLeaf,
     setAnalyticsCfgCliLeaf,
     getShadowCliLeaf,

package/src/subcommands/app/models.ts

@@ -1,23 +1,10 @@
-import {
-  CliLeaf,
-  CliNumberInput,
-  CliStringArrayInput,
-  CliStringInput
-} from '@alwaysai/alwayscli';
-import {
-  getAppModels,
-  readAppCfgFile,
-  removeModel,
-  replaceModels,
-  updateModels
-} from '../../application-control';
+import { CliLeaf, CliStringInput } from '@alwaysai/alwayscli';
+import { getAppModels, readAppCfgFile } from '../../application-control';
 import { DeviceAgentCloudConnection } from '../../cloud-connection/device-agent-cloud-connection';
 import sleep from '../../util/sleep';
-import { logger } from '../../util/logger';
 import {
-  buildUpdateProjectShadowMessage,
-  getShadowTopic,
-  validateShadowProjectsUpdateAll
+  buildUpdateShadowMessage,
+  getShadowTopic
 } from '@alwaysai/device-agent-schemas';
 
 export const showAppModelsCliLeaf = CliLeaf({
@@ -56,7 +43,6 @@ export const addModelCliLeaf = CliLeaf({
   async action(_, opts) {
     const { project: projectId, model, version } = opts;
     const deviceAgent = new DeviceAgentCloudConnection();
-    await deviceAgent.setupHandlers();
 
     // Update the shadow as a client
 
@@ -71,7 +57,7 @@ export const addModelCliLeaf = CliLeaf({
     deviceAgent.publisher.publish(
       getShadowTopic(deviceAgent.getClientId(), 'projects', 'update'),
       JSON.stringify(
-        buildUpdateProjectShadowMessage({
+        buildUpdateShadowMessage({
           clientId: 'client',
           desired: toDesire
         })
@@ -86,65 +72,3 @@ export const addModelCliLeaf = CliLeaf({
     await deviceAgent.stop();
   }
 });
-
-export const removeModelCliLeaf = CliLeaf({
-  name: 'remove-model',
-  description: 'Remove a model from an alwaysAI app',
-  namedInputs: {
-    project: CliStringInput({
-      description: 'Project ID',
-      required: true
-    }),
-    model: CliStringInput({
-      description: 'Model ID',
-      required: true
-    })
-  },
-  async action(_, opts) {
-    const { project, model } = opts;
-    // TODO: Replace with handleMessage()
-    await removeModel({ projectId: project, modelId: model });
-  }
-});
-
-export const replaceModelsCliLeaf = CliLeaf({
-  name: 'replace-models',
-  description: 'Replace all models of an alwaysAI app with new models',
-  namedInputs: {
-    project: CliStringInput({
-      description: 'Project Id',
-      required: true
-    }),
-    models: CliStringArrayInput({
-      description: 'One or more model IDs',
-      required: true
-    })
-  },
-  hidden: true,
-  async action(_, opts) {
-    const { project, models } = opts;
-    console.log(
-      'This command is deprecated and will be removed in a future release'
-    );
-    await replaceModels({ projectId: project, modelIds: models });
-  }
-});
-
-export const updateModelsCliLeaf = CliLeaf({
-  name: 'update-models',
-  description: 'Update all models for an alwaysAI app',
-  namedInputs: {
-    project: CliStringInput({
-      description: 'Project Id',
-      required: true
-    })
-  },
-  hidden: true,
-  async action(_, opts) {
-    const { project } = opts;
-    console.log(
-      'This command is deprecated and will be removed in a future release'
-    );
-    await updateModels({ projectId: project });
-  }
-});

package/src/subcommands/app/shadow.ts

@@ -14,10 +14,9 @@ export const getShadowCliLeaf = CliLeaf({
   async action(_, opts) {
     const { project } = opts;
     const deviceAgent = new DeviceAgentCloudConnection();
-    await deviceAgent.setupHandlers();
+    deviceAgent.shadowHandler.getProjectShadowUpdates();
 
-    // The Device Agent always gets the latest shadow and updates
-    // itself. 5 seconds should cover the initial response wait time
+    // 5 seconds should cover the response wait time
     await sleep(5000);
     while (deviceAgent.isCmdInProgress(project)) {
       await sleep(1000);
@@ -38,11 +37,13 @@ export const updateShadowCliLeaf = CliLeaf({
   async action(_, opts) {
     const { project } = opts;
     const deviceAgent = new DeviceAgentCloudConnection();
-    await deviceAgent.setupHandlers();
-    await deviceAgent.updateProjectShadow(project);
+    await deviceAgent.shadowHandler.updateProjectShadow(project);
 
     // 5 seconds should cover the response wait time
     await sleep(5000);
+    while (deviceAgent.isCmdInProgress(project)) {
+      await sleep(1000);
+    }
     await deviceAgent.stop();
   }
 });

package/src/subcommands/app/version.ts

@@ -2,6 +2,7 @@ import { CliLeaf, CliStringInput, CliTerseError } from '@alwaysai/alwayscli';
 import {
   AppVersionControlMessage,
   generateTxId,
+  getToDeviceTopic,
   keyMirrors
 } from '@alwaysai/device-agent-schemas';
 import { rollbackApp } from '../../application-control';
@@ -36,10 +37,26 @@ export const installAppCliLeaf = CliLeaf({
       description: 'Release Hash',
       required: false,
       hidden: true
+    }),
+    appCfg: CliStringInput({
+      description: 'Application Configuration',
+      required: false,
+      hidden: true
+    }),
+    envVars: CliStringInput({
+      description: 'Environment Variables configuration',
+      hidden: true,
+      required: false
     })
   },
   async action(_, opts) {
-    const { project, releaseHash, 'release-hash': releaseHashNew } = opts;
+    const {
+      project,
+      releaseHash,
+      'release-hash': releaseHashNew,
+      appCfg,
+      envVars
+    } = opts;
     if (releaseHash) {
       logger.warn(
         `--releaseHash is deprecated and will be removed in a future release. Please switch to --release-hash`
@@ -50,8 +67,7 @@ export const installAppCliLeaf = CliLeaf({
       throw new CliTerseError('--release-hash flag is required!');
     }
     const deviceAgent = new DeviceAgentCloudConnection();
-    await deviceAgent.setupHandlers();
-    const topic = deviceAgent.getToDeviceTopic();
+    const topic = getToDeviceTopic(deviceAgent.getClientId());
     const message: AppVersionControlMessage = {
       timestamp: '',
       topic,
@@ -60,7 +76,9 @@ export const installAppCliLeaf = CliLeaf({
       payload: {
         baseCommand: keyMirrors.appVersionControl.install,
         projectId: project,
-        appReleaseHash: releaseHashResolved
+        appReleaseHash: releaseHashResolved,
+        appCfg,
+        envVars
       }
     };
     await deviceAgent.handleMessage(topic, message);
@@ -83,8 +101,7 @@ export const uninstallAppCliLeaf = CliLeaf({
   async action(_, opts) {
     const { project } = opts;
     const deviceAgent = new DeviceAgentCloudConnection();
-    await deviceAgent.setupHandlers();
-    const topic = deviceAgent.getToDeviceTopic();
+    const topic = getToDeviceTopic(deviceAgent.getClientId());
     const message: AppVersionControlMessage = {
       timestamp: '',
       topic,

package/src/subcommands/config.ts

@@ -0,0 +1,42 @@
+import { CliBranch, CliLeaf, CliOneOfInput } from '@alwaysai/alwayscli';
+import { SYSTEM_IDS } from 'alwaysai/lib/constants';
+import { ALWAYSAI_SYSTEM_ID } from 'alwaysai/lib/environment';
+import { getSystemId, setSystemId } from 'alwaysai/lib/infrastructure';
+import { echo } from 'alwaysai/lib/util';
+import { ALWAYSAI_SHOW_HIDDEN } from '../environment';
+
+const show = CliLeaf({
+  name: 'show',
+  description: `Show your current configuration`,
+  action() {
+    if (ALWAYSAI_SYSTEM_ID) {
+      echo(
+        '[WARNING] System ID is overridden by environment variable "ALWAYSAI_SYSTEM_ID"'
+      );
+    }
+    return { systemId: getSystemId() };
+  }
+});
+
+const set = CliLeaf({
+  name: 'set',
+  description: 'Set an configuration value',
+  namedInputs: {
+    systemId: CliOneOfInput({ values: SYSTEM_IDS, required: true })
+  },
+  action(_, { systemId }) {
+    if (ALWAYSAI_SYSTEM_ID) {
+      echo(
+        'WARNING] System ID is overridden by environment variable "ALWAYSAI_SYSTEM_ID"'
+      );
+    }
+    return setSystemId(systemId);
+  }
+});
+
+export const configBranch = CliBranch({
+  name: 'config',
+  hidden: !ALWAYSAI_SHOW_HIDDEN,
+  description: `Show or set  configuration values`,
+  subcommands: [show, set]
+});

package/src/subcommands/device/clean.ts

@@ -1,20 +1,25 @@
 import { CliLeaf } from '@alwaysai/alwayscli';
-import { logger } from '../../util/logger';
-import { AgentConfigFile } from '../../infrastructure/agent-config';
 import { DeviceConfigFile } from 'alwaysai/lib/core/device';
 import {
-  APP_ROOT,
-  CREDENTIALS_FILE_PATH,
-  DEVICE_AGENT_CFG_PATH,
-  DEVICE_AGENT_DOCKER_COMPOSE_PATH
-} from '../../util/directories';
+  LOCAL_CERT_AND_KEY_DIR,
+  LocalLegacyDeviceCertificates
+} from 'alwaysai/lib/infrastructure';
+import { stringifyError } from 'alwaysai/lib/util';
+import { readdir } from 'fs/promises';
+import { uninstallApp } from '../../application-control';
+import { AgentConfigFile } from '../../infrastructure/agent-config';
 import {
   checkRabbitMQContainerRunning,
   stopRabbitMQContainer
 } from '../../local-connection/rabbitmq-connection';
-import { stopApp } from '../../application-control';
-import { LOCAL_CERT_AND_KEY_DIR } from 'alwaysai/lib/paths';
+import {
+  APP_ROOT,
+  CREDENTIALS_FILE_PATH,
+  getDeviceAgentConfigPath,
+  getDeviceAgentDockerComposePath
+} from '../../util/directories';
 import { safeRimraf } from '../../util/file';
+import { logger } from '../../util/logger';
 
 export const cleanCliLeaf = CliLeaf({
   name: 'clean',
@@ -27,23 +32,27 @@ export const cleanCliLeaf = CliLeaf({
       }
     } catch (e) {
       logger.error(
-        `You may need to manually stop the container by running docker-compose down in the following directory: ${DEVICE_AGENT_DOCKER_COMPOSE_PATH}`
+        `You may need to manually stop the container by running docker-compose down in the following directory: ${getDeviceAgentDockerComposePath()}`
+      );
+      logger.debug(
+        `Error in checking / stopping RabbitMQ container!\n${stringifyError(e)}`
       );
-      logger.debug(`Error in checking / stopping RabbitMQ container: ${e}`);
     }
 
-    await safeRimraf(DEVICE_AGENT_CFG_PATH);
+    await safeRimraf(getDeviceAgentConfigPath());
 
     logger.debug('Checking for alwaysAI applications still running');
     const apps = await AgentConfigFile().getApps();
-    const stopAppPromises: Promise<void>[] = [];
+    const uninstallAppPromises: Promise<void>[] = [];
     for (const app of apps) {
       const { projectId } = app;
-      stopAppPromises.push(stopApp({ projectId }));
+      uninstallAppPromises.push(uninstallApp({ projectId }));
     }
-    await Promise.all(stopAppPromises);
+    await Promise.all(uninstallAppPromises);
 
     await safeRimraf(LOCAL_CERT_AND_KEY_DIR);
+    const legacyCertificates = new LocalLegacyDeviceCertificates();
+    await safeRimraf(legacyCertificates.getCertificateDirectoryPath());
 
     logger.debug(`Removing ${AgentConfigFile().path}`);
     AgentConfigFile().remove();
@@ -51,8 +60,13 @@ export const cleanCliLeaf = CliLeaf({
     DeviceConfigFile().remove();
 
     await safeRimraf(CREDENTIALS_FILE_PATH);
-    await safeRimraf(APP_ROOT);
-
+    const files = await readdir(APP_ROOT);
+    logger.debug(`${APP_ROOT} folder contents ${JSON.stringify(files.length)}`);
+    // This won't handle hidden files, but will remove if truly empty, which is safer
+    if (files.length === 0) {
+      logger.debug('Applications directory is empty, removing.');
+      await safeRimraf(APP_ROOT);
+    }
     logger.info('Device configuration cleaned');
   }
 });

package/src/subcommands/device/index.ts

@@ -4,6 +4,7 @@ import { cleanCliLeaf } from './clean';
 import { initCliLeaf } from './init';
 import { restartCliLeaf } from './restart';
 import { refreshCliLeaf } from './refresh';
+import { migrateCliLeaf } from './migrate';
 
 export const deviceCliBranch = CliBranch({
   name: 'device',
@@ -13,6 +14,7 @@ export const deviceCliBranch = CliBranch({
     getInfoCliLeaf,
     refreshCliLeaf,
     cleanCliLeaf,
-    restartCliLeaf
+    restartCliLeaf,
+    migrateCliLeaf
   ]
 });

package/src/subcommands/device/init.ts

@@ -1,21 +1,21 @@
 import { CliLeaf, CliStringInput } from '@alwaysai/alwayscli';
 import { getTargetHardwareUuid } from 'alwaysai/lib/core/app';
-import { v4 as uuidv4 } from 'uuid';
-import { CliAuthenticationClient } from 'alwaysai/lib/infrastructure';
+import {
+  CliAuthenticationClient,
+  DEVICE_PRIVATE_KEY_FILE_PATH,
+  LOCAL_CERT_AND_KEY_DIR
+} from 'alwaysai/lib/infrastructure';
 import { existsSync } from 'fs';
+import { v4 as uuidv4 } from 'uuid';
+import { AWS_ROOT_CERTIFICATE_FILE_NAME } from '../../util/directories';
 import { httpClient, microServiceHttpClient } from '../../util/http-client';
-import {
-  AWS_ROOT_CERTIFICATE_FILE_NAME,
-  BOOTSTRAP_CERTIFICATES_DIR_PATH,
-  DEVICE_PRIVATE_KEY_FILE_PATH
-} from '../../util/directories';
 
+import { DeviceConfigFile } from 'alwaysai/lib/core/device';
 import { JsSpawner } from 'alwaysai/lib/util';
+import { getBootstrapCertificateDirectoryPath } from '../../infrastructure/device-certificate';
 import { writeTokenAndDeviceCfg } from '../../infrastructure/tokens-and-device-cfg';
 import { logger } from '../../util/logger';
-import { DeviceConfigFile } from 'alwaysai/lib/core/device';
 import { requireLoggedInAndPaidPlan } from '../../util/require-logged-in-and-paid-plan';
-import { LOCAL_CERT_AND_KEY_DIR } from 'alwaysai/lib/paths';
 
 export const initCliLeaf = CliLeaf({
   name: 'init',
@@ -79,14 +79,14 @@ export const initCliLeaf = CliLeaf({
       deviceUuid: response.deviceUuid
     });
 
-    await JsSpawner().mkdirp(BOOTSTRAP_CERTIFICATES_DIR_PATH());
+    await JsSpawner().mkdirp(getBootstrapCertificateDirectoryPath());
 
     await JsSpawner({ path: LOCAL_CERT_AND_KEY_DIR }).writeFile(
       AWS_ROOT_CERTIFICATE_FILE_NAME,
       ROOT_CERT_AWS
     );
     const certSpawner = JsSpawner({
-      path: BOOTSTRAP_CERTIFICATES_DIR_PATH()
+      path: getBootstrapCertificateDirectoryPath()
     });
     const writeCertFilePromises: Promise<void>[] = [];
     for (const key in response.claimCertificate) {

package/src/subcommands/device/migrate.ts

@@ -0,0 +1,20 @@
+import { CliLeaf } from '@alwaysai/alwayscli';
+import { logger } from '../../util/logger';
+import { requiredConfigFilesPresentAndValid } from '../../infrastructure/required-config-checks';
+import { migrateFromLegacyCertsAndTokens } from '../../infrastructure/legacy-migration/legacy-migration';
+
+export const migrateCliLeaf = CliLeaf({
+  name: 'migrate',
+  description: 'Migrate configuration to the latest',
+  namedInputs: {},
+  async action(_, opts) {
+    const filesAlreadyMigrated = await requiredConfigFilesPresentAndValid();
+    if (filesAlreadyMigrated) {
+      logger.info('Configuration already migrated');
+    } else {
+      logger.info('Attempting configuration file migration');
+      await migrateFromLegacyCertsAndTokens();
+      logger.info('Migration complete');
+    }
+  }
+});

package/src/subcommands/device/refresh.ts

@@ -3,6 +3,7 @@ import { writeTokenAndDeviceCfg } from '../../infrastructure/tokens-and-device-c
 import { logger } from '../../util/logger';
 import { DeviceConfigFile } from 'alwaysai/lib/core/device';
 
+// FIXME: Do we need this command now that tokens aren't used outside of provisioning step?
 export const refreshCliLeaf = CliLeaf({
   name: 'refresh',
   description: 'Refresh the device tokens',

package/src/subcommands/index.ts

@@ -1,11 +1,13 @@
 import { appCliBranch } from './app';
+import { configBranch } from './config';
 import { deviceCliBranch } from './device';
-import { loginCliLeaf } from './login';
 import { getModelPackageCliLeaf } from './get-model-package';
+import { loginCliLeaf } from './login';
 import { stopRabbitMQContainerCliLeaf } from './rabbitmq-connection';
 
 export const subcommands = [
   loginCliLeaf,
+  configBranch,
   appCliBranch,
   deviceCliBranch,
   getModelPackageCliLeaf,

package/src/util/aai-error.ts

@@ -0,0 +1,20 @@
+import { ErrorObject } from 'ajv';
+
+type ErrorOption = ErrorObject<string, Record<string, any>, unknown>;
+
+type Details = ErrorOption | ErrorOption[] | Error | null;
+
+interface Options {
+  cause?: Details;
+}
+
+class AaiError extends Error {
+  public detailed?: Details;
+  constructor(message: string, options?: Options) {
+    super(message);
+    this.name = 'AaiError';
+    this.detailed = options?.cause;
+  }
+}
+
+export default AaiError;

package/src/util/{cloud-mode-ready.ts → aws-regions.ts}

@@ -1,10 +1,3 @@
-import { existsSync } from 'fs';
-import {
-  AWS_ROOT_CERTIFICATE_FILE_PATH,
-  DEVICE_PRIVATE_KEY_FILE_PATH,
-  DEVICE_CERTIFICATE_FILE_PATH
-} from '../util/directories';
-
 const VALID_AWS_REGIONS = [
   // 'af-south-1', // Africa (Cape Town)
   // 'ap-east-1', // Asia Pacific (Hong Kong)
@@ -40,20 +33,3 @@ const VALID_AWS_REGIONS = [
 export function isValidAwsRegion(region: string): boolean {
   return VALID_AWS_REGIONS.includes(region);
 }
-
-export function cloudModeReady() {
-  let ready = true;
-  const requiredFiles = [
-    DEVICE_CERTIFICATE_FILE_PATH,
-    DEVICE_PRIVATE_KEY_FILE_PATH,
-    AWS_ROOT_CERTIFICATE_FILE_PATH
-  ];
-
-  for (const path of requiredFiles) {
-    if (!existsSync(path)) {
-      ready = false;
-      break;
-    }
-  }
-  return ready;
-}