@alwaysai/device-agent 0.1.0 → 0.1.2

package/src/application-control/environment-variables.test.ts

@@ -0,0 +1,171 @@
+import { AgentConfigFile } from '../infrastructure/agent-config';
+import { readDockerCompose, writeDockerCompose } from './config';
+import { getAllEnvs, setEnv } from './environment-variables';
+import { isAppStarted, restartApp } from './status';
+import { buildApp, requireAppReady } from './utils';
+
+jest.mock('./config');
+
+jest.mock('./utils');
+jest.mocked(requireAppReady).mockResolvedValue();
+jest.mocked(buildApp).mockResolvedValue();
+
+jest.mock('./status');
+jest.mocked(isAppStarted).mockResolvedValue(true);
+jest.mocked(restartApp).mockResolvedValue();
+
+jest.mock('../infrastructure/agent-config');
+const mockSetAppInstalling = jest.fn().mockResolvedValue({});
+const mockSetAppInstalled = jest.fn().mockResolvedValue({});
+const mockGetAppVersion = jest.fn().mockResolvedValue('test-version');
+jest.mocked(AgentConfigFile as jest.Mock).mockReturnValue({
+  setAppInstalling: mockSetAppInstalling,
+  setAppInstalled: mockSetAppInstalled,
+  getAppVersion: mockGetAppVersion
+});
+const projectId1 = 'test-project';
+
+describe('Test environment variable get and set', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+  describe('Test getAllEnvs()', () => {
+    test('get all envs with one service and no envs', async () => {
+      const compose = {
+        services: {
+          alwaysai: {}
+        }
+      };
+      jest.mocked(readDockerCompose).mockResolvedValue(compose);
+
+      const envVars = await getAllEnvs({ projectId: projectId1 });
+      expect(jest.mocked(readDockerCompose)).toBeCalledWith({
+        projectId: projectId1
+      });
+      expect(envVars).toEqual({
+        alwaysai: {}
+      });
+    });
+    // TODO: Add test with env file
+    test('get all envs with one service and two envs', async () => {
+      const compose = {
+        services: {
+          alwaysai: {
+            environment: ['TEST=1', 'TEST2=2']
+          }
+        }
+      };
+      jest.mocked(readDockerCompose).mockResolvedValue(compose);
+
+      const envVars = await getAllEnvs({ projectId: projectId1 });
+      expect(jest.mocked(readDockerCompose)).toBeCalledWith({
+        projectId: projectId1
+      });
+      expect(envVars).toEqual({
+        alwaysai: { TEST: '1', TEST2: '2' }
+      });
+    });
+    test('get all envs with two services and envs', async () => {
+      const compose = {
+        services: {
+          alwaysai: {
+            environment: ['TEST=3', 'TEST2=4']
+          },
+          edgeiq: {
+            environment: ['TEST3=5', 'TEST4=6']
+          }
+        }
+      };
+      jest.mocked(readDockerCompose).mockResolvedValue(compose);
+
+      const envVars = await getAllEnvs({ projectId: projectId1 });
+      expect(jest.mocked(readDockerCompose)).toBeCalledWith({
+        projectId: projectId1
+      });
+      expect(envVars).toEqual({
+        alwaysai: { TEST: '3', TEST2: '4' },
+        edgeiq: { TEST3: '5', TEST4: '6' }
+      });
+    });
+  });
+  describe('Test setEnv()', () => {
+    test('Set one env', async () => {
+      const compose = {
+        services: {
+          alwaysai: {}
+        }
+      };
+      jest.mocked(readDockerCompose).mockResolvedValue(compose);
+
+      const envVars = { alwaysai: { TEST: '1' } };
+      await setEnv({ projectId: projectId1, envVars });
+      expect(jest.mocked(readDockerCompose)).toBeCalledWith({
+        projectId: projectId1
+      });
+      expect(jest.mocked(writeDockerCompose)).toBeCalledWith({
+        projectId: projectId1,
+        dockerCompose: {
+          services: {
+            alwaysai: {
+              environment: ['TEST=1']
+            }
+          }
+        }
+      });
+    });
+    test('Update one env', async () => {
+      const environment = ['TEST1=1', 'TEST2=2'];
+      const compose = {
+        services: {
+          alwaysai: { environment }
+        }
+      };
+      jest.mocked(readDockerCompose).mockResolvedValue(compose);
+
+      const envVars = { alwaysai: { TEST1: '2' } };
+      await setEnv({ projectId: projectId1, envVars });
+      expect(jest.mocked(readDockerCompose)).toBeCalledWith({
+        projectId: projectId1
+      });
+      expect(jest.mocked(writeDockerCompose)).toBeCalledWith({
+        projectId: projectId1,
+        dockerCompose: {
+          services: {
+            alwaysai: {
+              environment: ['TEST1=1', 'TEST2=2', 'TEST1=2']
+            }
+          }
+        }
+      });
+    });
+    test('Update one env with two services', async () => {
+      const environment = ['TEST1=3', 'TEST2=4'];
+      const compose = {
+        services: {
+          alwaysai: { environment },
+          other: { environment: ['OTHER_TEST=1'] }
+        }
+      };
+      jest.mocked(readDockerCompose).mockResolvedValue(compose);
+
+      const envVars = { alwaysai: { TEST1: '2' } };
+      await setEnv({ projectId: projectId1, envVars });
+      expect(jest.mocked(readDockerCompose)).toBeCalledWith({
+        projectId: projectId1
+      });
+      expect(jest.mocked(writeDockerCompose)).toBeCalledWith({
+        projectId: projectId1,
+        dockerCompose: {
+          services: {
+            alwaysai: {
+              environment: ['TEST1=3', 'TEST2=4', 'TEST1=2']
+            },
+            other: {
+              environment: ['OTHER_TEST=1']
+            }
+          }
+        }
+      });
+    });
+  });
+});

package/src/application-control/environment-variables.ts

@@ -1,72 +1,131 @@
 import { JsSpawner } from 'alwaysai/lib/util';
-import { AgentConfigFile } from '../infrastructure/agent-config';
 import { readDockerCompose, writeDockerCompose } from './config';
-import { getAppDir } from './utils';
+import { buildApp, getAppDir, requireAppReady } from './utils';
+import { logger } from '../util/logger';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+import { isAppStarted, restartApp } from './status';
+
+export interface EnvVars {
+  [service: string]: {
+    [name: string]: string;
+  };
+}
+
+export async function setEnv(props: { projectId: string; envVars: EnvVars }) {
+  const { projectId, envVars } = props;
+  await requireAppReady({ projectId });
+  const appReleaseHash = await AgentConfigFile().getAppVersion({
+    projectId
+  });
+  await AgentConfigFile().setAppInstalling({
+    projectId,
+    version: appReleaseHash
+  });
 
-export async function setEnv(props: {
-  projectId: string;
-  vars: string[];
-  service?: string;
-}) {
-  const { projectId, vars } = props;
-  if (!(await AgentConfigFile().isAppReady({ projectId }))) {
-    throw new Error(`App ${projectId} is not ready!`);
-  }
   const composeParsed = await readDockerCompose({ projectId });
-  if ('services' in composeParsed) {
-    let services: string[] = Object.keys(composeParsed['services']);
-    if (props.service) {
-      if (services.includes(props.service)) {
-        services = [props.service];
-      } else {
-        throw new Error(`Service ${props.service} not found in ${services}`);
-      }
+  if (!('services' in composeParsed)) {
+    throw new Error(`Docker compose file for ${projectId} has no services!`);
+  }
+  const services: string[] = Object.keys(envVars);
+  for (const s of services) {
+    if (!Object.keys(composeParsed['services']).includes(s)) {
+      throw new Error(
+        `Service ${s} not found in ${JSON.stringify(
+          composeParsed['services'],
+          null,
+          2
+        )}`
+      );
     }
-    for (const s of services) {
-      const service = composeParsed['services'][s];
-      // The environment field overrides the env files, so by appending to
-      // the environment list we can assure the value will be used
-      if ('environment' in service) {
-        const environment: string[] = service['environment'];
-        composeParsed['services'][s]['environment'] = environment.concat(vars);
-      } else {
-        composeParsed['services'][s]['environment'] = vars;
-      }
+    const envVarList: string[] = [];
+    for (const envVar of Object.keys(envVars[s])) {
+      envVarList.push(`${envVar}=${envVars[s][envVar]}`);
+    }
+    const service = composeParsed['services'][s];
+    // The environment field overrides the env files, so by appending to
+    // the environment list we can assure the value will be used over
+    // the env_file.
+
+    // NOTE: We are only appending the new values to the end of the
+    // environment variable list, which will override but not replace
+    // previous instances of the same environment variable.
+    if ('environment' in service) {
+      const environment: string[] = service['environment'];
+      composeParsed['services'][s]['environment'] =
+        environment.concat(envVarList);
+    } else {
+      composeParsed['services'][s]['environment'] = envVarList;
     }
   }
   await writeDockerCompose({ projectId, dockerCompose: composeParsed });
+
+  const appDir = getAppDir(projectId);
+  await buildApp({ appDir });
+
+  await AgentConfigFile().setAppInstalled({
+    projectId,
+    version: appReleaseHash
+  });
+
+  if (await isAppStarted({ projectId })) {
+    await restartApp({ projectId });
+  }
+  logger.info(
+    `Updated environment variables for ${projectId}: ${JSON.stringify(
+      envVars,
+      null,
+      2
+    )}`
+  );
 }
 
-export async function getAllEnvs(props: { projectId: string }) {
+async function convertStringEnvsToKeyVal(stringEnvs: string[]) {
+  const envVars = {};
+  stringEnvs.forEach((env: string) => {
+    const keyVal = env.split('=');
+    envVars[keyVal[0]] = keyVal[1];
+  });
+  return envVars;
+}
+
+export async function getAllEnvs(props: {
+  projectId: string;
+}): Promise<EnvVars> {
   const { projectId } = props;
-  if (!(await AgentConfigFile().isAppReady({ projectId }))) {
-    throw new Error(`App ${projectId} is not ready!`);
-  }
+  await requireAppReady({ projectId });
   const appDir = getAppDir(projectId);
   const spawner = JsSpawner({ path: appDir });
-  const envVars = {};
+  const envVars: EnvVars = {};
   const composeParsed = await readDockerCompose({ projectId });
   if ('services' in composeParsed) {
     const services = Object.keys(composeParsed['services']);
     for (const s of services) {
-      envVars[s] = [];
+      envVars[s] = {};
       const service = composeParsed['services'][s];
+      // Read env_file first, since it is lowest on the hierarchy
       if ('env_file' in service) {
         const envFiles: string[] = service['env_file'];
         for (const ef of envFiles) {
-          envVars[s] = envVars[s].concat(
-            (await spawner.readFile(ef)).split('\n')
-          );
+          if (!(await spawner.exists(ef))) {
+            logger.error(
+              `Skipping env file ${ef} for service ${s}: not found!`
+            );
+            continue;
+          }
+          let envFileLines = (await spawner.readFile(ef)).split('\n');
+          // Filter out empty lines and comment lines
+          envFileLines = envFileLines.filter((v) => {
+            return v !== '' && !v.includes('#');
+          });
+          const newEnvVars = await convertStringEnvsToKeyVal(envFileLines);
+          envVars[s] = { ...envVars[s], ...newEnvVars };
         }
       }
       if ('environment' in service) {
         const environment: string[] = service['environment'];
-        envVars[s] = envVars[s].concat(environment);
+        const newEnvVars = await convertStringEnvsToKeyVal(environment);
+        envVars[s] = { ...envVars[s], ...newEnvVars };
       }
-      // Filter out empty lines and comment lines
-      envVars[s] = envVars[s].filter((v) => {
-        return v !== '' && !v.includes('#');
-      });
     }
   }
 

package/src/application-control/index.ts

@@ -1,9 +1,4 @@
-import {
-  readAppCfgFile,
-  updateAppCfg,
-  readDockerCompose,
-  writeDockerCompose
-} from './config';
+import { readAppCfgFile, updateAppCfg } from './config';
 import { installApp, uninstallApp } from './install';
 import { rollbackApp } from './backup';
 import {
@@ -14,13 +9,11 @@ import {
   restartApp
 } from './status';
 import { ModelDetails } from './types';
-import { getAllEnvs, setEnv } from './environment-variables';
+import { EnvVars, getAllEnvs, setEnv } from './environment-variables';
 
 export {
   readAppCfgFile,
   updateAppCfg,
-  readDockerCompose,
-  writeDockerCompose,
   installApp,
   uninstallApp,
   rollbackApp,
@@ -30,6 +23,7 @@ export {
   stopApp,
   restartApp,
   ModelDetails,
+  EnvVars,
   getAllEnvs,
   setEnv
 };

package/src/application-control/models.ts

@@ -13,8 +13,7 @@ import { join, dirname } from 'path';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import { copyDir } from '../util/copy-dir';
 import { runInDir } from '../util/run-in-dir';
-import { restartApp } from './status';
-
+import { isAppStarted, restartApp } from './status';
 import { ModelDetails } from './types';
 import {
   buildApp,
@@ -24,7 +23,7 @@ import {
 } from './utils';
 import { MODEL_JSON_FILE_NAME } from 'alwaysai/lib/core/model';
 import { APP_MODELS_DIRECTORY_NAME } from 'alwaysai/lib/constants';
-import { readAppCfgFile, writeAppCfgFile } from './config';
+import { writeAppCfgFile } from './config';
 import { AppConfig } from '@alwaysai/app-configuration-schemas';
 
 export async function getAppModels(props: { projectId: string }) {
@@ -158,27 +157,21 @@ export async function installModelsWithPresignedURLs(
 export async function updateModelsWithPresignedUrls(props: {
   projectId: string;
   modelInstallPayloads: ModelInstallPayload[];
-  appReleaseHash: string;
   newAppCfg: AppConfig;
 }) {
-  const { projectId, modelInstallPayloads, appReleaseHash, newAppCfg } = props;
+  const { projectId, modelInstallPayloads, newAppCfg } = props;
   logger.info(`Installing models for ${projectId}`);
   const spawner = JsSpawner();
   const appDir = getAppDir(projectId);
 
-  if (await AgentConfigFile().isAppPresent({ projectId })) {
-    if (!(await AgentConfigFile().isAppReady({ projectId }))) {
-      throw new Error('Application already has installation in progress!');
-    }
-    logger.info('Updating installed application');
-    await AgentConfigFile().setAppInstalling({
-      projectId,
-      version: appReleaseHash
-    });
-  } else {
-    throw new Error('Application is not installed!');
-  }
-  const ogAppCfg = await readAppCfgFile({ projectId });
+  await requireAppReady({ projectId });
+  const appReleaseHash = await AgentConfigFile().getAppVersion({
+    projectId
+  });
+  await AgentConfigFile().setAppInstalling({
+    projectId,
+    version: appReleaseHash
+  });
 
   const ogDir = path.join(appDir, APP_MODELS_DIRECTORY_NAME);
   // Copy all current models to restore dir in case of failure
@@ -206,19 +199,11 @@ export async function updateModelsWithPresignedUrls(props: {
       version: appReleaseHash
     });
 
-    await restartApp({ projectId });
+    if (await isAppStarted({ projectId })) {
+      await restartApp({ projectId });
+    }
 
     logger.info(`Models installed for project ${projectId}`);
-    /* Leave error handling to higher level so errors are sent to cloud
-  } catch (e) {
-    logger.error(
-      'Error updating app models from presigned URL, restoring models.',
-      e.message
-    );
-    await spawner.rimraf(ogDir);
-    await copyDir({ srcPath: restoreDir, destPath: ogDir });
-    await writeAppCfgFile({ projectId, appCfg: ogAppCfg });
-    */
   } finally {
     await spawner.rimraf(tmpDir);
     await spawner.rimraf(restoreDir);

package/src/application-control/status.ts

@@ -26,6 +26,8 @@ export async function getAppStatus(props: {
   };
   if (!(await AgentConfigFile().isAppReady({ projectId }))) {
     // App is being installed or updated
+    // FIXME: We may be able to get the service status even
+    // though an update is in progress
     return { appDetails, services: [] };
   }
 
@@ -95,6 +97,24 @@ export async function getAppStatus(props: {
   return { appDetails, services };
 }
 
+export async function isAppStarted(props: {
+  projectId: string;
+}): Promise<boolean> {
+  const { projectId } = props;
+  const appStatus = await getAppStatus({ projectId });
+  if (appStatus.services.length === 0) {
+    // Services list will be empty if app has not yet been started.
+    // NOTE: This depends on internal handling in getAppStatus()
+    return false;
+  }
+  for (const service of appStatus.services) {
+    if (service.state === keyMirrors.appState.stopped) {
+      return false;
+    }
+  }
+  return true;
+}
+
 export async function getAppLogs(props: {
   projectId: string;
   services?: string[];

package/src/application-control/utils.ts

@@ -28,7 +28,7 @@ export async function requireAppInstalled(props: { projectId: string }) {
   const { projectId } = props;
   // Ensure an app that is being installed or updated is not modified
   if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
-    throw new Error('Application is not installed');
+    throw new Error(`Application ${projectId} is not installed`);
   }
 }
 
@@ -36,7 +36,9 @@ export async function requireAppReady(props: { projectId: string }) {
   const { projectId } = props;
   await requireAppInstalled({ projectId });
   if (!(await AgentConfigFile().isAppReady({ projectId }))) {
-    throw new Error('Application is not done installing or updating');
+    throw new Error(
+      `Application ${projectId} is not done installing or updating`
+    );
   }
 }