@alwaysai/device-agent 0.0.12 → 0.0.14

package/src/application-control/config.ts

@@ -1,35 +1,97 @@
-import { AppConfig, AppJsonFile } from 'alwaysai/lib/core/app';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import { parse, stringify } from 'yaml';
-import { getAppDir } from './utils';
+import { buildApp, getAppDir } from './utils';
 import { JsSpawner } from 'alwaysai/lib/util';
 import compose from 'docker-compose';
 import { assign, merge } from 'lodash';
+import { AppJsonFile } from 'alwaysai/lib/core/app';
+import { AppConfig } from '@alwaysai/app-configuration-schemas';
+import { restartApp } from './status';
+import { logger } from '../util/logger';
 
-export async function readAppCfg(props: { projectId: string }): Promise<AppConfig> {
+export async function readAppCfgFile(props: {
+  projectId: string;
+}): Promise<AppConfig> {
   const { projectId } = props;
-  if (!(await AgentConfigFile().isAppReady({ projectId }))) {
-    throw new Error(`App ${projectId} is not ready!`);
+  if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
+    throw new Error(`App ${projectId} is not present!`);
   }
   const appDir = getAppDir(projectId);
   const appJson = AppJsonFile(appDir);
-  return appJson.read();
+  if (!appJson.exists()) {
+    throw new Error(`App config for ${projectId} doesn't exist!`);
+  }
+  try {
+    return appJson.read();
+  } catch (err) {
+    throw new Error(
+      `Error reading app config for ${projectId}:\n${err}\n${appJson.getErrors()}`
+    );
+  }
 }
 
-export async function writeAppCfg(props: { projectId: string; appCfg: AppConfig }) {
+export async function writeAppCfgFile(props: {
+  projectId: string;
+  appCfg: AppConfig;
+}) {
   const { projectId, appCfg } = props;
-  if (!(await AgentConfigFile().isAppReady({ projectId }))) {
-    throw new Error(`App ${projectId} is not ready!`);
+  if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
+    throw new Error(`App ${projectId} is not present!`);
   }
   const appDir = getAppDir(projectId);
   const appJson = AppJsonFile(appDir);
-  appJson.write(appCfg);
+  try {
+    appJson.write(appCfg);
+  } catch (err) {
+    throw new Error(
+      `Error writing app config for ${projectId}:\n${err}\n${appJson.getErrors()}`
+    );
+  }
 }
 
-export async function updateAppConfig(appDir: string, newAppCfg: AppConfig) {
-  const existingAppCfg = await readAppCfg({ projectId: appDir });
+export async function updateAppCfgFile(props: {
+  projectId: string;
+  newAppCfg: any;
+}) {
+  const { projectId, newAppCfg } = props;
+  const existingAppCfg = await readAppCfgFile({ projectId });
   const appCfg = assign(existingAppCfg, merge(existingAppCfg, newAppCfg));
-  await writeAppCfg({ projectId: appDir, appCfg });
+  await writeAppCfgFile({ projectId, appCfg });
+}
+
+export async function updateAppCfg(props: {
+  projectId: string;
+  appReleaseHash: string;
+  newAppCfg: AppConfig;
+}) {
+  const { projectId, appReleaseHash, newAppCfg } = props;
+  logger.info(`Updating app config for ${projectId}:${appReleaseHash}.`);
+  const appDir = getAppDir(projectId);
+
+  if (await AgentConfigFile().isAppPresent({ projectId })) {
+    if (!(await AgentConfigFile().isAppReady({ projectId }))) {
+      throw new Error('Application already has installation in progress!');
+    }
+    logger.info('Application is already installed, updating');
+    await AgentConfigFile().setAppInstalling({
+      projectId,
+      version: appReleaseHash
+    });
+  } else {
+    throw new Error('Application is not installed!');
+  }
+
+  await writeAppCfgFile({ projectId, appCfg: newAppCfg });
+  await buildApp({ appDir });
+
+  await AgentConfigFile().setAppInstalled({
+    projectId,
+    version: appReleaseHash
+  });
+
+  await restartApp({ projectId });
+
+  logger.info(`Updated app config and rebuilt ${projectId}:${appReleaseHash}.`);
 }
 
 export async function readDockerCompose(props: { projectId: string }) {

package/src/application-control/environment-variables.ts

@@ -1,6 +1,4 @@
 import { JsSpawner } from 'alwaysai/lib/util';
-import compose from 'docker-compose';
-import { parse, stringify } from 'yaml';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import { readDockerCompose, writeDockerCompose } from './config';
 import { getAppDir } from './utils';
@@ -56,7 +54,9 @@ export async function getAllEnvs(props: { projectId: string }) {
       if ('env_file' in service) {
         const envFiles: string[] = service['env_file'];
         for (const ef of envFiles) {
-          envVars[s] = envVars[s].concat((await spawner.readFile(ef)).split('\n'));
+          envVars[s] = envVars[s].concat(
+            (await spawner.readFile(ef)).split('\n')
+          );
         }
       }
       if ('environment' in service) {

package/src/application-control/index.ts

@@ -1,28 +1,29 @@
-import { readAppCfg, writeAppCfg, readDockerCompose, writeDockerCompose } from './config';
+import {
+  readAppCfgFile,
+  updateAppCfg,
+  readDockerCompose,
+  writeDockerCompose
+} from './config';
 import { installApp, uninstallApp } from './install';
 import { rollbackApp } from './backup';
 import {
-  listAppReleases,
-  listAppLatestRelease,
   getAppStatus,
   startApp,
   getAppLogs,
   stopApp,
-  restartApp,
+  restartApp
 } from './status';
 import { ModelDetails } from './types';
 import { getAllEnvs, setEnv } from './environment-variables';
 
 export {
-  readAppCfg,
-  writeAppCfg,
+  readAppCfgFile,
+  updateAppCfg,
   readDockerCompose,
   writeDockerCompose,
   installApp,
   uninstallApp,
   rollbackApp,
-  listAppReleases,
-  listAppLatestRelease,
   getAppStatus,
   startApp,
   getAppLogs,
@@ -30,16 +31,22 @@ export {
   restartApp,
   ModelDetails,
   getAllEnvs,
-  setEnv,
+  setEnv
 };
 
 // CLI-mode only
 import {
-  addModel,
   getAppModels,
   removeModel,
   replaceModels,
   updateModels,
+  updateModelsWithPresignedUrls
 } from './models';
 
-export { addModel, getAppModels, removeModel, replaceModels, updateModels };
+export {
+  getAppModels,
+  removeModel,
+  replaceModels,
+  updateModels,
+  updateModelsWithPresignedUrls
+};

package/src/application-control/install.ts

@@ -1,10 +1,11 @@
-import * as rimraf from 'rimraf';
+import { rimraf } from 'rimraf';
 import * as fs from 'fs';
 import * as path from 'path';
+import { buildDockerImage } from 'alwaysai/lib/util/docker';
 import { JsSpawner, Spawner } from 'alwaysai/lib/util';
 import { getAppDir, downloadPackageUsingPresignedUrl, buildApp } from './utils';
-import { AppDetailsPacket, ModelInstallPayload } from '@alwaysai/device-agent-schemas';
-import { BACKUP_EXT, createAppBackup } from './backup';
+import { AppDetailsPacket } from '@alwaysai/device-agent-schemas';
+import { BACKUP_EXT } from './backup';
 import { stopApp } from './status';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import { ProjectJsonFile } from 'alwaysai/lib/core/project';
@@ -12,14 +13,16 @@ import {
   getTargetHardwareType,
   AppJsonFile,
   TargetJsonFile,
+  appCleanDocker,
+  getPythonVenvPaths,
+  installPythonVenv,
+  installPythonReqs
 } from 'alwaysai/lib/core/app';
-import { appCleanDockerComponent, buildDocker } from 'alwaysai/lib/components/app';
 import {
-  appInstallComponent,
-  installVenv,
-} from 'alwaysai/lib/components/app/app-install-component';
-import { DOCKERFILE, DOCKER_IMAGE_ID_INITIAL_VALUE } from 'alwaysai/lib/constants';
-import { appReleasePullComponent } from 'alwaysai/lib/components/release';
+  DOCKERFILE,
+  DOCKER_IMAGE_ID_INITIAL_VALUE,
+  PYTHON_REQUIREMENTS_FILE_NAME
+} from 'alwaysai/lib/constants';
 import { runInDir } from '../util/run-in-dir';
 import { installModelsWithPresignedURLs } from './models';
 import { logger } from '../util/logger';
@@ -48,9 +51,10 @@ export async function getInstalledApps(): Promise<AppDetailsPacket[]> {
 export async function installApp(props: {
   projectId: string;
   appReleaseHash: string;
-  signedUrlsPayload?: SignedUrlPayloadType;
+  signedUrlsPayload: SignedUrlPayloadType;
 }): Promise<void> {
   const { projectId, appReleaseHash, signedUrlsPayload } = props;
+  logger.info(`Installing ${projectId}:${appReleaseHash}`);
 
   const appDir = getAppDir(projectId);
   const spawner = JsSpawner();
@@ -59,32 +63,27 @@ export async function installApp(props: {
       throw new Error('Application already has installation in progress!');
     }
     logger.info('Application is already installed, updating');
-    await AgentConfigFile().setAppInstalling({ projectId, version: appReleaseHash });
+    await AgentConfigFile().setAppInstalling({
+      projectId,
+      version: appReleaseHash
+    });
     await spawner.rimraf(appDir);
   } else {
-    await AgentConfigFile().setAppInstalling({ projectId, version: appReleaseHash });
+    await AgentConfigFile().setAppInstalling({
+      projectId,
+      version: appReleaseHash
+    });
   }
 
   await spawner.mkdirp(appDir);
 
   // download app package
   const localDest = path.join(appDir, `${path.basename(appReleaseHash)}.tgz`);
-  if (!signedUrlsPayload) {
-    await runInDir(
-      appReleasePullComponent,
-      [
-        {
-          yes: true,
-          project: projectId,
-          releaseHash: appReleaseHash,
-        },
-      ],
-      appDir,
-    );
-  } else {
-    const { appSignedUrl } = signedUrlsPayload.appInstallPayload;
-    await downloadPackageUsingPresignedUrl({ localDest, presignedUrl: appSignedUrl });
-  }
+  const { appSignedUrl } = signedUrlsPayload.appInstallPayload;
+  await downloadPackageUsingPresignedUrl({
+    localDest,
+    presignedUrl: appSignedUrl
+  });
 
   // Unpack app package and remove tar file
   await unPackApp({ spawner, localDest, appDir });
@@ -95,71 +94,73 @@ export async function installApp(props: {
   // NOTE: this process no longer checks project collaboration
   await checkValidProjectFiles({ appDir });
 
-  // install models, python venv, and docker image
-  if (!signedUrlsPayload) {
-    await runInDir(
-      appInstallComponent,
-      [
-        {
-          yes: true,
-          pull: true,
-          clean: true,
-          skipModels: false,
-          source: false,
-        },
-      ],
-      appDir,
-    );
-  } else {
-    await installAppBuildReqs({ appDir });
-    await installModelsWithPresignedURLs(
-      signedUrlsPayload.modelsInstallPayload,
-      path.join(appDir, 'models'),
-    );
-  }
+  await installAppBuildReqs({ appDir });
+  await installModelsWithPresignedURLs(
+    signedUrlsPayload.modelsInstallPayload,
+    path.join(appDir, 'models')
+  );
 
   await buildApp({ appDir });
 
-  await AgentConfigFile().setAppInstalled({ projectId, version: appReleaseHash });
+  await AgentConfigFile().setAppInstalled({
+    projectId,
+    version: appReleaseHash
+  });
 
-  logger.info(
-    `Installed app version ${appReleaseHash} from project ${projectId} to ${appDir}.`,
-  );
+  logger.info(`Completed installing ${projectId}:${appReleaseHash}`);
 }
 
 async function installAppBuildReqs(props: { appDir: string }) {
   const { appDir } = props;
   const targetJsonFile = TargetJsonFile(appDir);
-  const targetJson = targetJsonFile.readIfExists();
-  if (!targetJson) {
-    throw new Error('Target json file does not exist!');
+  const targetCfg = targetJsonFile.readIfExists();
+  if (!targetCfg || targetCfg.targetProtocol !== 'docker:') {
+    throw new Error('Target json file does not exist or is invalid!');
   }
 
   await runInDir(
     async () => {
-      const targetHostSpawner = targetJsonFile.readHostSpawner();
+      const hostSpawner = targetJsonFile.readHostSpawner();
 
-      await appCleanDockerComponent({ targetHostSpawner });
+      await appCleanDocker({ targetHostSpawner: hostSpawner });
 
-      await buildDocker({
-        targetJson,
-        targetJsonFile,
-        targetHostSpawner,
-        pull: true,
+      const dockerImageId = await buildDockerImage({
+        targetHostSpawner: hostSpawner,
+        targetHardware: targetCfg.targetHardware,
+        dockerImageTag: targetCfg.dockerImageId,
+        pullBaseImage: true,
+        logger
       });
-
-      await installVenv({
-        targetJson,
-        sourceSpawner: targetHostSpawner,
-        targetJsonFile,
+      targetCfg.dockerImageId = dockerImageId;
+      // FIXME: This should use async internally
+      targetJsonFile.update((cfg) => {
+        if (cfg.targetProtocol === 'docker:') {
+          cfg.dockerImageId = dockerImageId;
+        }
       });
+
+      const targetSpawner = targetJsonFile.readContainerSpawner();
+      const pythonVenvPaths = await getPythonVenvPaths({ targetCfg });
+      await installPythonVenv({ targetSpawner, pythonVenvPaths, logger });
+      if (await hostSpawner.exists(PYTHON_REQUIREMENTS_FILE_NAME)) {
+        await installPythonReqs({
+          reqFilePath: PYTHON_REQUIREMENTS_FILE_NAME,
+          targetSpawner,
+          pythonVenvPaths,
+          logger
+        });
+      }
     },
     [],
-    appDir,
+    appDir
   );
 }
 
-async function unPackApp(props: { spawner: Spawner; localDest: string; appDir: string }) {
+async function unPackApp(props: {
+  spawner: Spawner;
+  localDest: string;
+  appDir: string;
+}) {
   const { spawner, localDest, appDir } = props;
   await spawner.untar(fs.createReadStream(localDest), appDir);
   await spawner.rimraf(localDest);
@@ -182,25 +183,28 @@ async function checkValidProjectFiles({ appDir }) {
   TargetJsonFile(appDir).write({
     targetProtocol: 'docker:',
     dockerImageId: DOCKER_IMAGE_ID_INITIAL_VALUE,
-    targetHardware: await getTargetHardwareType({}),
+    targetHardware: await getTargetHardwareType({})
   });
 }
 
-export async function uninstallApp(props: { projectId: string }): Promise<void> {
+export async function uninstallApp(props: {
+  projectId: string;
+}): Promise<void> {
   const { projectId } = props;
+  logger.info(`Uninstalling ${projectId}`);
   if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
     logger.info(`Application ${projectId} not installed`);
     return;
   }
   try {
     await stopApp({ projectId });
-  } catch {
-    logger.info('Failed to stop app, may be left running...');
+  } catch (e) {
+    logger.warn(`Failed to stop ${projectId}, may be left running...\n${e}`);
   }
   await AgentConfigFile().setAppUninstalled({ projectId });
   // Delete application directory and backup
   const appDir = getAppDir(projectId);
-  rimraf.sync(appDir);
-  rimraf.sync(`${appDir}${BACKUP_EXT}`);
-  logger.info('Uninstalled', projectId);
+  await rimraf(appDir);
+  await rimraf(`${appDir}${BACKUP_EXT}`);
+  logger.info(`Completed uninstalling ${projectId}`);
 }