@alwaysai/device-agent 0.0.8 → 0.0.10

package/src/application-control/install.ts

@@ -1,13 +1,38 @@
 import * as rimraf from 'rimraf';
 import * as fs from 'fs';
 import * as path from 'path';
-import { JsSpawner } from 'alwaysai/lib/util';
+import { JsSpawner, Spawner } from 'alwaysai/lib/util';
 import { getAppDir, downloadPackageUsingPresignedUrl, buildApp } from './utils';
 import { AppDetailsPacket, ModelInstallPayload } from '@alwaysai/device-agent-schemas';
 import { BACKUP_EXT, createAppBackup } from './backup';
 import { stopApp } from './status';
 import { AgentConfigFile } from '../infrastructure/agent-config';
-import { runCliCmd } from '../util/run-cli-cmd';
+import { ProjectJsonFile } from 'alwaysai/lib/core/project';
+import {
+  getTargetHardwareType,
+  AppJsonFile,
+  TargetJsonFile,
+} from 'alwaysai/lib/core/app';
+import { appCleanDockerComponent, buildDocker } from 'alwaysai/lib/components/app';
+import {
+  appInstallComponent,
+  installVenv,
+} from 'alwaysai/lib/components/app/app-install-component';
+import { DOCKERFILE, DOCKER_IMAGE_ID_INITIAL_VALUE } from 'alwaysai/lib/constants';
+import { appReleasePullComponent } from 'alwaysai/lib/components/release';
+import { runInDir } from '../util/run-in-dir';
+import { installModelsWithPresignedURLs } from './models';
+
+type SignedUrlPayloadType = {
+  appInstallPayload: {
+    appSignedUrl: string;
+  };
+  modelsInstallPayload: {
+    id: string;
+    version: number;
+    modelSignedUrl: string;
+  }[];
+};
 
 export async function getInstalledApps(): Promise<AppDetailsPacket[]> {
   const apps = await AgentConfigFile().getApps();
@@ -22,16 +47,7 @@ export async function getInstalledApps(): Promise<AppDetailsPacket[]> {
 export async function installApp(props: {
   projectId: string;
   appReleaseHash: string;
-  signedUrlsPayload?: {
-    appInstallPayload: {
-      appSignedUrl: string;
-    };
-    modelsInstallPayload: {
-      id: string;
-      version: number;
-      modelSignedUrl: string;
-    }[];
-  };
+  signedUrlsPayload?: SignedUrlPayloadType;
 }): Promise<void> {
   const { projectId, appReleaseHash, signedUrlsPayload } = props;
 
@@ -41,9 +57,8 @@ export async function installApp(props: {
     if (!(await AgentConfigFile().isAppReady({ projectId }))) {
       throw new Error('Application already has installation in progress!');
     }
-    await AgentConfigFile().setAppInstalling({ projectId, version: appReleaseHash });
     console.log('Application is already installed, updating');
-    await createAppBackup({ projectId });
+    await AgentConfigFile().setAppInstalling({ projectId, version: appReleaseHash });
     await spawner.rimraf(appDir);
   } else {
     await AgentConfigFile().setAppInstalling({ projectId, version: appReleaseHash });
@@ -54,56 +69,51 @@ export async function installApp(props: {
   // download app package
   const localDest = path.join(appDir, `${path.basename(appReleaseHash)}.tgz`);
   if (!signedUrlsPayload) {
-    await runCliCmd({
-      cmd: [
-        'release',
-        'pull',
-        '--yes',
-        '--project',
-        projectId,
-        '--releaseHash',
-        appReleaseHash,
+    await runInDir(
+      appReleasePullComponent,
+      [
+        {
+          yes: true,
+          project: projectId,
+          releaseHash: appReleaseHash,
+        },
       ],
-      cwd: appDir,
-    });
+      appDir,
+    );
   } else {
     const { appSignedUrl } = signedUrlsPayload.appInstallPayload;
     await downloadPackageUsingPresignedUrl({ localDest, presignedUrl: appSignedUrl });
   }
 
   // Unpack app package and remove tar file
-  await spawner.untar(fs.createReadStream(localDest), appDir);
-  await spawner.rimraf(localDest);
+  await unPackApp({ spawner, localDest, appDir });
 
-  // write target json
-  await runCliCmd({
-    cmd: ['app', 'configure', '--yes', '--protocol', 'docker:', '--skip-model-sync'],
-    cwd: appDir,
-  });
+  // The following changes replace the runCliCmd for 'app configure'
+  // This part replaces findOrWriteProjectJsonFile() due to the
+  // calls that check the user is logged in and also for project collaboration
+  // NOTE: this process no longer checks project collaboration
+  await checkValidProjectFiles({ appDir });
 
   // install models, python venv, and docker image
   if (!signedUrlsPayload) {
-    await runCliCmd({
-      cmd: ['app', 'install', '--yes', '--clean', '--pull'],
-      cwd: appDir,
-    });
+    await runInDir(
+      appInstallComponent,
+      [
+        {
+          yes: true,
+          pull: true,
+          clean: true,
+          skipModels: false,
+          source: false,
+        },
+      ],
+      appDir,
+    );
   } else {
-    await runCliCmd({
-      cmd: ['app', 'install', '--yes', '--clean', '--pull', '--skip-models'],
-      cwd: appDir,
-    });
-    await Promise.all(
-      signedUrlsPayload.modelsInstallPayload.map(async (payload: ModelInstallPayload) => {
-        const modelDest = `${appDir}/models/${payload.id}`;
-        await spawner.mkdirp(modelDest);
-        const localDest = `${modelDest}/${payload.version}.tar.gz`;
-        await downloadPackageUsingPresignedUrl({
-          localDest,
-          presignedUrl: payload.modelSignedUrl,
-        });
-        await spawner.untar(fs.createReadStream(localDest), path.dirname(modelDest));
-        await spawner.rimraf(localDest);
-      }),
+    await installAppBuildReqs({ appDir });
+    await installModelsWithPresignedURLs(
+      signedUrlsPayload.modelsInstallPayload,
+      path.join(appDir, 'models'),
     );
   }
 
@@ -116,6 +126,65 @@ export async function installApp(props: {
   );
 }
 
+async function installAppBuildReqs(props: { appDir: string }) {
+  const { appDir } = props;
+  const targetJsonFile = TargetJsonFile(appDir);
+  const targetJson = targetJsonFile.readIfExists();
+  if (!targetJson) {
+    throw new Error('Target json file does not exist!');
+  }
+
+  await runInDir(
+    async () => {
+      const targetHostSpawner = targetJsonFile.readHostSpawner();
+
+      await appCleanDockerComponent({ targetHostSpawner });
+
+      await buildDocker({
+        targetJson,
+        targetJsonFile,
+        targetHostSpawner,
+        pull: true,
+      });
+
+      await installVenv({
+        targetJson,
+        sourceSpawner: targetHostSpawner,
+        targetJsonFile,
+      });
+    },
+    [],
+    appDir,
+  );
+}
+
+async function unPackApp(props: { spawner: Spawner; localDest: string; appDir: string }) {
+  const { spawner, localDest, appDir } = props;
+  await spawner.untar(fs.createReadStream(localDest), appDir);
+  await spawner.rimraf(localDest);
+}
+
+async function checkValidProjectFiles({ appDir }) {
+  if (!ProjectJsonFile(appDir).readIfExists()) {
+    throw new Error('Project JSON file does not exist!');
+  }
+
+  // check for app json file and app.py files
+  if (!AppJsonFile(appDir).readIfExists()) {
+    throw new Error('App JSON file does not exist!');
+  }
+
+  // write target json
+  if (!fs.existsSync(path.join(appDir, DOCKERFILE))) {
+    throw new Error('No Dockerfile found for application!');
+  }
+  TargetJsonFile(appDir).write({
+    targetProtocol: 'docker:',
+    dockerImageId: DOCKER_IMAGE_ID_INITIAL_VALUE,
+    targetHardware: await getTargetHardwareType({}),
+  });
+}
+
 export async function uninstallApp(props: { projectId: string }): Promise<void> {
   const { projectId } = props;
   if (!(await AgentConfigFile().isAppPresent({ projectId }))) {

package/src/application-control/models.ts

@@ -1,11 +1,26 @@
-import { JsSpawner } from 'alwaysai/lib/util';
-import { existsSync } from 'fs';
-import { join } from 'path';
+import { keyMirrors, ModelInstallPayload } from '@alwaysai/device-agent-schemas';
+import { app } from 'alwaysai/lib/components';
+import {
+  appModelsAddComponent,
+  appModelsRemoveAllComponent,
+  appModelsRemoveComponent,
+  appModelsUpdateComponent,
+} from 'alwaysai/lib/components/app';
+import { JsSpawner, logger } from 'alwaysai/lib/util';
+import { existsSync, createReadStream } from 'fs';
+import { join, dirname } from 'path';
 import { AgentConfigFile } from '../infrastructure/agent-config';
+import { copyDir } from '../util/copy-dir';
+import { runInDir } from '../util/run-in-dir';
+import { getAppStatus, restartApp } from './status';
 
-import { runCliCmd } from '../util/run-cli-cmd';
 import { ModelDetails } from './types';
-import { buildApp, getAppDir, requireAppInstalled } from './utils';
+import {
+  buildApp,
+  downloadPackageUsingPresignedUrl,
+  getAppDir,
+  requireAppInstalled,
+} from './utils';
 
 export async function getAppModels(props: { projectId: string }) {
   const { projectId } = props;
@@ -34,9 +49,11 @@ export async function addModel(props: { projectId: string; modelId: string }) {
   await requireAppInstalled({ projectId });
 
   const appDir = getAppDir(projectId);
-  await runCliCmd({
-    cmd: ['app', 'models', 'add', modelId],
-    cwd: appDir,
+  await appModelsAddComponent({
+    yes: false,
+    dir: appDir,
+    id: modelId,
+    addToProject: false,
   });
   await buildApp({ appDir });
 }
@@ -46,10 +63,18 @@ export async function removeModel(props: { projectId: string; modelId: string })
   await requireAppInstalled({ projectId });
 
   const appDir = getAppDir(projectId);
-  await runCliCmd({
-    cmd: ['app', 'models', 'remove', modelId, '--purge'],
-    cwd: appDir,
-  });
+
+  await runInDir(
+    appModelsRemoveComponent,
+    [
+      {
+        id: modelId,
+        purge: true,
+        removeFromProject: false,
+      },
+    ],
+    appDir,
+  );
 }
 
 export async function replaceModels(props: { projectId: string; modelIds: string[] }) {
@@ -57,14 +82,23 @@ export async function replaceModels(props: { projectId: string; modelIds: string
   await requireAppInstalled({ projectId });
 
   const appDir = getAppDir(projectId);
-  await runCliCmd({
-    cmd: ['app', 'models', 'remove-all', '--purge'],
-    cwd: appDir,
-  });
+
+  await runInDir(
+    appModelsRemoveAllComponent,
+    [
+      {
+        purge: true,
+        removeFromProject: false,
+      },
+    ],
+    appDir,
+  );
   for (const modelId of modelIds) {
-    await runCliCmd({
-      cmd: ['app', 'models', 'add', modelId],
-      cwd: appDir,
+    await appModelsAddComponent({
+      yes: false,
+      dir: appDir,
+      id: modelId,
+      addToProject: false,
     });
   }
   await buildApp({ appDir });
@@ -75,9 +109,85 @@ export async function updateModels(props: { projectId: string }) {
   await requireAppInstalled({ projectId });
 
   const appDir = getAppDir(projectId);
-  await runCliCmd({
-    cmd: ['app', 'models', 'update'],
-    cwd: appDir,
+  await appModelsUpdateComponent({
+    yes: false,
+    dir: appDir,
   });
   await buildApp({ appDir });
 }
+
+export async function installModelsWithPresignedURLs(
+  modelPayloads: ModelInstallPayload[],
+  targetDir: string,
+) {
+  const spawner = JsSpawner();
+  await Promise.all(
+    modelPayloads.map(async (payload: ModelInstallPayload) => {
+      const modelDest = `${targetDir}/${payload.id}`;
+      await spawner.mkdirp(modelDest);
+      const localDest = `${modelDest}/${payload.version}.tar.gz`;
+      await downloadPackageUsingPresignedUrl({
+        localDest,
+        presignedUrl: payload.modelSignedUrl,
+      });
+      await spawner.untar(createReadStream(localDest), dirname(modelDest));
+      await spawner.rimraf(localDest);
+    }),
+  );
+}
+
+export async function updateModelsWithPresignedUrls(
+  project: string,
+  modelInstallPayloads: ModelInstallPayload[],
+) {
+  // create temp dir with all untouched models
+  const spawner = JsSpawner();
+  const appDir = getAppDir(project);
+  const ogDir = `${appDir}/models`;
+  const tmpDir = `${ogDir}.tmp`;
+  const restoreDir = `${ogDir}.restore`;
+  await copyDir({ srcPath: ogDir, destPath: restoreDir });
+  await spawner.rimraf(tmpDir);
+
+  try {
+    // Move unchanged models
+    const existingModels = await getAppModels({ projectId: project });
+    for (const model of existingModels) {
+      if (
+        !modelInstallPayloads
+          .map((newModel: ModelInstallPayload) => newModel.id)
+          .includes(model.modelId)
+      ) {
+        // model does not need to be updated
+        await copyDir({
+          srcPath: `${ogDir}/${model.modelId}`,
+          destPath: `${tmpDir}/${model.modelId}`,
+        });
+      }
+    }
+
+    // install model packages
+    await installModelsWithPresignedURLs(modelInstallPayloads, tmpDir);
+    await spawner.rimraf(ogDir);
+    await copyDir({ srcPath: tmpDir, destPath: ogDir });
+    await spawner.rimraf(tmpDir);
+
+    await buildApp({ appDir: getAppDir(project) });
+
+    const appStatus = await getAppStatus({ projectId: project });
+    const appIsRunning =
+      appStatus.services.length &&
+      appStatus.services[0].state !== keyMirrors.appState.stopped;
+
+    if (appIsRunning) {
+      restartApp({ projectId: project });
+    }
+
+    console.log(`Models installed for project ${project}`);
+  } catch (e) {
+    console.error(e, 'Error updating app models from presigned URL, restoring models.');
+    await spawner.rimraf(tmpDir);
+    await spawner.rimraf(ogDir);
+    await copyDir({ srcPath: restoreDir, destPath: ogDir });
+  }
+}

package/src/application-control/utils.ts

@@ -1,14 +1,16 @@
 import compose from 'docker-compose';
 import * as path from 'path';
 import * as fs from 'fs';
-import { AAI_DIR } from 'alwaysai/lib/constants';
+import { AAI_DIR, TARGET_JSON_FILE_NAME } from 'alwaysai/lib/constants';
 
-import { runCliCmd } from '../util/run-cli-cmd';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import nodeFetch from 'node-fetch';
+import { TargetJsonFile } from 'alwaysai/lib/core/app';
+import { appDeployLinuxAndRemoteDevice } from 'alwaysai/lib/components/app';
+import { runInDir } from '../util/run-in-dir';
+import { JsSpawner } from 'alwaysai/lib/util';
 
 export const APP_ROOT = path.join(AAI_DIR, 'applications');
-const DOCKER_COMPOSE_FILENAME = 'docker-compose.yaml';
 
 export function getAppDir(projectId: string): string {
   return path.join(APP_ROOT, projectId);
@@ -27,15 +29,27 @@ export async function requireAppInstalled(props: { projectId: string }) {
 
 export async function buildApp(props: { appDir: string }) {
   const { appDir } = props;
-  console.log(appDir);
 
   // Build standalone image and docker-compose
-  if (!fs.existsSync(path.join(appDir, DOCKER_COMPOSE_FILENAME))) {
-    await runCliCmd({
-      cmd: ['app', 'deploy', '--yes', '--generateDockerCompose'],
-      cwd: appDir,
-    });
+  const targetJson = TargetJsonFile(appDir);
+  const targetConfig = targetJson.read();
+  if (targetConfig.targetProtocol !== 'docker:') {
+    throw new Error(`${TARGET_JSON_FILE_NAME} is not properly configured!`);
   }
+  await runInDir(
+    appDeployLinuxAndRemoteDevice,
+    [
+      {
+        yes: true,
+        generateDockerCompose: true,
+        logs: false,
+        stop: false,
+        targetJson,
+        targetConfig,
+      },
+    ],
+    appDir,
+  );
 
   const buildOut = await compose.buildAll({ cwd: appDir });
   console.log(buildOut);
@@ -46,10 +60,12 @@ export async function buildApp(props: { appDir: string }) {
   }
 }
 
-export async function downloadPackageUsingPresignedUrl({
-  localDest,
-  presignedUrl,
+export async function downloadPackageUsingPresignedUrl(props: {
+  localDest: string;
+  presignedUrl: string;
 }): Promise<void> {
+  console.log('downloading URL');
+  const { localDest, presignedUrl } = props;
   const response = await nodeFetch(presignedUrl);
   if (response.status !== 200) {
     // If the URL is invalid; I think we shouldn't get here with the new changes
@@ -67,3 +83,12 @@ export async function downloadPackageUsingPresignedUrl({
     stream.on('error', reject);
   });
 }
+
+export async function getAppConfig(project: string) {
+  const appCfgPath = path.join(getAppDir(project), 'alwaysai.app.json');
+  if (!fs.existsSync(appCfgPath)) {
+    throw new Error(`Application config not found for project ${project}`);
+  }
+
+  return JSON.parse(await JsSpawner().readFile(appCfgPath));
+}