@alwaysai/device-agent 0.0.7 → 0.0.9

package/src/application-control/install.ts

@@ -1,19 +1,42 @@
 import * as rimraf from 'rimraf';
 import * as fs from 'fs';
 import * as path from 'path';
-import nodeFetch, { Response } from 'node-fetch';
-import { JsSpawner } from 'alwaysai/lib/util';
-
-import { runCliCmd } from '../util/run-cli-cmd';
-import { buildApp, getAppDir } from './utils';
-import { AppDetails } from '@alwaysai/device-agent-schemas';
+import { JsSpawner, Spawner } from 'alwaysai/lib/util';
+import { getAppDir, downloadPackageUsingPresignedUrl, buildApp } from './utils';
+import { AppDetailsPacket, ModelInstallPayload } from '@alwaysai/device-agent-schemas';
 import { BACKUP_EXT, createAppBackup } from './backup';
 import { stopApp } from './status';
 import { AgentConfigFile } from '../infrastructure/agent-config';
+import { ProjectJsonFile } from 'alwaysai/lib/core/project';
+import {
+  getTargetHardwareType,
+  AppJsonFile,
+  TargetJsonFile,
+} from 'alwaysai/lib/core/app';
+import { appCleanDockerComponent, buildDocker } from 'alwaysai/lib/components/app';
+import {
+  appInstallComponent,
+  installVenv,
+} from 'alwaysai/lib/components/app/app-install-component';
+import { DOCKERFILE, DOCKER_IMAGE_ID_INITIAL_VALUE } from 'alwaysai/lib/constants';
+import { appReleasePullComponent } from 'alwaysai/lib/components/release';
+import { runInDir } from '../util/run-in-dir';
+import { installModelsWithPresignedURLs } from './models';
+
+type SignedUrlPayloadType = {
+  appInstallPayload: {
+    appSignedUrl: string;
+  };
+  modelsInstallPayload: {
+    id: string;
+    version: number;
+    modelSignedUrl: string;
+  }[];
+};
 
-export async function getInstalledApps(): Promise<AppDetails[]> {
+export async function getInstalledApps(): Promise<AppDetailsPacket[]> {
   const apps = await AgentConfigFile().getApps();
-  const appDetails: AppDetails[] = [];
+  const appDetails: AppDetailsPacket[] = [];
   for (const app of apps) {
     const { projectId, version } = app;
     appDetails.push({ projectId, version });
@@ -21,89 +44,145 @@ export async function getInstalledApps(): Promise<AppDetails[]> {
   return appDetails;
 }
 
-export type InstallAppProps = {
+export async function installApp(props: {
   projectId: string;
-  releaseHash: string;
-  presignedAppUrl?: string;
-};
-
-export async function installAppPackage({ localDest, presignedUrl }): Promise<void> {
-  const response = await nodeFetch(presignedUrl);
-  if (response.status !== 200) {
-    // If the URL is invalid; I think we shouldn't get here with the new changes
-    throw new Error(
-      `Status Code: ${response.status}, ${response.statusText}. Response: ${response.body}`,
-    );
-  }
-
-  /**
-   * Write the app package to the hash directory
-   */
-  const stream = response.body.pipe(fs.createWriteStream(localDest));
-  await new Promise((resolve, reject) => {
-    stream.on('finish', resolve);
-    stream.on('error', reject);
-  });
-}
+  appReleaseHash: string;
+  signedUrlsPayload?: SignedUrlPayloadType;
+}): Promise<void> {
+  const { projectId, appReleaseHash, signedUrlsPayload } = props;
 
-export async function installApp(props: InstallAppProps): Promise<void> {
-  const { projectId, releaseHash, presignedAppUrl } = props;
   const appDir = getAppDir(projectId);
   const spawner = JsSpawner();
   if (await AgentConfigFile().isAppPresent({ projectId })) {
     if (!(await AgentConfigFile().isAppReady({ projectId }))) {
       throw new Error('Application already has installation in progress!');
     }
-    await AgentConfigFile().setAppInstalling({ projectId, version: releaseHash });
     console.log('Application is already installed, updating');
-    await createAppBackup({ projectId });
+    await AgentConfigFile().setAppInstalling({ projectId, version: appReleaseHash });
     await spawner.rimraf(appDir);
   } else {
-    await AgentConfigFile().setAppInstalling({ projectId, version: releaseHash });
+    await AgentConfigFile().setAppInstalling({ projectId, version: appReleaseHash });
   }
 
-  /**
-   * Prep directory and fetch the app package
-   */
   await spawner.mkdirp(appDir);
-  const localDest = path.join(appDir, `${path.basename(releaseHash)}.tgz`);
-  if (!presignedAppUrl) {
-    // Download the application using the CLI
-    await runCliCmd({
-      cmd: [
-        'release',
-        'pull',
-        '--yes',
-        '--project',
-        projectId,
-        '--releaseHash',
-        releaseHash,
+
+  // download app package
+  const localDest = path.join(appDir, `${path.basename(appReleaseHash)}.tgz`);
+  if (!signedUrlsPayload) {
+    await runInDir(
+      appReleasePullComponent,
+      [
+        {
+          yes: true,
+          project: projectId,
+          releaseHash: appReleaseHash,
+        },
       ],
-      cwd: appDir,
-    });
+      appDir,
+    );
   } else {
-    await installAppPackage({ localDest, presignedUrl: presignedAppUrl });
+    const { appSignedUrl } = signedUrlsPayload.appInstallPayload;
+    await downloadPackageUsingPresignedUrl({ localDest, presignedUrl: appSignedUrl });
   }
 
-  /**
-   * Unpack app package
-   */
-  await spawner.untar(fs.createReadStream(localDest), appDir);
-  console.log(`Unpacked application to ${appDir}`);
+  // Unpack app package and remove tar file
+  await unPackApp({ spawner, localDest, appDir });
+
+  // The following changes replace the runCliCmd for 'app configure'
+  // This part replaces findOrWriteProjectJsonFile() due to the
+  // calls that check the user is logged in and also for project collaboration
+  // NOTE: this process no longer checks project collaboration
+  await checkValidProjectFiles({ appDir });
+
+  // install models, python venv, and docker image
+  if (!signedUrlsPayload) {
+    await runInDir(
+      appInstallComponent,
+      [
+        {
+          yes: true,
+          pull: true,
+          clean: true,
+          skipModels: false,
+          source: false,
+        },
+      ],
+      appDir,
+    );
+  } else {
+    await installAppBuildReqs({ appDir });
+    await installModelsWithPresignedURLs(
+      signedUrlsPayload.modelsInstallPayload,
+      path.join(appDir, 'models'),
+    );
+  }
 
-  /**
-   * Build app
-   * ToDo: migrate the model pull to use MQTT workflow
-   */
   await buildApp({ appDir });
 
-  /**
-   * Remove tar file
-   */
+  await AgentConfigFile().setAppInstalled({ projectId, version: appReleaseHash });
+
+  console.log(
+    `Installed app version ${appReleaseHash} from project ${projectId} to ${appDir}.`,
+  );
+}
+
+async function installAppBuildReqs(props: { appDir: string }) {
+  const { appDir } = props;
+  const targetJsonFile = TargetJsonFile(appDir);
+  const targetJson = targetJsonFile.readIfExists();
+  if (!targetJson) {
+    throw new Error('Target json file does not exist!');
+  }
+
+  await runInDir(
+    async () => {
+      const targetHostSpawner = targetJsonFile.readHostSpawner();
+
+      await appCleanDockerComponent({ targetHostSpawner });
+
+      await buildDocker({
+        targetJson,
+        targetJsonFile,
+        targetHostSpawner,
+        pull: true,
+      });
+
+      await installVenv({
+        targetJson,
+        sourceSpawner: targetHostSpawner,
+        targetJsonFile,
+      });
+    },
+    [],
+    appDir,
+  );
+}
+
+async function unPackApp(props: { spawner: Spawner; localDest: string; appDir: string }) {
+  const { spawner, localDest, appDir } = props;
+  await spawner.untar(fs.createReadStream(localDest), appDir);
   await spawner.rimraf(localDest);
+}
+
+async function checkValidProjectFiles({ appDir }) {
+  if (!ProjectJsonFile(appDir).readIfExists()) {
+    throw new Error('Project JSON file does not exist!');
+  }
 
-  await AgentConfigFile().setAppInstalled({ projectId, version: releaseHash });
-  console.log(`Installed ${projectId} to ${appDir}`);
+  // check for app json file and app.py files
+  if (!AppJsonFile(appDir).readIfExists()) {
+    throw new Error('App JSON file does not exist!');
+  }
+
+  // write target json
+  if (!fs.existsSync(path.join(appDir, DOCKERFILE))) {
+    throw new Error('No Dockerfile found for application!');
+  }
+  TargetJsonFile(appDir).write({
+    targetProtocol: 'docker:',
+    dockerImageId: DOCKER_IMAGE_ID_INITIAL_VALUE,
+    targetHardware: await getTargetHardwareType({}),
+  });
 }
 
 export async function uninstallApp(props: { projectId: string }): Promise<void> {

package/src/application-control/models.ts

@@ -1,14 +1,32 @@
-import { JsSpawner } from 'alwaysai/lib/util';
-import { existsSync } from 'fs';
-import { join } from 'path';
+import { keyMirrors, ModelInstallPayload } from '@alwaysai/device-agent-schemas';
+import { app } from 'alwaysai/lib/components';
+import {
+  appModelsAddComponent,
+  appModelsRemoveAllComponent,
+  appModelsRemoveComponent,
+  appModelsUpdateComponent,
+} from 'alwaysai/lib/components/app';
+import { JsSpawner, logger } from 'alwaysai/lib/util';
+import { existsSync, createReadStream } from 'fs';
+import { join, dirname } from 'path';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+import { copyDir } from '../util/copy-dir';
+import { runInDir } from '../util/run-in-dir';
+import { getAppStatus, restartApp } from './status';
 
-import { runCliCmd } from '../util/run-cli-cmd';
 import { ModelDetails } from './types';
-import { buildApp, getAppDir, requireAppInstalled } from './utils';
+import {
+  buildApp,
+  downloadPackageUsingPresignedUrl,
+  getAppDir,
+  requireAppInstalled,
+} from './utils';
 
 export async function getAppModels(props: { projectId: string }) {
   const { projectId } = props;
-  await requireAppInstalled({ projectId });
+  if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
+    throw new Error('Application is not installed');
+  }
 
   const appDir = getAppDir(projectId);
   const appCfgPath = join(appDir, 'alwaysai.app.json');
@@ -31,9 +49,11 @@ export async function addModel(props: { projectId: string; modelId: string }) {
   await requireAppInstalled({ projectId });
 
   const appDir = getAppDir(projectId);
-  await runCliCmd({
-    cmd: ['app', 'models', 'add', modelId],
-    cwd: appDir,
+  await appModelsAddComponent({
+    yes: false,
+    dir: appDir,
+    id: modelId,
+    addToProject: false,
   });
   await buildApp({ appDir });
 }
@@ -43,10 +63,18 @@ export async function removeModel(props: { projectId: string; modelId: string })
   await requireAppInstalled({ projectId });
 
   const appDir = getAppDir(projectId);
-  await runCliCmd({
-    cmd: ['app', 'models', 'remove', modelId, '--purge'],
-    cwd: appDir,
-  });
+
+  await runInDir(
+    appModelsRemoveComponent,
+    [
+      {
+        id: modelId,
+        purge: true,
+        removeFromProject: false,
+      },
+    ],
+    appDir,
+  );
 }
 
 export async function replaceModels(props: { projectId: string; modelIds: string[] }) {
@@ -54,14 +82,23 @@ export async function replaceModels(props: { projectId: string; modelIds: string
   await requireAppInstalled({ projectId });
 
   const appDir = getAppDir(projectId);
-  await runCliCmd({
-    cmd: ['app', 'models', 'remove-all', '--purge'],
-    cwd: appDir,
-  });
+
+  await runInDir(
+    appModelsRemoveAllComponent,
+    [
+      {
+        purge: true,
+        removeFromProject: false,
+      },
+    ],
+    appDir,
+  );
   for (const modelId of modelIds) {
-    await runCliCmd({
-      cmd: ['app', 'models', 'add', modelId],
-      cwd: appDir,
+    await appModelsAddComponent({
+      yes: false,
+      dir: appDir,
+      id: modelId,
+      addToProject: false,
     });
   }
   await buildApp({ appDir });
@@ -72,9 +109,85 @@ export async function updateModels(props: { projectId: string }) {
   await requireAppInstalled({ projectId });
 
   const appDir = getAppDir(projectId);
-  await runCliCmd({
-    cmd: ['app', 'models', 'update'],
-    cwd: appDir,
+  await appModelsUpdateComponent({
+    yes: false,
+    dir: appDir,
   });
   await buildApp({ appDir });
 }
+
+export async function installModelsWithPresignedURLs(
+  modelPayloads: ModelInstallPayload[],
+  targetDir: string,
+) {
+  const spawner = JsSpawner();
+  await Promise.all(
+    modelPayloads.map(async (payload: ModelInstallPayload) => {
+      const modelDest = `${targetDir}/${payload.id}`;
+      await spawner.mkdirp(modelDest);
+      const localDest = `${modelDest}/${payload.version}.tar.gz`;
+      await downloadPackageUsingPresignedUrl({
+        localDest,
+        presignedUrl: payload.modelSignedUrl,
+      });
+      await spawner.untar(createReadStream(localDest), dirname(modelDest));
+      await spawner.rimraf(localDest);
+    }),
+  );
+}
+
+export async function updateModelsWithPresignedUrls(
+  project: string,
+  modelInstallPayloads: ModelInstallPayload[],
+) {
+  // create temp dir with all untouched models
+  const spawner = JsSpawner();
+  const appDir = getAppDir(project);
+  const ogDir = `${appDir}/models`;
+  const tmpDir = `${ogDir}.tmp`;
+  const restoreDir = `${ogDir}.restore`;
+  await copyDir({ srcPath: ogDir, destPath: restoreDir });
+  await spawner.rimraf(tmpDir);
+
+  try {
+    // Move unchanged models
+    const existingModels = await getAppModels({ projectId: project });
+    for (const model of existingModels) {
+      if (
+        !modelInstallPayloads
+          .map((newModel: ModelInstallPayload) => newModel.id)
+          .includes(model.modelId)
+      ) {
+        // model does not need to be updated
+        await copyDir({
+          srcPath: `${ogDir}/${model.modelId}`,
+          destPath: `${tmpDir}/${model.modelId}`,
+        });
+      }
+    }
+
+    // install model packages
+    await installModelsWithPresignedURLs(modelInstallPayloads, tmpDir);
+    await spawner.rimraf(ogDir);
+    await copyDir({ srcPath: tmpDir, destPath: ogDir });
+    await spawner.rimraf(tmpDir);
+
+    await buildApp({ appDir: getAppDir(project) });
+
+    const appStatus = await getAppStatus({ projectId: project });
+    const appIsRunning =
+      appStatus.services.length &&
+      appStatus.services[0].state !== keyMirrors.appState.stopped;
+
+    if (appIsRunning) {
+      restartApp({ projectId: project });
+    }
+
+    console.log(`Models installed for project ${project}`);
+  } catch (e) {
+    console.error(e, 'Error updating app models from presigned URL, restoring models.');
+    await spawner.rimraf(tmpDir);
+    await spawner.rimraf(ogDir);
+    await copyDir({ srcPath: restoreDir, destPath: ogDir });
+  }
+}

package/src/application-control/status.ts

@@ -4,7 +4,12 @@ import { JsSpawner } from 'alwaysai/lib/util';
 
 import { runDockerLogin } from '../docker/docker-cmd';
 import { getAppDir, requireAppInstalled } from './utils';
-import { AppStateValue, ServiceStatus, AppState } from '@alwaysai/device-agent-schemas';
+import {
+  ServiceStatusPacket,
+  AppStatePacket,
+  keyMirrors,
+  AppStateValue,
+} from '@alwaysai/device-agent-schemas';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 
 export async function listAppReleases(props: { projectId: string }) {
@@ -22,7 +27,9 @@ export async function listAppLatestRelease(props: { projectId: string }) {
   return undefined;
 }
 
-export async function getAppStatus(props: { projectId: string }): Promise<AppState> {
+export async function getAppStatus(props: {
+  projectId: string;
+}): Promise<AppStatePacket> {
   const { projectId } = props;
   if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
     throw new Error('Application is not installed');
@@ -49,7 +56,7 @@ export async function getAppStatus(props: { projectId: string }): Promise<AppSta
     );
   }
 
-  const services: ServiceStatus[] = [];
+  const services: ServiceStatusPacket[] = [];
   const spawner = JsSpawner({ path: appDir });
   for (const name of composeServices.data.services) {
     // Get container name for service
@@ -60,7 +67,7 @@ export async function getAppStatus(props: { projectId: string }): Promise<AppSta
     });
     if (containerId === '') {
       // The service was not yet started or failed to start
-      services.push({ name, state: 'Stopped' });
+      services.push({ name, state: keyMirrors.appState.stopped });
     } else {
       const slashContainerName = await spawner.run({
         exe: 'docker',
@@ -75,14 +82,14 @@ export async function getAppStatus(props: { projectId: string }): Promise<AppSta
         }
         let state: AppStateValue;
         if (container.state.includes('Up')) {
-          state = 'Up';
+          state = keyMirrors.appState.up;
         } else if (
           container.state.includes('Stopped') ||
           container.state.includes('Exit')
         ) {
-          state = 'Stopped';
+          state = keyMirrors.appState.stopped;
         } else if (container.state.includes('Restarting')) {
-          state = 'Restarting';
+          state = keyMirrors.appState.restarting;
         } else {
           throw new Error(`Unknown state for service! (${container.state})`);
         }
@@ -106,8 +113,9 @@ export async function getAppStatus(props: { projectId: string }): Promise<AppSta
 export async function getAppLogs(props: {
   projectId: string;
   services?: string[];
+  args?: string[];
 }): Promise<NodeJS.ReadableStream> {
-  const { projectId, services } = props;
+  const { projectId, services, args } = props;
   await requireAppInstalled({ projectId });
 
   const appDir = getAppDir(projectId);
@@ -119,10 +127,12 @@ export async function getAppLogs(props: {
         return composeServices.data.services;
       })();
 
+  const argsList = args ? args : [];
+
   // Use direct command with spawner in order to get a readable stream
   return await JsSpawner().runStreaming({
     exe: 'docker-compose',
-    args: ['logs', '-f', ...serviceList],
+    args: ['logs', '-f', ...argsList, ...serviceList],
     cwd: appDir,
   });
 }

package/src/application-control/utils.ts

@@ -1,13 +1,16 @@
 import compose from 'docker-compose';
 import * as path from 'path';
 import * as fs from 'fs';
-import { AAI_DIR } from 'alwaysai/lib/constants';
+import { AAI_DIR, TARGET_JSON_FILE_NAME } from 'alwaysai/lib/constants';
 
-import { runCliCmd } from '../util/run-cli-cmd';
 import { AgentConfigFile } from '../infrastructure/agent-config';
+import nodeFetch from 'node-fetch';
+import { TargetJsonFile } from 'alwaysai/lib/core/app';
+import { appDeployLinuxAndRemoteDevice } from 'alwaysai/lib/components/app';
+import { runInDir } from '../util/run-in-dir';
+import { JsSpawner } from 'alwaysai/lib/util';
 
 export const APP_ROOT = path.join(AAI_DIR, 'applications');
-const DOCKER_COMPOSE_FILENAME = 'docker-compose.yaml';
 
 export function getAppDir(projectId: string): string {
   return path.join(APP_ROOT, projectId);
@@ -26,24 +29,66 @@ export async function requireAppInstalled(props: { projectId: string }) {
 
 export async function buildApp(props: { appDir: string }) {
   const { appDir } = props;
-  await runCliCmd({
-    cmd: ['app', 'configure', '--yes', '--protocol', 'docker:', '--skip-model-sync'],
-    cwd: appDir,
-  });
-  await runCliCmd({ cmd: ['app', 'install', '--yes', '--clean', '--pull'], cwd: appDir });
 
   // Build standalone image and docker-compose
-  if (!fs.existsSync(path.join(appDir, DOCKER_COMPOSE_FILENAME))) {
-    await runCliCmd({
-      cmd: ['app', 'deploy', '--yes', '--generateDockerCompose'],
-      cwd: appDir,
-    });
+  const targetJson = TargetJsonFile(appDir);
+  const targetConfig = targetJson.read();
+  if (targetConfig.targetProtocol !== 'docker:') {
+    throw new Error(`${TARGET_JSON_FILE_NAME} is not properly configured!`);
   }
+  await runInDir(
+    appDeployLinuxAndRemoteDevice,
+    [
+      {
+        yes: true,
+        generateDockerCompose: true,
+        logs: false,
+        stop: false,
+        targetJson,
+        targetConfig,
+      },
+    ],
+    appDir,
+  );
 
   const buildOut = await compose.buildAll({ cwd: appDir });
+  console.log(buildOut);
   if (buildOut.exitCode !== 0) {
     throw new Error(
       `Failed to build application! stdout=${buildOut.out} stderr=${buildOut.err}`,
     );
   }
 }
+
+export async function downloadPackageUsingPresignedUrl(props: {
+  localDest: string;
+  presignedUrl: string;
+}): Promise<void> {
+  console.log('downloading URL');
+  const { localDest, presignedUrl } = props;
+  const response = await nodeFetch(presignedUrl);
+  if (response.status !== 200) {
+    // If the URL is invalid; I think we shouldn't get here with the new changes
+    throw new Error(
+      `Status Code: ${response.status}, ${response.statusText}. Response: ${response.body}`,
+    );
+  }
+
+  /**
+   * Write the app package to the hash directory
+   */
+  const stream = response.body.pipe(fs.createWriteStream(localDest));
+  await new Promise((resolve, reject) => {
+    stream.on('finish', resolve);
+    stream.on('error', reject);
+  });
+}
+
+export async function getAppConfig(project: string) {
+  const appCfgPath = path.join(getAppDir(project), 'alwaysai.app.json');
+  if (!fs.existsSync(appCfgPath)) {
+    throw new Error(`Application config not found for project ${project}`);
+  }
+
+  return JSON.parse(await JsSpawner().readFile(appCfgPath));
+}