@alwaysai/device-agent 1.4.0 → 2.0.0

package/src/application-control/models.ts

@@ -1,30 +1,18 @@
-import * as path from 'path';
 import { ModelInstallPayload } from '@alwaysai/device-agent-schemas';
-import {
-  appModelsAddComponent,
-  appModelsRemoveAllComponent,
-  appModelsRemoveComponent,
-  appModelsUpdateComponent
-} from 'alwaysai/lib/components/app';
 import { JsSpawner } from 'alwaysai/lib/util';
 import { logger } from '../util/logger';
 import { existsSync, createReadStream } from 'fs';
 import { join, dirname } from 'path';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import { copyDir } from '../util/copy-dir';
-import { runInDir } from '../util/run-in-dir';
 import { isAppStarted, restartApp } from './status';
 import { ModelDetails } from './types';
-import {
-  buildApp,
-  downloadPackageUsingPresignedUrl,
-  getAppDir,
-  requireAppReady
-} from './utils';
+import { buildApp, getAppDir, requireAppReady } from './utils';
 import { MODEL_JSON_FILE_NAME } from 'alwaysai/lib/core/model';
 import { writeAppCfgFile } from './config';
-import { AppConfig } from '@alwaysai/app-configuration-schemas';
 import { APP_MODELS_DIRECTORY_NAME } from 'alwaysai/lib/paths';
+import { downloadToFile, pruneDir } from '../util/file';
+import { AppConfig } from '@alwaysai/app-configuration-schemas';
 
 export async function getAppModels(props: { projectId: string }) {
   const { projectId } = props;
@@ -48,75 +36,6 @@ export async function getAppModels(props: { projectId: string }) {
   return modelDetails;
 }
 
-export async function removeModel(props: {
-  projectId: string;
-  modelId: string;
-}) {
-  const { projectId, modelId } = props;
-  await requireAppReady({ projectId });
-
-  const appDir = getAppDir(projectId);
-
-  await runInDir(
-    appModelsRemoveComponent,
-    [
-      {
-        id: modelId,
-        purge: true,
-        removeFromProject: false
-      }
-    ],
-    appDir
-  );
-}
-
-export async function replaceModels(props: {
-  projectId: string;
-  modelIds: string[];
-}) {
-  const { projectId, modelIds } = props;
-  await requireAppReady({ projectId });
-
-  const appDir = getAppDir(projectId);
-
-  await runInDir(
-    appModelsRemoveAllComponent,
-    [
-      {
-        purge: true,
-        removeFromProject: false
-      }
-    ],
-    appDir
-  );
-  const modelsAddPromises: Promise<void>[] = [];
-  for (const modelId of modelIds) {
-    modelsAddPromises.push(
-      appModelsAddComponent({
-        yes: false,
-        dir: appDir,
-        id: modelId,
-        addToProject: false
-      })
-    );
-  }
-  await Promise.all(modelsAddPromises);
-
-  await buildApp({ appDir });
-}
-
-export async function updateModels(props: { projectId: string }) {
-  const { projectId } = props;
-  await requireAppReady({ projectId });
-
-  const appDir = getAppDir(projectId);
-  await appModelsUpdateComponent({
-    yes: false,
-    dir: appDir
-  });
-  await buildApp({ appDir });
-}
-
 export async function installModelsWithPresignedURLs(
   modelPayloads: ModelInstallPayload[],
   targetDir: string
@@ -127,12 +46,12 @@ export async function installModelsWithPresignedURLs(
     modelPayloads.map(async (payload: ModelInstallPayload) => {
       logger.info(`Installing ${payload.id}: ${payload.version}`);
       const version = payload.version;
-      const modelDest = path.join(targetDir, payload.id);
+      const modelDest = join(targetDir, payload.id);
       await spawner.mkdirp(modelDest);
-      const localDest = path.join(modelDest, `${payload.version}.tar.gz`);
-      await downloadPackageUsingPresignedUrl({
-        localDest,
-        presignedUrl: payload.modelSignedUrl
+      const localDest = join(modelDest, `${payload.version}.tar.gz`);
+      await downloadToFile({
+        path: localDest,
+        url: payload.modelSignedUrl
       });
       await spawner.untar(createReadStream(localDest), dirname(modelDest));
       await updateModelJson(modelDest, (modelJson) => ({
@@ -178,20 +97,21 @@ export async function updateModelsWithPresignedUrls(props: {
     version: appReleaseHash
   });
 
-  const ogDir = path.join(appDir, APP_MODELS_DIRECTORY_NAME);
-  // Copy all current models to restore dir in case of failure
-  const restoreDir = `${ogDir}.restore`;
-  await spawner.rimraf(restoreDir);
-  await copyDir({ srcPath: ogDir, destPath: restoreDir });
-
-  // Create temp dir to install new models
+  const ogDir = join(appDir, APP_MODELS_DIRECTORY_NAME);
   const tmpDir = `${ogDir}.tmp`;
 
   try {
     await spawner.rimraf(tmpDir);
     await copyDir({ srcPath: ogDir, destPath: tmpDir });
+
+    await pruneModels({
+      projectId,
+      appCfg: newAppCfg,
+      path: tmpDir
+    });
+
     await installModelsWithPresignedURLs(modelInstallPayloads, tmpDir);
-    // TODO: Purge outdated models
+
     await spawner.rimraf(ogDir);
     await copyDir({ srcPath: tmpDir, destPath: ogDir });
     await spawner.rimraf(tmpDir);
@@ -211,6 +131,28 @@ export async function updateModelsWithPresignedUrls(props: {
     logger.info(`Models installed for project ${projectId}`);
   } finally {
     await spawner.rimraf(tmpDir);
-    await spawner.rimraf(restoreDir);
   }
 }
+
+export async function pruneModels(props: {
+  projectId: string;
+  appCfg: AppConfig;
+  path?: string;
+}) {
+  const { projectId, appCfg, path } = props;
+
+  const modelsPath = path ?? join(getAppDir(projectId), 'models');
+
+  if (!existsSync(modelsPath)) {
+    logger.error(
+      `Attempted to prune models from ${modelsPath} for project app ${projectId} that does not exist or has no models dir.`
+    );
+    return;
+  }
+
+  await pruneDir({
+    path: modelsPath,
+    exclude: Object.keys(appCfg.models),
+    recurse: true
+  });
+}

package/src/application-control/status.ts

@@ -1,4 +1,8 @@
-import compose from 'docker-compose';
+import {
+  compose,
+  runStreamingDockerComposeCmd,
+  runDockerComposeCmd
+} from '../docker/docker-compose';
 import { JsSpawner } from 'alwaysai/lib/util';
 
 import { runDockerLogin } from '../docker/docker-cmd';
@@ -48,9 +52,8 @@ export async function getAppState(props: {
   const spawner = JsSpawner({ path: appDir });
   for (const name of composeServices.data.services) {
     // Get container name for service
-    const containerId = await spawner.run({
-      exe: 'docker-compose',
-      cwd: appDir,
+    const containerId = await runDockerComposeCmd({
+      dir: appDir,
       args: ['ps', '-q', name]
     });
     if (containerId === '') {
@@ -136,10 +139,9 @@ export async function getAppLogs(props: {
   const argsList = args || [];
 
   // Use direct command with spawner in order to get a readable stream
-  return await JsSpawner().runStreaming({
-    exe: 'docker-compose',
+  return await runStreamingDockerComposeCmd({
     args: ['logs', '-f', ...argsList, ...serviceList],
-    cwd: appDir
+    dir: appDir
   });
 }
 

package/src/application-control/utils.ts

@@ -1,9 +1,7 @@
-import compose from 'docker-compose';
+import { compose } from '../docker/docker-compose';
 import * as path from 'path';
-import * as fs from 'fs';
 
 import { AgentConfigFile } from '../infrastructure/agent-config';
-import { fetchWithTimeout } from '../util/fetch-with-timeout';
 import {
   getDockerComposeCmdForApp,
   TargetJsonFile,
@@ -77,29 +75,3 @@ export async function buildApp(props: { appDir: string }) {
     );
   }
 }
-
-export async function downloadPackageUsingPresignedUrl(props: {
-  localDest: string;
-  presignedUrl: string;
-}): Promise<void> {
-  const { localDest, presignedUrl } = props;
-  logger.debug(`Downloading package from ${presignedUrl}`);
-  let response: any;
-  try {
-    response = await fetchWithTimeout(presignedUrl);
-  } catch (e) {
-    const errorBody = e.type === 'aborted' ? e : await e.response.text();
-    throw new Error(
-      `downloadPackageUsingPresignedUrl: Error=${e}\n${errorBody}`
-    );
-  }
-
-  /**
-   * Write the app package to the hash directory
-   */
-  const stream = response.body.pipe(fs.createWriteStream(localDest));
-  await new Promise((resolve, reject) => {
-    stream.on('finish', resolve);
-    stream.on('error', reject);
-  });
-}

package/src/cloud-connection/bootstrap-provision.ts

@@ -1,10 +1,10 @@
+import {
+  getBootstrapCertificateFilePath,
+  getBootstrapPrivateKeyFilePath
+} from '../infrastructure/device-certificate';
 import { getIoTCoreEndpointUrl } from '../infrastructure/urls';
 import { rmBootstrapCertsAndClose } from '../util/clean-certs';
-import {
-  BOOTSTRAP_PRIVATE_KEY_FILE_PATH,
-  BOOTSTRAP_CERTIFICATE_FILE_PATH,
-  AWS_ROOT_CERTIFICATE_FILE_PATH
-} from '../util/directories';
+import { AWS_ROOT_CERTIFICATE_FILE_PATH } from '../util/directories';
 import { getDeviceUuid } from '../util/get-device-id';
 import { logger } from '../util/logger';
 import { BootstrapAgent } from './device-agent';
@@ -14,8 +14,8 @@ export async function bootstrapProvision() {
 
   const clientId = getDeviceUuid();
   const bootstrapConfig = {
-    keyPath: BOOTSTRAP_PRIVATE_KEY_FILE_PATH(),
-    certPath: BOOTSTRAP_CERTIFICATE_FILE_PATH(),
+    keyPath: getBootstrapPrivateKeyFilePath(),
+    certPath: getBootstrapCertificateFilePath(),
     caPath: AWS_ROOT_CERTIFICATE_FILE_PATH,
     clientId,
     host: getIoTCoreEndpointUrl()