@alwaysai/device-agent 0.0.11 → 0.0.13

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@alwaysai/device-agent",
   "description": "The alwaysAI Device Agent",
-  "version": "0.0.11",
+  "version": "0.0.13",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "publishConfig": {
@@ -20,18 +20,23 @@
     "clean": "rimraf lib packages",
     "build:clean": "npm run clean && npm run build",
     "build:watch": "tsc --project . --watch",
-    "lint": "tslint --project tsconfig.build.json",
+    "lint": "eslint . --ext .ts",
     "lint:fix": "npm run lint -- --fix",
+    "prettier-format": "prettier --config .prettierrc 'src/**/*.ts' --write",
     "test": "npm run test:unit",
     "test:unit": "jest --coverage src",
     "test:watch": "npm run test:unit -- --watch",
     "preversion": "npm test && npm run build:clean",
-    "postversion": "npm publish"
+    "postversion": "npm publish",
+    "publish-provisioning:dev": "node ./scripts/publish-provisioning-scripts.js dev",
+    "publish-provisioning:qa": "node ./scripts/publish-provisioning-scripts.js qa",
+    "publish-provisioning:prod": "node ./scripts/publish-provisioning-scripts.js prod"
   },
   "dependencies": {
-    "@alwaysai/alwayscli": "0.3.1",
-    "@alwaysai/config-nodejs": "0.1.0",
-    "@alwaysai/device-agent-schemas": "1.1.0",
+    "@alwaysai/alwayscli": "0.3.2",
+    "@alwaysai/app-configuration-schemas": "0.0.1",
+    "@alwaysai/config-nodejs": "0.3.0",
+    "@alwaysai/device-agent-schemas": "1.1.2",
     "@carnesen/coded-error": "0.4.0",
     "@types/mkdirp": "1.0.2",
     "@types/pump": "1.1.1",
@@ -39,7 +44,7 @@
     "@types/signal-exit": "3.0.1",
     "@types/tar": "6.1.1",
     "ajv": "8.11.0",
-    "alwaysai": "1.10.0",
+    "alwaysai": "1.12.3",
     "aws-iot-device-sdk": "2.2.12",
     "docker-compose": "0.23.17",
     "express": "4.17.3",
@@ -53,6 +58,7 @@
     "socket.io": "4.4.1",
     "tar": "6.1.11",
     "tree-kill": "1.2.2",
+    "uuid": "9.0.0",
     "winston": "3.3.3",
     "winston-daily-rotate-file": "4.5.5",
     "yaml": "2.1.1"
@@ -65,6 +71,8 @@
     "@types/jest": "27.0.3",
     "@types/node": "16.11.12",
     "@types/sinon": "10.0.6",
+    "@typescript-eslint/eslint-plugin": "^5.48.0",
+    "@typescript-eslint/parser": "^5.48.0",
     "aws-sdk": "^2.1046.0",
     "cp-cli": "2.0.0",
     "get-stream": "6.0.1",
@@ -74,7 +82,9 @@
     "ts-jest": "27.1.1",
     "ts-mock-imports": "1.3.8",
     "ts-node": "10.4.0",
-    "tslint": "5.20.1",
+    "eslint": "^8.31.0",
+    "eslint-config-prettier": "^8.6.0",
+    "eslint-plugin-prettier": "^4.2.1",
     "typescript": "^4.5.3"
   },
   "files": [

package/src/application-control/backup.ts

@@ -3,7 +3,6 @@ import * as rimraf from 'rimraf';
 import { copyDir } from '../util/copy-dir';
 import { buildApp, getAppDir } from './utils';
 import { AgentConfigFile } from '../infrastructure/agent-config';
-import { runInDir } from '../util/run-in-dir';
 import { logger } from '../util/logger';
 
 export const BACKUP_EXT = '.bak';
@@ -25,12 +24,18 @@ export async function rollbackApp(props: { projectId: string }) {
     throw new Error(`Backup doesn't exist for ${projectId}`);
   }
   logger.info(`Rolling back app ${projectId} to ${config.version}`);
-  await AgentConfigFile().setAppInstalling({ projectId, version: config.version });
+  await AgentConfigFile().setAppInstalling({
+    projectId,
+    version: config.version
+  });
   const appDir = getAppDir(projectId);
   const backupAppDir = `${appDir}${BACKUP_EXT}`;
   rimraf.sync(appDir);
   await copyDir({ srcPath: backupAppDir, destPath: appDir });
   await buildApp({ appDir });
-  await AgentConfigFile().setAppInstalled({ projectId, version: config.version });
+  await AgentConfigFile().setAppInstalled({
+    projectId,
+    version: config.version
+  });
   logger.info(`Rolled back app ${projectId} to ${config.version}`);
 }

package/src/application-control/config.ts

@@ -1,35 +1,97 @@
-import { AppConfig, AppJsonFile } from 'alwaysai/lib/core/app';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import { parse, stringify } from 'yaml';
-import { getAppDir } from './utils';
+import { buildApp, getAppDir } from './utils';
 import { JsSpawner } from 'alwaysai/lib/util';
 import compose from 'docker-compose';
 import { assign, merge } from 'lodash';
+import { AppJsonFile } from 'alwaysai/lib/core/app';
+import { AppConfig } from '@alwaysai/app-configuration-schemas';
+import { restartApp } from './status';
+import { logger } from '../util/logger';
 
-export async function readAppCfg(props: { projectId: string }): Promise<AppConfig> {
+export async function readAppCfgFile(props: {
+  projectId: string;
+}): Promise<AppConfig> {
   const { projectId } = props;
-  if (!(await AgentConfigFile().isAppReady({ projectId }))) {
-    throw new Error(`App ${projectId} is not ready!`);
+  if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
+    throw new Error(`App ${projectId} is not present!`);
   }
   const appDir = getAppDir(projectId);
   const appJson = AppJsonFile(appDir);
-  return appJson.read();
+  if (!appJson.exists()) {
+    throw new Error(`App config for ${projectId} doesn't exist!`);
+  }
+  try {
+    return appJson.read();
+  } catch (err) {
+    throw new Error(
+      `Error reading app config for ${projectId}:\n${err}\n${appJson.getErrors()}`
+    );
+  }
 }
 
-export async function writeAppCfg(props: { projectId: string; appCfg: AppConfig }) {
+export async function writeAppCfgFile(props: {
+  projectId: string;
+  appCfg: AppConfig;
+}) {
   const { projectId, appCfg } = props;
-  if (!(await AgentConfigFile().isAppReady({ projectId }))) {
-    throw new Error(`App ${projectId} is not ready!`);
+  if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
+    throw new Error(`App ${projectId} is not present!`);
   }
   const appDir = getAppDir(projectId);
   const appJson = AppJsonFile(appDir);
-  appJson.write(appCfg);
+  try {
+    appJson.write(appCfg);
+  } catch (err) {
+    throw new Error(
+      `Error writing app config for ${projectId}:\n${err}\n${appJson.getErrors()}`
+    );
+  }
 }
 
-export async function updateAppConfig(appDir: string, newAppCfg: AppConfig) {
-  const existingAppCfg = await readAppCfg({ projectId: appDir });
+export async function updateAppCfgFile(props: {
+  projectId: string;
+  newAppCfg: any;
+}) {
+  const { projectId, newAppCfg } = props;
+  const existingAppCfg = await readAppCfgFile({ projectId });
   const appCfg = assign(existingAppCfg, merge(existingAppCfg, newAppCfg));
-  await writeAppCfg({ projectId: appDir, appCfg });
+  await writeAppCfgFile({ projectId, appCfg });
+}
+
+export async function updateAppCfg(props: {
+  projectId: string;
+  appReleaseHash: string;
+  newAppCfg: AppConfig;
+}) {
+  const { projectId, appReleaseHash, newAppCfg } = props;
+  logger.info(`Updating app config for ${projectId}:${appReleaseHash}.`);
+  const appDir = getAppDir(projectId);
+
+  if (await AgentConfigFile().isAppPresent({ projectId })) {
+    if (!(await AgentConfigFile().isAppReady({ projectId }))) {
+      throw new Error('Application already has installation in progress!');
+    }
+    logger.info('Application is already installed, updating');
+    await AgentConfigFile().setAppInstalling({
+      projectId,
+      version: appReleaseHash
+    });
+  } else {
+    throw new Error('Application is not installed!');
+  }
+
+  await writeAppCfgFile({ projectId, appCfg: newAppCfg });
+  await buildApp({ appDir });
+
+  await AgentConfigFile().setAppInstalled({
+    projectId,
+    version: appReleaseHash
+  });
+
+  await restartApp({ projectId });
+
+  logger.info(`Updated app config and rebuilt ${projectId}:${appReleaseHash}.`);
 }
 
 export async function readDockerCompose(props: { projectId: string }) {

package/src/application-control/environment-variables.ts

@@ -1,6 +1,4 @@
 import { JsSpawner } from 'alwaysai/lib/util';
-import compose from 'docker-compose';
-import { parse, stringify } from 'yaml';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import { readDockerCompose, writeDockerCompose } from './config';
 import { getAppDir } from './utils';
@@ -56,7 +54,9 @@ export async function getAllEnvs(props: { projectId: string }) {
       if ('env_file' in service) {
         const envFiles: string[] = service['env_file'];
         for (const ef of envFiles) {
-          envVars[s] = envVars[s].concat((await spawner.readFile(ef)).split('\n'));
+          envVars[s] = envVars[s].concat(
+            (await spawner.readFile(ef)).split('\n')
+          );
         }
       }
       if ('environment' in service) {

package/src/application-control/index.ts

@@ -1,4 +1,9 @@
-import { readAppCfg, writeAppCfg, readDockerCompose, writeDockerCompose } from './config';
+import {
+  readAppCfgFile,
+  updateAppCfg,
+  readDockerCompose,
+  writeDockerCompose
+} from './config';
 import { installApp, uninstallApp } from './install';
 import { rollbackApp } from './backup';
 import {
@@ -8,14 +13,14 @@ import {
   startApp,
   getAppLogs,
   stopApp,
-  restartApp,
+  restartApp
 } from './status';
 import { ModelDetails } from './types';
 import { getAllEnvs, setEnv } from './environment-variables';
 
 export {
-  readAppCfg,
-  writeAppCfg,
+  readAppCfgFile,
+  updateAppCfg,
   readDockerCompose,
   writeDockerCompose,
   installApp,
@@ -30,7 +35,7 @@ export {
   restartApp,
   ModelDetails,
   getAllEnvs,
-  setEnv,
+  setEnv
 };
 
 // CLI-mode only
@@ -40,6 +45,14 @@ import {
   removeModel,
   replaceModels,
   updateModels,
+  updateModelsWithPresignedUrls
 } from './models';
 
-export { addModel, getAppModels, removeModel, replaceModels, updateModels };
+export {
+  addModel,
+  getAppModels,
+  removeModel,
+  replaceModels,
+  updateModels,
+  updateModelsWithPresignedUrls
+};

package/src/application-control/install.ts

@@ -3,22 +3,28 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { JsSpawner, Spawner } from 'alwaysai/lib/util';
 import { getAppDir, downloadPackageUsingPresignedUrl, buildApp } from './utils';
-import { AppDetailsPacket, ModelInstallPayload } from '@alwaysai/device-agent-schemas';
-import { BACKUP_EXT, createAppBackup } from './backup';
+import { AppDetailsPacket } from '@alwaysai/device-agent-schemas';
+import { BACKUP_EXT } from './backup';
 import { stopApp } from './status';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import { ProjectJsonFile } from 'alwaysai/lib/core/project';
 import {
   getTargetHardwareType,
   AppJsonFile,
-  TargetJsonFile,
+  TargetJsonFile
 } from 'alwaysai/lib/core/app';
-import { appCleanDockerComponent, buildDocker } from 'alwaysai/lib/components/app';
+import {
+  appCleanDockerComponent,
+  buildDocker
+} from 'alwaysai/lib/components/app';
 import {
   appInstallComponent,
-  installVenv,
+  installVenv
 } from 'alwaysai/lib/components/app/app-install-component';
-import { DOCKERFILE, DOCKER_IMAGE_ID_INITIAL_VALUE } from 'alwaysai/lib/constants';
+import {
+  DOCKERFILE,
+  DOCKER_IMAGE_ID_INITIAL_VALUE
+} from 'alwaysai/lib/constants';
 import { appReleasePullComponent } from 'alwaysai/lib/components/release';
 import { runInDir } from '../util/run-in-dir';
 import { installModelsWithPresignedURLs } from './models';
@@ -51,6 +57,7 @@ export async function installApp(props: {
   signedUrlsPayload?: SignedUrlPayloadType;
 }): Promise<void> {
   const { projectId, appReleaseHash, signedUrlsPayload } = props;
+  logger.info(`Installing ${projectId}:${appReleaseHash}`);
 
   const appDir = getAppDir(projectId);
   const spawner = JsSpawner();
@@ -59,10 +66,16 @@ export async function installApp(props: {
       throw new Error('Application already has installation in progress!');
     }
     logger.info('Application is already installed, updating');
-    await AgentConfigFile().setAppInstalling({ projectId, version: appReleaseHash });
+    await AgentConfigFile().setAppInstalling({
+      projectId,
+      version: appReleaseHash
+    });
     await spawner.rimraf(appDir);
   } else {
-    await AgentConfigFile().setAppInstalling({ projectId, version: appReleaseHash });
+    await AgentConfigFile().setAppInstalling({
+      projectId,
+      version: appReleaseHash
+    });
   }
 
   await spawner.mkdirp(appDir);
@@ -76,14 +89,17 @@ export async function installApp(props: {
         {
           yes: true,
           project: projectId,
-          releaseHash: appReleaseHash,
-        },
+          releaseHash: appReleaseHash
+        }
       ],
-      appDir,
+      appDir
     );
   } else {
     const { appSignedUrl } = signedUrlsPayload.appInstallPayload;
-    await downloadPackageUsingPresignedUrl({ localDest, presignedUrl: appSignedUrl });
+    await downloadPackageUsingPresignedUrl({
+      localDest,
+      presignedUrl: appSignedUrl
+    });
   }
 
   // Unpack app package and remove tar file
@@ -105,26 +121,27 @@ export async function installApp(props: {
           pull: true,
           clean: true,
           skipModels: false,
-          source: false,
-        },
+          source: false
+        }
       ],
-      appDir,
+      appDir
     );
   } else {
     await installAppBuildReqs({ appDir });
     await installModelsWithPresignedURLs(
       signedUrlsPayload.modelsInstallPayload,
-      path.join(appDir, 'models'),
+      path.join(appDir, 'models')
     );
   }
 
   await buildApp({ appDir });
 
-  await AgentConfigFile().setAppInstalled({ projectId, version: appReleaseHash });
+  await AgentConfigFile().setAppInstalled({
+    projectId,
+    version: appReleaseHash
+  });
 
-  logger.info(
-    `Installed app version ${appReleaseHash} from project ${projectId} to ${appDir}.`,
-  );
+  logger.info(`Completed installing ${projectId}:${appReleaseHash}`);
 }
 
 async function installAppBuildReqs(props: { appDir: string }) {
@@ -145,21 +162,25 @@ async function installAppBuildReqs(props: { appDir: string }) {
         targetJson,
         targetJsonFile,
         targetHostSpawner,
-        pull: true,
+        pull: true
       });
 
       await installVenv({
         targetJson,
         sourceSpawner: targetHostSpawner,
-        targetJsonFile,
+        targetJsonFile
       });
     },
     [],
-    appDir,
+    appDir
   );
 }
 
-async function unPackApp(props: { spawner: Spawner; localDest: string; appDir: string }) {
+async function unPackApp(props: {
+  spawner: Spawner;
+  localDest: string;
+  appDir: string;
+}) {
   const { spawner, localDest, appDir } = props;
   await spawner.untar(fs.createReadStream(localDest), appDir);
   await spawner.rimraf(localDest);
@@ -182,12 +203,15 @@ async function checkValidProjectFiles({ appDir }) {
   TargetJsonFile(appDir).write({
     targetProtocol: 'docker:',
     dockerImageId: DOCKER_IMAGE_ID_INITIAL_VALUE,
-    targetHardware: await getTargetHardwareType({}),
+    targetHardware: await getTargetHardwareType({})
   });
 }
 
-export async function uninstallApp(props: { projectId: string }): Promise<void> {
+export async function uninstallApp(props: {
+  projectId: string;
+}): Promise<void> {
   const { projectId } = props;
+  logger.info(`Uninstalling ${projectId}`);
   if (!(await AgentConfigFile().isAppPresent({ projectId }))) {
     logger.info(`Application ${projectId} not installed`);
     return;
@@ -195,12 +219,12 @@ export async function uninstallApp(props: { projectId: string }): Promise<void>
   try {
     await stopApp({ projectId });
   } catch {
-    logger.info('Failed to stop app, may be left running...');
+    logger.warn(`Failed to stop ${projectId}, may be left running...`);
   }
   await AgentConfigFile().setAppUninstalled({ projectId });
   // Delete application directory and backup
   const appDir = getAppDir(projectId);
   rimraf.sync(appDir);
   rimraf.sync(`${appDir}${BACKUP_EXT}`);
-  logger.info('Uninstalled', projectId);
+  logger.info(`Completed uninstalling ${projectId}`);
 }