@alwaysai/device-agent 0.0.11 → 0.0.13

package/src/application-control/models.ts

@@ -1,10 +1,10 @@
-import { keyMirrors, ModelInstallPayload } from '@alwaysai/device-agent-schemas';
-import { app } from 'alwaysai/lib/components';
+import * as path from 'path';
+import { ModelInstallPayload } from '@alwaysai/device-agent-schemas';
 import {
   appModelsAddComponent,
   appModelsRemoveAllComponent,
   appModelsRemoveComponent,
-  appModelsUpdateComponent,
+  appModelsUpdateComponent
 } from 'alwaysai/lib/components/app';
 import { JsSpawner } from 'alwaysai/lib/util';
 import { logger } from '../util/logger';
@@ -13,15 +13,19 @@ import { join, dirname } from 'path';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import { copyDir } from '../util/copy-dir';
 import { runInDir } from '../util/run-in-dir';
-import { getAppStatus, restartApp } from './status';
+import { restartApp } from './status';
 
 import { ModelDetails } from './types';
 import {
   buildApp,
   downloadPackageUsingPresignedUrl,
   getAppDir,
-  requireAppInstalled,
+  requireAppInstalled
 } from './utils';
+import { MODEL_JSON_FILE_NAME } from 'alwaysai/lib/core/model';
+import { APP_MODELS_DIRECTORY_NAME } from 'alwaysai/lib/constants';
+import { readAppCfgFile, updateAppCfgFile, writeAppCfgFile } from './config';
+import { AppConfig } from '@alwaysai/app-configuration-schemas';
 
 export async function getAppModels(props: { projectId: string }) {
   const { projectId } = props;
@@ -54,12 +58,15 @@ export async function addModel(props: { projectId: string; modelId: string }) {
     yes: false,
     dir: appDir,
     id: modelId,
-    addToProject: false,
+    addToProject: false
   });
   await buildApp({ appDir });
 }
 
-export async function removeModel(props: { projectId: string; modelId: string }) {
+export async function removeModel(props: {
+  projectId: string;
+  modelId: string;
+}) {
   const { projectId, modelId } = props;
   await requireAppInstalled({ projectId });
 
@@ -71,14 +78,17 @@ export async function removeModel(props: { projectId: string; modelId: string })
       {
         id: modelId,
         purge: true,
-        removeFromProject: false,
-      },
+        removeFromProject: false
+      }
     ],
-    appDir,
+    appDir
   );
 }
 
-export async function replaceModels(props: { projectId: string; modelIds: string[] }) {
+export async function replaceModels(props: {
+  projectId: string;
+  modelIds: string[];
+}) {
   const { projectId, modelIds } = props;
   await requireAppInstalled({ projectId });
 
@@ -89,17 +99,17 @@ export async function replaceModels(props: { projectId: string; modelIds: string
     [
       {
         purge: true,
-        removeFromProject: false,
-      },
+        removeFromProject: false
+      }
     ],
-    appDir,
+    appDir
   );
   for (const modelId of modelIds) {
     await appModelsAddComponent({
       yes: false,
       dir: appDir,
       id: modelId,
-      addToProject: false,
+      addToProject: false
     });
   }
   await buildApp({ appDir });
@@ -112,83 +122,117 @@ export async function updateModels(props: { projectId: string }) {
   const appDir = getAppDir(projectId);
   await appModelsUpdateComponent({
     yes: false,
-    dir: appDir,
+    dir: appDir
   });
   await buildApp({ appDir });
 }
 
 export async function installModelsWithPresignedURLs(
   modelPayloads: ModelInstallPayload[],
-  targetDir: string,
+  targetDir: string
 ) {
   const spawner = JsSpawner();
+  await spawner.mkdirp(targetDir);
   await Promise.all(
     modelPayloads.map(async (payload: ModelInstallPayload) => {
+      logger.info(`Installing ${payload.id}: ${payload.version}`);
+      const version = payload.version;
       const modelDest = `${targetDir}/${payload.id}`;
       await spawner.mkdirp(modelDest);
       const localDest = `${modelDest}/${payload.version}.tar.gz`;
       await downloadPackageUsingPresignedUrl({
         localDest,
-        presignedUrl: payload.modelSignedUrl,
+        presignedUrl: payload.modelSignedUrl
       });
       await spawner.untar(createReadStream(localDest), dirname(modelDest));
+      await updateModelJson(modelDest, (modelJson) => ({
+        ...modelJson,
+        version
+      }));
       await spawner.rimraf(localDest);
-    }),
+    })
   );
+
+  async function readModelJson(dir: string) {
+    const filePath = spawner.resolvePath(dir, MODEL_JSON_FILE_NAME);
+    const output = await spawner.readFile(filePath);
+    const parsed = JSON.parse(output);
+    return parsed;
+  }
+
+  async function updateModelJson(dir: string, updater: (current: any) => any) {
+    const parsed = await readModelJson(dir);
+    const updated = updater(parsed);
+    const filePath = spawner.resolvePath(dir, MODEL_JSON_FILE_NAME);
+    const serialized = JSON.stringify(updated, null, 2);
+    await spawner.writeFile(filePath, serialized);
+  }
 }
 
-export async function updateModelsWithPresignedUrls(
-  project: string,
-  modelInstallPayloads: ModelInstallPayload[],
-) {
-  // create temp dir with all untouched models
+export async function updateModelsWithPresignedUrls(props: {
+  projectId: string;
+  modelInstallPayloads: ModelInstallPayload[];
+  appReleaseHash: string;
+  newAppCfg: AppConfig;
+}) {
+  const { projectId, modelInstallPayloads, appReleaseHash, newAppCfg } = props;
+  logger.info(`Installing models for ${projectId}`);
   const spawner = JsSpawner();
-  const appDir = getAppDir(project);
-  const ogDir = `${appDir}/models`;
-  const tmpDir = `${ogDir}.tmp`;
+  const appDir = getAppDir(projectId);
+
+  if (await AgentConfigFile().isAppPresent({ projectId })) {
+    if (!(await AgentConfigFile().isAppReady({ projectId }))) {
+      throw new Error('Application already has installation in progress!');
+    }
+    logger.info('Updating installed application');
+    await AgentConfigFile().setAppInstalling({
+      projectId,
+      version: appReleaseHash
+    });
+  } else {
+    throw new Error('Application is not installed!');
+  }
+  const ogAppCfg = await readAppCfgFile({ projectId });
+
+  const ogDir = path.join(appDir, APP_MODELS_DIRECTORY_NAME);
+  // Copy all current models to restore dir in case of failure
   const restoreDir = `${ogDir}.restore`;
+  await spawner.rimraf(restoreDir);
   await copyDir({ srcPath: ogDir, destPath: restoreDir });
-  await spawner.rimraf(tmpDir);
 
-  try {
-    // Move unchanged models
-    const existingModels = await getAppModels({ projectId: project });
-    for (const model of existingModels) {
-      if (
-        !modelInstallPayloads
-          .map((newModel: ModelInstallPayload) => newModel.id)
-          .includes(model.modelId)
-      ) {
-        // model does not need to be updated
-        await copyDir({
-          srcPath: `${ogDir}/${model.modelId}`,
-          destPath: `${tmpDir}/${model.modelId}`,
-        });
-      }
-    }
+  // Create temp dir to install new models
+  const tmpDir = `${ogDir}.tmp`;
 
-    // install model packages
+  try {
+    await spawner.rimraf(tmpDir);
+    await copyDir({ srcPath: ogDir, destPath: tmpDir });
     await installModelsWithPresignedURLs(modelInstallPayloads, tmpDir);
+    // TODO: Purge outdated models
     await spawner.rimraf(ogDir);
     await copyDir({ srcPath: tmpDir, destPath: ogDir });
     await spawner.rimraf(tmpDir);
 
-    await buildApp({ appDir: getAppDir(project) });
+    await writeAppCfgFile({ projectId, appCfg: newAppCfg });
+    await buildApp({ appDir });
 
-    const appStatus = await getAppStatus({ projectId: project });
-    const appIsRunning =
-      appStatus.services.length &&
-      appStatus.services[0].state !== keyMirrors.appState.stopped;
+    await AgentConfigFile().setAppInstalled({
+      projectId,
+      version: appReleaseHash
+    });
 
-    if (appIsRunning) {
-      restartApp({ projectId: project });
-    }
+    await restartApp({ projectId });
 
-    logger.info(`Models installed for project ${project}`);
+    logger.info(`Models installed for project ${projectId}`);
   } catch (e) {
-    logger.error(e, 'Error updating app models from presigned URL, restoring models.');
-    await spawner.rimraf(tmpDir);
+    logger.error(
+      'Error updating app models from presigned URL, restoring models.',
+      e
+    );
     await spawner.rimraf(ogDir);
     await copyDir({ srcPath: restoreDir, destPath: ogDir });
+    await writeAppCfgFile({ projectId, appCfg: ogAppCfg });
+  } finally {
+    await spawner.rimraf(tmpDir);
+    await spawner.rimraf(restoreDir);
   }
 }

package/src/application-control/status.ts

@@ -8,7 +8,7 @@ import {
   ServiceStatusPacket,
   AppStatePacket,
   keyMirrors,
-  AppStateValue,
+  AppStateValue
 } from '@alwaysai/device-agent-schemas';
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import { logger } from '../util/logger';
@@ -38,7 +38,7 @@ export async function getAppStatus(props: {
 
   const appDetails = {
     projectId,
-    version: await AgentConfigFile().getAppVersion({ projectId }),
+    version: await AgentConfigFile().getAppVersion({ projectId })
   };
   if (!(await AgentConfigFile().isAppReady({ projectId }))) {
     // App is being installed or updated
@@ -53,7 +53,7 @@ export async function getAppStatus(props: {
 
   if (status.exitCode !== 0) {
     throw new Error(
-      `Failed to get application status! stdout=${status.out} stderr=${status.err}`,
+      `Failed to get application status! stdout=${status.out} stderr=${status.err}`
     );
   }
 
@@ -64,7 +64,7 @@ export async function getAppStatus(props: {
     const containerId = await spawner.run({
       exe: 'docker-compose',
       cwd: appDir,
-      args: ['ps', '-q', name],
+      args: ['ps', '-q', name]
     });
     if (containerId === '') {
       // The service was not yet started or failed to start
@@ -72,7 +72,7 @@ export async function getAppStatus(props: {
     } else {
       const slashContainerName = await spawner.run({
         exe: 'docker',
-        args: ['inspect', '-f', '{{.Name}}', containerId],
+        args: ['inspect', '-f', '{{.Name}}', containerId]
       });
       const containerName = slashContainerName.substring(1);
 
@@ -102,8 +102,8 @@ export async function getAppStatus(props: {
           `Unable to find container for ${name} in ${JSON.stringify(
             status.data.services,
             null,
-            2,
-          )}`,
+            2
+          )}`
         );
       }
     }
@@ -134,7 +134,7 @@ export async function getAppLogs(props: {
   return await JsSpawner().runStreaming({
     exe: 'docker-compose',
     args: ['logs', '-f', ...argsList, ...serviceList],
-    cwd: appDir,
+    cwd: appDir
   });
 }
 
@@ -148,19 +148,19 @@ export async function startApp(props: {
   const appDir = getAppDir(projectId);
   if (dockerLoginToken !== undefined) {
     const result = await runDockerLogin({ token: dockerLoginToken });
-    logger.info(result);
+    logger.debug(`docker login: ${result}`);
   }
 
   // TODO: Check if app is running
   // Start app
   const upOut = await compose.upAll({ cwd: appDir });
-  logger.info(upOut);
+  logger.debug(`docker-compose up: ${JSON.stringify(upOut, null, 2)}`);
   if (upOut.exitCode !== 0) {
     throw new Error(
-      `Failed to start application! stdout=${upOut.out} stderr=${upOut.err}`,
+      `Failed to start application! stdout=${upOut.out} stderr=${upOut.err}`
     );
   }
-  logger.info('Started', projectId);
+  logger.info(`Started ${projectId}`);
 }
 
 export async function stopApp(props: { projectId: string }): Promise<void> {
@@ -168,16 +168,21 @@ export async function stopApp(props: { projectId: string }): Promise<void> {
   await requireAppInstalled({ projectId });
 
   const appDir = getAppDir(projectId);
-  // TODO: Check if app is running
-  // Stop app
-  const output = await compose.down({ cwd: appDir });
-  logger.info(output);
-  if (output.exitCode !== 0) {
-    throw new Error(
-      `Failed to stop application! stdout=${output.out} stderr=${output.err}`,
-    );
+  const appStatus = await getAppStatus({ projectId });
+  const appIsRunning =
+    appStatus.services.length &&
+    appStatus.services[0].state !== keyMirrors.appState.stopped;
+  if (appIsRunning) {
+    // Stop app
+    const output = await compose.down({ cwd: appDir });
+    logger.debug(`docker-compose down: ${JSON.stringify(output, null, 2)}`);
+    if (output.exitCode !== 0) {
+      throw new Error(
+        `Failed to stop application! stdout=${output.out} stderr=${output.err}`
+      );
+    }
+    logger.info(`Stopped ${projectId}`);
   }
-  logger.info('Stopped', projectId);
 }
 
 export async function restartApp(props: { projectId: string }): Promise<void> {

package/src/application-control/utils.ts

@@ -1,17 +1,15 @@
 import compose from 'docker-compose';
 import * as path from 'path';
 import * as fs from 'fs';
-import { AAI_DIR, TARGET_JSON_FILE_NAME } from 'alwaysai/lib/constants';
+import { TARGET_JSON_FILE_NAME } from 'alwaysai/lib/constants';
 
 import { AgentConfigFile } from '../infrastructure/agent-config';
 import nodeFetch from 'node-fetch';
 import { TargetJsonFile } from 'alwaysai/lib/core/app';
 import { appDeployLinuxAndRemoteDevice } from 'alwaysai/lib/components/app';
 import { runInDir } from '../util/run-in-dir';
-import { JsSpawner } from 'alwaysai/lib/util';
 import { logger } from '../util/logger';
-
-export const APP_ROOT = path.join(AAI_DIR, 'applications');
+import { APP_ROOT } from '../util/directories';
 
 export function getAppDir(projectId: string): string {
   return path.join(APP_ROOT, projectId);
@@ -46,17 +44,17 @@ export async function buildApp(props: { appDir: string }) {
         logs: false,
         stop: false,
         targetJson,
-        targetConfig,
-      },
+        targetConfig
+      }
     ],
-    appDir,
+    appDir
   );
 
   const buildOut = await compose.buildAll({ cwd: appDir });
-  logger.info(buildOut);
+  logger.debug(`docker-compose build: ${JSON.stringify(buildOut, null, 2)}`);
   if (buildOut.exitCode !== 0) {
     throw new Error(
-      `Failed to build application! stdout=${buildOut.out} stderr=${buildOut.err}`,
+      `Failed to build application! stdout=${buildOut.out} stderr=${buildOut.err}`
     );
   }
 }
@@ -65,13 +63,13 @@ export async function downloadPackageUsingPresignedUrl(props: {
   localDest: string;
   presignedUrl: string;
 }): Promise<void> {
-  logger.info('downloading URL');
   const { localDest, presignedUrl } = props;
+  logger.info(`Downloading app package from ${presignedUrl}`);
   const response = await nodeFetch(presignedUrl);
   if (response.status !== 200) {
     // If the URL is invalid; I think we shouldn't get here with the new changes
     throw new Error(
-      `Status Code: ${response.status}, ${response.statusText}. Response: ${response.body}`,
+      `Status Code: ${response.status}, ${response.statusText}. Response: ${response.body}`
     );
   }
 
@@ -84,12 +82,3 @@ export async function downloadPackageUsingPresignedUrl(props: {
     stream.on('error', reject);
   });
 }
-
-export async function getAppConfig(project: string) {
-  const appCfgPath = path.join(getAppDir(project), 'alwaysai.app.json');
-  if (!fs.existsSync(appCfgPath)) {
-    throw new Error(`Application config not found for project ${project}`);
-  }
-
-  return JSON.parse(await JsSpawner().readFile(appCfgPath));
-}

package/src/cloud-connection/app-install-status.ts

@@ -0,0 +1,62 @@
+import { AppInstallStatusPacket } from '@alwaysai/device-agent-schemas';
+import { AppInstallStatusValue } from '@alwaysai/device-agent-schemas/lib/constants';
+
+export class AppInstallStatus {
+  private appReleaseHash: string;
+  private status: AppInstallStatusValue;
+  private message?: string;
+
+  constructor(appReleaseHash, status, message?) {
+    this.appReleaseHash = appReleaseHash;
+    this.status = status;
+    this.message = message;
+  }
+
+  public getAppReleaseHash() {
+    return this.appReleaseHash;
+  }
+
+  public update(status, message?) {
+    this.status = status;
+    this.message = message;
+  }
+
+  public getStatusPacket(): AppInstallStatusPacket {
+    return {
+      status: this.status,
+      message: this.message,
+      appReleaseHash: this.appReleaseHash
+    };
+  }
+}
+
+export class AppInstallStatusManager {
+  private appList: AppInstallStatus[] = [];
+
+  public update(appReleaseHash, status, message?) {
+    // Update status if existing
+    // TODO: Reimplement this as a map
+    for (const app of this.appList) {
+      if (app.getAppReleaseHash() === appReleaseHash) {
+        app.update(status, message);
+        return;
+      }
+    }
+    // App was not found, so add to list
+    const appInstallStatus = new AppInstallStatus(
+      appReleaseHash,
+      status,
+      message
+    );
+    this.appList.push(appInstallStatus);
+  }
+
+  public getStatusPacket(appReleaseHash) {
+    for (const app of this.appList) {
+      if (app.getAppReleaseHash() === appReleaseHash) {
+        return app.getStatusPacket();
+      }
+    }
+    throw new Error(`No status for ${appReleaseHash}`);
+  }
+}

package/src/cloud-connection/bootstrap-provision.ts

@@ -0,0 +1,40 @@
+import { logger } from 'alwaysai/lib/util';
+import { getIoTCoreEndpointUrl } from '../infrastructure/urls';
+import { rmBootstrapCertsAndClose } from '../util/clean-certs';
+import {
+  BOOTSTRAP_DEVICE_PRIVATE_KEY_FILE_PATH,
+  BOOTSTRAP_DEVICE_CERTIFICATE_FILE_PATH,
+  AWS_ROOT_CERTIFICATE_FILE_PATH
+} from '../util/directories';
+import { getDeviceUuid } from '../util/get-device-id';
+import { BootstrapAgent } from './device-agent';
+
+export function bootstrapProvision() {
+  setTimeout(rmBootstrapCertsAndClose, 60000);
+
+  const clientId = getDeviceUuid();
+  const bootstrapConfig = {
+    keyPath: BOOTSTRAP_DEVICE_PRIVATE_KEY_FILE_PATH(),
+    certPath: BOOTSTRAP_DEVICE_CERTIFICATE_FILE_PATH(),
+    caPath: AWS_ROOT_CERTIFICATE_FILE_PATH,
+    clientId,
+    host: getIoTCoreEndpointUrl()
+  };
+
+  const bootstrapAgent = new BootstrapAgent(bootstrapConfig);
+  bootstrapAgent.subscribeToAllTopics();
+
+  bootstrapAgent.publishMessage('$aws/certificates/create/json', '');
+
+  bootstrapAgent.device.on('connect', () => {
+    logger.info('Your device is being provisioned');
+  });
+
+  bootstrapAgent.device.on('message', (topic: string, payload: string) => {
+    bootstrapAgent.handleAwsCertificateTopics(topic, payload);
+  });
+
+  bootstrapAgent.device.on('packetsend', (packet: any) => {
+    logger.debug(`Sending packet: ${JSON.stringify({ packet }, null, 2)}`);
+  });
+}