@alwaysai/device-agent 1.3.1 → 1.4.0

package/src/application-control/environment-variables.ts

@@ -41,24 +41,27 @@ export async function setEnv(props: { projectId: string; envVars: EnvVars }) {
 
     const service = composeParsed['services'][s];
     const oldEnv: string[] | undefined = service['environment'];
+    const envVarsForService = envVars[s];
 
     const newEnvVarsObj = {};
     oldEnv?.forEach((envVarStr: string) => {
       const envVarSplit = envVarStr.split('=');
       const key = envVarSplit[0];
       const value = envVarSplit[1];
-      newEnvVarsObj[key] = value;
+      if (envVarsForService[key] !== null) {
+        newEnvVarsObj[key] = value;
+      }
     });
 
-    for (const envVar of Object.keys(envVars[s])) {
-      newEnvVarsObj[envVar] = envVars[s][envVar];
+    for (const envVar of Object.keys(envVarsForService)) {
+      if (envVarsForService[envVar] !== null) {
+        newEnvVarsObj[envVar] = envVarsForService[envVar];
+      }
     }
 
     const envVarList: string[] = [];
     for (const envVar of Object.keys(newEnvVarsObj)) {
-      envVarList.push(
-        `${envVar}=${newEnvVarsObj[envVar] ? newEnvVarsObj[envVar] : ''}`
-      );
+      envVarList.push(`${envVar}=${newEnvVarsObj[envVar]}`);
     }
 
     service['environment'] = envVarList;

package/src/application-control/install.ts

@@ -113,12 +113,18 @@ export async function installApp(props: {
   const DACfg = await createAppCfgDeviceAgentConnection();
   await updateAppCfgFile({ projectId, newAppCfg: DACfg });
 
-  await installAppBuildReqs({ appDir });
+  // Clean up original dependency files (models, virtual env)
+  const targetJsonFile = TargetJsonFile(appDir);
+  const hostSpawner = targetJsonFile.readHostSpawner();
+  await appCleanDocker({ targetHostSpawner: hostSpawner });
+
+  // Download model files
   await installModelsWithPresignedURLs(
     signedUrlsPayload.modelsInstallPayload,
     path.join(appDir, 'models')
   );
 
+  await installAppBuildReqs({ appDir });
   await buildApp({ appDir });
 
   await AgentConfigFile().setAppInstalled({
@@ -153,8 +159,6 @@ async function installAppBuildReqs(props: { appDir: string }) {
     async () => {
       const hostSpawner = targetJsonFile.readHostSpawner();
 
-      await appCleanDocker({ targetHostSpawner: hostSpawner });
-
       const dockerImageId = await buildDockerImage({
         targetHostSpawner: hostSpawner,
         targetHardware: targetCfg.targetHardware,

package/src/application-control/models.ts

@@ -89,14 +89,19 @@ export async function replaceModels(props: {
     ],
     appDir
   );
+  const modelsAddPromises: Promise<void>[] = [];
   for (const modelId of modelIds) {
-    await appModelsAddComponent({
-      yes: false,
-      dir: appDir,
-      id: modelId,
-      addToProject: false
-    });
+    modelsAddPromises.push(
+      appModelsAddComponent({
+        yes: false,
+        dir: appDir,
+        id: modelId,
+        addToProject: false
+      })
+    );
   }
+  await Promise.all(modelsAddPromises);
+
   await buildApp({ appDir });
 }
 

package/src/application-control/status.ts

@@ -126,14 +126,14 @@ export async function getAppLogs(props: {
 
   const appDir = getAppDir(projectId);
 
-  const serviceList = services
-    ? services
-    : await (async function () {
-        const composeServices = await compose.configServices({ cwd: appDir });
-        return composeServices.data.services;
-      })();
-
-  const argsList = args ? args : [];
+  const serviceList =
+    services ||
+    (await (async function () {
+      const composeServices = await compose.configServices({ cwd: appDir });
+      return composeServices.data.services;
+    })());
+
+  const argsList = args || [];
 
   // Use direct command with spawner in order to get a readable stream
   return await JsSpawner().runStreaming({

package/src/cloud-connection/device-agent-cloud-connection.ts

@@ -1,59 +1,57 @@
 // eslint-disable-next-line
-const awsIot = require("aws-iot-device-sdk");
-import { getIoTCoreEndpointUrl } from '../infrastructure/urls';
-import { existsSync } from 'fs';
-import {
-  BOOTSTRAP_PRIVATE_KEY_FILE_PATH,
-  AWS_ROOT_CERTIFICATE_FILE_PATH,
-  BOOTSTRAP_CERTIFICATES_DIR_PATH,
-  DEVICE_PRIVATE_KEY_FILE_PATH,
-  DEVICE_CERTIFICATE_FILE_PATH
-} from '../util/directories';
+const awsIot = require('aws-iot-device-sdk');
 import {
-  keyMirrors,
-  SignedUrlsRequestPayload,
-  getToDeviceTopic,
   AppInstallResponsePayload,
-  validateToDeviceAgentMessage,
-  ToDeviceAgentMessage,
-  ToCloudMessage,
   AppStateControlPayload,
   AppVersionControlInstallPayload,
   AppVersionControlUninstallPayload,
   DeviceActionPayload,
+  ModelsInstallResponsePayload,
+  SignedUrlsRequestPayload,
+  ToCloudMessage,
+  ToDeviceAgentMessage,
+  getToDeviceTopic,
   buildSignedUrlsRequestMessage,
   buildToClientStatusResponseMessage,
-  StatusResponsePayload
+  StatusResponsePayload,
+  keyMirrors,
+  validateToDeviceAgentMessage
 } from '@alwaysai/device-agent-schemas';
-import { getDeviceUuid } from '../util/get-device-id';
-import { logger } from '../util/logger';
-import { cloudModeReady } from '../util/cloud-mode-ready';
-import { AgentConfigFile } from '../infrastructure/agent-config';
+import { existsSync } from 'fs';
 import {
+  installApp,
+  restartApp,
   startApp,
   stopApp,
-  restartApp,
-  updateModelsWithPresignedUrls,
-  installApp,
   uninstallApp,
   updateAppCfg,
-  setEnv
+  updateModelsWithPresignedUrls
 } from '../application-control';
-import { ShadowHandler, ShadowTopics, ShadowUpdate } from './shadow-handler';
-import { secureTunnelNotifyHandler } from '../secure-tunneling/index';
-import { Publisher } from './publisher';
-import { bootstrapProvision } from './bootstrap-provision';
-import { PassthroughHandler, runChannel } from './passthrough-handler';
+import { createAppBackup, rollbackApp } from '../application-control/backup';
+import { reboot } from '../device-control/device-control';
 import { ALWAYSAI_ANALYTICS_PASSTHROUGH } from '../environment';
-import { ModelsInstallResponsePayload } from '@alwaysai/device-agent-schemas';
+import { AgentConfigFile } from '../infrastructure/agent-config';
+import { getIoTCoreEndpointUrl } from '../infrastructure/urls';
+import { SecureTunnelHandlerSingleton } from '../secure-tunneling/secure-tunneling';
+import { cloudModeReady } from '../util/cloud-mode-ready';
+import {
+  AWS_ROOT_CERTIFICATE_FILE_PATH,
+  BOOTSTRAP_CERTIFICATES_DIR_PATH,
+  BOOTSTRAP_PRIVATE_KEY_FILE_PATH,
+  DEVICE_CERTIFICATE_FILE_PATH,
+  DEVICE_PRIVATE_KEY_FILE_PATH
+} from '../util/directories';
+import { getDeviceUuid } from '../util/get-device-id';
+import { logger } from '../util/logger';
 import sleep from '../util/sleep';
-import { createAppBackup, rollbackApp } from '../application-control/backup';
+import { bootstrapProvision } from './bootstrap-provision';
+import { LiveUpdatesHandler } from './live-updates-handler';
+import { PassthroughHandler, runChannel } from './passthrough-handler';
+import { Publisher } from './publisher';
+import { ShadowHandler, ShadowUpdate } from './shadow-handler';
 import { TransactionManager } from './transaction-manager';
-import { reboot } from '../device-control/device-control';
-
 import { exec } from 'child_process';
 import { promisify } from 'util';
-import { LiveUpdatesHandler } from './live-updates-handler';
 
 const exec_promise = promisify(exec);
 
@@ -69,6 +67,8 @@ export class DeviceAgentCloudConnection {
   private port = 8883;
   private readonly toDeviceTopic = getToDeviceTopic(this.clientId);
   private readonly secureTunnelNotifyTopic = `$aws/things/${this.clientId}/tunnels/notify`;
+  private readonly secureTunnelHandler =
+    SecureTunnelHandlerSingleton.getInstance();
   // FIXME: Add support for multiple simultaneous project updates
   private appCfgUpdateQueue: ShadowUpdate[] = [];
 
@@ -144,11 +144,10 @@ export class DeviceAgentCloudConnection {
       appInstallPayload,
       modelsInstallPayload
     };
-    await this.atomicApplicationUpdate(
-      installApp,
-      [{ projectId, appReleaseHash, signedUrlsPayload }],
-      projectId
-    );
+    await this.atomicApplicationUpdate(async () => {
+      this.shadowHandler.clearProjectShadow(projectId);
+      await installApp({ projectId, appReleaseHash, signedUrlsPayload });
+    }, projectId);
     return true;
   };
 
@@ -165,23 +164,25 @@ export class DeviceAgentCloudConnection {
     const projectId = payload.modelsInstallResponse.projectId;
     if (appCfgUpdate) {
       await this.atomicApplicationUpdate(
-        updateModelsWithPresignedUrls,
-        [
-          {
+        async () =>
+          await updateModelsWithPresignedUrls({
             projectId,
             modelInstallPayloads: payload.modelsInstallResponse.newModels,
             newAppCfg: appCfgUpdate.newAppCfg
-          }
-        ],
+          }),
         projectId
       );
     }
 
     if (envVarUpdate) {
       await this.atomicApplicationUpdate(
-        setEnv,
-        [{ projectId, envVars: envVarUpdate.envVars }],
-        projectId
+        async () =>
+          await this.shadowHandler.updateProjectEnvVars({
+            projectId,
+            envVars: envVarUpdate.envVars
+          }),
+        projectId,
+        true
       );
     }
     return true;
@@ -214,7 +215,7 @@ export class DeviceAgentCloudConnection {
   private async atomicApplicationUninstall(projectId: string) {
     try {
       await uninstallApp({ projectId });
-      this.shadowHandler.clearAppConfig(projectId);
+      this.shadowHandler.clearProjectShadow(projectId);
     } catch (e) {
       logger.error(`Failed to uninstall ${projectId}: ${e.message}`);
       throw e;
@@ -222,11 +223,11 @@ export class DeviceAgentCloudConnection {
   }
 
   // eslint-disable-next-line
-  private async atomicApplicationUpdate<T extends any[], R extends any>(
-    func: (...args: T) => R,
-    args: T,
-    projectId: string
-  ) {
+  private async atomicApplicationUpdate <F extends () => any>(
+    func: F,
+    projectId: string,
+    skipUpdateShadow?: boolean
+  ): Promise<ReturnType<F>> {
     // First try to create a backup, so that there is one available if something goes wrong in the next try:catch.
     if (await AgentConfigFile().isAppPresent({ projectId })) {
       try {
@@ -239,9 +240,9 @@ export class DeviceAgentCloudConnection {
     }
 
     try {
-      const out: R = await func(...args);
-      this.shadowHandler.clearAppConfig(projectId);
-      await this.shadowHandler.updateProjectShadow(projectId);
+      const out: ReturnType<F> = await func();
+      if (!skipUpdateShadow)
+        await this.shadowHandler.updateProjectShadow(projectId);
       return out;
     } catch (errorAppUpdate) {
       logger.error(
@@ -256,9 +257,9 @@ export class DeviceAgentCloudConnection {
       } catch (errorRollbackApp) {
         // and if that fails, uninstall the app as a last resort.
         try {
-          await uninstallApp({ projectId });
-        } finally {
-          this.shadowHandler.clearAppConfig(projectId);
+          await this.atomicApplicationUninstall(projectId);
+        } catch {
+          // atomicApplicationUninstall handles failing, so there's nothing to handle here.
         }
         logger.error(
           `Application update failed, rolled back to previous version: ${errorAppUpdate}`
@@ -273,15 +274,14 @@ export class DeviceAgentCloudConnection {
     }
   }
 
-  private handleAppConfigUpdate = async (
+  private handleProjectShadowConfigUpdate = async (
     update: ShadowUpdate,
     txId: string
   ): Promise<boolean> => {
     const { projectId, appCfgUpdate, envVarUpdate } = update;
 
     if (
-      appCfgUpdate &&
-      appCfgUpdate.updatedModels &&
+      appCfgUpdate?.updatedModels &&
       Object.keys(appCfgUpdate.updatedModels).length
     ) {
       // When there are model updates request signed URLs and wait to apply config changes
@@ -311,27 +311,64 @@ export class DeviceAgentCloudConnection {
 
     if (appCfgUpdate) {
       await this.atomicApplicationUpdate(
-        updateAppCfg,
-        [
-          {
+        async () =>
+          await updateAppCfg({
             projectId,
             newAppCfg: appCfgUpdate.newAppCfg
-          }
-        ],
+          }),
         projectId
       );
     }
 
     if (envVarUpdate) {
       await this.atomicApplicationUpdate(
-        setEnv,
-        [{ projectId, envVars: envVarUpdate.envVars }],
-        projectId
+        async () =>
+          await this.shadowHandler.updateProjectEnvVars({
+            projectId,
+            envVars: envVarUpdate.envVars
+          }),
+        projectId,
+        true
       );
     }
     return true;
   };
 
+  private async handleProjectShadowMessage(topic: string, message: any) {
+    const shadowUpdates = await this.shadowHandler.handleProjectShadow({
+      topic,
+      payload: message,
+      clientToken: message.clientToken
+    });
+    if (shadowUpdates.length) {
+      const shadowUpdatePromises: Promise<void>[] = [];
+      for (const shadowUpdate of shadowUpdates) {
+        const projectId = shadowUpdate.projectId;
+        const txId = shadowUpdate.txId;
+        shadowUpdatePromises.push(
+          this.txnMgr
+            .runTransactionStep({
+              func: () =>
+                this.handleProjectShadowConfigUpdate(shadowUpdate, txId),
+              projectId,
+              txId,
+              start: true,
+              stepName: topic
+            })
+            .catch((e: Error) => {
+              logger.error(
+                `There was an issue updating project shadow config: ${JSON.stringify(
+                  e
+                )}`
+              );
+            })
+        );
+      }
+
+      await Promise.all(shadowUpdatePromises);
+    }
+  }
+
   /*=================================================================
                             Public interface
   =================================================================*/
@@ -359,11 +396,11 @@ export class DeviceAgentCloudConnection {
 
     this.subscribe(this.toDeviceTopic);
     this.subscribe(this.secureTunnelNotifyTopic);
-    this.subscribe(this.shadowHandler.shadowTopics.projects.getAccepted);
-    this.subscribe(this.shadowHandler.shadowTopics.projects.getRejected);
-    this.subscribe(this.shadowHandler.shadowTopics.projects.updateDelta);
-    this.subscribe(this.shadowHandler.shadowTopics.projects.updateAccepted);
-    this.subscribe(this.shadowHandler.shadowTopics.projects.updateRejected);
+    for (const topic of this.shadowHandler.projectShadowTopics) {
+      this.subscribe(topic);
+    }
+    this.subscribe(this.shadowHandler.shadowTopics.secureTunnel.updateDelta);
+    this.subscribe(this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted);
   }
 
   public getClientId(): string {
@@ -374,10 +411,6 @@ export class DeviceAgentCloudConnection {
     return this.toDeviceTopic;
   }
 
-  public getShadowTopics(): ShadowTopics {
-    return this.shadowHandler.shadowTopics;
-  }
-
   public isCmdInProgress(projectId: string): boolean {
     return this.txnMgr.isOngoingTransactionForProjectID(projectId);
   }
@@ -412,7 +445,8 @@ export class DeviceAgentCloudConnection {
       live_state_updates,
       app_install_response,
       models_install_response,
-      status_response
+      status_response,
+      device_action
     } = keyMirrors.toDeviceAgentMessageType;
     switch (message.messageType) {
       case app_state_control: {
@@ -520,8 +554,20 @@ export class DeviceAgentCloudConnection {
         }
         break;
       }
-      case keyMirrors.toDeviceAgentMessageType.device_action: {
+      case device_action: {
         try {
+          const statusResponsePayload: StatusResponsePayload = {
+            status: keyMirrors.statusResponse.in_progress
+          };
+          const statusResponseMessage = buildToClientStatusResponseMessage(
+            this.clientId,
+            statusResponsePayload,
+            txId
+          );
+          this.publisher.publishToClient(statusResponseMessage);
+
+          await this.handleDeviceAction(message.payload);
+
           const successStatusResponsePayload: StatusResponsePayload = {
             status: keyMirrors.statusResponse.success
           };
@@ -532,8 +578,6 @@ export class DeviceAgentCloudConnection {
               txId
             );
           this.publisher.publishToClient(successStatusResponseMessage);
-
-          await this.handleDeviceAction(message.payload);
         } catch (e) {
           logger.error(
             `There was a problem performing device action '${message.payload.action}': ${e.message}`
@@ -570,50 +614,36 @@ export class DeviceAgentCloudConnection {
     logger.debug(
       `Received message: ${JSON.stringify({ topic, message }, null, 2)}`
     );
-    switch (topic) {
-      case this.shadowHandler.shadowTopics.projects.getAccepted:
-      case this.shadowHandler.shadowTopics.projects.updateAccepted: {
-        const shadowUpdates = await this.shadowHandler.handleShadowTopic({
-          topic,
-          payload: message.state,
-          clientToken: message.clientToken
-        });
-        if (shadowUpdates.length) {
-          for (const shadowUpdate of shadowUpdates) {
-            const projectId = shadowUpdate.projectId;
-            const txId = shadowUpdate.txId;
-            try {
-              await this.txnMgr.runTransactionStep({
-                func: () => this.handleAppConfigUpdate(shadowUpdate, txId),
-                projectId,
-                txId,
-                start: true,
-                stepName: topic
-              });
-            } catch (e) {
-              logger.error(`Error handling shadow message: ${e.message}`);
-            }
-          }
-        }
-        break;
-      }
-      case this.shadowHandler.shadowTopics.projects.getRejected:
-      case this.shadowHandler.shadowTopics.projects.updateDelta:
-      case this.shadowHandler.shadowTopics.projects.updateRejected:
-        // Not handling these for now
-        break;
-      case this.toDeviceTopic:
-        await this.handleDeviceAgentMessage({
-          topic,
-          message
-        });
-        break;
-
-      case this.secureTunnelNotifyTopic:
-        await secureTunnelNotifyHandler(message);
-        break;
-      default:
-        logger.error(`Unexpected topic, ignoring! ${topic}`);
+    // ProjectShadow messages
+    if (this.shadowHandler.projectShadowTopics.includes(topic)) {
+      await this.handleProjectShadowMessage(topic, message);
+    } else if (topic === this.toDeviceTopic) {
+      await this.handleDeviceAgentMessage({
+        topic,
+        message
+      });
+      // SecureTunnelNotify messages
+    } else if (topic === this.secureTunnelNotifyTopic) {
+      await this.secureTunnelHandler.secureTunnelNotifyHandler(message);
+      // SecureTunnel messages
+    } else if (
+      topic === this.shadowHandler.shadowTopics.secureTunnel.updateDelta
+    ) {
+      logger.info(`Received secure tunnel update: ${message}`);
+      const reported = await this.secureTunnelHandler.syncShadowToDeviceState(
+        message
+      );
+      this.publisher.publish(
+        this.shadowHandler.shadowTopics.secureTunnel.update,
+        JSON.stringify({ state: { reported } })
+      );
+    } else if (
+      topic === this.shadowHandler.shadowTopics.secureTunnel.deleteAccepted
+    ) {
+      logger.info(`Received secure tunnel deleteAccepted: ${message}`);
+      await this.secureTunnelHandler.destroy();
+    } else {
+      logger.error(`Unexpected topic, ignoring! ${topic}`);
     }
   }
 
@@ -710,9 +740,9 @@ export class DeviceAgentCloudConnection {
   }
 
   public async stop() {
-    // FIXME: This method is currently only used by the CLI, and shadow messages
-    // can be lost since we aren't waiting for responses so sleep for a short
-    // time to receive them
+    // This method is currently only used by the CLI, and shadow messages can be
+    // lost since we aren't waiting for responses so sleep for a short time to
+    // receive them
     await sleep(1000);
     this.device.end();
   }

package/src/cloud-connection/live-updates-handler.ts

@@ -7,15 +7,16 @@ import {
   buildAppStateMessage,
   buildDeviceStatsMessage,
   StatusResponsePayload,
-  buildToClientStatusResponseMessage
+  buildToClientStatusResponseMessage,
+  ToClientMessageTypeValue
 } from '@alwaysai/device-agent-schemas';
 import { getAppLogs } from '../application-control';
 import { logger } from '../util/logger';
 import sleep from '../util/sleep';
 import { Publisher } from './publisher';
 import { getAppStatePayload, getDeviceStatsPayload } from './messages';
-import { ToClientMessageTypeValue } from '@alwaysai/device-agent-schemas';
 import { ALWAYSAI_LIVE_UPDATES_TIMEOUT_MS } from '../environment';
+import { read } from 'fs';
 
 const LIVE_UPDATES_TIMEOUT = ALWAYSAI_LIVE_UPDATES_TIMEOUT_MS
   ? parseInt(ALWAYSAI_LIVE_UPDATES_TIMEOUT_MS)
@@ -42,14 +43,41 @@ export class LiveUpdatesHandler {
   private appLogStreams = new Set<string>();
   private transactionStatuses = new Set<string>();
 
+  private async getAppLogsWithRetry(
+    projectId: string
+  ): Promise<NodeJS.ReadableStream | null> {
+    // Retry starting logs until it starts successfully or is terminated
+    while (this.appLogStreams.has(projectId)) {
+      try {
+        return await getAppLogs({
+          projectId,
+          args: ['--tail', '100', '--no-log-prefix']
+        });
+      } catch (e) {
+        logger.info(
+          `Failed to start app logs, retrying in 1 second. Error: ${e}`
+        );
+        await sleep(1000);
+      }
+    }
+    // Case where logs were disabled prior to connecting
+    return null;
+  }
+
   private async startAppLogStream(projectId: string, txId: string) {
     logger.info(`Starting log stream for ${projectId}`);
 
     this.appLogStreams.add(projectId);
-    const readable = await getAppLogs({
-      projectId,
-      args: ['--tail', '100', '--no-log-prefix']
-    });
+
+    const readable = await this.getAppLogsWithRetry(projectId);
+
+    if (readable === null) {
+      logger.info(
+        `App log stream terminated for project ${projectId} prior to starting`
+      );
+      return;
+    }
+
     readable.on('data', async (chunk: Buffer) => {
       if (!this.appLogStreams.has(projectId)) {
         // why doesn't typescript know about this function?

package/src/cloud-connection/messages.ts

@@ -12,15 +12,14 @@ import {
 import { AgentConfigFile } from '../infrastructure/agent-config';
 
 export async function getAppStatePayload(): Promise<AppStatePayload> {
-  const appState: AppState[] = [];
+  const appStatePromises: Promise<AppState>[] = [];
   const apps = await AgentConfigFile().getApps();
   for (const app of apps) {
     const projectId = app.projectId;
-    const status = await getAppState({ projectId });
-    appState.push(status);
+    appStatePromises.push(getAppState({ projectId }));
   }
   const appStatePayload = {
-    appState: appState
+    appState: await Promise.all(appStatePromises)
   };
   return appStatePayload;
 }