@alwaysai/device-agent 1.3.1 → 1.5.0

package/src/cloud-connection/live-updates-handler.test.ts

@@ -1,5 +1,12 @@
 import { LiveUpdatesHandler } from './live-updates-handler';
 import { Publisher } from './publisher';
+import sleep from '../util/sleep';
+import { getDeviceStatsPayload } from './messages';
+
+jest.mock('../util/sleep');
+jest.mock('./messages');
+
+jest.mocked(getDeviceStatsPayload).mockResolvedValue({});
 
 global.setTimeout = jest.fn() as unknown as typeof setTimeout;
 
@@ -18,64 +25,163 @@ const testFalseToggles = {
   appState: false
 };
 
-const mockClient = jest.fn();
+const mockClient = {
+  publish: jest.fn()
+};
 const clientId = 'test-client';
 const emptyTxId = '';
 
-// NOTE: this was the way I found to mock private class functions
-const mockStartPublishingLiveUpdates = jest.spyOn(
-  LiveUpdatesHandler.prototype as any,
-  'startPublishingLiveUpdates'
-);
-mockStartPublishingLiveUpdates.mockResolvedValue(null);
-
 describe('Test Live Updates Handler', () => {
   let liveUpdatesHandler: LiveUpdatesHandler;
   let publisher: Publisher;
 
   beforeEach(() => {
+    mockClient.publish = jest.fn();
+    jest.mocked(sleep).mockImplementation(async () => {
+      return;
+    });
     publisher = new Publisher(mockClient, clientId);
     liveUpdatesHandler = new LiveUpdatesHandler(publisher, clientId);
     jest.clearAllMocks();
   });
 
+  test('enable device stats', async () => {
+    mockClient.publish = jest.fn().mockImplementation(async () => {
+      liveUpdatesHandler.disableDeviceStatsLiveUpdates();
+    });
+
+    const enable = { deviceStats: true };
+    const promises = await liveUpdatesHandler.handleToggles(enable, emptyTxId);
+    await Promise.all(promises);
+
+    expect(jest.mocked(setTimeout)).toBeCalledTimes(1);
+    expect(mockClient.publish).toBeCalledTimes(1);
+    expect(jest.mocked(sleep)).toBeCalledTimes(1);
+  });
+
+  test('enable app state', async () => {
+    mockClient.publish = jest.fn().mockImplementation(async () => {
+      liveUpdatesHandler.disableAppStateLiveUpdates();
+    });
+
+    const enable = { appState: true };
+    const promises = await liveUpdatesHandler.handleToggles(enable, emptyTxId);
+    await Promise.all(promises);
+
+    expect(jest.mocked(setTimeout)).toBeCalledTimes(1);
+    expect(mockClient.publish).toBeCalledTimes(1);
+    expect(jest.mocked(sleep)).toBeCalledTimes(1);
+  });
+
   test('ignore subsequent enables', async () => {
-    void liveUpdatesHandler.handleToggles(testTrueToggles, emptyTxId);
-    // called twice, once for device stats, once for app state
-    expect(mockStartPublishingLiveUpdates).toBeCalledTimes(2);
-    // restartLiveUpdatesTimeout is always called once when handleToggles is called
+    // Block on the first sleep call
+    let doneWaiting;
+    const waitPromise = new Promise(function (resolve) {
+      doneWaiting = resolve;
+    });
+
+    let startSleeping;
+    const sleepPromise = new Promise(function (resolve) {
+      startSleeping = resolve;
+    });
+    jest.mocked(sleep).mockImplementation(async () => {
+      startSleeping();
+      await waitPromise;
+    });
+
+    const enable = { deviceStats: true };
+    await liveUpdatesHandler.handleToggles(enable, emptyTxId);
+    await sleepPromise;
+
     expect(jest.mocked(setTimeout)).toBeCalledTimes(1);
+    expect(mockClient.publish).toBeCalledTimes(1);
+    expect(jest.mocked(sleep)).toBeCalledTimes(1);
 
-    // Second call -> should not call startPublishingLiveUpdates should not be called
+    // Second call
     jest.clearAllMocks();
-    void liveUpdatesHandler.handleToggles(testTrueToggles, emptyTxId);
-    expect(mockStartPublishingLiveUpdates).toBeCalledTimes(0);
+
+    await liveUpdatesHandler.handleToggles(enable, emptyTxId);
+
     expect(jest.mocked(setTimeout)).toHaveBeenCalledTimes(1);
+    expect(mockClient.publish).toBeCalledTimes(0);
+    expect(jest.mocked(sleep)).toBeCalledTimes(0);
+
+    // Disable and clean up
+    liveUpdatesHandler.disableDeviceStatsLiveUpdates();
+    doneWaiting();
   });
 
   test('test disable live updates', async () => {
+    let doneWaiting;
+    const waitPromise = new Promise(function (resolve) {
+      doneWaiting = resolve;
+    });
+
+    let startSleeping;
+    const sleepPromise = new Promise(function (resolve) {
+      startSleeping = resolve;
+    });
+
+    // Wait for two sleep calls, one for Device Stats and one for App State
+    jest
+      .mocked(sleep)
+      .mockImplementationOnce(async () => {
+        await waitPromise;
+      })
+      .mockImplementationOnce(async () => {
+        startSleeping();
+        await waitPromise;
+      });
+
     // Test calling handleToggles one time, enabling it
-    void liveUpdatesHandler.handleToggles(testTrueToggles, emptyTxId);
-    expect(mockStartPublishingLiveUpdates).toBeCalledTimes(2);
+    await liveUpdatesHandler.handleToggles(testTrueToggles, emptyTxId);
+    await sleepPromise;
+
     expect(jest.mocked(setTimeout)).toHaveBeenCalledTimes(1);
+    expect(jest.mocked(sleep)).toBeCalledTimes(2);
     expect(liveUpdatesHandler.getDeviceStatsLiveUpdates()).toBe(true);
     expect(liveUpdatesHandler.getAppStateLiveUpdates()).toBe(true);
     expect(liveUpdatesHandler.getAppLogsLiveUpdates()).toBe(false);
 
     // Check to see that attributes are properly set to false when disabled
     jest.clearAllMocks();
-    void liveUpdatesHandler.handleToggles(testFalseToggles, emptyTxId);
-    expect(mockStartPublishingLiveUpdates).toBeCalledTimes(0);
+    await liveUpdatesHandler.handleToggles(testFalseToggles, emptyTxId);
+
     expect(jest.mocked(setTimeout)).toHaveBeenCalledTimes(1);
+    expect(jest.mocked(sleep)).toBeCalledTimes(0);
     expect(liveUpdatesHandler.getDeviceStatsLiveUpdates()).toBe(false);
     expect(liveUpdatesHandler.getAppStateLiveUpdates()).toBe(false);
     expect(liveUpdatesHandler.getAppLogsLiveUpdates()).toBe(false);
+
+    doneWaiting();
   });
 
   test('timeout turns off live updates', async () => {
     jest.useFakeTimers({ legacyFakeTimers: true });
+    let doneWaiting;
+    const waitPromise = new Promise(function (resolve) {
+      doneWaiting = resolve;
+    });
+
+    let startSleeping;
+    const sleepPromise = new Promise(function (resolve) {
+      startSleeping = resolve;
+    });
+
+    // Wait for two sleep calls, one for Device Stats and one for App State
+    jest
+      .mocked(sleep)
+      .mockImplementationOnce(async () => {
+        await waitPromise;
+      })
+      .mockImplementationOnce(async () => {
+        startSleeping();
+        await waitPromise;
+      });
+
+    await liveUpdatesHandler.handleToggles(testTrueToggles, emptyTxId);
+    await sleepPromise;
 
-    void liveUpdatesHandler.handleToggles(testTrueToggles, emptyTxId);
     expect(liveUpdatesHandler.getDeviceStatsLiveUpdates()).toBe(true);
     expect(liveUpdatesHandler.getAppStateLiveUpdates()).toBe(true);
     expect(liveUpdatesHandler.getAppLogsLiveUpdates()).toBe(false);
@@ -85,5 +191,40 @@ describe('Test Live Updates Handler', () => {
     expect(liveUpdatesHandler.getDeviceStatsLiveUpdates()).toBe(false);
     expect(liveUpdatesHandler.getAppStateLiveUpdates()).toBe(false);
     expect(liveUpdatesHandler.getAppLogsLiveUpdates()).toBe(false);
+
+    doneWaiting();
+  });
+
+  test("failure doesn't kill publish loop", async () => {
+    // Block on the first sleep call
+    let doneWaiting;
+    const waitPromise = new Promise(function (resolve) {
+      doneWaiting = resolve;
+    });
+
+    let startSleeping;
+    const sleepPromise = new Promise(function (resolve) {
+      startSleeping = resolve;
+    });
+    jest
+      .mocked(sleep)
+      .mockImplementationOnce(async () => {
+        throw new Error('Test error!');
+      })
+      .mockImplementationOnce(async () => {
+        startSleeping();
+        await waitPromise;
+      });
+
+    const enable = { deviceStats: true };
+    await liveUpdatesHandler.handleToggles(enable, emptyTxId);
+    await sleepPromise;
+
+    expect(jest.mocked(setTimeout)).toBeCalledTimes(1);
+    expect(mockClient.publish).toBeCalledTimes(2);
+    expect(jest.mocked(sleep)).toBeCalledTimes(2);
+
+    liveUpdatesHandler.disableDeviceStatsLiveUpdates();
+    doneWaiting();
   });
 });

package/src/cloud-connection/live-updates-handler.ts

@@ -7,19 +7,19 @@ import {
   buildAppStateMessage,
   buildDeviceStatsMessage,
   StatusResponsePayload,
-  buildToClientStatusResponseMessage
+  buildToClientStatusResponseMessage,
+  ToClientMessageTypeValue
 } from '@alwaysai/device-agent-schemas';
 import { getAppLogs } from '../application-control';
 import { logger } from '../util/logger';
 import sleep from '../util/sleep';
 import { Publisher } from './publisher';
 import { getAppStatePayload, getDeviceStatsPayload } from './messages';
-import { ToClientMessageTypeValue } from '@alwaysai/device-agent-schemas';
 import { ALWAYSAI_LIVE_UPDATES_TIMEOUT_MS } from '../environment';
 
 const LIVE_UPDATES_TIMEOUT = ALWAYSAI_LIVE_UPDATES_TIMEOUT_MS
   ? parseInt(ALWAYSAI_LIVE_UPDATES_TIMEOUT_MS)
-  : 60000;
+  : 80000;
 
 export class LiveUpdatesHandler {
   private publisher: Publisher;
@@ -42,14 +42,41 @@ export class LiveUpdatesHandler {
   private appLogStreams = new Set<string>();
   private transactionStatuses = new Set<string>();
 
+  private async getAppLogsWithRetry(
+    projectId: string
+  ): Promise<NodeJS.ReadableStream | null> {
+    // Retry starting logs until it starts successfully or is terminated
+    while (this.appLogStreams.has(projectId)) {
+      try {
+        return await getAppLogs({
+          projectId,
+          args: ['--tail', '100', '--no-log-prefix']
+        });
+      } catch (e) {
+        logger.info(
+          `Failed to start app logs, retrying in 1 second. Error: ${e}`
+        );
+        await sleep(1000);
+      }
+    }
+    // Case where logs were disabled prior to connecting
+    return null;
+  }
+
   private async startAppLogStream(projectId: string, txId: string) {
     logger.info(`Starting log stream for ${projectId}`);
 
     this.appLogStreams.add(projectId);
-    const readable = await getAppLogs({
-      projectId,
-      args: ['--tail', '100', '--no-log-prefix']
-    });
+
+    const readable = await this.getAppLogsWithRetry(projectId);
+
+    if (readable === null) {
+      logger.info(
+        `App log stream terminated for project ${projectId} prior to starting`
+      );
+      return;
+    }
+
     readable.on('data', async (chunk: Buffer) => {
       if (!this.appLogStreams.has(projectId)) {
         // why doesn't typescript know about this function?
@@ -107,22 +134,11 @@ export class LiveUpdatesHandler {
     return -1;
   }
 
-  private setLiveUpdates(toggles: LiveStateUpdatesTogglePayload) {
-    if (toggles.deviceStats !== undefined) {
-      this.liveUpdatesAlive.device_stats = toggles.deviceStats;
-    }
-    if (toggles.appState !== undefined) {
-      this.liveUpdatesAlive.app_state = toggles.appState;
-    }
-  }
-
   private restartLiveUpdatesTimeout() {
     clearTimeout(this.liveUpdatesTimeout);
     this.liveUpdatesTimeout = setTimeout(() => {
-      this.setLiveUpdates({
-        deviceStats: false,
-        appState: false
-      });
+      this.disableAppStateLiveUpdates();
+      this.disableDeviceStatsLiveUpdates();
       this.appLogStreams.clear();
     }, LIVE_UPDATES_TIMEOUT);
   }
@@ -133,17 +149,17 @@ export class LiveUpdatesHandler {
     txId: string
   ) {
     logger.info(`Turned on live updates for ${messageType}`);
-    try {
-      while (this.continuePublishing(messageType, txId)) {
+    while (this.continuePublishing(messageType, txId)) {
+      try {
         const message = await getMessage();
         this.publisher.publishToClient(message, logger.silly);
 
         await sleep(this.getLiveUpdatesInterval(messageType));
+      } catch (e) {
+        logger.error(
+          `Error publishing live updates for ${messageType}: ${e.message}`
+        );
       }
-    } catch (e) {
-      logger.error(
-        `Error publishing live updates for ${messageType}: ${e.message}`
-      );
     }
     logger.info(`Turned off live updates for ${messageType}`);
   }
@@ -160,14 +176,24 @@ export class LiveUpdatesHandler {
     );
   }
 
-  public getDeviceStatsLiveUpdates() {
+  public getDeviceStatsLiveUpdates(): boolean {
     return this.liveUpdatesAlive.device_stats;
   }
 
-  public getAppStateLiveUpdates() {
+  public disableDeviceStatsLiveUpdates() {
+    logger.info('Disabled live updates for device_stats');
+    this.liveUpdatesAlive.device_stats = false;
+  }
+
+  public getAppStateLiveUpdates(): boolean {
     return this.liveUpdatesAlive.app_state;
   }
 
+  public disableAppStateLiveUpdates() {
+    logger.info('Disabled live updates for app_state');
+    this.liveUpdatesAlive.app_state = false;
+  }
+
   public getAppLogsLiveUpdates() {
     return this.liveUpdatesAlive.app_logs;
   }
@@ -211,12 +237,14 @@ export class LiveUpdatesHandler {
     const { deviceStats, appState, appLogs } = toggles;
     this.restartLiveUpdatesTimeout();
 
+    const promises: Promise<void>[] = [];
+
     if (deviceStats !== undefined) {
       const currentDeviceStats = this.getDeviceStatsLiveUpdates();
       this.liveUpdatesAlive.device_stats = deviceStats;
       if (deviceStats && currentDeviceStats !== true) {
         // Don't wait for this call to finish since it loops until disabled
-        void this.startPublishingLiveUpdates(
+        const deviceStatsPromise = this.startPublishingLiveUpdates(
           keyMirrors.toClientMessageType.device_stats,
           async () => {
             const payload = await getDeviceStatsPayload();
@@ -224,6 +252,7 @@ export class LiveUpdatesHandler {
           },
           txId
         );
+        promises.push(deviceStatsPromise);
       }
     }
 
@@ -232,7 +261,7 @@ export class LiveUpdatesHandler {
       this.liveUpdatesAlive.app_state = appState;
       if (appState && currentAppState !== true) {
         // Don't wait for this call to finish since it loops until disabled
-        void this.startPublishingLiveUpdates(
+        const appStatePromise = this.startPublishingLiveUpdates(
           keyMirrors.toClientMessageType.app_state,
           async () => {
             const payload = await getAppStatePayload();
@@ -240,6 +269,7 @@ export class LiveUpdatesHandler {
           },
           txId
         );
+        promises.push(appStatePromise);
       }
     }
 
@@ -247,10 +277,12 @@ export class LiveUpdatesHandler {
       const currentAppLogs = this.getAppLogsLiveUpdates();
       if (appLogs.toggle && currentAppLogs !== true) {
         // Don't wait for this call to finish since it loops until disabled
-        void this.startAppLogStream(appLogs.projectId, txId);
+        const appLogPromise = this.startAppLogStream(appLogs.projectId, txId);
+        promises.push(appLogPromise);
       } else {
         this.appLogStreams.delete(appLogs.projectId);
       }
     }
+    return promises;
   }
 }

package/src/cloud-connection/messages.ts

@@ -12,15 +12,14 @@ import {
 import { AgentConfigFile } from '../infrastructure/agent-config';
 
 export async function getAppStatePayload(): Promise<AppStatePayload> {
-  const appState: AppState[] = [];
+  const appStatePromises: Promise<AppState>[] = [];
   const apps = await AgentConfigFile().getApps();
   for (const app of apps) {
     const projectId = app.projectId;
-    const status = await getAppState({ projectId });
-    appState.push(status);
+    appStatePromises.push(getAppState({ projectId }));
   }
   const appStatePayload = {
-    appState: appState
+    appState: await Promise.all(appStatePromises)
   };
   return appStatePayload;
 }

package/src/cloud-connection/passthrough-handler.ts

@@ -1,11 +1,13 @@
-// eslint-disable-next-line
-const amqp = require('amqplib');
+import * as amqp from 'amqplib';
 import {
   LOCAL_CONNECTION_HOST,
   LOCAL_CONNECTION_PORT,
   LOCAL_CONNECTION_ROUTING_KEY
 } from '../local-connection/constants';
-import { setupRabbitMQContainer } from '../local-connection/rabbitmq-connection';
+import {
+  stopRabbitMQContainer,
+  setupRabbitMQContainer
+} from '../local-connection/rabbitmq-connection';
 import { logger } from '../util/logger';
 import sleep from '../util/sleep';
 import { Publisher } from './publisher';
@@ -16,21 +18,84 @@ const MAX_LOCAL_CONNECTION_ATTEMPTS = 10;
 
 export class PassthroughHandler {
   public publisher: Publisher;
-  public connection;
-  public channel;
+  public connection: amqp.Connection | undefined;
+  public channel: amqp.Channel;
   public packetQueue;
 
   constructor(publisher: Publisher) {
     this.publisher = publisher;
   }
 
-  public async setup() {
-    await setupRabbitMQContainer();
+  runChannel = async () => {
+    logger.debug('Beginning to consume packets');
+    await this.channel.consume(
+      this.packetQueue,
+      (msg) => {
+        // NOTE: this needs to be an arrow function and then the whole contents of processPublish are below
+        if (msg?.content !== undefined) {
+          const packet = JSON.parse(msg.content.toString());
+          messageQueue.push({ packet, msg });
+          while (messageQueue.length > 0) {
+            const entry = messageQueue.shift();
+            const { packet, msg } = entry;
+            try {
+              const parsedPacket = JSON.parse(packet);
+              if (parsedPacket?.['action']) {
+                switch (parsedPacket['action']) {
+                  case 'analytics':
+                    ackQueue.push(msg);
+                    // FIXME: put real topic here
+                    this.publisher.publishToCloudWithAck(
+                      packet,
+                      (errOrResp) => {
+                        while (ackQueue.length > 0) {
+                          const msg = ackQueue.shift();
+                          if (errOrResp === true) {
+                            this.channel.ack(msg); // acknowledge, allow queue to discard
+                          } else if (errOrResp === false) {
+                            this.channel.reject(msg, true); // reject and requeue
+                          }
+                        }
+                      }
+                    );
+                    break;
+                  case 'heartbeat':
+                    this.channel.ack(msg);
+                    logger.silly(
+                      `Heartbeat package received & acknowledged: ${packet}`
+                    );
+                    break;
+                  default:
+                    this.channel.ack(msg);
+                    logger.debug(
+                      `Unknown 'action' package received & acknowledged: ${packet}`
+                    );
+                    break;
+                }
+              } else {
+                this.channel.ack(msg);
+                logger.debug(
+                  `Received & acknowledged a RabbitMQ Package of unknown structure: ${parsedPacket}`
+                );
+              }
+            } catch (e) {
+              logger.error(`There was a problem parsing RabbitMQ packet ${e}`);
+              this.channel.ack(msg);
+              logger.debug(`Problematic packet was acknowledged`);
+            }
+          }
+        }
+      },
+      {
+        noAck: false // When true, RabbitMQ deletes message as soon as it is consumed
+      }
+    );
+  };
+
+  async establishLocalConnection(): Promise<void> {
     let connectAttempts = 0;
     let connected = false;
-    logger.debug(
-      `Setting up alwaysAI Local Connection on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
-    );
+    logger.debug(`Establishing local connection...`);
     while (
       connectAttempts <= MAX_LOCAL_CONNECTION_ATTEMPTS &&
       this.connection === undefined
@@ -40,6 +105,13 @@ export class PassthroughHandler {
           `amqp://${LOCAL_CONNECTION_HOST}:${LOCAL_CONNECTION_PORT}`
         );
         this.channel = await this.connection.createChannel();
+        this.connection.on('error', async () => {
+          logger.error(`Local connection failed. Attempting to reconnect...`);
+          await stopRabbitMQContainer();
+          this.connection = undefined;
+          await this.setup();
+        });
+
         connected = true;
       } catch (e) {
         const timeTillNextAttemptMs = 1000 + 1000 * connectAttempts;
@@ -53,84 +125,34 @@ export class PassthroughHandler {
       }
     }
     if (connected === true) {
-      this.channel.prefetch(1); // This ensures we only get one packet at a time! This appears to have prevented throttling
+      await this.channel.prefetch(1); // This ensures we only get one packet at a time! This appears to have prevented throttling
       this.packetQueue = `${LOCAL_CONNECTION_ROUTING_KEY}`;
       await this.channel.assertQueue(this.packetQueue, {
         durable: true
       });
+      logger.info(`Local connection established.`);
     } else {
       throw new Error(
         'Unable to establish connection to alwaysAI Local Connection, please try restarting Device Agent.'
       );
     }
   }
-}
 
-function processPublish(passthroughHandler: PassthroughHandler) {
-  while (messageQueue.length > 0) {
-    const entry = messageQueue.shift();
-    const { packet, msg } = entry;
+  public async setup() {
+    logger.debug(
+      `Setting up alwaysAI Local Connection on host: ${LOCAL_CONNECTION_HOST} and channel key: ${LOCAL_CONNECTION_ROUTING_KEY}`
+    );
+    await setupRabbitMQContainer();
     try {
-      const parsedPacket = JSON.parse(packet);
-      if (parsedPacket && parsedPacket['action']) {
-        switch (parsedPacket['action']) {
-          case 'analytics':
-            ackQueue.push(msg);
-            // FIXME: put real topic here
-            passthroughHandler.publisher.publishToCloudWithAck(
-              packet,
-              (errOrResp) => {
-                while (ackQueue.length > 0) {
-                  const msg = ackQueue.shift();
-                  if (errOrResp === true) {
-                    passthroughHandler.channel.ack(msg); // acknowledge, allow queue to discard
-                  } else if (errOrResp === false) {
-                    passthroughHandler.channel.reject(msg, true); // reject and requeue
-                  }
-                }
-              }
-            );
-            break;
-          case 'heartbeat':
-            passthroughHandler.channel.ack(msg);
-            logger.debug(
-              `Heartbeat package received & acknowledged: ${packet}`
-            );
-            break;
-          default:
-            passthroughHandler.channel.ack(msg);
-            logger.debug(
-              `Unknown 'action' package received & acknowledged: ${packet}`
-            );
-            break;
-        }
-      } else {
-        passthroughHandler.channel.ack(msg);
-        logger.debug(
-          `Received & acknowledged a RabbitMQ Package of unknown structure: ${parsedPacket}`
-        );
-      }
-    } catch (e) {
-      logger.error(`There was a problem parsing RabbitMQ packet ${e}`);
-      passthroughHandler.channel.ack(msg);
-      logger.debug(`Problematic packet was acknowledged`);
+      await this.establishLocalConnection();
+      await this.runChannel();
+    } catch (error) {
+      logger.error(
+        `There was a problem maintaining RabbitMQ connection: ${error}`
+      );
+      throw new Error(
+        `There was a problem maintaining RabbitMQ connection: ${error}`
+      );
     }
   }
 }
-
-export async function runChannel(passthroughHandler: PassthroughHandler) {
-  logger.debug('Beginning to consume packets');
-  passthroughHandler.channel.consume(
-    passthroughHandler.packetQueue,
-    function (msg) {
-      if (msg.content !== undefined) {
-        const packet = JSON.parse(msg.content.toString());
-        messageQueue.push({ packet, msg });
-        processPublish(passthroughHandler);
-      }
-    },
-    {
-      noAck: false // When true, RabbitMQ deletes message as soon as it is consumed
-    }
-  );
-}