@ikonintegration/ikapi 3.0.18 → 3.1.0

package/README.md

@@ -50,7 +50,13 @@ export const handler = async (event, context) => {
       sensitiveFilteringKeywords: [], //replaced default blacklist set
     },
     //Queue
-    publisher: { region: 'SNS-REGION' }
+    publisher: { region: 'SNS-REGION' },
+    //Resources tracker
+    resourcesTracker: { //by default, it will auto track (lambda, ecs, ddb)
+      busName: 'busName',
+      region: 'region',
+      appName: 'appName',
+    }
   };)).handleEvent(event, context));
 }
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ikonintegration/ikapi",
-  "version": "3.0.18",
+  "version": "3.1.0",
   "description": "",
   "main": "index.js",
   "module": "main.js",
@@ -17,6 +17,7 @@
     "@aws-sdk/credential-provider-node": "^3.72.0",
     "@aws-sdk/node-http-handler": "^3.58.0",
     "@aws-sdk/util-dynamodb": "^3.72.0",
+    "@ikonintegration/mod-resources-tracker-client": "0.0.1",
     "abind": "^1.0.5",
     "bluebird": "^3.7.2",
     "email-templates": "^8.0.8",

package/src/API/IKResponse.js

@@ -22,7 +22,7 @@ export default class IKResponse {
   getBody() { return this._body; }
   appendIntoBody(key, value) { this._body[key] = value; }
   appendHeader(key, value) { this._headers[key] = value; }
-  async build(context, transaction, isBatch) {
+  async build(context, transaction, doNotSucceed) {
     //Stream support
     if (this._streamingOut) return;
     if (this._isStream) return this._pipe(context);
@@ -42,7 +42,7 @@ export default class IKResponse {
     //Check for transaction response proxy
     if (transaction.responseProxy) await transaction.responseProxy(b);
     //Batch does not succeed directly just on upper transaction (which will should be a batch)
-    if (!isBatch) context.succeed(b);
+    if (!doNotSucceed) context.succeed(b);
   }
   _pipe(context) {
     //Check if not streaming

package/src/BaseEvent/IKProcess.js

@@ -10,6 +10,7 @@ import IKCacheRedis from '../Cache/Redis/IKRedis';
 import IKLogger from "../Logger/IKLogger";
 import IKValidation from '../Validation/IKValidation';
 import IKPublisher from '../Publisher/IKPublisher';
+import IKExecutionTracker from '../Tracker/IKExecutionTracker';
 //
 export default class IKProcess {
   constructor(config, interval) {
@@ -20,6 +21,7 @@ export default class IKProcess {
     this.validator = new IKValidation(config.validation);
     this.db = this._getDBDriver();
     this.cache = this._getCacheDriver();
+    if (config.resourcesTracker) this.tracker = new IKExecutionTracker(config.resourcesTracker, context, this.db);
   }
 
   //Main interface
@@ -49,6 +51,8 @@ export default class IKProcess {
       this.logger.exception(e);
       //Rollback DB
       if (this.db) await this.db.rollback();
+      //Tracker
+      if (this.tracker) await this.tracker.stopTracking();
     } return executionFailed;
   }
   /* DB drivers support */

package/src/BaseEvent/IKTransaction.js

@@ -10,6 +10,7 @@ import IKCacheRedis from '../Cache/Redis/IKRedis';
 import IKLogger from "../Logger/IKLogger";
 import IKValidation from '../Validation/IKValidation';
 import IKPublisher from '../Publisher/IKPublisher';
+import IKExecutionTracker from '../Tracker/IKExecutionTracker';
 //Request
 import IKRequest from '../API/IKRequest';
 import IKResponse, { IKBadRequestResponseWithRollback } from '../API/IKResponse';
@@ -20,12 +21,11 @@ export default class IKTransaction {
     this._event = event;
     this._context = context;
     this._config = config;
-    //queue support
-    this._isBatch = _isBatch;
-    this._resp = null;
-    //Step function support
+    //response behaviour flags
+    this._syncReturn = _isBatch;
     this._retrowErrors = _retrowErrors; /* retrow internal errors */
-    //When set, this will be called with the response context right before calling the context suceed/fail
+    this._resp = null;
+      //When set, this will be called with the response context right before calling the context suceed/fail - useful for IOing the resp for example.
     this.responseProxy = null;
     //
     this.logger = new IKLogger(config.logger, Utils.logLevel(), (context.awsRequestId ? context.awsRequestId : (event.requestContext ? event.requestContext.requestId : 'unknown')));
@@ -33,6 +33,7 @@ export default class IKTransaction {
     this.publisher = new IKPublisher(config.publisher);
     this.validator = new IKValidation(config.validation);
     this.db = this._getDBDriver();
+    if (config.resourcesTracker) this.tracker = new IKExecutionTracker(config.resourcesTracker, context, this.db);
     this.cache = this._getCacheDriver();
   }
 
@@ -43,7 +44,7 @@ export default class IKTransaction {
         return await this._execute(executionFunc);
       });
     });
-    if (this._isBatch) return this._resp;
+    if (this._syncReturn) return this._resp;
   }
   //Executions
   async _execute(executionFunc) {
@@ -54,11 +55,11 @@ export default class IKTransaction {
       this._resp = await executionFunc(this);
       //Answer client
       if (this._resp && this._resp instanceof IKResponse) {
-        await this._resp.build(this._context, this, this._isBatch);
+        await this._resp.build(this._context, this, this._syncReturn);
         executionFailed = !!(this._resp.getBody() && this._resp.getBody().rollback);
       } else {
         this._resp = this._getErrorResponse(IKGlobals.ErrorResponseInvalidServerResponse, IKGlobals.ErrorCode_APIError)
-        await this._resp.build(this._context, this, this._isBatch);
+        await this._resp.build(this._context, this, this._syncReturn);
         this.logger.error("Invalid response object from main request code.");
       }
     } catch (e) { /*EXECUTION FAIL*/
@@ -69,7 +70,7 @@ export default class IKTransaction {
       //envelope exception?
       if (executionFailed) {
         this._resp = this._getErrorResponse(IKGlobals.ErrorResponseUnhandledError, IKGlobals.ErrorCode_APIError);
-        await this._resp.build(this._context, this, this._isBatch);
+        await this._resp.build(this._context, this, this._syncReturn);
       }
     } return executionFailed;
   }
@@ -101,6 +102,8 @@ export default class IKTransaction {
     try {
       await safeExecution();
       await this.logger.flushLogs();
+      //Tracker
+      if (this.tracker) await this.tracker.stopTracking();
     } catch (e) {
       this.logger.error('Exception when flushing logs.');
       this.logger.exception(e);

package/src/Database/DDB/IKDB.js

@@ -16,6 +16,7 @@ export default class IKDB_DDB extends IKDB {
     super(config, transaction);
     this.tableName = config.tableName;
     this.region = config.region;
+    this.stats = { readUnits: 0, writeUnits: 0 };
     //
     // const localConsole = (transaction ? transaction.logger : console);
     // if (config && this.tableName) { localConsole.debug(`Using table: ${this.tableName} on region: ${this.region}`); }
@@ -37,8 +38,9 @@ export default class IKDB_DDB extends IKDB {
       // localConsole.debug("Starting remote database connection");
       //initialize connection
       this.connection = new DynamoDB({ 
-        region: this.region,
+        region: this.region, maxAttempts: IKGlobals.DDBMaxAttempts,
         requestHandler: new NodeHttpHandler({
+          connectionTimeout: IKGlobals.DDBHttpTimeout, socketTimeout: IKGlobals.DDBHttpTimeout,
           httpsAgent: new Agent({ keepAlive: false, maxSockets: 50, rejectUnauthorized: true }),
         })
       });
@@ -47,4 +49,8 @@ export default class IKDB_DDB extends IKDB {
       DDB_CONN_HASH = sha1(this.region);
     }
   }
+  getTrackerStats() {
+    if (!(this.stats.readUnits > 0 || this.state.writeUnits > 0)) return null;
+    return { tableName: this.tableName, ...this.stats };
+  }
 }

package/src/Database/DDB/IKDBQueryBatchWrite.js

@@ -23,6 +23,7 @@ export default class IKDBQueryBatchWrite extends IKDBBaseQuery {
     localConsole.log('Batch writting items: ', query);
     const resp = await dbManager.connection.batchWriteItem(query);
     // localConsole.debug('Raw result: ', resp);
+    this._incrementTrackerStats(resp, dbManager);
     return resp;
   }
 
@@ -56,4 +57,8 @@ export default class IKDBQueryBatchWrite extends IKDBBaseQuery {
       expression.RequestItems[dbTableName].push({DeleteRequest: item});
     } return expression;
   }
+  /* tracker support */
+  _incrementTrackerStats(resp, dbManager) {
+    if (resp && resp.ConsumedCapacity && resp.ConsumedCapacity.CapacityUnits) dbManager.stats.writeUnits += parseFloat(resp.ConsumedCapacity.CapacityUnits);
+  }
 }

package/src/Database/DDB/IKDBQueryDelete.js

@@ -19,6 +19,7 @@ export default class IKDBQueryUpdate extends IKDBBaseQuery {
     localConsole.log("Deleting item: ", query);
     let resp = await dbManager.connection.deleteItem(query);
     // localConsole.debug('Raw result: ', resp);
+    this._incrementTrackerStats(resp, dbManager);
     return resp;
   }
 
@@ -26,4 +27,8 @@ export default class IKDBQueryUpdate extends IKDBBaseQuery {
     return { ...super._rawQuery(dbTableName, isTransaction),
              ...(isTransaction ? {} : {ReturnValues: "ALL_OLD" /*request all deleted values*/ })};
   }
+  /* tracker support */
+  _incrementTrackerStats(resp, dbManager) {
+    if (resp && resp.ConsumedCapacity && resp.ConsumedCapacity.CapacityUnits) dbManager.stats.writeUnits += parseFloat(resp.ConsumedCapacity.CapacityUnits);
+  }
 }

package/src/Database/DDB/IKDBQueryGet.js

@@ -37,6 +37,12 @@ export default class IKDBQueryGet extends IKDBBaseQuery {
     localConsole.log('Querying: ', query, appendingItems ? appendingItems.length : null);
     const resp = await dbManager.connection.query(query);
     // localConsole.debug('Raw result: ', resp);
+    this._incrementTrackerStats(resp, dbManager);
     return resp;
   }
+
+  /* tracker support */
+  _incrementTrackerStats(resp, dbManager) {
+    if (resp && resp.ConsumedCapacity && resp.ConsumedCapacity.CapacityUnits) dbManager.stats.readUnits += parseFloat(resp.ConsumedCapacity.CapacityUnits);
+  }
 }

package/src/Database/DDB/IKDBQueryPut.js

@@ -61,6 +61,7 @@ export default class IKDBQueryPut extends IKDBBaseQuery {
     localConsole.log('Putting item: ', query);
     const resp = await dbManager.connection.putItem(query);
     // localConsole.debug('Raw result: ', resp);
+    this._incrementTrackerStats(resp, dbManager);
     return resp;
   }
 
@@ -79,4 +80,8 @@ export default class IKDBQueryPut extends IKDBBaseQuery {
       }
     } return expression;
   }
+  /* tracker support */
+  _incrementTrackerStats(resp, dbManager) {
+    if (resp && resp.ConsumedCapacity && resp.ConsumedCapacity.CapacityUnits) dbManager.stats.writeUnits += parseFloat(resp.ConsumedCapacity.CapacityUnits);
+  }
 }

package/src/Database/DDB/IKDBQueryScan.js

@@ -35,6 +35,11 @@ export default class IKDBQueryScan extends IKDBBaseQuery {
     localConsole.log('Scanning: ', query, appendingItems ? appendingItems.length : 0);
     const resp = await dbManager.connection.scan(query);
     // localConsole.debug('Raw result: ', resp);
+    this._incrementTrackerStats(resp, dbManager);
     return resp;
   }
+  /* tracker support */
+  _incrementTrackerStats(resp, dbManager) {
+    if (resp && resp.ConsumedCapacity && resp.ConsumedCapacity.CapacityUnits) dbManager.stats.readUnits += parseFloat(resp.ConsumedCapacity.CapacityUnits);
+  }
 }

package/src/Database/DDB/IKDBQueryUpdate.js

@@ -96,6 +96,7 @@ export default class IKDBQueryUpdate extends IKDBBaseQuery {
     localConsole.log("Updating item: ", query);
     let resp = await dbManager.connection.updateItem(query);
     // localConsole.debug('Raw result: ', resp);
+    this._incrementTrackerStats(resp, dbManager);
     return resp;
   }
 
@@ -211,4 +212,8 @@ export default class IKDBQueryUpdate extends IKDBBaseQuery {
       }
     } return expression;
   }
+  /* tracker support */
+  _incrementTrackerStats(resp, dbManager) {
+    if (resp && resp.ConsumedCapacity && resp.ConsumedCapacity.CapacityUnits) dbManager.stats.writeUnits += parseFloat(resp.ConsumedCapacity.CapacityUnits);
+  }
 }

package/src/Database/PSQL/IKDB.js

@@ -17,6 +17,8 @@ export default class IKDB_PSQL extends IKDB {
   async commit() { return await (await this._getConnection()).query('COMMIT'); }
   async rollback() { return await (await this._getConnection()).query('ROLLBACK'); }
   async runQuery(qry) { return await qry.run(await this._getConnection()); }
+  //tracker support
+  getTrackerStats() { return null; }
   //Public 
   async sanitize(val) { return val; }
 

package/src/Database/Prototype/IKDB.js

@@ -15,4 +15,7 @@ export default class IKDB {
 
   //main interface
   async runQuery(qry) { return await qry.run(this); }
+
+  //tracker support
+  getTrackerStats() { return null; }
 }

package/src/IKEventProcessor.js

@@ -7,7 +7,7 @@ export default class IKEventProcessor {
     this.event = event;
     this.context = context;
     this.apiConfig = config;
-    this.isBatch = isBatch;
+    this._syncReturn = isBatch;
   }
   async processEvent(execution, doNotDecodeMessage) {
     const resp = await this._processRawEvent(execution, doNotDecodeMessage);
@@ -21,7 +21,7 @@ export default class IKEventProcessor {
         //Map records with decoded message when required
         const decodedRecords = this.event.Records.map((eventRecord) => (doNotDecodeMessage ? eventRecord.body : JSON.parse(eventRecord.body)));
         //If is batch, return execution 
-        if (this.isBatch) return await execution(transaction, decodedRecords);
+        if (this._syncReturn) return await execution(transaction, decodedRecords);
         //for each available event
         for (let eventRecord of decodedRecords) {
           //Call execution

package/src/IKGlobals.js

@@ -6,6 +6,9 @@ IKGlobals.DBDrivers = {
 IKGlobals.CacheDrivers = {
   REDIS: 'REDIS'
 };
+//DDB Client 
+IKGlobals.DDBHttpTimeout = 15000;
+IKGlobals.DDBMaxAttempts = 3;
 //Error messages
 IKGlobals.ErrorResponseValidationFail = 'Input validation failed: '; //400
 IKGlobals.ErrorResponseInvalidServerResponse = 'No valid response, this is a system error.'; //400

package/src/Logger/IKLogger.js

@@ -9,7 +9,7 @@ const LOG_STRINGS = ['DEBUG', 'INFO', 'WARN', 'ERROR'];
 const PURE_CONSOLE = (console.flushLogs ? console.origin : console);
 const DEFAULT_LOG_FUNCTION = PURE_CONSOLE.log.bind(PURE_CONSOLE);
 //
-const blacklist = ['password','phonenumber','code','resetCode','recaptchaToken','token','mfa','REFRESH_TOKEN','SECRET_HASH','SecretHash','AccessToken','UserCode','paymentMethodNonce'];
+const blacklist = ['password','phonenumber'/*,'code'*/,'resetCode','recaptchaToken','token','mfa','REFRESH_TOKEN','SECRET_HASH','SecretHash','AccessToken','UserCode','paymentMethodNonce'];
 //
 export default class IKLogger {
   constructor(_config, _LOG_LEVEL, transactionID) {
@@ -81,7 +81,7 @@ export default class IKLogger {
     if (Utils.isOffline()) {
       return ` [${this._timestamp()} - ${LOG_STRINGS[level]}] [${caller}] ${msg.join(" ")}`;
     } else if (Utils.isHybridlessContainer() && this._transactionID) {
-      return (isRaw ? '' : ` ${this._transactionID}`) + ` [${LOG_STRINGS[level]}] [${caller}] ${this._supressSensitiveInfo(msg.join(" "))}`;
+      return (isRaw ? '' : ` ${this._transactionID}`) + ` [${LOG_STRINGS[level]}] [${caller}] ${(msg.map(this._supressSensitiveInfo).join(" "))}`;
     } else {
       return ` [${LOG_STRINGS[level]}] [${caller}] ${msg.join(" ")}`;
     }
@@ -127,10 +127,10 @@ export default class IKLogger {
       Object.keys(value).forEach(function (elt, i, array) {
         const match = blacklist.find((f) => elt.toLowerCase().includes(f.toLowerCase()));
         if (match) value[elt] = '**SUPRESSED_SENSITIVE_DATA**';
-        else value[elt] = supress(value[elt]);
+        else value[elt] = value[elt];
       });
       return value;
-    } else if (Array.isArray(value)) return value.map(v=>supress(v));
+    } else if (Array.isArray(value)) return value.map(v=>this._supressSensitiveInfo(v));
     return value;
 	}
 }

package/src/Tracker/IKExecutionTracker.js

@@ -0,0 +1,57 @@
+import { RTClient } from '@ikonintegration/mod-resources-tracker-client';
+//
+import Utils from "../API/IKUtils";
+//
+const fs = reuiqre('fs');
+//
+export default class IKExecutionTracker {
+  constructor(config, context, db) {
+    this.context = context;
+    this.db = db;
+    this.startedOn = Date.now();
+    if (config) this.client = this._getClient(config);
+    this.claimers = {};
+  }
+  /* Public */
+  appendClaimer(value, key) { this.claimers[value] = key; }
+  appendEvent(event) { this.client.appendEvent(event); }
+  //
+  async stopTracking() {
+    try {
+      //Check for client and claimers
+      if (!this.client || Object.keys(this.claimers).length == 0) return; //disabled
+      //detect execution and log (ECS or lambda)
+      this._detectExecutionAndLog();
+      //detect DB and log (DDB only for now)
+      this._collectDDBStats();
+      //drain stats
+      await this.client.drain();
+    } catch (e) { console.error(`Error while tracking execution: ${e}`); }
+  }
+  /* private tracker */
+  _detectExecutionAndLog() {
+    //Is running on container?
+    if (Utils.isHybridlessContainer()) {
+      //ECS, get metadata info and log if applicable and succeeded on it
+      const context = this._getECSMetadataObject();
+      if (context && context.containerInstanceArn) {
+        const extraDetails = { startTime: this.startedOn, endTime: Date.now(), taskConfig: { cpu: process.env.CPU, memory: process.env.MEMORY } };
+        this.client.appendEvent(RTClient.newECSExecutionEvent(context.ecsTaskId, context.containerInstanceArn, extraDetails));
+      }
+    } else this.client.appendEvent(RTClient.newLambdaInvocationEvent(this.context.awsRequestId)); //lambda logging
+  }
+  _collectDDBStats() {
+    const stats = this.db ? this.db.getTrackerStats() : null;
+    if (stats) this.client.appendEvent(RTClient.newDynamoEvent(stats.tableName, stats.readUnits, stats,writeUnits));
+  }
+  /* Private */
+  _getClient(config) {
+    return new RTClient({ busName: config.busName, region: config.region, source: config.appName });
+  }
+  /* ECS helper */
+  _getECSMetadataObject() {
+    try { return JSON.parse(fs.readFileSync(process.env.ECS_CONTAINER_METADATA_FILE)); }
+    catch (e) { console.error('Unable to retrieve metadata object: ' + e); }
+    return null;
+  }
+}