@mhubcloud/tms_core 2.0.0-beta.0 → 2.0.0-beta.2

package/tests/GoogleAnalytics4.js

@@ -2,7 +2,7 @@ const publisher = require('../dist/index.js').publisher.GoogleAnalytics4;
 const defaultPublisher = require('../dist/index.js').DefaultPublisher;
 const utils = require('../dist/index.js').utils;
 var assert = require('assert');
-const testUtils = require('./testUtils');
+const testUtils = require('../dist/index.js').testUtils;
 
 describe('test GoogleAnalytics4 publisher', function () {
   it('purchase', async function () {

package/tests/Piano.js

@@ -1,8 +1,8 @@
 const publisher = require('../dist/index.js').publisher.Piano;
 const defaultPublisher = require('../dist/index.js').DefaultPublisher;
 const utils = require('../dist/index.js').utils;
+const testUtils = require('../dist/index.js').testUtils;
 var assert = require('assert');
-const testUtils = require('./testUtils');
 
 describe('test Piano publisher', function () {
   it('purchase', async function () {

package/tests/UniversalAnalytics.js

@@ -2,7 +2,7 @@ const publisher = require('../dist/index.js').publisher.UniversalAnalytics;
 const defaultPublisher = require('../dist/index.js').DefaultPublisher;
 const utils = require('../dist/index.js').utils;
 var assert = require('assert');
-const testUtils = require('./testUtils');
+const testUtils = require('../dist/index.js').testUtils;
 
 describe('test UniversalAnalytics publisher', function () {
   it('pageview', async function () {

package/js/lib/defaultPublisher.js

@@ -1,116 +0,0 @@
-//Default common implementation of each publisher's index.js
-const utils = require("./utils");
-
-/**
- * Implements the Azure Function, called for set of events received via Event Hub
- * @param {FunctionContext} log Azure Function context.log.warn
- * @param {Object[]} eventHubMessages Array of messages received from Event Hub
- * @param {string} source Name of the publisher (it's folder name in which it's implemented)
- * @param {PlatformPublisher} publisher Publisher implementation instance (from <publisher>/publisher.js)
- * @param {Trigger} trigger Publisher-specific trigger function (from <publisher>/customizable/trigger.js)
- * @returns {Promise<void|ReqResTestData[]>} When run in test mode (@param context.test is true), returns collected array of requests (otherwise) sent to platform. When additionaly run as test with publication (@param context.tests and @param context.testPublish) the response contains both - request and response data.
- */
-module.exports = async function DefaultPublisher(azureFunctionContext, eventHubMessages, source, publisher, trigger) {
-
-    azureFunctionContext.log.warn(`JavaScript eventhub trigger function called for message array [${eventHubMessages.length}]`);
-    let mi = 0;
-    
-    let processMessagePromises = [];
-    for (const message of eventHubMessages) {
-        processMessagePromises.push(processMessage(message, azureFunctionContext.log.warn, azureFunctionContext.test, azureFunctionContext.debug, azureFunctionContext.testPublish, source, publisher, trigger, mi++));
-    }
-    let results = await Promise.allSettled(processMessagePromises);
-    for (const result of results) {
-        if(result.status === "rejected"){
-            utils.logException(azureFunctionContext.log.warn, result.reason);
-        }
-    }
-}
-
-/**
- * Processes single message
- * @param {Object} message A message received from Event Hub
- * @param {FunctionContext} log Azure Function context.log.warn
- * @param {FunctionContext} test Azure Function context.test
- * @param {string} source Name of the publisher (it's folder name in which it's implemented)
- * @param {PlatformPublisher} platfromPublisher Publisher implementation instance (from <publisher>/publisher.js)
- * @param {Trigger} trigger Publisher-specific trigger function (from <publisher>/customizable/trigger.js)
- * @param {number} mi Messae index (0-based), only for logging purposes
- * @returns {Promise<boolean|ReqResTestData[]>} In normal mode returns true/false indicating if the message has been successfully published to any platform. In (unit) test mode returns array of publication requests (or even results if it's not unit test), fulse just in case the were no publishers configured.
- */
-async function processMessage(message, log, test, debug, testPublish, source, platfromPublisher, trigger, mi) {
-
-    let ret = test ? [] : false;
-    if (debug)
-        log(`Processing message №${mi + 1} ${JSON.stringify(message, null, "")}`);
-
-    const data = platfromPublisher.parse(message, context);
-    const logProp = {
-        source: source,
-        data: JSON.stringify(data)
-    };
-
-    /** @type {EventPublisher[]} */
-    const publishers = trigger(data);
-    if (!publishers?.length) {
-        utils.log2(log, 1, "Message excluded by filter (no publishers determined)", logProp);
-        return false;
-    }
-
-    let prevPublisher = null;
-    let prevMappedData = null;
-    let prevValidionResult = null;
-    let passedAnyValidation = false;
-    let published = false;
-    for (const eventPublisher of publishers) {
-        if (prevPublisher === eventPublisher ? prevValidionResult : (prevValidionResult = eventPublisher.validate(data))) { //Optimization
-            passedAnyValidation = true;
-        } else {
-            utils.log2(log, 1, "Message excluded by validation", logProp);
-            continue;
-        }
-
-        const mapped = logProp.mapped = prevMappedData = (prevPublisher === eventPublisher ? prevMappedData : await eventPublisher.map(data)); //Optimization
-
-        prevPublisher = eventPublisher;
-
-        const res = await publishMessage(log, mapped, eventPublisher.config, test, testPublish, platfromPublisher, message.event.code + (message.event.name ? "/"+ message.event.name : ""));
-        if (res) published = true;
-        if (test && res instanceof Array) ret.push(...res);
-        if (res && !test) ret = true;
-    }
-
-    if (!passedAnyValidation) {
-        utils.log2(log, 2, "Message excluded by all publishers' validations", logProp);
-        return false;
-    }
-    if (!published) {
-        utils.log2(log, 2, "The message has not been published to any publisher configuration", logProp);
-        return false;
-    }
-
-    return ret;
-}
-
-/**
- * Publishes a message to configured publisher
- * @param {Object} mapped Mapped message data
- * @param {Object} config Platform-specific platform publisher configuration
- * @param {FunctionContext} test Azure Function context.test 
- * @param {PlatformPublisher} platfromPublisher Publisher implementation instance (from <publisher>/publisher.js)
- * @param {string} event for which logic was triggered
- * @returns {Promise<boolean|ReqResTestData[]>} In normal mode returns true/false indicating if the message has been successfully published to any platform. In (unit) test mode returns array of publication requests (or even results if it's not unit test), fulse just in case the were no publishers configured.
- */
-async function publishMessage(log, mapped, config, test, testPublish, platfromPublisher, event) {
-    let ret = test ? [] : true;
-    if (test)
-        ret.push({ req: mapped, config: config });
-
-    if (!test || testPublish) {
-        const res = await platfromPublisher.publish(log, config, mapped, event);
-        if (context.test) ret[ret.length - 1].res = res;
-        return ret; //true
-    } else {
-        return ret; //true
-    }
-}

package/js/lib/utils.js

@@ -1,349 +0,0 @@
-const crypto = require('crypto');
-const fs = require('fs');
-const https = require('https');
-const Ajv = require('ajv');
-
-let aiClient;
-if (process.env.APPINSIGHTS_INSTRUMENTATIONKEY) {
-  const appInsights = require('applicationinsights');
-  appInsights
-    .setup() // Set einvironmet variable APPINSIGHTS_INSTRUMENTATIONKEY, so setup() can be called w/o parameters
-    .setAutoCollectDependencies(false)
-    .setAutoCollectConsole(true, true)
-    .setSendLiveMetrics(false)
-    .setAutoCollectRequests(false)
-    .setAutoCollectPerformance(false)
-    .setUseDiskRetryCaching(false)
-    .start();
-  aiClient = appInsights.defaultClient;
-}
-
-/**
- * Base class for Event Publisher classes
- * Facilitates registration and configuration
- */
-class EventPublisherBase {
-  /**
-   * Create a  new insatnce of the @see EventPublisherBase class
-   * @param {Object} config Publisher configuration (in platform-specific format)
-   */
-  constructor(config) {
-    this.config = config;
-  }
-
-  /**
-   * Validates input data that they can be passed to mapper
-   * @param {Object} data Input data to validate
-   * @returns {boolean} Validation result
-   */
-  validate(data) {
-    return true; // Override this method in custom publisher for custom validation
-  }
-
-  /**
-   * Registers a new instance of @see EventPublisherBase -derived class with array of publishers
-   * @param {EventPublisher[]} publishers Tha publishers array to register with
-   * @param {Object} config Configuration to pass to new instance
-   * Call this static method on @see EventPublisherBase -derived class.
-   * It will create its instance and pass @param config to its constructor.
-   * Than it will add resulting instance of @param publishers.
-   */
-  static register(publishers, config) {
-    publishers.push(
-      new this(
-        config,
-      ) /* Trick: Creates instance of the specific derived class `register` has been called on */,
-    );
-  }
-}
-
-module.exports = {
-  EventPublisherBase: EventPublisherBase,
-
-  /**
-   * Provides access to Application Insights logging (event reporting)
-   * @type {appInsights.TelemetryClient|null}
-   */
-  appInsights: aiClient || null,
-
-  /**
-   * Computes SHA-256 hash of a string
-   * @param {string} email The vaue to hash
-   * @returns {string} @param email SHA-256 hash; returns email if it is 𝑓𝑎𝑙𝑠𝑒 (null, undefined etc.)
-   */
-  encodeEmail: function encodeEmail(email) {
-    if (!email) {
-      return email;
-    }
-    return crypto
-      .createHash('sha256')
-      .update(email.toLowerCase(), 'ascii')
-      .digest('hex')
-      .toLowerCase();
-  },
-
-  /**
-   * Handles HTTP response from node-fetch
-   * @param {*} response Node-fetch response object
-   * @param {string} event for which response is
-   * @returns {Object} Deserialized response JSON
-   * Throws exception in case of response error code
-   */
-  handleResponse: async function handleResponse(response, event) {
-    if (!response.ok) {
-      throw new Error(
-        `HTTP Error code: ${
-          response.status
-        }, event: ${event}, body: ${await response.text()}`,
-      );
-    }
-    const body = await response.text();
-    try {
-      return JSON.parse(body);
-    } catch (ex) {
-      return body;
-    }
-  },
-
-  /**
-   * Creates a deep copy of given object removing any properties or array items with undefined value
-   * @param {*} obj Objec or array to copy. If value is neither array nor object it's returned as passed.
-   * @returns Deep copy of @param obj (providing array or object has bee passed in). Any properties or array item with undefined value are skipped when making the copy, recursively. Returns @param obj untouched if neither array nor object has been passed in.
-   */
-  removeUndefined: function removeUndefined(obj) {
-    let ret;
-    if (obj instanceof Array) {
-      ret = [];
-      for (let i = 0; i < obj.length; i++) {
-        if (obj[i] !== undefined) {
-          ret.push(removeUndefined(obj[i]));
-        }
-      }
-    } else if (obj instanceof Object) {
-      ret = {};
-      for (const prp in obj) {
-        if (obj.hasOwnProperty(prp) && obj[prp] !== undefined) {
-          ret[prp] = removeUndefined(obj[prp]);
-        }
-      }
-    } else {
-      return obj;
-    }
-    return ret;
-  },
-
-  /**
-   * Validates an object against the schema it states or given schema
-   * @param {Object} data An object to validate. The shema must be provided in its `$schema` property if ot passed in @param schema
-   * @param {string} schemaSource Name of a folder to look for schemas. Schemas JSON format are looked for in subfolder `customizable/schemas` of given folder.
-   * @param {string} schema Full URL id (i.e. URL starting with https://schema.measurementhub.net/), or just relative part of that URL to resolve schem in @param schemaSource`/customizable/schemas`
-   * @returns {boolean} Validation result, true if successful, Exception if unsuccessfull
-   */
-  validate: function validate(data, schemaSource, schema = null) {
-    schema = schema || data.$schema;
-    if (!schema) {
-      throw "Data doesn't have schema and schema not passed";
-    }
-    if (
-      !schema.startsWith('https://schema.measurementhub.net/') &&
-      (schema.startsWith('http://') || schema.startsWith('https://'))
-    ) {
-      throw `Schema ${data.$schema} is not supported, only https://schema.measurementhub.net/... and schemas identified by relative URL shemas are supported`;
-    }
-    let parts = schema.split('/');
-    if (parts.includes('..')) {
-      // Directory traversal attack protection
-      throw `It's invalid for schema URI to contain '..' segment, such as in ${schema}`;
-    }
-    if (schema.startsWith('http://') || schema.startsWith('https://')) {
-      parts = parts.slice(3);
-    }
-
-    const schemaLocation = `${schemaSource}/customizable/schemas/${parts.join(
-      '/',
-    )}.json`;
-    if (!fs.existsSync(schemaLocation)) {
-      throw `Schema file ${schemaLocation} not found`;
-    }
-
-    let schemaStr;
-    try {
-      schemaStr = fs.readFileSync(schemaLocation).toString();
-    } catch (ex) {
-      throw `Failed to load schema ${schema}`;
-    }
-
-    let schemaObj;
-    try {
-      schemaObj = JSON.parse(schemaStr);
-    } catch (ex) {
-      throw `Failed to deserialize ${schema}`;
-    }
-
-    const ajv = new Ajv({ allErrors: true });
-    let validate;
-    try {
-      validate = ajv.compile(schemaObj);
-    } catch (ex) {
-      throw `Failed to compile schema ${schema}`;
-    }
-
-    const valid = validate(data);
-    if (!valid) {
-      throw `Schema vaidation failed for schema ${schema}, ${
-        validate.errors.length
-      } errors: ${JSON.stringify(validate.errors)}`;
-    }
-    return true;
-  },
-
-  /**
-   * Enhances header before sending test request to secondary target (like Pipedream)
-   * @param {Object} headers Request header dictionary. This contains the original headers and will be ehnanced.
-   * @param {string} origUrl Original URL the request has been sent to
-   * @param {*} origRequest The entire original request options
-   * @param {*} origResponse Response recievend from original request
-   */
-  prepareTestRequestHeaders: function prepareTestRequestHeaders(
-    headers,
-    origUrl,
-    origRequest,
-    origResponse,
-  ) {
-    headers['x-mhub-orig-url'] = origUrl;
-    headers['x-mhub-orig-method'] = origRequest.method;
-    headers['x-mhub-orig-response-status-code'] = origResponse.status;
-    headers['x-mhub-orig-response-status-message'] = origResponse.statusText;
-  },
-
-  /**
-   * Extracts ad code from standardly structured event data (in legacy or new format)
-   * @param {{ads:(LegacyAd|Ad)[]}} data Event data to extract ad code from. As you can see the `ads` field is accepted in 2 formats the first being legacy and the later being new one.
-   * @param {string} adPlatform value of `data.ads.adPlatform` field to filter ad codes by platform (this is for new style data; ignored for legacy data)
-   * @param {string} adCode value of `data.ads.adType` field to filter items in `data.ads` by (this is for legacy style data); als used for new style data to identify the main parameter from `data.ads.adCodes` to obtain
-   * @param {(code: AdCodeInfo, ad: LegacyAd|Ad) => string|any} format If this custom function is provided it forats the code returned, so the function does not return object but string formated by this function. But nothing prevents it from returning object if the consumer can hanlde it.
-   * @returns {AdCodeInfo|string|null|any} Ad code and time when it was recorded of newest event (by `visitAt`) with given ad code from @param data.ads; when @param format is provided returns string result of that function.
-   */
-  extractAdCode: function extractAdCode(
-    data,
-    adPlatform,
-    adCode,
-    format = null,
-  ) {
-    const ad = data.ads
-      .filter(
-        (x) =>
-          x.adType === adCode /* legacy*/ ||
-          x.adPlatform === adPlatform /* new*/,
-      )
-      .sort((a, b) => b.visitAt - a.visitAt)[0];
-    const code =
-      ad?.adCode /* legacy*/ || (ad?.adCodes ? ad.adCodes[adCode] : null);
-    if (code) {
-      const ret = { code: code, visitAt: ad.visitAt };
-      if (format) {
-        return format(ret, ad);
-      }
-      return ret;
-    }
-    return null;
-  },
-
-  /**
-   * Logs message to Azure Function context and to Application Insights (if configured)
-   * @param {FunctionContext} log Azure Function context.log.warn
-   * @param {0|1|2|3|4} level Application Insights log level according to https://docs.microsoft.com/en-us/dotnet/api/microsoft.applicationinsights.datacontracts.severitylevel?view=azure-dotnet
-   * @param {string} message The message to log
-   * @param {Object} properties Properties to pass to Application Insights trace
-   */
-  log2: function log2(log, level, message, properties) {
-    log(message);
-    aiClient?.trackTrace({
-      message: message,
-      severityLevel: level,
-      properties: properties,
-    });
-  },
-
-  /**
-   * Logs exception to Azure Function context and to Application Insights (if configured)
-   * @param {FunctionContext} log Azure Function context.log.warn
-   * @param {object} exception
-   */
-  logException: function logException(log, exception) {
-    log(exception);
-    aiClient?.trackException({ exception: exception });
-    // raise exception if it is running under mocha test
-    if (typeof global.it === 'function') {
-      throw exception;
-    }
-  },
-
-  /**
-   * For http request. This function is for edge cases. Use Fetch if possible.
-   * @param {object} params (hostname, port, path, method)
-   * @param {object} postData
-   * @returns
-   */
-  httpRequest: function httpRequest(params, postData) {
-    return new Promise(function (resolve, reject) {
-      var req = https.request(params, function (res) {
-        // reject on bad status
-        if (res.statusCode < 200 || res.statusCode >= 300) {
-          return reject(new Error('statusCode=' + res.statusCode));
-        }
-        // cumulate data
-        var body = [];
-        res.on('data', function (chunk) {
-          body.push(chunk);
-        });
-        // resolve on end
-        res.on('end', function () {
-          try {
-            if (body.length > 0) {
-              body = JSON.parse(Buffer.concat(body).toString());
-            }
-          } catch (e) {
-            reject(e);
-          }
-          resolve(body);
-        });
-      });
-      // reject on request error
-      req.on('error', function (err) {
-        // This is not a "Second reject", just a different sort of failure
-        reject(err);
-      });
-      if (postData) {
-        req.write(postData);
-      }
-      // IMPORTANT
-      req.end();
-    });
-  },
-
-  /**
-   * Transforms an object to be flat, for instance for GA4
-   * @param {object} event Event that is received
-   * @param {sring} separator Separator that will be used to unnest the structure. The default separator is underscore _
-   * @returns
-   */
-  flatten: function flatten(event, separator) {
-    return Object.assign(
-      {},
-      ...(function _flatten(o, separator = '_', parent) {
-        return [].concat(
-          ...Object.keys(o).map((key) => {
-            if (typeof o[key] === 'object' && o[key] !== null) {
-              const newParent = parent ? parent + separator + key : key;
-              return _flatten(o[key], separator, newParent);
-            } else {
-              const newKey = parent ? parent + separator + key : key;
-              return { [newKey]: o[key] };
-            }
-          }),
-        );
-      })(event, separator),
-    );
-  },
-};

package/js/publishers/AffiliatePort.js

@@ -1,78 +0,0 @@
-const fetch = require('node-fetch');
-const utils = require('../lib/utils');
-const SOURCE = 'f_AffiliatePort';
-
-module.exports = {
-  /**
-   * Processed data comming from EventHub to data for further processing
-   * @param {Object} data The data comming from EventHub
-   * @returns {Object} Data for further processing
-   */
-  parse: function parse(data, context) {
-    // ConsumerBase.cs:ConsumeOneEventAsync
-    data.enqueuedAt = context.test ? 1600000000000 : Date.now();
-    return data;
-  },
-
-  /**
-   * Publishes data to AffiliatePort
-   * @param {{id: string, access_token: string}} config Publisher configuration
-   * @param {Object} data Data in publisher-friendly format to be published
-   * @param {string} event for which is it triggered
-   */
-  publish: async function publish(log, config, data, event) {
-    data = utils.removeUndefined(data);
-    const requestOptions: PlatformRequestOptions = {
-      method: 'GET',
-    };
-
-    for (let product of data.products) {
-      // https://tracking.affiliateclub.cz/aff_lsr?offer_id=OFFER_ID&adv_sub=TID_SUB_ID&amount=AMOUNT&transaction_id=TRANSACTION_ID&adv_sub2=CATEGORY&adv_sub3=MASTER_TYPE
-      const url =
-        'https://tracking.affiliateclub.cz/aff_lsr?offer_id=' +
-        encodeURIComponent(config.offerId) +
-        '&adv_sub=TID_' +
-        data.orderId +
-        '&amount=' +
-        product.quantity * product.price +
-        '&transaction_id=' +
-        data.transactionID +
-        '&adv_sub2=' +
-        product.category +
-        '&adv_sub3=' +
-        product.masterType;
-
-      const resp = await fetch(url, reqOpts);
-
-      if (process.env.PUBLISHER_TESTURL) {
-        try {
-          utils.prepareTestRequestHeaders(reqOpts.headers, url, reqOpts, resp);
-          await fetch(process.env.PUBLISHER_TESTURL + query, reqOpts);
-        } catch {}
-      }
-
-      await utils.handleResponse(resp, event);
-
-      log(`AffiliatePort - ${event} - sent`);
-    }
-  },
-
-  /**
-   * Validates an object against given schema
-   * @param {Object} data An object to validate.
-   * @param {string} schema Name of the schema. That's name of a file with `.schema.json` extension in `customizable/schemas` subfolder of current publisher folder
-   * @returns {boolean} Validation result, true if successful, false if unsuccessfull (in which case more info is written to console)
-   */
-  validate: function validate(data, schema) {
-    if (!schema) {
-      throw new Error('Schema parameters not provided');
-    }
-    if (schema.includes('/')) {
-      throw new Error(
-        'Schema must be provided as schema name, not schema path',
-      );
-    }
-    // More validation done by utils.validate
-    return utils.validate(data, SOURCE, schema);
-  },
-};

package/js/publishers/CJ.js

@@ -1,68 +0,0 @@
-const fetch = require('node-fetch');
-const utils = require("../lib/utils");
-const SOURCE = "f_CJAffiliate";
-
-module.exports = {
-    /**
-     * Processed data comming from EventHub to data for further processing
-     * @param {Object} data The data comming from EventHub
-     * @param {FunctionContext} test Azure Function context.test
-     * @returns {Object} Data for further processing
-     */
-    parse: function parse(data, test) {
-        //ConsumerBase.cs:ConsumeOneEventAsync
-        data.enqueuedAt = test ? 1600000000000 : Date.now();
-        return data;
-    },
-
-    /**
-     * Publishes data to Facebook
-     * @param {FunctionContext} log Azure Function context.log.warn
-     * @param {{cid: string|number, type: string|number}} config Publisher configuration
-     * @param {Object} data Data in publisher-friendly format to be published
-     * @returns {*} Data returned by platform, if possible JSON-deserialized (otherwise as string)
-     * @param {string} event for which is it triggered
-     */
-    publish: async function publish(log, config, data, event) {
-        if(!data.cjevent){
-            log(`CJ - ${event} - skipped because CJEVENT is missing`);
-            return false;
-        }
-        
-        data = utils.removeUndefined(data);
-        const baseUrl = "https://www.kdukvh.com/u";
-        let dt = "?cid=" + encodeURI(config.cid);
-        dt += "&type=" + encodeURI(config.type);
-        for (let prp in data) {
-            if (data.hasOwnProperty(prp) && data[prp] !== undefined) {
-                dt += "&" + encodeURI(prp) + "=" + encodeURI(data[prp] === null ? "" : data[prp].toString());
-            }
-        }
-        const reqOpts = { method: "get", headers: {} /*Needed by test call*/ };
-        const resp = await fetch(baseUrl + dt, reqOpts);
-
-        if (process.env.PUBLISHER_TESTURL) {
-            try {
-                utils.prepareTestRequestHeaders(reqOpts.headers, baseUrl + dt, reqOpts, resp);
-                await fetch(process.env.PUBLISHER_TESTURL + dt, reqOpts);
-            } catch { }
-        }
-
-        await utils.handleResponse(resp, event);
-
-        log(`CJ - ${event} - sent`);
-    },
-
-    /**
-    * Validates an object against given schema
-    * @param {Object} data An object to validate.
-    * @param {string} schema Name of the schema. That's name of a file with `.schema.json` extension in `customizable/schemas` subfolder of current publisher folder
-    * @returns {boolean} Validation result, true if successful, false if unsuccessfull (in which case more info is written to console)
-    */
-    validate: function validate(data, schema) {
-        if (!schema) throw new Error("Schema parameters not provided");
-        if (schema.indexOf("/") >= 0) throw new Error("Schema must be provided as schema name, not schema path");
-        //More validation done by utils.validate
-        return utils.validate(data, SOURCE, schema);
-    }
-};