fiftyone.pipeline.engines.fiftyone 4.4.3 → 4.4.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fiftyone.pipeline.engines.fiftyone",
-  "version": "4.4.3",
+  "version": "4.4.4",
   "description": "Shared base functions for implementing 51Degrees engines for the 51Degrees Pipeline API",
   "main": "index.js",
   "directories": {
@@ -11,8 +11,8 @@
   },
   "author": "51Degrees <support@51Degrees.com>",
   "dependencies": {
-    "fiftyone.pipeline.core": "^4.4.3",
-    "fiftyone.pipeline.engines": "^4.4.3"
+    "fiftyone.pipeline.core": "^4.4.4",
+    "fiftyone.pipeline.engines": "^4.4.4"
   },
   "devDependencies": {
     "eslint": "^6.8.0",

package/shareUsage.js

@@ -34,10 +34,18 @@ const Engine = engines.Engine;
 const ShareUsageEvidenceKeyFilter = require('./shareUsageEvidenceKeyFilter');
 const ShareUsageTracker = require('./shareUsageTracker');
 const zlib = require('zlib');
-const https = require('https');
+const url = require('url');
 
 const os = require('os');
 
+/**
+ * The maximum length of a piece of evidence's value which can be
+ * added to the usage data being sent.
+ */
+const SHARE_USAGE_MAX_EVIDENCE_LENGTH = 512;
+
+const SHARE_USAGE_VERSION = '1.1';
+
 /**
  * @typedef {import('fiftyone.pipeline.core').FlowData} FlowData
  */
@@ -71,7 +79,8 @@ class ShareUsage extends Engine {
    * @param {Array} options.headerBlacklist By default, all HTTP headers
    * (except a few, such as Cookies) are shared. Individual headers can 
    * be excluded from sharing by adding them to this list.
-   * @param {number} options.sharePercentage percentage of requests to share.
+   * @param {number} options.sharePercentage approximate proportion of
+   * requests to be shared. 1 = 100%, 0.5 = 50%, etc..
    */
   constructor (
     {
@@ -80,7 +89,8 @@ class ShareUsage extends Engine {
       cookie,
       queryWhitelist,
       headerBlacklist,
-      sharePercentage = 100
+      sharePercentage = 1,
+      endpoint = 'https://devices-v4.51degrees.com/new.ashx'
     } = {}) {
     super(...arguments);
 
@@ -101,9 +111,21 @@ class ShareUsage extends Engine {
 
     this.sharePercentage = sharePercentage;
 
-    this.sharePercentageCounter = 0;
-
     this.shareData = [];
+
+    if (endpoint.includes('https://') === false &&
+      endpoint.includes('http://') == false) {
+      endpoint = 'https://' + endpoint;
+    }
+    this.endpoint = url.parse(endpoint);
+    switch (this.endpoint.protocol) {
+      case 'http:':
+        this.http = require('http');
+        break;
+      case 'https:':
+        this.http = require('https');
+        break;
+    }
   }
 
   /**
@@ -113,71 +135,156 @@ class ShareUsage extends Engine {
    * @param {FlowData} flowData flowData to process
    */
   processInternal (flowData) {
-    this.sharePercentageCounter += 1;
 
-    if (this.sharePercentageCounter === this.sharePercentage) {
-      this.sharePercentageCounter = 0;
-    }
-
-    if (this.sharePercentageCounter !== 0) {
-      return;
-    }
+    if (Math.random() <= this.sharePercentage) {
+      const cacheKey = this.evidenceKeyFilter
+        .filterEvidence(flowData.evidence.getAll());
 
-    const cacheKey = this.evidenceKeyFilter
-      .filterEvidence(flowData.evidence.getAll());
+      const share = this.tracker.track(cacheKey);
+      if (share) {
+        this.tracker.put(cacheKey);
 
-    const share = this.tracker.track(cacheKey);
+        this.addToShareUsage(this.getDataFromEvidence(flowData));
+      }
+    }
+  }
 
-    if (share) {
-      this.tracker.put(cacheKey);
+  /**
+   * Creates a ShareUsageData instance populated from the evidence
+   * within the flow data provided.
+   * @param {FlowData} flowData the flow data containing the evidence to use
+   * @returns a new ShareUsageData instance, populated from the evidence
+   * provided 
+   */
+   getDataFromEvidence(flowData) {
+    const data = new ShareUsageData();
+
+    Object.keys(flowData.evidence.getAll()).forEach(key => {
+      const value = flowData.evidence.get(key);
+      if (key === 'server.client-ip') {
+        // The client IP is dealt with separately for backwards
+        // compatibility purposes.
+        data.clientIP = value;
+      } else if (key === 'query.session-id') {
+        // The SessionID is dealt with separately.
+        data.sessionId = value;
+      } else if (key === 'query.sequence') {
+        // The Sequence is dealt with separately.
+        var sequence = parseInt(value);
+
+        if (isNaN(sequence) === false) {
+            data.sequence = sequence;
+        }
+        else {
+          this._log('error',
+            `The value '${value}' could not be parsed to an integer.`);
+        }
+      } else {
+        // Check if we can send this piece of evidence
+        if (this.evidenceKeyFilter.filterEvidenceKey(key)) {
+            data.tryAddToData(key, value);
+        }
+      }
+    });
+    return data;
+  }
 
-      this.addToShareUsage(cacheKey);
+  getConstantXml() {
+    if (!this.constantXml) {
+      const versions = this.getPackageVersions();
+      const coreVersion = versions['fiftyone.pipeline.core'];
+      const osVersion = `${new ReplacedString(process.platform).result} ${new ReplacedString(os.release()).result}`;
+      const nodeVersion = new ReplacedString(process.versions.node).result;
+
+      let xml = '';
+
+      // The version number of the Pipeline API
+      xml += `<Version>${coreVersion}</Version>`;
+      // Write Pipeline information
+      // The product name
+      xml += '<Product>Pipeline</Product>';
+      // The flow elements in the current pipeline
+      this.getFlowElements().forEach(element => {
+        xml += `<FlowElement>${element}</FlowElement>`;
+      })
+      xml += '<Language>Node.JS</Language>';
+      // The software language version
+      xml += `<LanguageVersion>${nodeVersion}</LanguageVersion>`;
+      // The OS name and version
+      xml += `<Platform>${osVersion}</Platform>`;
+      this.constantXml = xml;
     }
+    return this.constantXml;
   }
 
   /**
    * Internal method which adds to the share usage bundle (generating XML)
    *
-   * @param {object} key key value store of current
+   * @param {object} data key value store of current
    * evidence in FlowData (filtered by the ShareUsageEvidenceKeyFilter)
    */
-  addToShareUsage (key) {
+  addToShareUsage (data) {
     let xml = '';
-
-    xml += '<device>';
-
-    xml += '<Version>4</Version>';
-
-    xml += '<Language>Node.JS</Language>';
-
-    xml += '<LanguageVersion>' + process.version + '</LanguageVersion>';
-
-    xml += '<Platform>' + process.platform + ' ' + os.release() + '</Platform>';
-
-    for (const item in key) {
-      const parts = item.split('.');
-
-      const prefix = parts[0];
-      const name = parts[1];
-      const value = key[item];
-
-      xml += `<${prefix} name="${name}">${value}</${prefix}>`;
-    }
-
+    
+    xml += '<Device>';
+
+    // --- write invariant data
+    xml += this.getConstantXml();
+
+    // --- write variable data
+    // The SessionID used to track a series of requests
+    xml += `<SessionId>${data.sessionId}</SessionId>`;
+    // The sequence number of the request in a series of requests.
+    xml += `<Sequence>${data.sequence}</Sequence>`;
+    // The client IP of the request
+    xml += `<ClientIP>${data.clientIP}</ClientIP>`;
+    
+    // The UTC date/time this entry was written
     const date = new Date()
       .toISOString()
-      .substr(0, 19)
+      .substring(0, 19)
       .replace('T', ' ')
       .split(' ')
       .join(':');
+    xml += `<DateSent>${date}</DateSent>`;
+
+    // Write all other evidence data that has been included.
+    Object.keys(data.evidenceData).forEach(categoryKey => {
+      const categoryValue = data.evidenceData[categoryKey];
+      Object.keys(categoryValue).forEach(entryKey => {
+        const entryValue = categoryValue[entryKey];
+        const replacedString = new ReplacedString(entryValue);
+        // Write start element
+        if (categoryKey.length > 0) {
+          xml += `<${categoryKey} Name="${entryKey}"`;
+        } else {
+          xml += `<${entryKey}`;
+        }
+        // Write any attributes
+        if (replacedString.replaced) {
+          xml += ' replaced="true"';
+        }
+        if (replacedString.truncated) {
+          xml += ' truncated="true"';
+        }
+        // End the start element
+        xml += '>';
+        // Write the value
+        xml += replacedString.result;
+        // Write end element
+        if (categoryKey.length > 0) {
+          xml += `</${categoryKey}>`;
+        } else {
+          xml += `</${entryKey}>`;
+        }
+      });
+    });
 
-    xml += '<dateSent>' + date + '</dateSent>';
-
-    xml += '</device>';
+    xml += '</Device>';
 
     this.shareData.push(xml);
-
-    if (this.shareData.length === this.requestedPackageSize) {
+    
+    if (this.shareData.length >= this.requestedPackageSize) {
       this.sendShareUsage();
     }
   }
@@ -186,33 +293,167 @@ class ShareUsage extends Engine {
    * Internal method to send the share usage bundle to the 51Degrees servers
    */
   sendShareUsage () {
-    const data = '<devices>' + this.shareData.join() + '</devices>';
+    const usageEngine = this;
+    var shareData = this.shareData;
+    this.shareData = [];
+    var data = `<Devices version="${SHARE_USAGE_VERSION}">${shareData.join()}</Devices>`;
 
     const options = {
-      hostname: 'devices-v4.51degrees.com',
-      path: '/new.ashx',
+      hostname: this.endpoint.hostname,
+      path: this.endpoint.pathname,
+      port: this.endpoint.port,
       method: 'POST',
-      headers: { 'Content-Encoding': 'gzip', 'Content-Type': 'text/xml' }
+      headers: { 'Content-Encoding': 'gzip', 'Content-Type': 'text/xml; charset=utf-8' }
     };
 
     zlib.gzip(data, function (err, buffer) {
       if (err) {
-        this.pipeline.emit('warning', err);
+        usageEngine._log('warning', err);
       }
-
-      var req = https.request(options, function (res) {
-        // Reset sharedata collection
-
-        this.shareData = [];
+      var req = usageEngine.http.request(options, function (res) {
+        usageEngine._log('debug', `Usage data sent. Response code ${res.statusCode}`);
       });
       req.on('error', function (e) {
-        this.pipeline.emit('warning', e);
+        usageEngine._log('warning', e);
       });
-
       req.write(buffer);
       req.end();
     });
   }
+
+  
+    /**
+     * Return a list of FlowElements in the pipeline.
+     * If the list is null then populate from the pipeline.
+     * If there are multiple or no pipelines then log an error.
+     * @returns list of flow elements
+     */
+    getFlowElements() {
+        if (!this.flowElements) {
+            if (this.pipelines.length === 1) {
+                const list = [];
+                for (const [key, value] of Object.entries(this.pipelines[0].flowElements)) {
+                  list.push(value.constructor.name);
+                }
+                this.flowElements = list;
+            } else {
+                // This element has somehow been registered to too
+                // many (or zero) pipelines.
+                // This means we cannot know the flow elements that
+                // make up the pipeline so a warning is logged
+                // but otherwise, the system can continue as normal.
+                this._log('warn', 'Share usage element registered ' +
+                    `to ${this.pipelines.length > 0 ? "too many" : "no"}` +
+                    ' pipelines. Unable to send share usage information.');
+                this.flowElements = [];
+            }
+        }
+        return this.flowElements;
+    }
+    
+    /**
+     * Get the versions of all packages installed in 'node_modules'.
+     * @returns package versions keyed on name
+     */
+    getPackageVersions() {
+      var fs = require('fs');
+      var dirs = fs.readdirSync('node_modules');
+      var data = {};
+      dirs.forEach(function(dir) {
+          try {
+            var file = dir + '/package.json';
+            var json = require(file);
+            var name = json.name;
+            var version = json.version;
+            data[name] = version;
+          } catch(err) {
+            // Do nothing here
+          }
+      });
+      return data;
+
+    }
+}
+
+// a set of valid XML character values (ignoring valid controls x09, x0a, x0d, x85)
+const VALID_XML_CHARS = 
+  Array.from({length: parseInt('0x7F', 16) - parseInt('0x20', 16)}, (v, k) => k + parseInt('0x20', 16))
+  .concat(
+    Array.from({length: parseInt('0x100', 16) - parseInt('0x40', 16)}, (v, k) => k + parseInt('0x40', 16)));
+
+// an array describing whether a character value is valid
+const IS_VALID_XML_CHAR = __getIsValidCharMap();
+
+function __getIsValidCharMap() {
+  const maxChar = parseInt('0x100', 16);
+  const isValidChar = {};
+  for (var c = 0; c <= maxChar; c++) {
+    isValidChar[c] = VALID_XML_CHARS.includes(c);
+  }
+  return isValidChar;
 }
 
+/**
+ * Replace characters that cause problems in XML with the "Replacement character"
+ */
+class ReplacedString {
+
+  constructor(text) {
+    this.result = '';
+    this.replaced = false;
+    this.truncated = false;
+    if (text) {
+      const escapedText = text
+        .replace(/&/g, '&amp;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;');
+      const length = Math.min(escapedText.length, SHARE_USAGE_MAX_EVIDENCE_LENGTH);
+      this.truncated = escapedText.length > SHARE_USAGE_MAX_EVIDENCE_LENGTH;
+      Array.from(escapedText.substring(0, length))
+        .map(c => {
+          if (c.charCodeAt(0) < Object.keys(IS_VALID_XML_CHAR).length && IS_VALID_XML_CHAR[c.charCodeAt(0)]) {
+            this.result += c;
+          } else {
+            this.result += String.fromCharCode(parseInt('0xFFFD', 16));
+            this.replaced = true;
+          }
+        });
+    }
+  }
+}
+/**
+ * Internal class that is used to store details of data in memory
+ * prior to it being sent to 51Degrees.
+ */
+class ShareUsageData {
+
+  constructor() {
+    this.evidenceData = {};
+    this.sessionId = '';
+    this.clientIp = '';
+    this.sequence = '';
+  }
+
+  tryAddToData(key, value) {
+    // Get the category and field names from the evidence key.
+    let category = '';
+    let field = key;
+
+    const firstSeparator = key.indexOf('.');
+    if (firstSeparator > 0) {
+        category = key.substring(0, firstSeparator);
+        field = key.substring(firstSeparator + 1);
+    }
+
+    // Add the evidence to the dictionary.
+    let categoryDict = this.evidenceData[category];
+    if (!categoryDict) {
+      categoryDict = {};
+      this.evidenceData[category] = categoryDict;
+    }
+    categoryDict[field] = value.toString();
+  }
+}
 module.exports = ShareUsage;

package/shareUsageEvidenceKeyFilter.js

@@ -48,9 +48,17 @@ class ShareUsageEvidenceKeyFilter extends EvidenceKeyFilter {
   constructor ({ cookie, queryWhitelist = [], headerBlacklist = [] }) {
     super(...arguments);
 
-    this.queryWhitelist = queryWhitelist;
-    this.headerBlacklist = headerBlacklist;
+    this.includedQueryStringParams = queryWhitelist;
+    this.blockedHttpHeaders = headerBlacklist;
     this.cookie = cookie;
+    if (!cookie &&
+      (!queryWhitelist || queryWhitelist.length === 0) &&
+      (!headerBlacklist || headerBlacklist.length === 0)) {
+      this.shareAll = true;
+    }
+    else {
+      this.shareAll = false;
+    }
   }
 
   /**
@@ -63,39 +71,39 @@ class ShareUsageEvidenceKeyFilter extends EvidenceKeyFilter {
     const prefix = key.toLowerCase().split('.')[0];
     const suffix = key.toLowerCase().split('.')[1];
 
-    const allowed = ['header', 'cookie', 'query'];
-
-    // First filter out anything that isn't in the allowed list
-
-    if (!allowed.includes(prefix)) {
-      return false;
-    } else {
-      // Filter out all cookies that aren't either 51D prefixed
-      // or the tracking cookie
-
-      if (prefix === 'cookie' && (
-        prefix.indexOf('51D') !== 0 || suffix === this.cookie
-      )
-      ) {
-        return false;
+    let result = this.shareAll;
+
+    if (!this.shareAll) {
+      if (prefix === 'header') {
+        // Add the header to the list if the header name does not 
+        // appear in the list of blocked headers.
+        result = this.blockedHttpHeaders
+          .includes(suffix) === false;
+      } else if (prefix === 'cookie') {
+          // Only add cookies that start with the 51Degrees cookie 
+          // prefix.
+          result = suffix.indexOf('51d_') === 0 ||
+          (this.includeSession && suffix === this.cookie);
+      } else if (prefix === 'session') {
+          // Only add session values that start with the 51Degrees
+          // cookie prefix.
+          result = suffix.indexOf('51d_') === 0;
+      } else if (prefix === 'query') {
+          // If no query string parameter filter was specified 
+          // then share all of them.
+          // Otherwise, only include query string parameters that 
+          // start with 51d_ or that have been specified in 
+          // the constructor.
+          result = this.includedQueryStringParams === null ||
+          suffix.indexOf('51d_') === 0 ||
+            this.includedQueryStringParams.includes(suffix);
+      } else {
+          // Add anything that is not a cookie, header, session 
+          // variable or query string parameter.
+          result = true;
       }
-
-      // Filter out any query evidence not in whitelist
-
-      if (prefix === 'query' && !this.queryWhitelist.includes(suffix)) {
-        return false;
-      }
-
-      // Filter out any header evidence in blacklist
-
-      if (prefix === 'header' && this.headerBlacklist.includes(suffix)) {
-        return false;
-      }
-
-      // Passed filtering tests, track
-
-      return true;
     }
+    return result;
   }
 }
 

package/tests/shareUsage.test.js

@@ -0,0 +1,465 @@
+/* *********************************************************************
+ * This Original Work is copyright of 51 Degrees Mobile Experts Limited.
+ * Copyright 2022 51 Degrees Mobile Experts Limited, Davidson House,
+ * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
+ *
+ * This Original Work is licensed under the European Union Public Licence
+ * (EUPL) v.1.2 and is subject to its terms as set out below.
+ *
+ * If a copy of the EUPL was not distributed with this file, You can obtain
+ * one at https://opensource.org/licenses/EUPL-1.2.
+ *
+ * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
+ * amended by the European Commission) shall be deemed incompatible for
+ * the purposes of the Work and the provisions of the compatibility
+ * clause in Article 5 of the EUPL shall not apply.
+ *
+ * If using the Work as, or as part of, a network application, by
+ * including the attribution notice(s) required under Article 5 of the EUPL
+ * in the end user terms of the application under an appropriate heading,
+ * such notice(s) shall fulfill the requirements of that article.
+ * ********************************************************************* */
+
+const { FlowData, PipelineBuilder } = require('fiftyone.pipeline.core');
+const ShareUsage = require('../shareUsage');
+const http = require('http');
+const os = require('os');
+const zlib = require('zlib');
+
+const shareUsageVersion = '1.1';
+
+// Data received by the local usage receiver.
+let received = [];
+// Local server used to capture usage sent.
+let server = null;
+
+/**
+ * Local server used to capture usage data sent.
+ * @param req the request
+ * @param res the response
+ */
+const requestListener = function (req, res) {
+  var data = '';
+  // Unzip the data.
+  var gunzip = zlib.createGunzip();
+  req.pipe(gunzip);
+  gunzip.on('data', function(chunk) {
+    data += chunk;
+  });
+  gunzip.on('error', (e) => fail(e));
+  // Return an OK response.
+  gunzip.on('end', () => {
+    received.push(data);
+    res.writeHead(200);
+    res.end();
+  });
+}
+
+beforeEach(() => {
+  received = [];
+  // Set up the local server. This is HTTP rather than HTTPS to avoid
+  // having to use local certificates.
+  server = http.createServer(requestListener);
+  server.listen(8080);
+});
+
+afterEach(() => {
+  server.close();
+});
+
+/**
+ * Check that data is received from the share usage element.
+ */
+test('share usage - data received', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData = pipeline.createFlowData();
+  
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received.length).toBe(1);
+      done();  
+    }, 2000);
+  })
+});
+
+/**
+ * Check that the data received from the share usage element is correct.
+ */
+test('share usage - data correct', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData = pipeline.createFlowData();
+  
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received[0]).not.toBeNull();
+      expect(received[0].length).toBeGreaterThan(0);
+      expect(received[0]).toMatch(new RegExp(`^<Devices version="${shareUsageVersion}"><Device>.*`));
+      expect(received[0]).toMatch(new RegExp('.*</Device></Devices>$'));
+      expect(received[0]).toContain('<Language>Node.JS</Language>');
+      expect(received[0]).toContain(`<LanguageVersion>${process.versions.node}</LanguageVersion>`);
+      expect(received[0]).toContain(`<Platform>${process.platform} ${os.release()}</Platform>`);
+      expect(received[0]).toContain('<FlowElement>ShareUsage</FlowElement>');
+      done();
+    }, 2000);
+  })
+});
+
+/**
+ * Check that the data received from the share usage element contains the
+ * headers which were in the evidence.
+ */
+test('share usage - includes header', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const uaValue = 'some user agent..';
+  const flowData = pipeline.createFlowData();
+  flowData.evidence.add('header.user-agent', uaValue);
+
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received[0]).toContain(`<header Name="user-agent">${uaValue}</header>`);
+      done();
+    }, 2000);
+  })
+});
+
+/**
+ * Check that the data received from the share usage element includes
+ * the client IP from the evidence.
+ */
+test('share usage - includes client ip', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const ip = '1.2.3.4';
+  const flowData = pipeline.createFlowData();
+  flowData.evidence.add('server.client-ip', ip);
+
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received.length).toBe(1);
+      expect(received[0]).toContain(`<ClientIP>${ip}</ClientIP>`);
+      done();
+    }, 2000);
+  })
+});
+
+/**
+ * Check that a single event is not sent when the requested package
+ * size is 2.
+ */
+test('share usage - two events - first event', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 2,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData = pipeline.createFlowData();
+
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received.length).toBe(0);
+      done();
+    }, 2000);
+  })
+});
+
+/**
+ * Check that data is sent in a single request when a second request is
+ * sent and the requested package size is 2.
+ */
+test('share usage - two events - second event', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 2,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData1 = pipeline.createFlowData();
+  flowData1.evidence.add('header.user-agent', 'ua 1');
+  const flowData2 = pipeline.createFlowData();
+  flowData2.evidence.add('header.user-agent', 'ua 2');
+
+  flowData1.process().then(() => {
+    flowData2.process().then(() => {
+      setTimeout(() => {
+        expect(received.length).toBe(1);
+        expect(received[0]).toContain('ua 1');
+        expect(received[0]).toContain('ua 2');
+        done();
+      }, 2000);
+    });
+  });
+});
+
+/**
+ * Check that the share usage element respects the headerBlacklist.
+ * When header names are added to the list, values for them should
+ * not be included in the data shared.
+ */
+test('share usage - ignore headers', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/',
+    headerBlacklist: [ 'x-forwarded-for', 'forwarded-for' ]
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData = pipeline.createFlowData();
+  flowData.evidence.add('header.user-agent', 'some user agent');
+  flowData.evidence.add('header.x-forwarded-for', '5.6.7.8');
+  flowData.evidence.add('header.forwarded-for', '2001::');
+
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received.length).toBe(1);
+      expect(received[0]).not.toContain('x-forwarded-for');
+      expect(received[0]).not.toContain('forwarded-for');
+      done();
+    }, 2000);
+  });
+});
+
+test('share usage - queue cleared', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData = pipeline.createFlowData();
+  flowData.evidence.add('header.user-agent', 'some user agent');
+
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received.length).toBe(1);
+      expect(usageEngine.shareData.length).toBe(0);
+      done();
+    }, 2000);
+  });
+});
+
+/**
+ * Check that session and sequence values are included.
+ */
+test('share usage - session and sequence', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData = pipeline.createFlowData();
+  flowData.evidence.add('query.session-id', '1');
+  flowData.evidence.add('query.sequence', '1');
+
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received.length).toBe(1);
+      expect(received[0]).toContain('<SessionId>1</SessionId>');
+      expect(received[0]).toContain('<Sequence>1</Sequence>');
+      done();
+    }, 2000);
+  });
+});
+
+/**
+ * Check that invalid characters are escaped correctly.
+ */
+test('share usage - invalid char', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const replacementChar = String.fromCharCode(parseInt('0xFFFD', 16));
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData = pipeline.createFlowData();
+  flowData.evidence.add('header.user-agent', '1ƌ2');
+
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received.length).toBe(1);
+      expect(received[0]).not.toContain('ƌ');
+      expect(received[0]).toContain(replacementChar);
+      expect(received[0]).toContain('replaced="true"');
+      expect(received[0]).toContain(`<header Name="user-agent" replaced="true">1${replacementChar}2</header>`);
+      done();
+    }, 2000);
+  });
+});
+
+/**
+ * Check that long values are truncated correctly.
+ */
+test('share usage - truncated value', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const ua = 'X'.repeat(1000);
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData = pipeline.createFlowData();
+  flowData.evidence.add('header.user-agent', ua);
+
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received.length).toBe(1);
+      expect(received[0]).toContain('truncated="true"');
+      expect(received[0]).not.toContain(ua);
+      expect(received[0]).toContain(`<header Name="user-agent" truncated="true">${'X'.repeat(512)}</header>`);
+      done();
+    }, 2000);
+  });
+});
+
+/**
+ * Check that the queue of data is correctly reset, and used for the
+ * next set of requests.
+ */
+test('share usage - send  more than once', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData1 = pipeline.createFlowData();
+  flowData1.evidence.add('header.user-agent', 'ua 1');
+  const flowData2 = pipeline.createFlowData();
+  flowData2.evidence.add('header.user-agent', 'ua 2');
+
+  flowData1.process().then(() => {
+    flowData2.process().then(() => {
+      setTimeout(() => {
+        expect(usageEngine.shareData.length).toBe(0);
+        expect(received.length).toBe(2);
+        expect(received[0]).toContain('ua 1');
+        expect(received[0]).not.toContain('ua 2');
+        expect(received[1]).toContain('ua 2');
+        expect(received[1]).not.toContain('ua 1');
+        done();
+      }, 2000);
+    });
+  });
+});
+
+/**
+ * Check that small portion sharing is done correctly.
+ */
+test('share usage - low percentage', function(done) {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 10,
+    sharePercentage: 0.1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  var requiredEvents = 0;
+  while (received.length == 0 && requiredEvents < 10000) {
+    let flowData = pipeline.createFlowData();
+    flowData.evidence.add('server.client-ip', '1.2.3.4');
+    flowData.evidence.add('header.user-agent', `ua ${requiredEvents}`);
+    usageEngine.processInternal(flowData);
+    requiredEvents++;
+  }
+
+  // Wait for a few seconds to make sure the requests from usage
+  // sharing have completed as they are asynchronous.
+  setTimeout(() => {
+    // On average, the number of required events should be around
+    // 100. However, as it's chance based it can vary
+    // significantly. We only want to catch any gross errors so just
+    // make sure the value is of the expected order of magnitude.
+    expect(received.length).toBeGreaterThan(10);
+    expect(received.length).toBeLessThan(1000);
+    done();    
+  }, (3000));
+});
+
+/**
+ * Check that XML characters are escaped correctly.
+ */
+ test('share usage - XML characters', done => {
+  const usageEngine = new ShareUsage({
+    requestedPackageSize: 1,
+    endpoint: 'http://127.0.0.1:8080/'
+  });
+
+  const pipeline = new PipelineBuilder()
+    .add(usageEngine)
+    .build();
+
+  const flowData = pipeline.createFlowData();
+  flowData.evidence.add('header.user-agent', '"\'&><');
+
+  flowData.process().then(() => {
+    setTimeout(() => {
+      expect(received.length).toBe(1);
+      expect(received[0]).toContain('&quot;');
+      expect(received[0]).toContain('&apos;');
+      expect(received[0]).toContain('&lt;');
+      expect(received[0]).toContain('&gt;');
+      expect(received[0]).toContain('&amp;');
+      done();
+    }, 2000);
+  });
+});