@hyperwatch/hyperwatch 3.8.2 → 3.9.1

package/scripts/compare-parser.js

@@ -1,97 +0,0 @@
-const path = require('path');
-
-const useragent = require('../src/lib/useragent');
-
-const date = new Date().toISOString().slice(0, 10);
-// const date = '2020-09-18';
-
-const filename = path.join(__dirname, '../data/ua', `${date}-parsed.json`);
-
-let uas;
-try {
-  uas = require(filename);
-} catch (err) {
-  uas = [];
-}
-
-// Transform Hyperwatch to their UAP Core equivalent
-const mapping = {
-  'Coc Coc Browser': 'Coc Coc',
-  'DuckDuckBot-Https': 'DuckDuckBot',
-  'Google Search App': 'Google',
-  'Mobile Safari UI/WKWebView': 'Safari WebView',
-  'Chrome Mobile WebView': 'Chrome WebView',
-  'Mobile Safari': 'Safari Mobile',
-  'WordPress.com': 'WordPress',
-  'python-requests': 'Python Requests',
-  'Android WebView': 'Android',
-  PaleMoon: 'Pale Moon',
-};
-
-// Transform UAP Core to their Hyperwatch equivalent
-const uapCoreMapping = {
-  'Chrome Mobile WebView': 'Chrome WebView',
-  'Chrome Mobile': 'Chrome',
-  'Edge Mobile': 'Edge',
-  'Firefox Alpha': 'Firefox',
-  'Firefox Beta': 'Firefox',
-  'Firefox Mobile': 'Firefox',
-  'Mail.RU_Bot': 'Mail.RU Bot',
-  'Opera Mobile': 'Opera',
-  'Samsung Internet': 'SamsungBrowser',
-  bingbot: 'BingBot',
-  coccocbot: 'Coc Coc Bot',
-  GmailImageProxy: 'GoogleImageProxy',
-  LINE: 'Line',
-  'Chrome Mobile iOS': 'Chrome',
-  'QQ Browser': 'QQBrowser',
-  'UC Browser': 'UCBrowser',
-  'Firefox iOS': 'Firefox',
-  'QQ Browser Mobile': 'QQBrowser',
-};
-
-for (const [ua, json] of Object.entries(uas)) {
-  if (!json) {
-    continue;
-  }
-  if (!json.family) {
-    console.log(`No result`, { hw: null }, ua);
-    continue;
-  }
-
-  // Transform Hyperwatch to their UAP Core equivalent
-  if (json.family && mapping[json.family]) {
-    json.family = mapping[json.family];
-  }
-
-  // if (['Dalvik'].includes(json.family)) {
-  //   continue;
-  // }
-
-  // Parse with UAP core only
-  const agent = useragent.parse(
-    ua,
-    /* enableCore */ false,
-    /* enableExtra */ false,
-    /* enableUapCore */ true
-  );
-  const result = agent.toJSON();
-
-  // if (['Electron'].includes(result.family)) {
-  //   continue;
-  // }
-
-  // Transform UAP Core result to its Hyperwatch equivalent
-  if (result.family && uapCoreMapping[result.family]) {
-    result.family = uapCoreMapping[result.family];
-  }
-
-  if (!result.family) {
-    console.log(`New:`, { hw: json.family });
-    continue;
-  }
-
-  if (result.family !== json.family) {
-    console.log('Diff', { uap: result.family, hw: json.family });
-  }
-}

package/scripts/hostname.js

@@ -1,23 +0,0 @@
-const fetch = require('node-fetch');
-
-fetch('http://localhost:3009/addresses?raw=1&limit=1000')
-  .then((response) => response.json())
-  .then((entries) => {
-    const hostnames = entries
-      .filter((entry) => entry.address && entry.address.hostname)
-      .filter((entry) => !entry.address.type)
-      .filter((entry) => entry.identity || entry.opencollective)
-      .map((entry) => entry.address.hostname)
-      .map((hostname) => {
-        const parts = hostname.split('.');
-        let extension = parts.pop();
-        let domain = parts.pop();
-        if (['com', 'net', 'on', 'ne', 'co', 'in'].includes(domain)) {
-          extension = `${domain}.${extension}`;
-          domain = parts.pop();
-        }
-        return `${domain}.${extension}`;
-      });
-
-    console.log([...new Set(hostnames)]);
-  });

package/scripts/test-update.js

@@ -1,21 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-
-const fetch = require('node-fetch'); // eslint-disable-line node/no-unpublished-require
-const yaml = require('yamlparser'); // eslint-disable-line node/no-unpublished-require
-
-fetch(
-  'https://raw.githubusercontent.com/ua-parser/uap-core/master/tests/test_ua.yaml'
-)
-  .then((response) => response.text())
-
-  .then((uaYaml) => {
-    fs.writeFileSync(path.resolve(__dirname, '..', 'test', 'ua.yml'), uaYaml);
-
-    const ua = yaml.eval(uaYaml);
-
-    fs.writeFileSync(
-      path.resolve(__dirname, '..', 'test', 'ua.json'),
-      JSON.stringify(ua.test_cases, null, 2)
-    );
-  });

package/scripts/update-uas.js

@@ -1,30 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-
-const { pick } = require('lodash');
-
-const useragent = require('../src/lib/useragent');
-
-const date = new Date().toISOString().slice(0, 10);
-// const date = '2020-09-18';
-
-const filename = path.join(__dirname, '../data/ua', `${date}.json`);
-const parsedFilename = path.join(
-  __dirname,
-  '../data/ua',
-  `${date}-parsed.json`
-);
-
-const uas = require(filename);
-
-const fields = ['family', 'major', 'minor', 'patch', 'patch_minor'];
-
-const parsedUas = {};
-
-for (const ua of uas) {
-  const agent = useragent.parse(ua);
-
-  parsedUas[ua] = pick(agent.toJSON(), fields);
-}
-
-fs.writeFileSync(parsedFilename, JSON.stringify(parsedUas, null, 2));

package/src/modules/user-agent.js

@@ -1,31 +0,0 @@
-const api = require('../app/api');
-const { Aggregator } = require('../lib/aggregator');
-const pipeline = require('../lib/pipeline');
-
-const identifier = (log) => log.getIn(['address', 'value']);
-
-function fill(log) {
-  if (!log.hasIn(['address', 'value'])) {
-    log = log.setIn(['address', 'value'], log.getIn(['request', 'address']));
-  }
-  return log;
-}
-
-function load() {
-  pipeline.getNode('main').map(fill).registerNode('main');
-}
-
-function beforeStart() {
-  const aggregator = new Aggregator();
-
-  aggregator.setIdentifier(identifier);
-
-  pipeline.getNode('main').map((log) => aggregator.processLog(log));
-
-  api.registerAggregator('userAgents', aggregator);
-}
-
-module.exports = {
-  load,
-  beforeStart,
-};

package/src/plugins/browser.js

@@ -1,35 +0,0 @@
-const { fromJS } = require('immutable');
-
-function augment(log) {
-  const browser = {};
-
-  if (log.hasIn(['request', 'headers', 'dnt'])) {
-    browser.dnt = true;
-  }
-
-  if (log.hasIn(['request', 'headers', 'accept-language'])) {
-    browser.dnt = true;
-  }
-
-  const parser = require('accept-language-parser');
-
-  const languages = parser.parse('en-GB,en;q=0.8');
-
-  const agentFamily = log.getIn(['useragent', 'family']);
-  // console.log({ agentFamily });
-
-  if (agentFamily && browsers.includes(agentFamily)) {
-    const browser = {};
-
-    if (agentFamily === 'Chrome') {
-      browser.check = checkChrome(log);
-    }
-    log = log.set('browser', fromJS(browser));
-  }
-
-  return log;
-}
-
-module.exports = {
-  augment,
-};

package/src/plugins/reputation.js

@@ -1,168 +0,0 @@
-const { fromJS } = require('immutable');
-
-const browsers = [
-  'Firefox',
-  'Firefox Mobile',
-  'Firefox iOS',
-  'Chrome',
-  'Chrome Mobile',
-  'Chrome Mobile iOS',
-  'Chrome Mobile WebView',
-  'Edge',
-  'Safari',
-  'Mobile Safari',
-  'Opera',
-  'Opera Mobile',
-  'Chromium',
-  'Samsung Internet',
-  'Epiphany',
-  'Yandex Browser',
-  'MiuiBrowser',
-  'UC Browser',
-];
-
-function checkChrome(log) {
-  // console.log(log.getIn(['request', 'headers']).toJS());
-
-  const check = {
-    version: null,
-    accept: null,
-    language: null,
-    charset: null,
-  };
-
-  const accepts = {
-    'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8':
-      'html',
-    'image/webp,image/apng,image/*,*/*;q=0.8': 'images',
-  };
-
-  const major = log.getIn(['useragent', 'major']);
-  if (!major) {
-    // console.log('Missing Version');
-    check['version'] = false;
-  } else if (Number(major) >= 81) {
-    check['version'] = true;
-  }
-
-  // const agent = log.getIn(['agent']);
-
-  // console.log(agent.major);
-
-  const accept = log.getIn(['request', 'headers', 'accept']);
-  if (!accept) {
-    // console.log('Missing Accept');
-    check['accept'] = false;
-  } else if (!accepts[accept]) {
-    // console.log('Unnkown Accept', accept);
-    check['accept'] = false;
-  } else {
-    check['accept'] = true;
-  }
-
-  const acceptLanguage = log.getIn(['request', 'headers', 'accept-language']);
-  if (!acceptLanguage) {
-    // console.log('Missing Accept-Language');
-    check['language'] = false;
-  } else {
-    if (/^(.*);q=0.9$/i.test(acceptLanguage)) {
-      // console.log(
-      //   'Expected Accept-Language header: 1 language',
-      //   acceptLanguage
-      // );
-      check['language'] = true;
-    } else if (/^(.*);q=0.9(.*);q=0.8$/i.test(acceptLanguage)) {
-      // console.log(
-      //   'Expected Accept-Language header: 2 languages',
-      //   acceptLanguage
-      // );
-      check['language'] = true;
-    } else if (/^(.*);q=0.9(.*);q=0.8(.*);q=0.7$/i.test(acceptLanguage)) {
-      // console.log(
-      //   'Expected Accept-Language header: 3 languages',
-      //   acceptLanguage
-      // );
-      check['language'] = true;
-    } else if (
-      /^(.*);q=0.9(.*);q=0.8(.*);q=0.7(.*);q=0.6/i.test(acceptLanguage)
-    ) {
-      // console.log(
-      //   'Expected Accept-Language header: 4+ languages',
-      //   acceptLanguage
-      // );
-      check['language'] = true;
-    } else {
-      // console.log('Other Accept-Language header', acceptLanguage);
-      check['language'] = false;
-    }
-  }
-
-  const acceptCharset = log.getIn(['request', 'headers', 'accept-charset']);
-  if (acceptCharset) {
-    // console.log('Accept-Charset detected', acceptCharset);
-    check['charset'] = false;
-  } else {
-    check['charset'] = true;
-  }
-
-  const secFetchDest = log.getIn(['request', 'headers', 'sec-fetch-dest']);
-  if (secFetchDest) {
-    // console.log('Accept-Charset detected', acceptCharset);
-    check['sec'] = true;
-  } else {
-    check['sec'] = false;
-  }
-
-  // 'sec-fetch-dest': 'image',
-  // 'sec-fetch-mode': 'no-cors',
-  // 'sec-fetch-site': 'cross-site',
-
-  // const acceptEncoding = log.getIn(['request', 'headers', 'accept-encoding']);
-  // if (acceptCharset) {
-  //   console.log('Accept-Encoding detected');
-  // }
-
-  // console.log('Accept', log.getIn(['request', 'headers', 'accept']));
-  // console.log(
-  //   'Accept-Language',
-  //   log.getIn(['request', 'headers', 'accept-language'])
-  // );
-  // console.log(
-  //   'Accept-Charset',
-  //   log.getIn(['request', 'headers', 'accept-charset'])
-  // );
-  // console.log(
-  //   'Accept-Encoding',
-  //   log.getIn(['request', 'headers', 'accept-encoding'])
-  // );
-
-  return check;
-
-  // console.log('DNT', log.getIn(['request', 'headers', 'dnt']));
-}
-
-function augment(log) {
-  // const agent = log.getIn(['agent']);
-
-  // console.log(agent);
-
-  const agentFamily = log.getIn(['useragent', 'family']);
-  // console.log({ agentFamily });
-
-  if (agentFamily && browsers.includes(agentFamily)) {
-    const browser = {};
-    if (log.hasIn(['request', 'headers', 'dnt'])) {
-      browser.dnt = true;
-    }
-    if (agentFamily === 'Chrome') {
-      browser.check = checkChrome(log);
-    }
-    log = log.set('browser', fromJS(browser));
-  }
-
-  return log;
-}
-
-module.exports = {
-  augment,
-};