@opentermsarchive/engine 0.15.0

package/src/archivist/fetcher/fullDomFetcher.js

@@ -0,0 +1,84 @@
+import puppeteer from 'puppeteer';
+import puppeteerExtra from 'puppeteer-extra';
+import stealthPlugin from 'puppeteer-extra-plugin-stealth';
+
+import { FetchDocumentError } from './errors.js';
+
+puppeteerExtra.use(stealthPlugin());
+
+let browser;
+
+export default async function fetch(url, cssSelectors, config) {
+  let page;
+  let response;
+  const selectors = [].concat(cssSelectors);
+
+  if (!browser) {
+    throw new Error('The headless browser should be controlled manually with "launchHeadlessBrowser" and "stopHeadlessBrowser".');
+  }
+
+  try {
+    page = await browser.newPage();
+
+    await page.setDefaultNavigationTimeout(config.navigationTimeout);
+    await page.setExtraHTTPHeaders({ 'Accept-Language': config.language });
+
+    response = await page.goto(url, { waitUntil: 'networkidle0' });
+
+    if (!response) {
+      throw new FetchDocumentError(`Response is empty when trying to fetch '${url}'`);
+    }
+
+    const statusCode = response.status();
+
+    if (statusCode < 200 || (statusCode >= 300 && statusCode !== 304)) {
+      throw new FetchDocumentError(`Received HTTP code ${statusCode} when trying to fetch '${url}'`);
+    }
+
+    const waitForSelectorsPromises = selectors.map(selector => page.waitForSelector(selector, { timeout: config.waitForElementsTimeout }));
+
+    // We expect all elements to be present on the page…
+    await Promise.all(waitForSelectorsPromises).catch(error => {
+      if (error.name == 'TimeoutError') {
+        // however, if they are not, this is not considered as an error since selectors may be out of date
+        // and the whole content of the page should still be returned.
+        return;
+      }
+
+      throw error;
+    });
+
+    return {
+      mimeType: 'text/html',
+      content: await page.content(),
+    };
+  } catch (error) {
+    if (error instanceof puppeteer.errors.TimeoutError) {
+      throw new FetchDocumentError(`Timed out after ${config.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
+    }
+    throw new FetchDocumentError(error.message);
+  } finally {
+    if (page) {
+      await page.close();
+    }
+  }
+}
+
+export async function launchHeadlessBrowser() {
+  if (browser) {
+    return browser;
+  }
+
+  browser = await puppeteerExtra.launch({ headless: true });
+
+  return browser;
+}
+
+export async function stopHeadlessBrowser() {
+  if (!browser) {
+    return;
+  }
+
+  await browser.close();
+  browser = null;
+}

package/src/archivist/fetcher/htmlOnlyFetcher.js

@@ -0,0 +1,62 @@
+import convertBody from '@opentermsarchive/fetch-charset-detection'; // eslint-disable-line import/no-unresolved
+import AbortController from 'abort-controller';
+// https://github.com/node-fetch/fetch-charset-detection/issues/247
+import HttpProxyAgent from 'http-proxy-agent';
+import HttpsProxyAgent from 'https-proxy-agent';
+import nodeFetch, { AbortError } from 'node-fetch';
+
+import { FetchDocumentError } from './errors.js';
+
+export default async function fetch(url, configuration) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), configuration.navigationTimeout);
+
+  const nodeFetchOptions = {
+    signal: controller.signal,
+    credentials: 'include',
+    headers: { 'Accept-Language': configuration.language },
+  };
+
+  if (url.startsWith('https:') && process.env.HTTPS_PROXY) {
+    nodeFetchOptions.agent = new HttpsProxyAgent(process.env.HTTPS_PROXY);
+  } else if (url.startsWith('http:') && process.env.HTTP_PROXY) {
+    nodeFetchOptions.agent = new HttpProxyAgent(process.env.HTTP_PROXY);
+  }
+
+  let response;
+
+  try {
+    response = await nodeFetch(url, nodeFetchOptions);
+
+    if (!response.ok) {
+      throw new FetchDocumentError(`Received HTTP code ${response.status} when trying to fetch '${url}'`);
+    }
+
+    const mimeType = response.headers.get('content-type');
+    const responseBuffer = await response.arrayBuffer();
+    let content;
+
+    if (mimeType.startsWith('text/')) {
+      content = convertBody(responseBuffer, response.headers);
+    } else {
+      content = Buffer.from(responseBuffer);
+    }
+
+    if (!content) {
+      throw new FetchDocumentError(`Received an empty content when fetching '${url}'`);
+    }
+
+    return {
+      mimeType,
+      content,
+    };
+  } catch (error) {
+    if (error instanceof AbortError) {
+      throw new FetchDocumentError(`Timed out after ${configuration.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
+    }
+
+    throw new FetchDocumentError(error.message);
+  } finally {
+    clearTimeout(timeout);
+  }
+}

package/src/archivist/fetcher/index.js

@@ -0,0 +1,35 @@
+import config from 'config';
+
+import fetchFullDom from './fullDomFetcher.js';
+import fetchHtmlOnly from './htmlOnlyFetcher.js';
+
+export { launchHeadlessBrowser, stopHeadlessBrowser } from './fullDomFetcher.js';
+export { FetchDocumentError } from './errors.js';
+
+/**
+ * Fetch a resource from the network, returning a promise which is fulfilled once the response is available
+ *
+ * @param {Object} params - Fetcher parameters
+ * @param {string} params.url - URL of the resource you want to fetch
+ * @param {boolean} [params.executeClientScripts] - Enable execution of client scripts. When set to `true`, this property loads the page in a headless browser to load all assets and execute client scripts before returning its content
+ * @param {string|Array} [params.cssSelectors] - List of CSS selectors to await when loading the resource in a headless browser. Can be a CSS selector or an array of CSS selectors. Only relevant when `executeClientScripts` is enabled
+ * @param {Object} [params.config] - Fetcher configuration
+ * @param {number} [params.config.navigationTimeout] - Maximum time (in milliseconds) to wait before considering the fetch failed
+ * @param {string} [params.config.language] - Language (in [ISO 639-1 format](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)) to be passed in request headers
+ * @param {number} [params.config.waitForElementsTimeout] - Maximum time (in milliseconds) to wait for selectors to exist on page before considering the fetch failed. Only relevant when `executeClientScripts` is enabled
+ * @returns {Promise} @returns {Promise<Object>} Promise which will be resolved with an object containing the `mimeType` and the `content` of the URL as string or Buffer
+ */
+export default async function fetch({
+  url, executeClientScripts, cssSelectors,
+  config: {
+    navigationTimeout = config.get('fetcher.navigationTimeout'),
+    language = config.get('fetcher.language'),
+    waitForElementsTimeout = config.get('fetcher.waitForElementsTimeout'),
+  } = {},
+}) {
+  if (executeClientScripts) {
+    return fetchFullDom(url, cssSelectors, { navigationTimeout, language, waitForElementsTimeout });
+  }
+
+  return fetchHtmlOnly(url, { navigationTimeout, language });
+}

package/src/archivist/fetcher/index.test.js

@@ -0,0 +1,239 @@
+import fs from 'fs';
+import http from 'http';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import chai from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import iconv from 'iconv-lite';
+
+import fetch, { launchHeadlessBrowser, stopHeadlessBrowser, FetchDocumentError } from './index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const { expect } = chai;
+const SERVER_PORT = 8976;
+
+chai.use(chaiAsPromised);
+
+const termsHTML = '<!DOCTYPE html><html><head><meta charset="UTF-8"><title>First provider TOS</title></head><body><h1>Terms of service</h1><p>Dapibus quis diam sagittis</p></body></html>';
+const termsWithOtherCharsetHTML = '<!DOCTYPE html><html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1251"><title>TOS на първия доставчик</title></head><body><h1>Условия за ползване</h1><p>Dapibus quis diam sagittis</p></body></html>';
+
+describe('Fetcher', function () {
+  this.timeout(10000);
+
+  before(launchHeadlessBrowser);
+
+  after(stopHeadlessBrowser);
+
+  describe('#fetch', () => {
+    let temporaryServer;
+    let expectedPDFContent;
+
+    before(done => {
+      temporaryServer = http.createServer((request, response) => {
+        if (request.url === '/') {
+          response.writeHead(200, { 'Content-Type': 'text/html' }).write(termsHTML);
+        }
+        if (request.url === '/other-charset') {
+          response.writeHead(200, { 'Content-Type': 'text/html' }).write(iconv.encode(termsWithOtherCharsetHTML, 'windows-1251'));
+        }
+        if (request.url == '/404') {
+          response.writeHead(404, { 'Content-Type': 'text/html' }).write('<!DOCTYPE html><html><body>404</body></html>');
+        }
+        if (request.url == '/terms.pdf') {
+          expectedPDFContent = fs.readFileSync(path.resolve(__dirname, '../../../test/fixtures/terms.pdf'));
+
+          response.writeHead(200, { 'Content-Type': 'application/pdf' }).write(expectedPDFContent);
+        }
+
+        return response.end();
+      }).listen(SERVER_PORT);
+
+      done();
+    });
+
+    after(() => {
+      if (temporaryServer) {
+        temporaryServer.close();
+      }
+    });
+
+    describe('Available URL', () => {
+      context('when html page is available', () => {
+        let content;
+        let mimeType;
+        const url = `http://127.0.0.1:${SERVER_PORT}`;
+
+        context('when expected selectors are present', () => {
+          before(async () => {
+            ({ content, mimeType } = await fetch({ url, selectors: 'body' }));
+          });
+
+          it('returns the web page content of the given URL', async () => {
+            expect(content).to.equal(termsHTML);
+          });
+
+          it('returns the MIME type of the given URL', async () => {
+            expect(mimeType).to.equal('text/html');
+          });
+
+          context('with client script enabled', () => {
+            before(async () => {
+              ({ content, mimeType } = await fetch({ url, selectors: 'body', executeClientScripts: true }));
+            });
+
+            it('returns the web page content of the given URL', async () => {
+              expect(content).to.equal(termsHTML);
+            });
+
+            it('returns the MIME type of the given URL', async () => {
+              expect(mimeType).to.equal('text/html');
+            });
+          });
+        });
+
+        context('when expected selectors are not present', () => {
+          const NOT_PRESENT_SELECTOR = 'h2';
+
+          before(async () => {
+            ({ content, mimeType } = await fetch({ url, selectors: NOT_PRESENT_SELECTOR }));
+          });
+
+          it('returns the web page content of the given URL', async () => {
+            expect(content).to.equal(termsHTML);
+          });
+
+          it('returns the MIME type of the given URL', async () => {
+            expect(mimeType).to.equal('text/html');
+          });
+
+          context('with client script enabled', () => {
+            before(async () => {
+              ({ content, mimeType } = await fetch({ url, selectors: NOT_PRESENT_SELECTOR, executeClientScripts: true }));
+            });
+
+            it('returns the web page content of the given URL', async () => {
+              expect(content).to.equal(termsHTML);
+            });
+
+            it('returns the MIME type of the given URL', async () => {
+              expect(mimeType).to.equal('text/html');
+            });
+          });
+        });
+      });
+
+      context('when html page is in different charset', () => {
+        let content;
+        const url = `http://127.0.0.1:${SERVER_PORT}/other-charset`;
+
+        context('when expected selectors are present', () => {
+          before(async () => {
+            ({ content } = await fetch({ url, selectors: 'body' }));
+          });
+
+          it('returns the web page content of the given URL', async () => {
+            expect(content).to.equal(termsWithOtherCharsetHTML);
+          });
+        });
+      });
+
+      context('when url targets a PDF file', () => {
+        let content;
+        let mimeType;
+        const pdfUrl = `http://127.0.0.1:${SERVER_PORT}/terms.pdf`;
+
+        before(async () => {
+          ({ content, mimeType } = await fetch({ url: pdfUrl }));
+        });
+
+        it('returns a buffer for PDF content', async () => {
+          expect(content).to.be.an.instanceOf(Buffer);
+        });
+
+        it('returns a blob with the file type', async () => {
+          expect(mimeType).to.equal('application/pdf');
+        });
+
+        it('returns a blob with the file content', async () => {
+          expect(content.equals(expectedPDFContent)).to.be.true;
+        });
+      });
+    });
+
+    describe('Error handling', () => {
+      const url404 = `http://127.0.0.1:${SERVER_PORT}/404`;
+
+      context('when web page is not available', () => {
+        it('throws a FetchDocumentError error', async () => {
+          await expect(fetch({ url: url404 })).to.be.rejectedWith(FetchDocumentError, /404/);
+        });
+
+        context('with client script enabled', () => {
+          it('throws a FetchDocumentError error', async () => {
+            await expect(fetch({ url: url404, executeClientScripts: true, cssSelectors: 'body' })).to.be.rejectedWith(FetchDocumentError, /404/);
+          });
+        });
+      });
+
+      context('when server is not resolved', () => {
+        const notAvailableUrl = 'https://not.available.example';
+
+        it('throws a FetchDocumentError error', async () => {
+          await expect(fetch({ url: notAvailableUrl })).to.be.rejectedWith(FetchDocumentError);
+        });
+
+        context('with client script enabled', () => {
+          it('throws a FetchDocumentError error', async () => {
+            await expect(fetch({ url: notAvailableUrl, executeClientScripts: true })).to.be.rejectedWith(FetchDocumentError);
+          });
+        });
+      });
+
+      describe('when there is a certificate error', () => {
+        context('when website has a self signed certificate', () => {
+          const selfSignedSslUrl = 'https://self-signed.badssl.com/';
+
+          it('throws a FetchDocumentError error', async () => {
+            await expect(fetch({ url: selfSignedSslUrl })).to.be.rejectedWith(FetchDocumentError);
+          });
+
+          context('with client script enabled', () => {
+            it('throws a FetchDocumentError error', async () => {
+              await expect(fetch({ url: selfSignedSslUrl, executeClientScripts: true })).to.be.rejectedWith(FetchDocumentError);
+            });
+          });
+        });
+
+        context('when website has an expired certificate', () => {
+          const expiredSslUrl = 'https://expired.badssl.com/';
+
+          it('throws a FetchDocumentError error', async () => {
+            await expect(fetch({ url: expiredSslUrl })).to.be.rejectedWith(FetchDocumentError);
+          });
+
+          context('with client script enabled', () => {
+            it('throws a FetchDocumentError error', async () => {
+              await expect(fetch({ url: expiredSslUrl, executeClientScripts: true })).to.be.rejectedWith(FetchDocumentError);
+            });
+          });
+        });
+
+        context('when website has an untrusted root certificate', () => {
+          const untrustedRootSslUrl = 'https://untrusted-root.badssl.com/';
+
+          it('throws a FetchDocumentError error', async () => {
+            await expect(fetch({ url: untrustedRootSslUrl })).to.be.rejectedWith(FetchDocumentError);
+          });
+
+          context('with client script enabled', () => {
+            it('throws a FetchDocumentError error', async () => {
+              await expect(fetch({ url: untrustedRootSslUrl, executeClientScripts: true })).to.be.rejectedWith(FetchDocumentError);
+            });
+          });
+        });
+      });
+    });
+  });
+});

package/src/archivist/filter/exports.js

@@ -0,0 +1,3 @@
+import filter from './index.js';
+
+export default filter;

package/src/archivist/filter/index.js

@@ -0,0 +1,178 @@
+import url from 'url';
+
+import ciceroMark from '@accordproject/markdown-cicero';
+import mardownPdf from '@accordproject/markdown-pdf';
+import TurndownService from '@opentermsarchive/turndown';
+import turndownPluginGithubFlavouredMarkdown from 'joplin-turndown-plugin-gfm';
+import jsdom from 'jsdom';
+
+import { InaccessibleContentError } from '../errors.js';
+
+const { JSDOM } = jsdom;
+const turndownService = new TurndownService();
+
+turndownService.use(turndownPluginGithubFlavouredMarkdown.gfm);
+
+export const LINKS_TO_CONVERT_SELECTOR = 'a[href]:not([href^="#"]):not([href=""])';
+
+const { PdfTransformer } = mardownPdf;
+const { CiceroMarkTransformer } = ciceroMark;
+
+const ciceroMarkTransformer = new CiceroMarkTransformer();
+
+/**
+ * Filter document content and convert it to Markdown
+ *
+ * @param {Object} params - Filter parameters
+ * @param {string|Buffer} params.content - Content to filter: a buffer containing PDF data in case mimetype associated is PDF or a DOM dump of an HTML page given as a string
+ * @param {string} params.mimeType - MIME type of the given content
+ * @param {string} params.pageDeclaration - see {@link ./src/archivist/services/pageDeclaration.js}
+ * @returns {Promise<string>} Promise which is fulfilled once the content is filtered and converted in Markdown. The promise will resolve into a string containing the filtered content in Markdown format
+*/
+export default async function filter({ content, mimeType, pageDeclaration }) {
+  if (mimeType == 'application/pdf') {
+    return filterPDF({ content });
+  }
+
+  return filterHTML({
+    content,
+    pageDeclaration,
+  });
+}
+
+export async function filterHTML({ content, pageDeclaration }) {
+  const {
+    location,
+    contentSelectors = [],
+    noiseSelectors = [],
+    filters: serviceSpecificFilters = [],
+  } = pageDeclaration;
+
+  const jsdomInstance = new JSDOM(content, {
+    url: location,
+    virtualConsole: new jsdom.VirtualConsole(),
+  });
+  const { document: webPageDOM } = jsdomInstance.window;
+
+  for (const filterFunction of serviceSpecificFilters) {
+    try {
+      /* eslint-disable no-await-in-loop */
+      // We want this to be made in series
+      await filterFunction(webPageDOM, {
+        fetch: location,
+        select: contentSelectors,
+        remove: noiseSelectors,
+        filter: serviceSpecificFilters.map(filter => filter.name),
+      });
+      /* eslint-enable no-await-in-loop */
+    } catch (error) {
+      throw new InaccessibleContentError(`The filter function "${filterFunction.name}" failed: ${error}`);
+    }
+  }
+
+  remove(webPageDOM, noiseSelectors); // remove function works in place
+
+  const domFragment = select(webPageDOM, contentSelectors);
+
+  if (!domFragment.children.length) {
+    throw new InaccessibleContentError(`The provided selector "${contentSelectors}" has no match in the web page at '${location}'`);
+  }
+
+  convertRelativeURLsToAbsolute(domFragment, location);
+
+  domFragment.querySelectorAll('script, style').forEach(node => node.remove());
+
+  // clean code from common changing patterns - initially for Windstream
+  domFragment.querySelectorAll('a[href*="/email-protection"]').forEach(node => {
+    if (node.href.match(/((.*?)\/email-protection#)[0-9a-fA-F]+/gim)) {
+      node.href = `${node.href.split('#')[0]}#removed`;
+    }
+  });
+
+  const markdownContent = transform(domFragment);
+
+  if (!markdownContent) {
+    throw new InaccessibleContentError(`The provided selector "${contentSelectors}" matches an empty content in the web page at '${location}'`);
+  }
+
+  return markdownContent;
+}
+
+export async function filterPDF({ content: pdfBuffer }) {
+  try {
+    const ciceroMarkdown = await PdfTransformer.toCiceroMark(pdfBuffer);
+
+    return ciceroMarkTransformer.toMarkdown(ciceroMarkdown);
+  } catch (error) {
+    if (error.parserError) {
+      throw new InaccessibleContentError("Can't parse PDF file");
+    }
+
+    throw error;
+  }
+}
+
+function selectRange(document, rangeSelector) {
+  const { startBefore, startAfter, endBefore, endAfter } = rangeSelector;
+
+  const selection = document.createRange();
+  const startNode = document.querySelector(startBefore || startAfter);
+  const endNode = document.querySelector(endBefore || endAfter);
+
+  if (!startNode) {
+    throw new InaccessibleContentError(`The "start" selector has no match in document in: ${JSON.stringify(rangeSelector)}`);
+  }
+
+  if (!endNode) {
+    throw new InaccessibleContentError(`The "end" selector has no match in document in: ${JSON.stringify(rangeSelector)}`);
+  }
+
+  selection[startBefore ? 'setStartBefore' : 'setStartAfter'](startNode);
+  selection[endBefore ? 'setEndBefore' : 'setEndAfter'](endNode);
+
+  return selection;
+}
+
+export function convertRelativeURLsToAbsolute(document, baseURL) {
+  Array.from(document.querySelectorAll(LINKS_TO_CONVERT_SELECTOR)).forEach(link => {
+    link.href = url.resolve(baseURL, link.href);
+  });
+}
+
+// Works in place
+function remove(webPageDOM, noiseSelectors) {
+  const rangeSelections = [];
+  const nodes = [];
+
+  [].concat(noiseSelectors).forEach(selector => {
+    if (typeof selector === 'object') {
+      rangeSelections.push(selectRange(webPageDOM, selector));
+    } else {
+      nodes.push(...webPageDOM.querySelectorAll(selector));
+    }
+  });
+
+  // Removing range selections still works even if the starting or ending node is deleted. So, start by removing all nodes selected by a direct CSS selector, then delete all contents selections.
+  nodes.forEach(node => node.remove());
+  rangeSelections.forEach(rangeSelection => rangeSelection.deleteContents());
+}
+
+function select(webPageDOM, contentSelectors) {
+  const result = webPageDOM.createDocumentFragment();
+
+  [].concat(contentSelectors).forEach(selector => {
+    if (typeof selector === 'object') {
+      const rangeSelection = selectRange(webPageDOM, selector);
+
+      result.appendChild(rangeSelection.cloneContents());
+    } else {
+      webPageDOM.querySelectorAll(selector).forEach(element => result.appendChild(element.cloneNode(true)));
+    }
+  });
+
+  return result;
+}
+
+function transform(domFragment) {
+  return turndownService.turndown(domFragment);
+}