@opentermsarchive/engine 0.15.0

package/scripts/dataset/publish/index.js

@@ -0,0 +1,39 @@
+import fsApi from 'fs';
+import path from 'path';
+import url from 'url';
+
+import config from 'config';
+import dotenv from 'dotenv';
+import { Octokit } from 'octokit';
+
+import * as readme from '../assets/README.template.js';
+
+dotenv.config();
+
+export default async function publish({ archivePath, releaseDate, stats }) {
+  const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
+
+  const [ owner, repo ] = url.parse(config.get('dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
+
+  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+
+  const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({
+    owner,
+    repo,
+    tag_name: tagName,
+    name: readme.title({ releaseDate }),
+    body: readme.body(stats),
+  });
+
+  await octokit.rest.repos.uploadReleaseAsset({
+    data: fsApi.readFileSync(archivePath),
+    headers: {
+      'content-type': 'application/zip',
+      'content-length': fsApi.statSync(archivePath).size,
+    },
+    name: path.basename(archivePath),
+    url: uploadUrl,
+  });
+
+  return releaseUrl;
+}

package/scripts/declarations/lint/index.js

@@ -0,0 +1,36 @@
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import config from 'config';
+import { ESLint } from 'eslint';
+
+import DeclarationUtils from '../utils/index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const declarationsPath = config.get('services.declarationsPath');
+const instancePath = path.resolve(declarationsPath, '../');
+const ESLINT_CONFIG_PATH = path.join(__dirname, '../../../.eslintrc.yaml');
+
+const lintDeclarations = async ({ services, modified }) => {
+  console.log(`Linting declaration files in ${instancePath}`);
+  let servicesToValidate = services || ['*'];
+
+  if (modified) {
+    const declarationUtils = new DeclarationUtils(instancePath);
+
+    servicesToValidate = await declarationUtils.getModifiedServices();
+  }
+
+  for (const service of servicesToValidate) {
+    /* eslint-disable no-await-in-loop */
+    const lintResults = await new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true })
+      .lintFiles(path.join(declarationsPath, `${service}.*`));
+
+    await ESLint.outputFixes(lintResults);
+    console.log(lintResults.map(lintResult => `${path.basename(lintResult.filePath)} linted`).join('\n'));
+    /* eslint-enable no-await-in-loop */
+  }
+};
+
+export default lintDeclarations;

package/scripts/declarations/utils/index.js

@@ -0,0 +1,81 @@
+import path from 'path';
+
+import DeepDiff from 'deep-diff';
+import simpleGit from 'simple-git';
+
+export default class DeclarationUtils {
+  constructor(instancePath, defaultBranch = 'remotes/origin/main') {
+    this.git = simpleGit(instancePath, { maxConcurrentProcesses: 1 });
+    this.defaultBranch = defaultBranch;
+  }
+
+  static filePathToServiceId = filePath => path.parse(filePath.replace(/\.history|\.filters/, '')).name;
+
+  async getJSONFile(path, ref) {
+    try {
+      return JSON.parse(await this.git.show([`${ref}:${path}`]));
+    } catch (e) {
+      return {};
+    }
+  }
+
+  async getModifiedData() {
+    const modifiedFilePathsAsString = (await this.git.diff([ '--diff-filter=d', '--name-only', this.defaultBranch, 'HEAD', '--', './declarations' ])).trim();
+
+    const modifiedFilePaths = modifiedFilePathsAsString ? modifiedFilePathsAsString.split('\n') : [];
+
+    return { modifiedFilePaths, modifiedServiceIds: Array.from(new Set(modifiedFilePaths.map(DeclarationUtils.filePathToServiceId))) };
+  }
+
+  async getModifiedServices() {
+    const { modifiedServiceIds } = await this.getModifiedData();
+
+    return modifiedServiceIds;
+  }
+
+  async getModifiedServiceDocumentTypes() {
+    const { modifiedFilePaths, modifiedServiceIds } = await this.getModifiedData();
+    const servicesDocumentTypes = {};
+
+    await Promise.all(modifiedFilePaths.map(async modifiedFilePath => {
+      const serviceId = DeclarationUtils.filePathToServiceId(modifiedFilePath);
+
+      if (!modifiedFilePath.endsWith('.json')) {
+        // Here we should compare AST of both files to detect on which function
+        // change has been made, and then find which document type depends on this
+        // function.
+        // As this is a complicated process, we will just send back all document types
+        const declaration = await this.getJSONFile(`declarations/${serviceId}.json`, this.defaultBranch);
+
+        return Object.keys(declaration.documents);
+      }
+
+      const defaultFile = await this.getJSONFile(modifiedFilePath, this.defaultBranch);
+      const modifiedFile = await this.getJSONFile(modifiedFilePath, 'HEAD');
+
+      const diff = DeepDiff.diff(defaultFile, modifiedFile);
+
+      if (!diff) {
+        // This can happen if only a lint has been applied to a document
+        return;
+      }
+
+      const modifiedDocumentTypes = diff.reduce((acc, { path }) => {
+        if (modifiedFilePath.includes('.history')) {
+          acc.add(path[0]);
+        } else if (path[0] == 'documents') {
+          acc.add(path[1]);
+        }
+
+        return acc;
+      }, new Set());
+
+      servicesDocumentTypes[serviceId] = Array.from(new Set([ ...servicesDocumentTypes[serviceId] || [], ...modifiedDocumentTypes ]));
+    }));
+
+    return {
+      services: modifiedServiceIds,
+      servicesDocumentTypes,
+    };
+  }
+}

package/scripts/declarations/validate/definitions.js

@@ -0,0 +1,63 @@
+const definitions = {
+  location: {
+    type: 'string',
+    format: 'uri',
+    description: 'The URL where the document can be found',
+  },
+  pdfLocation: {
+    type: 'string',
+    pattern: '^https?://.+.[pP][dD][fF](\\?.+)?$',
+    description: 'The URL where the document can be found',
+  },
+  executeClientScripts: {
+    type: 'boolean',
+    description: 'Execute client-side JavaScript loaded by the document before accessing the content, in case the DOM modifications are needed to access the content.',
+  },
+  selectors: {
+    description: 'Selector(s) that targets element to include',
+    oneOf: [
+      { $ref: '#/definitions/cssSelector' },
+      { $ref: '#/definitions/range' },
+      {
+        type: 'array',
+        items: { oneOf: [{ $ref: '#/definitions/cssSelector' }, { $ref: '#/definitions/range' }] },
+      },
+    ],
+  },
+  contentSelectors: { $ref: '#/definitions/selectors' },
+  noiseSelectors: { $ref: '#/definitions/selectors' },
+  filters: {
+    type: 'array',
+    items: {
+      type: 'string',
+      pattern: '^.+$',
+      description: 'Filter function name',
+    },
+  },
+  validUntil: {
+    type: 'string',
+    format: 'date-time',
+  },
+  cssSelector: {
+    type: 'string',
+    pattern: '^.+$',
+    description: 'A CSS selector',
+  },
+  range: {
+    type: 'object',
+    properties: {
+      startBefore: { $ref: '#/definitions/cssSelector' },
+      startAfter: { $ref: '#/definitions/cssSelector' },
+      endBefore: { $ref: '#/definitions/cssSelector' },
+      endAfter: { $ref: '#/definitions/cssSelector' },
+    },
+    oneOf: [
+      { required: [ 'startBefore', 'endBefore' ] },
+      { required: [ 'startBefore', 'endAfter' ] },
+      { required: [ 'startAfter', 'endBefore' ] },
+      { required: [ 'startAfter', 'endAfter' ] },
+    ],
+  },
+};
+
+export default definitions;

package/scripts/declarations/validate/index.mocha.js

@@ -0,0 +1,262 @@
+import fsApi from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import Ajv from 'ajv';
+import { expect } from 'chai';
+import config from 'config';
+import { ESLint } from 'eslint';
+import jsonSourceMap from 'json-source-map';
+
+import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from '../../../src/archivist/fetcher/index.js';
+import filter from '../../../src/archivist/filter/index.js';
+import * as services from '../../../src/archivist/services/index.js';
+import DeclarationUtils from '../utils/index.js';
+
+import serviceHistorySchema from './service.history.schema.js';
+import serviceSchema from './service.schema.js';
+
+const fs = fsApi.promises;
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT_PATH = path.resolve(__dirname, '../../../');
+const ESLINT_CONFIG_PATH = path.join(ROOT_PATH, '.eslintrc.yaml');
+
+const MIN_DOC_LENGTH = 100;
+const SLOW_DOCUMENT_THRESHOLD = 10 * 1000; // number of milliseconds after which a document fetch is considered slow
+
+const eslint = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: false });
+const eslintWithFix = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true });
+
+const declarationsPath = path.resolve(ROOT_PATH, config.get('services.declarationsPath'));
+const instancePath = path.resolve(declarationsPath, '../');
+
+export default async options => {
+  const schemaOnly = options.schemaOnly || false;
+  let servicesToValidate = options.services || [];
+  const documentTypes = options.documentTypes || [];
+  let servicesDocumentTypes = {};
+
+  const serviceDeclarations = await services.loadWithHistory(servicesToValidate);
+
+  if (!servicesToValidate.length) {
+    servicesToValidate = Object.keys(serviceDeclarations);
+  }
+
+  if (options.modified) {
+    const declarationUtils = new DeclarationUtils(instancePath);
+
+    ({ services: servicesToValidate, servicesDocumentTypes } = await declarationUtils.getModifiedServiceDocumentTypes());
+  }
+
+  describe('Service declarations validation', async function () {
+    this.timeout(30000);
+    this.slow(SLOW_DOCUMENT_THRESHOLD);
+
+    servicesToValidate.forEach(serviceId => {
+      const service = serviceDeclarations[serviceId];
+      const filePath = path.join(declarationsPath, `${serviceId}.json`);
+      const historyFilePath = path.join(declarationsPath, `${serviceId}.history.json`);
+
+      before(launchHeadlessBrowser);
+
+      after(stopHeadlessBrowser);
+
+      context(serviceId, async () => {
+        before(async function () {
+          if (!service) {
+            console.log('      (Tests skipped as declaration has been archived)');
+            this.skip();
+          }
+        });
+
+        it('valid declaration schema', async () => {
+          const declaration = JSON.parse(await fs.readFile(filePath));
+
+          assertValid(serviceSchema, declaration);
+        });
+
+        it('valid declaration file format', async () => {
+          await lintFile(filePath);
+        });
+
+        if (service && service.hasHistory()) {
+          it('valid history declaration schema', async () => {
+            const declarationHistory = JSON.parse(await fs.readFile(historyFilePath));
+
+            assertValid(serviceHistorySchema, declarationHistory);
+          });
+
+          it('valid history declaration file format', async () => {
+            await lintFile(path.join(declarationsPath, `${serviceId}.history.json`));
+          });
+        }
+
+        const filtersFilePath = path.join(declarationsPath, `${serviceId}.filters.js`);
+
+        if (fsApi.existsSync(filtersFilePath)) {
+          it('valid filters file format', async () => {
+            await lintFile(filtersFilePath);
+          });
+        }
+
+        const filtersHistoryFilePath = path.join(declarationsPath, `${serviceId}.filters.history.js`);
+
+        if (fsApi.existsSync(filtersHistoryFilePath)) {
+          it('valid filters history file format', async () => {
+            await lintFile(filtersHistoryFilePath);
+          });
+        }
+
+        if (!schemaOnly && service) {
+          service.getDocumentTypes()
+            .filter(documentType => {
+              if (servicesDocumentTypes[serviceId] && servicesDocumentTypes[serviceId].length > 0) {
+                return servicesDocumentTypes[serviceId].includes(documentType);
+              }
+
+              if (documentTypes.length > 0) {
+                return documentTypes.includes(documentType);
+              }
+
+              return true;
+            })
+            .forEach(type => {
+              describe(type, () => {
+                const documentDeclaration = service.getDocumentDeclaration(type);
+
+                documentDeclaration.pages.forEach(page => {
+                  let content;
+                  let filteredContent;
+                  let mimeType;
+
+                  context(page.location, () => {
+                    before(async function () {
+                      if (!documentDeclaration) {
+                        console.log('      (Tests skipped as declaration has been archived)');
+                        this.skip();
+                      }
+                    });
+
+                    it('fetchable URL', async () => {
+                      const { location, executeClientScripts } = page;
+                      const document = await fetch({
+                        url: location,
+                        executeClientScripts,
+                        cssSelectors: page.cssSelectors,
+                        config: config.get('fetcher'),
+                      });
+
+                      content = document.content;
+                      mimeType = document.mimeType;
+                    });
+
+                    it('selector matches an element in the web page', async function checkSelector() {
+                      if (!content) {
+                        console.log('          [Tests skipped as URL is not fetchable]');
+                        this.skip();
+                      }
+
+                      filteredContent = await filter({ content, pageDeclaration: page, mimeType });
+
+                      expect(filteredContent).to.not.be.empty;
+                    });
+
+                    it(`filtered content has at least ${MIN_DOC_LENGTH} characters`, async function checkContentLength() {
+                      if (!content) {
+                        console.log('          [Tests skipped as URL is not fetchable]');
+                        this.skip();
+                      }
+
+                      if (!filteredContent) {
+                        console.log('          [Tests skipped as content cannot be filtered]');
+                        this.skip();
+                      }
+
+                      expect(filteredContent.length).to.be.greaterThan(MIN_DOC_LENGTH);
+                    });
+
+                    it('content is consistent when fetched and filtered twice in a row', async function checkContentConsistency() {
+                      this.slow(SLOW_DOCUMENT_THRESHOLD * 2);
+
+                      if (!content) {
+                        console.log('          [Tests skipped as URL is not fetchable]');
+                        this.skip();
+                      }
+
+                      if (!filteredContent) {
+                        console.log('          [Tests skipped as content cannot be filtered]');
+                        this.skip();
+                      }
+
+                      const document = await fetch({
+                        url: page.location,
+                        executeClientScripts: page.executeClientScripts,
+                        cssSelectors: page.cssSelectors,
+                        config: config.get('fetcher'),
+                      });
+                      const secondFilteredContent = await filter({ content: document.content, pageDeclaration: page, mimeType: document.mimeType });
+
+                      expect(secondFilteredContent).to.equal(filteredContent);
+                    });
+                  });
+                });
+              });
+            });
+        }
+      });
+    });
+  });
+
+  run();
+};
+
+const validator = new Ajv({
+  allErrors: true,
+  jsonPointers: true,
+});
+
+function assertValid(schema, subject) {
+  const valid = validator.validate(schema, subject);
+
+  if (!valid) {
+    const errorPointers = new Set();
+    let errorMessage = '';
+    const sourceMap = jsonSourceMap.stringify(subject, null, 2);
+    const jsonLines = sourceMap.json.split('\n');
+
+    validator.errors.forEach(error => {
+      console.log('error', error);
+      errorMessage += `\n\n${validator.errorsText([error])}`;
+      const errorPointer = sourceMap.pointers[error.dataPath];
+
+      if (errorPointer) {
+        errorMessage += `\n> ${jsonLines
+          .slice(errorPointer.value.line, errorPointer.valueEnd.line)
+          .join('\n> ')}`;
+        errorPointers.add(errorPointer);
+      } else {
+        errorMessage += ' (in entire file)\n';
+      }
+    });
+
+    errorMessage += `\n\n${errorPointers.size} features have errors in total`;
+
+    throw new Error(errorMessage);
+  }
+}
+
+async function lintFile(filePath) {
+  const [lintResult] = await eslint.lintFiles(filePath);
+
+  if (!lintResult.errorCount) {
+    return;
+  }
+
+  // Create a new instance of linter with option `fix` set to true to get a fixed output.
+  // It is not possible to use only a linter with this option enabled because when this option is set, if it can fix errors, it considers that there are no errors and returns `0` for the `errorCount`.
+  // So use two linters to have access both to `errorCount` and fix `output` variables.
+  const [lintResultFixed] = await eslintWithFix.lintFiles(filePath);
+
+  expect(lintResult.source).to.equal(lintResultFixed.output, `${path.basename(filePath)} is not properly formatted. Use the lint script to format it correctly.\n`);
+}

package/scripts/declarations/validate/service.history.schema.js

@@ -0,0 +1,86 @@
+import { DOCUMENT_TYPES } from '../../../src/archivist/services/index.js';
+
+import definitions from './definitions.js';
+
+const AVAILABLE_TYPES_NAME = Object.keys(DOCUMENT_TYPES);
+
+const documentsProperties = () => {
+  const result = {};
+
+  AVAILABLE_TYPES_NAME.forEach(type => {
+    result[type] = {
+      type: 'array',
+      items: {
+        oneOf: [
+          { $ref: '#/definitions/singlePageDocumentHistory' },
+          { $ref: '#/definitions/multiPageDocumentHistory' },
+          { $ref: '#/definitions/pdfDocumentHistory' },
+        ],
+      },
+    };
+  });
+
+  return result;
+};
+
+const schema = {
+  type: 'object',
+  additionalProperties: false,
+  title: 'Service declaration history',
+  properties: documentsProperties(),
+  propertyNames: { enum: AVAILABLE_TYPES_NAME },
+  definitions: {
+    ...definitions,
+    pdfDocumentHistory: {
+      type: 'object',
+      additionalProperties: false,
+      required: [ 'fetch', 'validUntil' ],
+      properties: {
+        fetch: { $ref: '#/definitions/pdfLocation' },
+        validUntil: { $ref: '#/definitions/validUntil' },
+      },
+    },
+    singlePageDocumentHistory: {
+      type: 'object',
+      additionalProperties: false,
+      required: [ 'fetch', 'select', 'validUntil' ],
+      properties: {
+        fetch: { $ref: '#/definitions/location' },
+        select: { $ref: '#/definitions/contentSelectors' },
+        filter: { $ref: '#/definitions/filters' },
+        remove: { $ref: '#/definitions/noiseSelectors' },
+        executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
+        validUntil: { $ref: '#/definitions/validUntil' },
+      },
+    },
+    multiPageDocumentHistory: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['combine'],
+      properties: {
+        combine: {
+          type: 'array',
+          items: {
+            type: 'object',
+            additionalProperties: false,
+            required: ['fetch'],
+            properties: {
+              fetch: { $ref: '#/definitions/location' },
+              select: { $ref: '#/definitions/contentSelectors' },
+              filter: { $ref: '#/definitions/filters' },
+              remove: { $ref: '#/definitions/noiseSelectors' },
+              executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
+            },
+          },
+        },
+        select: { $ref: '#/definitions/contentSelectors' },
+        filter: { $ref: '#/definitions/filters' },
+        remove: { $ref: '#/definitions/noiseSelectors' },
+        executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
+        validUntil: { $ref: '#/definitions/validUntil' },
+      },
+    },
+  },
+};
+
+export default schema;

package/scripts/declarations/validate/service.schema.js

@@ -0,0 +1,91 @@
+import { DOCUMENT_TYPES } from '../../../src/archivist/services/index.js';
+
+import definitions from './definitions.js';
+
+const AVAILABLE_TYPES_NAME = Object.keys(DOCUMENT_TYPES);
+
+const documentsProperties = () => {
+  const result = {};
+
+  AVAILABLE_TYPES_NAME.forEach(type => {
+    result[type] = {
+      oneOf: [
+        { $ref: '#/definitions/singlePageDocument' },
+        { $ref: '#/definitions/multiPageDocument' },
+        { $ref: '#/definitions/pdfDocument' },
+      ],
+    };
+  });
+
+  return result;
+};
+
+const schema = {
+  type: 'object',
+  additionalProperties: false,
+  title: 'Service declaration',
+  required: [ 'name', 'documents' ],
+  properties: {
+    name: {
+      type: 'string',
+      title: 'Service public name',
+      examples: ['Facebook'],
+    },
+    documents: {
+      type: 'object',
+      properties: documentsProperties(),
+      propertyNames: { enum: AVAILABLE_TYPES_NAME },
+    },
+    importedFrom: {
+      type: 'string',
+      title: 'Imported from',
+      examples: [
+        'https://github.com/tosdr/tosback2/blob/5acac7abb5e967cfafd124a5e275f98f6ecd423e/rules/4shared.com.xml',
+      ],
+    },
+  },
+  definitions: {
+    ...definitions,
+    pdfDocument: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['fetch'],
+      properties: { fetch: { $ref: '#/definitions/pdfLocation' } },
+    },
+    page: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['fetch'],
+      properties: {
+        fetch: { $ref: '#/definitions/location' },
+        select: { $ref: '#/definitions/contentSelectors' },
+        filter: { $ref: '#/definitions/filters' },
+        remove: { $ref: '#/definitions/noiseSelectors' },
+        executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
+      },
+    },
+    singlePageDocument: {
+      allOf: [
+        { $ref: '#/definitions/page' },
+        { required: [ 'fetch', 'select' ] },
+      ],
+    },
+    multiPageDocument: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['combine'],
+      properties: {
+        combine: {
+          type: 'array',
+          items: { $ref: '#/definitions/page' },
+        },
+        select: { $ref: '#/definitions/contentSelectors' },
+        filter: { $ref: '#/definitions/filters' },
+        remove: { $ref: '#/definitions/noiseSelectors' },
+        executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
+      },
+    },
+  },
+};
+
+export default schema;