@opentermsarchive/engine 0.20.0 → 0.22.0

package/README.md

@@ -184,7 +184,7 @@ npx ota track --services "<service_id>" ["<service_id>"...]
 ##### Track specific terms of specific services
 
 ```sh
-npx ota track --services "<service_id>" ["<service_id>"...] --termsTypes "<terms_type>" ["<terms_type>"...]
+npx ota track --services "<service_id>" ["<service_id>"...] --terms-types "<terms_type>" ["<terms_type>"...]
 ```
 
 ##### Track documents four times a day
@@ -196,7 +196,7 @@ npx ota track --schedule
 #### `ota validate`
 
 ```sh
-npx ota validate [--services <service_id>...] [--termsTypes <terms_type>...]
+npx ota validate [--services <service_id>...] [--terms-types <terms_type>...]
 ```
 
 Check that all declarations allow recording a snapshot and a version properly.
@@ -206,7 +206,7 @@ If one or several `<service_id>` are provided, check only those services.
 ##### Validate schema only
 
 ```sh
-npx ota validate --schema-only [--services <service_id>...] [--termsTypes <terms_type>...]
+npx ota validate --schema-only [--services <service_id>...] [--terms-types <terms_type>...]
 ```
 
 Check that all declarations are readable by the engine.
@@ -227,6 +227,38 @@ Automatically correct formatting mistakes and ensure that all declarations are s
 
 If one or several `<service_id>` are provided, check only those services.
 
+#### `ota dataset`
+
+Export the versions dataset into a ZIP file and publish it to GitHub releases.
+
+The dataset title and the URL of the versions repository are defined in the [configuration](#configuring).
+
+To export the dataset into a local ZIP file:
+
+```sh
+npx ota dataset [--file <filename>]
+```
+
+To export the dataset into a ZIP file and publish it on GitHub releases:
+
+```sh
+GITHUB_TOKEN=ghp_XXXXXXXXX npx ota dataset --publish
+```
+
+The `GITHUB_TOKEN` can also be defined in a [`.env` file](#environment-variables).
+
+To export, publish the dataset and remove the local copy that was created after it has been uploaded:
+
+```sh
+GITHUB_TOKEN=ghp_XXXXXXXXX npx ota dataset --publish --remove-local-copy
+```
+
+To schedule export, publishing and local copy removal:
+
+```sh
+GITHUB_TOKEN=ghp_XXXXXXXXX npx ota dataset --schedule --publish --remove-local-copy
+```
+
 ### API
 
 Once added as a dependency, the engine exposes a JavaScript API that can be called in your own code. The following modules are available.
@@ -277,10 +309,6 @@ import pageDeclaration from '@opentermsarchive/engine/page-declaration';
 
 The `PageDeclaration` format is defined [in source code](./src/archivist/services/pageDeclaration.js).
 
-### Dataset generation
-
-See the [`dataset` script documentation](./scripts/dataset/README.md).
-
 ## Configuring
 
 ### Configuration file

package/bin/ota-dataset.js

@@ -0,0 +1,33 @@
+#! /usr/bin/env node
+import './env.js';
+
+import { program } from 'commander';
+import cron from 'croner';
+
+import { release } from '../scripts/dataset/index.js';
+import logger from '../src/logger/index.js';
+
+program
+  .name('ota dataset')
+  .description('Export the versions dataset into a ZIP file and optionally publish it to GitHub releases')
+  .option('-f, --file <filename>', 'file name of the generated dataset')
+  .option('-p, --publish', 'publish dataset to GitHub releases on versions repository. Mandatory authentication to GitHub is provided through the `GITHUB_TOKEN` environment variable')
+  .option('-r, --remove-local-copy', 'remove local copy of dataset after publishing. Works only in combination with --publish option')
+  .option('--schedule', 'schedule automatic dataset generation');
+
+const { schedule, publish, removeLocalCopy, file: fileName } = program.parse().opts();
+
+const options = {
+  fileName,
+  shouldPublish: publish,
+  shouldRemoveLocalCopy: removeLocalCopy,
+};
+
+if (!schedule) {
+  await release(options);
+} else {
+  logger.info('The scheduler is running…');
+  logger.info('Dataset will be published every Monday at 08:30 in the timezone of this machine');
+
+  cron('30 8 * * MON', () => release(options));
+}

package/bin/ota-track.js

@@ -14,7 +14,7 @@ program
   .name('ota track')
   .description('Retrieve declared documents, record snapshots, extract versions and publish the resulting records')
   .option('-s, --services [serviceId...]', 'service IDs of services to track')
-  .option('-t, --termsType [termsType...]', 'terms types to track')
+  .option('-t, --terms-types [termsType...]', 'terms types to track')
   .option('-r, --refilter-only', 'refilter existing snapshots with latest declarations and engine, without recording new snapshots')
   .option('--schedule', 'schedule automatic document tracking');
 

package/bin/ota-validate.js

@@ -21,7 +21,7 @@ program
   .name('ota validate')
   .description('Run a series of tests to check the validity of document declarations')
   .option('-s, --services [serviceId...]', 'service IDs of services to validate')
-  .option('-t, --termsTypes [termsType...]', 'terms types to validate')
+  .option('-t, --terms-types [termsType...]', 'terms types to validate')
   .option('-m, --modified', 'target only services modified in the current git branch')
   .option('-o, --schema-only', 'much faster check of declarations, but does not check that the documents are actually accessible');
 

package/bin/ota.js

@@ -13,4 +13,5 @@ program
   .command('track', 'Track the current terms of services according to provided declarations')
   .command('validate', 'Run a series of tests to check the validity of document declarations')
   .command('lint', 'Check format and stylistic errors in declarations and auto fix them')
+  .command('dataset', 'Export the versions dataset into a ZIP file and optionally publish it to GitHub releases')
   .parse(process.argv);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "0.20.0",
+  "version": "0.22.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://github.com/ambanum/OpenTermsArchive#readme",
   "bugs": {
@@ -30,8 +30,8 @@
     ".eslintrc.yaml"
   ],
   "scripts": {
-    "dataset:generate": "node scripts/dataset/main.js",
-    "dataset:release": "node scripts/dataset/main.js --publish --remove-local-copy",
+    "dataset:generate": "node bin/ota.js dataset",
+    "dataset:release": "node bin/ota.js dataset --publish --remove-local-copy",
     "dataset:scheduler": "npm run dataset:release -- --schedule",
     "declarations:lint": "node bin/ota.js lint",
     "declarations:validate": "node bin/ota.js validate",
@@ -100,6 +100,9 @@
     "sinon": "^12.0.1",
     "sinon-chai": "^3.7.0"
   },
+  "peerDependencies": {
+    "@opentermsarchive/terms-types": "~0.1.1"
+  },
   "engines": {
     "node": ">=16.0.0"
   }

package/src/archivist/services/index.js

@@ -1,7 +1,8 @@
 import fsApi from 'fs';
 import path from 'path';
-import { fileURLToPath, pathToFileURL } from 'url';
+import { pathToFileURL } from 'url';
 
+import TERMS_TYPES from '@opentermsarchive/terms-types';
 import config from 'config';
 
 import DocumentDeclaration from './documentDeclaration.js';
@@ -9,10 +10,9 @@ import PageDeclaration from './pageDeclaration.js';
 import Service from './service.js';
 
 const fs = fsApi.promises;
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const declarationsPath = path.resolve(process.cwd(), config.get('services.declarationsPath'));
 
-export const DOCUMENT_TYPES = JSON.parse(fsApi.readFileSync(path.resolve(__dirname, './documentTypes.json')));
+export const DOCUMENT_TYPES = TERMS_TYPES;
 
 export async function load(servicesIdsToLoad = []) {
   let servicesIds = await getDeclaredServicesIds();

package/src/index.js

@@ -6,7 +6,7 @@ import logger from './logger/index.js';
 import Notifier from './notifier/index.js';
 import Tracker from './tracker/index.js';
 
-export default async function track({ services = [], termsType: documentTypes, refilterOnly, schedule }) {
+export default async function track({ services = [], termsTypes: documentTypes, refilterOnly, schedule }) {
   const archivist = new Archivist({ recorderConfig: config.get('recorder') });
 
   archivist.attach(logger);

package/scripts/dataset/README.md

@@ -1,37 +0,0 @@
-# Dataset release
-
-Export the versions dataset into a ZIP file and publish it to GitHub releases.
-
-## Configuring
-
-You can change the configuration in the appropriate config file in the `config` folder. See the [main README](../../README.md#configuring) for documentation on using the configuration file.
-
-## Running
-
-To export the dataset into a local ZIP file:
-
-```sh
-node scripts/dataset/main.js [$filename]
-```
-
-To export the dataset into a ZIP file and publish it on GitHub releases:
-
-```sh
-node scripts/dataset/main.js --publish
-```
-
-To export, publish the dataset and remove the local copy that was created after it has been uploaded:
-
-```sh
-node scripts/dataset/main.js --publish --remove-local-copy
-```
-
-To schedule export, publishing and local copy removal:
-
-```sh
-node scripts/dataset/main.js --schedule --publish --remove-local-copy
-```
-
-## Adding renaming rules
-
-See the [renamer module documentation](../utils/renamer/README.md).

package/scripts/dataset/main.js

@@ -1,25 +0,0 @@
-import cron from 'croner';
-
-import logger from './logger/index.js';
-
-import { release } from './index.js';
-
-const args = process.argv.slice(2);
-const argsWithoutOptions = args.filter(arg => !arg.startsWith('--'));
-const [fileName] = argsWithoutOptions;
-const shouldSchedule = args.includes('--schedule');
-
-const options = {
-  fileName,
-  shouldPublish: args.includes('--publish'),
-  shouldRemoveLocalCopy: args.includes('--remove-local-copy'),
-};
-
-if (!shouldSchedule) {
-  release(options);
-} else {
-  logger.info('The scheduler is running…');
-  logger.info('Dataset will be published at 08:30 on every Monday');
-
-  cron('30 8 * * MON', () => release(options));
-}

package/src/archivist/services/documentTypes.json

@@ -1,386 +0,0 @@
-{
-  "Terms of Service": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "end user’s service usage"
-    }
-  },
-  "Privacy Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "end user’s personal data"
-    }
-  },
-  "Imprint": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "everyone",
-      "object": "identification of content author and hosting service for official inquiries"
-    }
-  },
-  "Trackers Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "all tracking technologies, including cookies, session storage, fingerprints…"
-    }
-  },
-  "Parent Organization Terms": {
-    "commitment": {
-      "writer": "parent organization",
-      "audience": "end user",
-      "object": "products and services usage"
-    }
-  },
-  "Parent Organization Privacy Policy": {
-    "commitment": {
-      "writer": "parent organization",
-      "audience": "end user",
-      "object": "personal data"
-    }
-  },
-  "Developer Terms": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "developer",
-      "object": "APIs and programmatic access to content"
-    }
-  },
-  "Community Guidelines": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "public behaviour"
-    }
-  },
-  "Community Guidelines - Self-harm": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "content-publishing user",
-      "object": "content depicting, encouraging or glorifying suicide, eating disorders, self-harm"
-    }
-  },
-  "Community Guidelines - Hate Speech": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "content-publishing user",
-      "object": "content or behavior that threatens, incites violence against, dehumanizes an individual or a group on the basis of protected attributes such as race, ethnicity, gender, sexual orientation…"
-    }
-  },
-  "Community Guidelines - Child Exploitation": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "content-publishing user",
-      "object": "material that features or promotes the exploitation of minors (sexual, emotional…)"
-    }
-  },
-  "Community Guidelines - Violence Incitement": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "content-publishing user",
-      "object": "content threatening, inciting or facilitating offline violence"
-    }
-  },
-  "Community Guidelines - Violent Organizations": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "accounts for individuals or groups that engage in or promote offline violence or organized criminal activity"
-    }
-  },
-  "Community Guidelines - Spam": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "content-publishing user",
-      "object": "untargeted, irrelevant, obviously unwanted, unauthorized commercial or promotional content"
-    }
-  },
-  "Community Guidelines - Platform Manipulation": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "everyone",
-      "object": "behaviour aimed at artificially amplifying or suppressing content visibility (fake engagement, inauthentic behaviour…)"
-    }
-  },
-  "Community Guidelines - Regulated Goods": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "everyone",
-      "object": "selling, buying, or facilitating transactions in illegal or regulated goods or services (drugs, medication, weapons…)"
-    }
-  },
-  "Community Guidelines - Harassment": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "content-publishing user",
-      "object": "content containing or engaging others to use abusive language, doxxing, intimidation against an individual"
-    }
-  },
-  "Community Guidelines - Privacy Violations": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "content-publishing user",
-      "object": "content containing or inciting sharing people's private information or media without their permission"
-    }
-  },
-  "Community Guidelines - Misinformation": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "content-publishing user",
-      "object": "demonstrably false or misleading information which may lead to physical harm, electoral processes disruption or other systemic harms"
-    }
-  },
-  "Community Guidelines - Adult Nudity": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "content-publishing user",
-      "object": "consensually produced and distributed media containing adult genitalia or nipples"
-    }
-  },
-  "Community Guidelines - Sexual Solicitation": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "expressions of desire, requests for a romantic or sexual relationship"
-    }
-  },
-  "Community Guidelines - Intellectual Property": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "content-publishing user",
-      "object": "content containing copyrighted works"
-    }
-  },
-  "Deceased Users": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "relatives of a deceased user",
-      "object": "handling of information of a deceased individual"
-    }
-  },
-  "Acceptable Use Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "acceptable and unacceptable usage"
-    }
-  },
-  "Restricted Use Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "restrictions on specific usage"
-    }
-  },
-  "Commercial Terms": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "commercial partners, content creators in a platform",
-      "object": "business or commercial usage"
-    }
-  },
-  "Copyright Claims Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "copyrighted works rights holders",
-      "object": "how copyright complaints (DMCA…) will be handled"
-    }
-  },
-  "Law Enforcement Guidelines": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "law enforcement authorities",
-      "object": "account records access"
-    }
-  },
-  "Human Rights Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "shareholders",
-      "object": "human rights"
-    }
-  },
-  "In-App Purchases Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "in-app and other virtual goods purchase usage"
-    }
-  },
-  "Review Guidelines": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "developer",
-      "object": "review process of provided content"
-    }
-  },
-  "Brand Guidelines": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "developer",
-      "object": "usage of the service provider’s brand"
-    }
-  },
-  "Quality Guidelines": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "quality of content produced by means of the service"
-    }
-  },
-  "Data Controller Agreement": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "data controllers (in the sense of GDPR)",
-      "object": "end user personal data"
-    }
-  },
-  "Data Processor Agreement": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "data controllers (in the sense of GDPR)",
-      "object": "status of data processor (in the sense of GDPR) for the service provider"
-    }
-  },
-  "User Consent Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "data controllers (in the sense of GDPR)",
-      "object": "user consent collection and storage"
-    }
-  },
-  "Closed Captioning Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "closed captioning on streamed content"
-    }
-  },
-  "Seller Warranty": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "protection of end users that act as sellers"
-    }
-  },
-  "Single Sign-On Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "connecting user account to other services"
-    }
-  },
-  "Vulnerability Disclosure Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "vulnerability reporter",
-      "object": "how to report a security vulnerability or issue"
-    }
-  },
-  "Live Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "end user’s live service usage"
-    }
-  },
-  "Ad Publishing Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "end user",
-      "object": "acceptable and unacceptable usage"
-    }
-  },
-  "Complaints Policy": {
-    "commitment": {
-      "writer": "service provider",
-      "audience": "anyone",
-      "object": "how generic complaints will be handled"
-    }
-  },
-  "Conditions of Carriage": {
-    "commitment": {
-      "writer": "transportation operator (airline, railway, bus…)",
-      "audience": "passenger",
-      "object": "benefits and limitations associated with the transportation being provided"
-    }
-  },
-  "General Conditions of Sale": {
-    "commitment": {
-      "writer": "paid-for goods or service provider",
-      "audience": "buyer",
-      "object": "warranty, delivery, return of goods or services bought through a monetary transaction"
-    }
-  },
-  "Marketplace Buyers Conditions": {
-    "commitment": {
-      "writer": "online marketplace",
-      "audience": "buyer",
-      "object": "buying through a monetary transaction goods or services offered by sellers other than the marketplace itself"
-    }
-  },
-  "Marketplace Sellers Conditions": {
-    "commitment": {
-      "writer": "online marketplace",
-      "audience": "seller",
-      "object": "selling through a monetary transaction goods or services to buyers other than the marketplace itself"
-    }
-  },
-  "Ranking Parameters Description": {
-    "commitment": {
-      "writer": "search engine or intermediation service provider",
-      "audience": "users presented with a list of ranked results, or contributors of results that are ranked",
-      "object": "parameters determining ranking and the reasons for their relative importance"
-    },
-    "references": {
-      "Open Terms Archive Discussion": "https://github.com/ambanum/OpenTermsArchive/discussions/902",
-      "Article 5 of P2B Regulation 2019/1150": "https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=celex%3A32019R1150#d1e868-57-1",
-      "Article 6a of Directive 2011/83/EU": "https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02011L0083-20220528#tocId12",
-      "Point (m) of Article 2(1) of Directive 2005/29/EC": "https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:02005L0029-20220528#tocId5"
-    }
-  },
-  "Premium Partner Conditions": {
-    "commitment": {
-      "writer": "intermediation service provider",
-      "audience": "sellers of goods or services",
-      "object": "eligibility criteria, limitations and benefits of a privileged seller status programme"
-    },
-    "references": {
-      "Open Terms Archive Discussion": "https://github.com/ambanum/OpenTermsArchive/discussions/922"
-    }
-  },
-  "Platform to Business Notice": {
-    "commitment": {
-      "writer": "search engine or intermediation service provider",
-      "audience": "business user",
-      "object": "complaints handling, mediation, transparency and other rights and information mandated by the P2B regulation"
-    },
-    "references": {
-      "Open Terms Archive Discussion": "https://github.com/ambanum/OpenTermsArchive/discussions/940",
-      "P2B Regulation 2019/1150": "https://eur-lex.europa.eu/eli/reg/2019/1150/oj"
-    }
-  },
-  "Business Mediation Policy": {
-    "commitment": {
-      "writer": "intermediation service provider",
-      "audience": "business users",
-      "object": "eligibility and process of mediation after internal complaints handling failed"
-    },
-    "references": {
-      "Open Terms Archive discussion": "https://github.com/ambanum/OpenTermsArchive/discussions/933",
-      "P2B Regulation 2019/1150, Article 12": "https://eur-lex.europa.eu/eli/reg/2019/1150/oj#d1e1148-57-1"
-    }
-  },
-  "Business Privacy Policy": {
-    "commitment": {
-      "writer": "intermediation service provider",
-      "audience": "business user",
-      "object": "personal data of business users and of people acting on their behalf"
-    },
-    "references": {
-      "Open Terms Archive discussion": "https://github.com/ambanum/OpenTermsArchive/discussions/923"
-    }
-  }
-}