@opentermsarchive/engine 0.18.2 → 0.19.0

package/README.md

@@ -150,17 +150,17 @@ This quick example aimed at letting you try the engine quickly. Most likely, you
 
 ### CLI
 
-Once the engine module is installed as a dependency within another module, the following commands are available.
+Once the engine module is installed as a dependency within another module, the `ota` command with the following subcommands is available.
 
 In these commands:
 
 - **`<service_id>`** is the case sensitive name of the service declaration file without the extension. For example, for `Twitter.json`, the service ID is `Twitter`.
 - **`<terms_type>`** is the property name used under the `documents` property in the declaration to declare a terms. For example, in the getting started declaration, the terms type declared is `Privacy Policy`.
 
-#### `ota-track`
+#### `ota track`
 
 ```sh
-npx ota-track
+npx ota track
 ```
 
 [Track](#tracking-documents) the current terms of services according to provided declarations.
@@ -172,31 +172,31 @@ The declarations, snapshots and versions paths are defined in the [configuration
 ##### Recap of available options
 
 ```sh
-npx ota-track --help
+npx ota track --help
 ```
 
 ##### Track terms of specific services
 
 ```sh
-npx ota-track --services "<service_id>" ["<service_id>"...]
+npx ota track --services "<service_id>" ["<service_id>"...]
 ```
 
 ##### Track specific terms of specific services
 
 ```sh
-npx ota-track --services "<service_id>" ["<service_id>"...] --documentTypes "<terms_type>" ["<terms_type>"...]
+npx ota track --services "<service_id>" ["<service_id>"...] --termsTypes "<terms_type>" ["<terms_type>"...]
 ```
 
 ##### Track documents four times a day
 
 ```sh
-npx ota-track --schedule
+npx ota track --schedule
 ```
 
-#### `ota-validate-declarations`
+#### `ota validate`
 
 ```sh
-npx ota-validate-declarations [--services <service_id>...]
+npx ota validate [--services <service_id>...] [--termsTypes <terms_type>...]
 ```
 
 Check that all declarations allow recording a snapshot and a version properly.
@@ -206,7 +206,7 @@ If one or several `<service_id>` are provided, check only those services.
 ##### Validate schema only
 
 ```sh
-npx ota-validate-declarations --schema-only [--services <service_id>...]
+npx ota validate --schema-only [--services <service_id>...] [--termsTypes <terms_type>...]
 ```
 
 Check that all declarations are readable by the engine.
@@ -215,10 +215,10 @@ Allows for a much faster check of declarations, but does not check that the docu
 
 If one or several `<service_id>` are provided, check only those services.
 
-#### `ota-lint-declarations`
+#### `ota lint`
 
 ```sh
-npx ota-lint-declarations [--services <service_id>...]
+npx ota lint [--services <service_id>...]
 ```
 
 Normalise the format of declarations.

package/bin/env.js

@@ -0,0 +1,11 @@
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+process.env.SUPPRESS_NO_CONFIG_WARNING = 'y'; // Caller don't get "no config files" warnings if it does define configuration files
+
+const OTA_CONFIG_DIR = path.resolve(`${__dirname}./../config/`);
+const PROCESS_CONFIG_DIR = path.resolve(`${process.cwd()}/config/`);
+
+process.env.NODE_CONFIG_DIR = OTA_CONFIG_DIR + path.delimiter + PROCESS_CONFIG_DIR; // Ensure OTA config is loaded and loaded before caller config

package/bin/{lint-declarations.js → ota-lint.js}

@@ -1,10 +1,6 @@
 #! /usr/bin/env node
-// makes it easy to lint all files relative to one service ID, which would have been
-// more difficult to achieve using an eslint based command directly defined in the package.json.
-// It also ensures that the same version of eslint is used in the OpenTermsArchive core and declarations repositories.
-import './.env.js'; // Workaround to ensure `SUPPRESS_NO_CONFIG_WARNING` is set before config is imported
+import './env.js';
 
-import fs from 'fs';
 import path from 'path';
 import { fileURLToPath, pathToFileURL } from 'url';
 
@@ -12,15 +8,12 @@ import { program } from 'commander';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
-const { version } = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url)).toString());
-
 program
-  .name('ota-lint-declarations')
+  .name('ota lint')
   .description('Check format and stylistic errors in declarations and auto fix them')
-  .version(version)
-  .option('-s, --services [serviceId...]', 'service IDs of services to handle')
+  .option('-s, --services [serviceId...]', 'service IDs of services to lint')
   .option('-m, --modified', 'to only lint modified services already commited to git');
 
-const lintDeclarations = (await import(pathToFileURL(path.resolve(__dirname, '../scripts/declarations/lint/index.js')))).default;
+const lintDeclarations = (await import(pathToFileURL(path.resolve(__dirname, '../scripts/declarations/lint/index.js')))).default; // load asynchronously to ensure env.js is loaded before
 
 lintDeclarations(program.parse().opts());

package/bin/ota-track.js

@@ -0,0 +1,21 @@
+#! /usr/bin/env node
+import './env.js';
+
+import path from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+
+import { program } from 'commander';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const track = (await import(pathToFileURL(path.resolve(__dirname, '../src/index.js')))).default; // load asynchronously to ensure env.js is loaded before
+
+program
+  .name('ota track')
+  .description('Retrieve declared documents, record snapshots, extract versions and publish the resulting records')
+  .option('-s, --services [serviceId...]', 'service IDs of services to track')
+  .option('-t, --termsType [termsType...]', 'terms types to track')
+  .option('-r, --refilter-only', 'refilter existing snapshots with latest declarations and engine, without recording new snapshots')
+  .option('--schedule', 'schedule automatic document tracking');
+
+track(program.parse(process.argv).opts());

package/bin/{validate-declarations.js → ota-validate.js}

@@ -1,15 +1,12 @@
 #! /usr/bin/env node
-import './.env.js'; // Workaround to ensure `SUPPRESS_NO_CONFIG_WARNING` is set before config is imported
+import './env.js';
 
-import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
 
 import { program } from 'commander';
 import Mocha from 'mocha';
 
-const { version } = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url)).toString());
-
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const VALIDATE_PATH = path.resolve(__dirname, '../scripts/declarations/validate/index.mocha.js');
@@ -21,13 +18,12 @@ process.on('unhandledRejection', reason => {
 });
 
 program
-  .name('ota-validate-declarations')
+  .name('ota validate')
   .description('Run a series of tests to check the validity of document declarations')
-  .version(version)
-  .option('-s, --services [serviceId...]', 'service IDs of services to handle')
-  .option('-d, --documentTypes [documentType...]', 'document types to handle')
-  .option('-m, --modified', 'to only lint modified services already commited to git')
-  .option('-so, --schema-only', 'only refilter exisiting snapshots with last declarations and engine\'s updates');
+  .option('-s, --services [serviceId...]', 'service IDs of services to validate')
+  .option('-t, --termsTypes [termsType...]', 'terms types to validate')
+  .option('-m, --modified', 'target only services modified in the current git branch')
+  .option('-o, --schema-only', 'much faster check of declarations, but does not check that the documents are actually accessible');
 
 const mocha = new Mocha({
   delay: true, // as the validation script performs an asynchronous load before running the tests, the execution of the tests are delayed until run() is called

package/bin/ota.js

@@ -0,0 +1,16 @@
+#! /usr/bin/env node
+import './env.js';
+
+import fs from 'fs';
+
+import { program } from 'commander';
+
+const { description, version } = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url)).toString());
+
+program
+  .description(description)
+  .version(version)
+  .command('track', 'Track the current terms of services according to provided declarations')
+  .command('validate', 'Run a series of tests to check the validity of document declarations')
+  .command('lint', 'Check format and stylistic errors in declarations and auto fix them')
+  .parse(process.argv);

package/config/production.json

@@ -1,7 +1,4 @@
 {
-  "services": {
-    "declarationsPath": "../declarations/declarations"
-  },
   "recorder": {
     "versions": {
       "storage": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "0.18.2",
+  "version": "0.19.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://github.com/ambanum/OpenTermsArchive#readme",
   "bugs": {
@@ -20,9 +20,7 @@
     "./page-declaration": "./src/archivist/services/pageDeclaration.js"
   },
   "bin": {
-    "ota-lint-declarations": "./bin/lint-declarations.js",
-    "ota-track": "./bin/track.js",
-    "ota-validate-declarations": "./bin/validate-declarations.js"
+    "ota": "./bin/ota.js"
   },
   "files": [
     "bin",
@@ -35,13 +33,13 @@
     "dataset:generate": "node scripts/dataset/main.js",
     "dataset:release": "node scripts/dataset/main.js --publish --remove-local-copy",
     "dataset:scheduler": "npm run dataset:release -- --schedule",
-    "declarations:lint": "node ./bin/lint-declarations.js",
-    "declarations:validate": "node ./bin/validate-declarations.js",
+    "declarations:lint": "node bin/ota.js lint",
+    "declarations:validate": "node bin/ota.js validate",
     "declarations:validate:schema": "npm run declarations:validate -- --schema-only",
     "lint": "eslint src test scripts bin",
     "lint:fix": "npm run lint -- --fix",
     "refilter": "npm start -- --refilter-only",
-    "start": "node --max-http-header-size=32768 bin/track.js",
+    "start": "node --max-http-header-size=32768 bin/ota.js track",
     "start:scheduler": "npm start -- --schedule",
     "test": "cross-env NODE_ENV=test mocha --recursive \"./src/**/*.test.js\" \"./scripts/**/*.test.js\" --exit",
     "posttest": "npm run lint",

package/scripts/declarations/validate/index.mocha.js

@@ -34,7 +34,7 @@ const instancePath = path.resolve(declarationsPath, '../');
 export default async options => {
   const schemaOnly = options.schemaOnly || false;
   let servicesToValidate = options.services || [];
-  const documentTypes = options.documentTypes || [];
+  const documentTypes = options.termsTypes || [];
   let servicesDocumentTypes = {};
 
   const serviceDeclarations = await services.loadWithHistory(servicesToValidate);

package/src/archivist/fetcher/env.js

@@ -0,0 +1 @@
+process.env.SUPPRESS_NO_CONFIG_WARNING = 'y'; // Caller don't get "no config files" warnings if it does define configuration files

package/src/archivist/fetcher/exports.js

@@ -1,4 +1,4 @@
-import '../../../bin/.env.js'; // Workaround to ensure `SUPPRESS_NO_CONFIG_WARNING` is set before config is imported
+import './env.js'; // Workaround to ensure `SUPPRESS_NO_CONFIG_WARNING` is set before config is imported
 
 import fs from 'fs';
 import path from 'path';

package/src/index.js

@@ -6,7 +6,7 @@ import logger from './logger/index.js';
 import Notifier from './notifier/index.js';
 import Tracker from './tracker/index.js';
 
-export default async function track({ services = [], documentTypes, refilterOnly, schedule }) {
+export default async function track({ services = [], termsType: documentTypes, refilterOnly, schedule }) {
   const archivist = new Archivist({ recorderConfig: config.get('recorder') });
 
   archivist.attach(logger);

package/bin/.env.js

@@ -1,10 +0,0 @@
-import path from 'path';
-import { fileURLToPath } from 'url';
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-process.env.SUPPRESS_NO_CONFIG_WARNING = 'y'; // Caller don't get "no config files" warnings if it does define configuration files
-
-const OTA_CONFIG_DIR = path.resolve(__dirname + "./../config/");
-const PROCESS_CONFIG_DIR = path.resolve(process.cwd() + "/config/");
-
-process.env["NODE_CONFIG_DIR"] = OTA_CONFIG_DIR + path.delimiter + PROCESS_CONFIG_DIR; // Ensure OTA config is loaded and loaded before caller config

package/bin/track.js

@@ -1,25 +0,0 @@
-#! /usr/bin/env node
-import './.env.js'; // Workaround to ensure `SUPPRESS_NO_CONFIG_WARNING` is set before config is imported
-
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath, pathToFileURL } from 'url';
-
-import { program } from 'commander';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-const { description, version } = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url)).toString());
-
-program
-  .name('ota-track')
-  .description(description)
-  .version(version)
-  .option('-s, --services [serviceId...]', 'service IDs of services to handle')
-  .option('-d, --documentTypes [documentType...]', 'terms types to handle')
-  .option('-r, --refilter-only', 'only refilter exisiting snapshots with last declarations and engine\'s updates')
-  .option('--schedule', 'schedule automatic document tracking');
-
-const track = (await import(pathToFileURL(path.resolve(__dirname, '../src/index.js')))).default;
-
-track(program.parse(process.argv).opts());