@opentermsarchive/engine 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/env.js +2 -0
- package/bin/ota-dataset.js +1 -1
- package/bin/ota-serve.js +1 -1
- package/config/ci.json +4 -2
- package/config/default.json +53 -55
- package/config/production.json +21 -19
- package/config/test.json +46 -40
- package/package.json +4 -4
- package/scripts/dataset/assets/README.template.js +2 -2
- package/scripts/dataset/export/index.js +1 -1
- package/scripts/dataset/export/index.test.js +2 -2
- package/scripts/dataset/index.js +1 -1
- package/scripts/dataset/publish/index.js +2 -5
- package/scripts/declarations/lint/index.mocha.js +1 -1
- package/scripts/declarations/validate/index.mocha.js +3 -3
- package/scripts/history/migrate-services.js +8 -8
- package/scripts/history/update-to-full-hash.js +3 -3
- package/scripts/import/index.js +6 -6
- package/scripts/import/loadCommits.js +4 -4
- package/scripts/rewrite/initializer/index.js +1 -1
- package/scripts/rewrite/rewrite-snapshots.js +6 -6
- package/scripts/rewrite/rewrite-versions.js +5 -5
- package/src/archivist/fetcher/index.js +3 -3
- package/src/archivist/index.test.js +16 -16
- package/src/archivist/recorder/index.test.js +2 -2
- package/src/archivist/recorder/repositories/git/git.test.js +3 -3
- package/src/archivist/recorder/repositories/git/index.test.js +13 -13
- package/src/archivist/recorder/repositories/mongo/index.test.js +7 -7
- package/src/archivist/services/index.js +1 -1
- package/src/{api → collection-api}/logger.js +5 -5
- package/src/{api → collection-api}/routes/docs.test.js +1 -1
- package/src/{api → collection-api}/routes/services.test.js +1 -1
- package/src/{api → collection-api}/routes/versions.js +1 -1
- package/src/{api → collection-api}/routes/versions.test.js +2 -2
- package/src/{api → collection-api}/server.js +2 -2
- package/src/index.js +7 -7
- package/src/logger/index.js +9 -11
- package/src/notifier/index.js +3 -5
- package/src/reporter/github.js +1 -1
- /package/src/{api → collection-api}/middlewares/errors.js +0 -0
- /package/src/{api → collection-api}/middlewares/logger.js +0 -0
- /package/src/{api → collection-api}/routes/docs.js +0 -0
- /package/src/{api → collection-api}/routes/index.js +0 -0
- /package/src/{api → collection-api}/routes/services.js +0 -0
package/bin/env.js
CHANGED
package/bin/ota-dataset.js
CHANGED
|
@@ -11,7 +11,7 @@ program
|
|
|
11
11
|
.name('ota dataset')
|
|
12
12
|
.description('Export the versions dataset into a ZIP file and optionally publish it to GitHub releases')
|
|
13
13
|
.option('-f, --file <filename>', 'file name of the generated dataset')
|
|
14
|
-
.option('-p, --publish', 'publish dataset to GitHub releases on versions repository. Mandatory authentication to GitHub is provided through the `
|
|
14
|
+
.option('-p, --publish', 'publish dataset to GitHub releases on versions repository. Mandatory authentication to GitHub is provided through the `OTA_ENGINE_GITHUB_TOKEN` environment variable')
|
|
15
15
|
.option('-r, --remove-local-copy', 'remove local copy of dataset after publishing. Works only in combination with --publish option')
|
|
16
16
|
.option('--schedule', 'schedule automatic dataset generation');
|
|
17
17
|
|
package/bin/ota-serve.js
CHANGED
|
@@ -6,4 +6,4 @@ import { fileURLToPath, pathToFileURL } from 'url';
|
|
|
6
6
|
|
|
7
7
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
8
8
|
|
|
9
|
-
await import(pathToFileURL(path.resolve(__dirname, '../src/api/server.js'))); // load asynchronously to ensure env.js is loaded before
|
|
9
|
+
await import(pathToFileURL(path.resolve(__dirname, '../src/collection-api/server.js'))); // load asynchronously to ensure env.js is loaded before
|
package/config/ci.json
CHANGED
package/config/default.json
CHANGED
|
@@ -1,65 +1,63 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
"
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
"
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
|
|
10
|
-
"
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
"
|
|
15
|
-
|
|
2
|
+
"@opentermsarchive/engine": {
|
|
3
|
+
"services": {
|
|
4
|
+
"declarationsPath": "./declarations"
|
|
5
|
+
},
|
|
6
|
+
"recorder": {
|
|
7
|
+
"versions": {
|
|
8
|
+
"storage": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"git": {
|
|
11
|
+
"path": "./data/versions",
|
|
12
|
+
"publish": false,
|
|
13
|
+
"snapshotIdentiferTemplate": "./data/snapshots/%SNAPSHOT_ID",
|
|
14
|
+
"author": {
|
|
15
|
+
"name": "Open Terms Archive Bot",
|
|
16
|
+
"email": "bot@opentermsarchive.org"
|
|
17
|
+
}
|
|
16
18
|
}
|
|
17
19
|
}
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
20
|
+
},
|
|
21
|
+
"snapshots": {
|
|
22
|
+
"storage": {
|
|
23
|
+
"type": "git",
|
|
24
|
+
"git": {
|
|
25
|
+
"path": "./data/snapshots",
|
|
26
|
+
"publish": false,
|
|
27
|
+
"author": {
|
|
28
|
+
"name": "Open Terms Archive Bot",
|
|
29
|
+
"email": "bot@opentermsarchive.org"
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
"mongo": {
|
|
33
|
+
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
34
|
+
"database": "open-terms-archive",
|
|
35
|
+
"collection": "snapshots"
|
|
29
36
|
}
|
|
30
|
-
},
|
|
31
|
-
"mongo": {
|
|
32
|
-
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
33
|
-
"database": "open-terms-archive",
|
|
34
|
-
"collection": "snapshots"
|
|
35
37
|
}
|
|
36
38
|
}
|
|
37
|
-
}
|
|
38
|
-
},
|
|
39
|
-
"fetcher": {
|
|
40
|
-
"waitForElementsTimeout": 10000,
|
|
41
|
-
"navigationTimeout": 30000,
|
|
42
|
-
"language": "en"
|
|
43
|
-
},
|
|
44
|
-
"logger": {
|
|
45
|
-
"smtp": {
|
|
46
|
-
"host": "smtp-relay.sendinblue.com",
|
|
47
|
-
"username": "admin@opentermsarchive.org"
|
|
48
39
|
},
|
|
49
|
-
"
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
40
|
+
"fetcher": {
|
|
41
|
+
"waitForElementsTimeout": 10000,
|
|
42
|
+
"navigationTimeout": 30000,
|
|
43
|
+
"language": "en"
|
|
44
|
+
},
|
|
45
|
+
"logger": {
|
|
46
|
+
"smtp": {
|
|
47
|
+
"host": "smtp-relay.sendinblue.com",
|
|
48
|
+
"username": "admin@opentermsarchive.org"
|
|
49
|
+
},
|
|
50
|
+
"sendMailOnError": false
|
|
51
|
+
},
|
|
52
|
+
"notifier": {
|
|
53
|
+
"sendInBlue": {
|
|
54
|
+
"updatesListId": 850,
|
|
55
|
+
"updateTemplateId": 7
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
"dataset": {
|
|
59
|
+
"title": "sandbox",
|
|
60
|
+
"versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox"
|
|
55
61
|
}
|
|
56
|
-
},
|
|
57
|
-
"dataset": {
|
|
58
|
-
"title": "sandbox",
|
|
59
|
-
"versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox"
|
|
60
|
-
},
|
|
61
|
-
"api": {
|
|
62
|
-
"port": 3000,
|
|
63
|
-
"basePath": "/api"
|
|
64
62
|
}
|
|
65
63
|
}
|
package/config/production.json
CHANGED
|
@@ -1,27 +1,29 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
"
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
|
|
2
|
+
"@opentermsarchive/engine": {
|
|
3
|
+
"recorder": {
|
|
4
|
+
"versions": {
|
|
5
|
+
"storage": {
|
|
6
|
+
"git": {
|
|
7
|
+
"path": "../versions",
|
|
8
|
+
"publish": true
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
},
|
|
12
|
+
"snapshots": {
|
|
13
|
+
"storage": {
|
|
14
|
+
"git": {
|
|
15
|
+
"path": "../snapshots",
|
|
16
|
+
"publish": true
|
|
17
|
+
}
|
|
8
18
|
}
|
|
9
19
|
}
|
|
10
20
|
},
|
|
11
|
-
"
|
|
12
|
-
"
|
|
13
|
-
"
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
}
|
|
21
|
+
"logger": {
|
|
22
|
+
"sendMailOnError": {
|
|
23
|
+
"to": "admin@opentermsarchive.org",
|
|
24
|
+
"from": "noreply@opentermsarchive.org",
|
|
25
|
+
"sendWarnings": false
|
|
17
26
|
}
|
|
18
27
|
}
|
|
19
|
-
},
|
|
20
|
-
"logger": {
|
|
21
|
-
"sendMailOnError": {
|
|
22
|
-
"to": "admin@opentermsarchive.org",
|
|
23
|
-
"from": "noreply@opentermsarchive.org",
|
|
24
|
-
"sendWarnings": false
|
|
25
|
-
}
|
|
26
28
|
}
|
|
27
29
|
}
|
package/config/test.json
CHANGED
|
@@ -1,49 +1,55 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
"
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
"
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
"
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
"
|
|
14
|
-
|
|
2
|
+
"@opentermsarchive/engine": {
|
|
3
|
+
"services": {
|
|
4
|
+
"declarationsPath": "./test/services"
|
|
5
|
+
},
|
|
6
|
+
"recorder": {
|
|
7
|
+
"versions": {
|
|
8
|
+
"storage": {
|
|
9
|
+
"git": {
|
|
10
|
+
"path": "./test/data/versions",
|
|
11
|
+
"publish": false,
|
|
12
|
+
"snapshotIdentiferTemplate": "https://github.com/OpenTermsArchive/sandbox-snapshots/commit/%SNAPSHOT_ID",
|
|
13
|
+
"author": {
|
|
14
|
+
"name": "Open Terms Archive Testing Bot",
|
|
15
|
+
"email": "bot@opentermsarchive.org"
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"mongo": {
|
|
19
|
+
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
20
|
+
"database": "open-terms-archive-test",
|
|
21
|
+
"collection": "versions"
|
|
15
22
|
}
|
|
16
|
-
},
|
|
17
|
-
"mongo": {
|
|
18
|
-
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
19
|
-
"database": "open-terms-archive-test",
|
|
20
|
-
"collection": "versions"
|
|
21
23
|
}
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
24
|
+
},
|
|
25
|
+
"snapshots": {
|
|
26
|
+
"storage": {
|
|
27
|
+
"git": {
|
|
28
|
+
"path": "./test/data/snapshots",
|
|
29
|
+
"publish": false,
|
|
30
|
+
"author": {
|
|
31
|
+
"name": "Open Terms Archive Testing Bot",
|
|
32
|
+
"email": "bot@opentermsarchive.org"
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"mongo": {
|
|
36
|
+
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
37
|
+
"database": "open-terms-archive-test",
|
|
38
|
+
"collection": "snapshots"
|
|
32
39
|
}
|
|
33
|
-
},
|
|
34
|
-
"mongo": {
|
|
35
|
-
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
36
|
-
"database": "open-terms-archive-test",
|
|
37
|
-
"collection": "snapshots"
|
|
38
40
|
}
|
|
39
41
|
}
|
|
42
|
+
},
|
|
43
|
+
"fetcher": {
|
|
44
|
+
"waitForElementsTimeout": 1000
|
|
45
|
+
},
|
|
46
|
+
"dataset": {
|
|
47
|
+
"title": "sandbox",
|
|
48
|
+
"versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox-versions"
|
|
49
|
+
},
|
|
50
|
+
"collection-api": {
|
|
51
|
+
"port": 3000,
|
|
52
|
+
"basePath": "/collection-api"
|
|
40
53
|
}
|
|
41
|
-
},
|
|
42
|
-
"fetcher": {
|
|
43
|
-
"waitForElementsTimeout": 1000
|
|
44
|
-
},
|
|
45
|
-
"dataset": {
|
|
46
|
-
"title": "sandbox",
|
|
47
|
-
"versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox-versions"
|
|
48
54
|
}
|
|
49
55
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opentermsarchive/engine",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "Tracks and makes visible changes to the terms of online services",
|
|
5
5
|
"homepage": "https://opentermsarchive.org",
|
|
6
6
|
"bugs": {
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"declarations:validate:schema": "npm run declarations:validate -- --schema-only",
|
|
41
41
|
"lint": "eslint src test scripts bin",
|
|
42
42
|
"lint:fix": "npm run lint -- --fix",
|
|
43
|
-
"start": "node --max-http-header-size=32768 bin/ota.js track",
|
|
43
|
+
"start": "node -r dotenv/config --max-http-header-size=32768 bin/ota.js track",
|
|
44
44
|
"start:api": "node bin/ota.js serve",
|
|
45
45
|
"start:scheduler": "npm start -- --schedule",
|
|
46
46
|
"test": "cross-env NODE_ENV=test mocha --recursive \"./src/**/*.test.js\" \"./scripts/**/*.test.js\" --exit",
|
|
@@ -72,7 +72,7 @@
|
|
|
72
72
|
"eslint-plugin-import": "^2.29.0",
|
|
73
73
|
"eslint-plugin-json-format": "^2.0.1",
|
|
74
74
|
"eslint-plugin-no-only-tests": "^3.1.0",
|
|
75
|
-
"express": "^4.
|
|
75
|
+
"express": "^4.19.2",
|
|
76
76
|
"express-async-errors": "^3.1.1",
|
|
77
77
|
"fs-extra": "^10.0.0",
|
|
78
78
|
"helmet": "^6.0.1",
|
|
@@ -97,7 +97,7 @@
|
|
|
97
97
|
"sib-api-v3-sdk": "^8.2.1",
|
|
98
98
|
"simple-git": "^3.8.0",
|
|
99
99
|
"swagger-jsdoc": "^6.2.8",
|
|
100
|
-
"swagger-ui-express": "
|
|
100
|
+
"swagger-ui-express": "^5.0.0",
|
|
101
101
|
"winston": "^3.3.3",
|
|
102
102
|
"winston-mail": "^2.0.0"
|
|
103
103
|
},
|
|
@@ -12,7 +12,7 @@ ${body({ servicesCount, firstVersionDate, lastVersionDate })}`;
|
|
|
12
12
|
export function title({ releaseDate }) {
|
|
13
13
|
releaseDate = releaseDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
|
|
14
14
|
|
|
15
|
-
const title = config.get('dataset.title');
|
|
15
|
+
const title = config.get('@opentermsarchive/engine.dataset.title');
|
|
16
16
|
|
|
17
17
|
return `${title} — ${releaseDate} dataset`;
|
|
18
18
|
}
|
|
@@ -21,7 +21,7 @@ export function body({ servicesCount, firstVersionDate, lastVersionDate }) {
|
|
|
21
21
|
firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
|
|
22
22
|
lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
|
|
23
23
|
|
|
24
|
-
const versionsRepositoryURL = config.get('dataset.versionsRepositoryURL');
|
|
24
|
+
const versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL');
|
|
25
25
|
|
|
26
26
|
return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}.
|
|
27
27
|
|
|
@@ -17,7 +17,7 @@ const fs = fsApi.promises;
|
|
|
17
17
|
const ARCHIVE_FORMAT = 'zip'; // for supported formats, see https://www.archiverjs.com/docs/archive-formats
|
|
18
18
|
|
|
19
19
|
export default async function generate({ archivePath, releaseDate }) {
|
|
20
|
-
const versionsRepository = await RepositoryFactory.create(config.get('recorder.versions.storage')).initialize();
|
|
20
|
+
const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage')).initialize();
|
|
21
21
|
|
|
22
22
|
const archive = await initializeArchive(archivePath);
|
|
23
23
|
|
|
@@ -48,8 +48,8 @@ describe('Export', () => {
|
|
|
48
48
|
before(async function () {
|
|
49
49
|
this.timeout(10000);
|
|
50
50
|
repository = new GitRepository({
|
|
51
|
-
...config.get('recorder.versions.storage.git'),
|
|
52
|
-
path: path.resolve(__dirname, '../../../', config.get('recorder.versions.storage.git.path')),
|
|
51
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
52
|
+
path: path.resolve(__dirname, '../../../', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path')),
|
|
53
53
|
});
|
|
54
54
|
|
|
55
55
|
await repository.initialize();
|
package/scripts/dataset/index.js
CHANGED
|
@@ -9,7 +9,7 @@ import publishRelease from './publish/index.js';
|
|
|
9
9
|
|
|
10
10
|
export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }) {
|
|
11
11
|
const releaseDate = new Date();
|
|
12
|
-
const archiveName = fileName || `dataset-${config.get('dataset.title')}-${releaseDate.toISOString().replace(/T.*/, '')}`;
|
|
12
|
+
const archiveName = fileName || `dataset-${config.get('@opentermsarchive/engine.dataset.title')}-${releaseDate.toISOString().replace(/T.*/, '')}`;
|
|
13
13
|
const archivePath = `${path.basename(archiveName, '.zip')}.zip`; // allow to pass filename or filename.zip as the archive name and have filename.zip as the result name
|
|
14
14
|
|
|
15
15
|
logger.info('Start exporting dataset…');
|
|
@@ -3,17 +3,14 @@ import path from 'path';
|
|
|
3
3
|
import url from 'url';
|
|
4
4
|
|
|
5
5
|
import config from 'config';
|
|
6
|
-
import dotenv from 'dotenv';
|
|
7
6
|
import { Octokit } from 'octokit';
|
|
8
7
|
|
|
9
8
|
import * as readme from '../assets/README.template.js';
|
|
10
9
|
|
|
11
|
-
dotenv.config();
|
|
12
|
-
|
|
13
10
|
export default async function publish({ archivePath, releaseDate, stats }) {
|
|
14
|
-
const octokit = new Octokit({ auth: process.env.
|
|
11
|
+
const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
|
|
15
12
|
|
|
16
|
-
const [ owner, repo ] = url.parse(config.get('dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
|
|
13
|
+
const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
|
|
17
14
|
|
|
18
15
|
const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
|
|
19
16
|
|
|
@@ -16,7 +16,7 @@ const ESLINT_CONFIG_PATH = path.join(ROOT_PATH, '.eslintrc.yaml');
|
|
|
16
16
|
const eslint = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: false });
|
|
17
17
|
const eslintWithFix = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true });
|
|
18
18
|
|
|
19
|
-
const declarationsPath = path.resolve(process.cwd(), config.get('services.declarationsPath'));
|
|
19
|
+
const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
|
|
20
20
|
const instancePath = path.resolve(declarationsPath, '../');
|
|
21
21
|
|
|
22
22
|
export default async options => {
|
|
@@ -19,7 +19,7 @@ const fs = fsApi.promises;
|
|
|
19
19
|
const MIN_DOC_LENGTH = 100;
|
|
20
20
|
const SLOW_DOCUMENT_THRESHOLD = 10 * 1000; // number of milliseconds after which a document fetch is considered slow
|
|
21
21
|
|
|
22
|
-
const declarationsPath = path.resolve(process.cwd(), config.get('services.declarationsPath'));
|
|
22
|
+
const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
|
|
23
23
|
const instancePath = path.resolve(declarationsPath, '../');
|
|
24
24
|
|
|
25
25
|
export default async options => {
|
|
@@ -114,7 +114,7 @@ export default async options => {
|
|
|
114
114
|
url: location,
|
|
115
115
|
executeClientScripts,
|
|
116
116
|
cssSelectors: sourceDocument.cssSelectors,
|
|
117
|
-
config: config.get('fetcher'),
|
|
117
|
+
config: config.get('@opentermsarchive/engine.fetcher'),
|
|
118
118
|
}));
|
|
119
119
|
});
|
|
120
120
|
|
|
@@ -160,7 +160,7 @@ export default async options => {
|
|
|
160
160
|
url: sourceDocument.location,
|
|
161
161
|
executeClientScripts: sourceDocument.executeClientScripts,
|
|
162
162
|
cssSelectors: sourceDocument.cssSelectors,
|
|
163
|
-
config: config.get('fetcher'),
|
|
163
|
+
config: config.get('@opentermsarchive/engine.fetcher'),
|
|
164
164
|
}));
|
|
165
165
|
const secondFilteredContent = await extract(sourceDocument);
|
|
166
166
|
|
|
@@ -41,22 +41,22 @@ const counters = {
|
|
|
41
41
|
from: {
|
|
42
42
|
snapshots: {
|
|
43
43
|
source: new GitRepository({
|
|
44
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
44
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
45
45
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.snapshots}`),
|
|
46
46
|
}),
|
|
47
47
|
destination: new GitRepository({
|
|
48
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
48
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
49
49
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.snapshots}-migrated`),
|
|
50
50
|
}),
|
|
51
51
|
logger: winston.createLogger({ transports: [ new (winston.transports.File)({ filename: `${__dirname}/logs/${CONFIG.from.snapshots}.log` }), new winston.transports.Console() ], format }),
|
|
52
52
|
},
|
|
53
53
|
versions: {
|
|
54
54
|
source: new GitRepository({
|
|
55
|
-
...config.get('recorder.versions.storage.git'),
|
|
55
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
56
56
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.versions}`),
|
|
57
57
|
}),
|
|
58
58
|
destination: new GitRepository({
|
|
59
|
-
...config.get('recorder.versions.storage.git'),
|
|
59
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
60
60
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.versions}-migrated`),
|
|
61
61
|
prefixMessageToSnapshotId: CONFIG.from.prefixMessageToSnapshotId,
|
|
62
62
|
}),
|
|
@@ -66,22 +66,22 @@ const counters = {
|
|
|
66
66
|
to: {
|
|
67
67
|
snapshots: {
|
|
68
68
|
source: new GitRepository({
|
|
69
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
69
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
70
70
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.snapshots}`),
|
|
71
71
|
}),
|
|
72
72
|
destination: new GitRepository({
|
|
73
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
73
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
74
74
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.snapshots}-migrated`),
|
|
75
75
|
}),
|
|
76
76
|
logger: winston.createLogger({ transports: [ new (winston.transports.File)({ filename: `${__dirname}/logs/${CONFIG.to.snapshots}.log` }), new winston.transports.Console() ], format }),
|
|
77
77
|
},
|
|
78
78
|
versions: {
|
|
79
79
|
source: new GitRepository({
|
|
80
|
-
...config.get('recorder.versions.storage.git'),
|
|
80
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
81
81
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.versions}`),
|
|
82
82
|
}),
|
|
83
83
|
destination: new GitRepository({
|
|
84
|
-
...config.get('recorder.versions.storage.git'),
|
|
84
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
85
85
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.versions}-migrated`),
|
|
86
86
|
prefixMessageToSnapshotId: CONFIG.to.prefixMessageToSnapshotId,
|
|
87
87
|
}),
|
|
@@ -15,18 +15,18 @@ const ROOT_PATH = path.resolve(__dirname, '../../');
|
|
|
15
15
|
console.time('Total time');
|
|
16
16
|
|
|
17
17
|
const versionsRepository = new GitRepository({
|
|
18
|
-
...config.get('recorder.versions.storage.git'),
|
|
18
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
19
19
|
path: path.resolve(ROOT_PATH, './data/france-elections-versions'),
|
|
20
20
|
});
|
|
21
21
|
|
|
22
22
|
const versionsTargetRepository = new GitRepository({
|
|
23
|
-
...config.get('recorder.versions.storage.git'),
|
|
23
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
24
24
|
prefixMessageToSnapshotId: 'This version was recorded after filtering snapshot https://github.com/OpenTermsArchive/france-elections-snapshots/commit/',
|
|
25
25
|
path: path.resolve(ROOT_PATH, './data/france-elections-versions-hash-updated-test'),
|
|
26
26
|
});
|
|
27
27
|
|
|
28
28
|
const snapshotsRepository = new GitRepository({
|
|
29
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
29
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
30
30
|
path: path.resolve(ROOT_PATH, './data/france-elections-snapshots'),
|
|
31
31
|
});
|
|
32
32
|
|
package/scripts/import/index.js
CHANGED
|
@@ -53,15 +53,15 @@ let client;
|
|
|
53
53
|
}());
|
|
54
54
|
|
|
55
55
|
async function initialize() {
|
|
56
|
-
client = new MongoClient(config.get('import.mongo.connectionURI'));
|
|
56
|
+
client = new MongoClient(config.get('@opentermsarchive/engine.import.mongo.connectionURI'));
|
|
57
57
|
|
|
58
58
|
await client.connect();
|
|
59
|
-
const db = client.db(config.get('import.mongo.database'));
|
|
59
|
+
const db = client.db(config.get('@opentermsarchive/engine.import.mongo.database'));
|
|
60
60
|
|
|
61
|
-
snapshotsCollection = db.collection(config.get('import.mongo.snapshotsCollection'));
|
|
62
|
-
commitsCollection = db.collection(config.get('import.mongo.commitsCollection'));
|
|
61
|
+
snapshotsCollection = db.collection(config.get('@opentermsarchive/engine.import.mongo.snapshotsCollection'));
|
|
62
|
+
commitsCollection = db.collection(config.get('@opentermsarchive/engine.import.mongo.commitsCollection'));
|
|
63
63
|
|
|
64
|
-
sourceRepository = new Git({ path: path.resolve(ROOT_PATH, config.get('import.sourcePath')) });
|
|
64
|
+
sourceRepository = new Git({ path: path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.import.sourcePath')) });
|
|
65
65
|
|
|
66
66
|
await sourceRepository.initialize();
|
|
67
67
|
await renamer.loadRules();
|
|
@@ -119,7 +119,7 @@ function queueDrainHandler(totalToTreat) {
|
|
|
119
119
|
|
|
120
120
|
async function getCommitContent({ sha, serviceId, termsType, extension }) {
|
|
121
121
|
const start = performance.now();
|
|
122
|
-
const url = `https://raw.githubusercontent.com/${config.get('import.githubRepository')}/${sha}/${encodeURI(serviceId)}/${encodeURI(termsType)}.${extension}`;
|
|
122
|
+
const url = `https://raw.githubusercontent.com/${config.get('@opentermsarchive/engine.import.githubRepository')}/${sha}/${encodeURI(serviceId)}/${encodeURI(termsType)}.${extension}`;
|
|
123
123
|
const response = await nodeFetch(url);
|
|
124
124
|
const end = performance.now();
|
|
125
125
|
|
|
@@ -54,13 +54,13 @@ let client;
|
|
|
54
54
|
}());
|
|
55
55
|
|
|
56
56
|
async function initialize() {
|
|
57
|
-
client = new MongoClient(config.get('import.mongo.connectionURI'));
|
|
57
|
+
client = new MongoClient(config.get('@opentermsarchive/engine.import.mongo.connectionURI'));
|
|
58
58
|
|
|
59
59
|
await client.connect();
|
|
60
|
-
const db = client.db(config.get('import.mongo.database'));
|
|
60
|
+
const db = client.db(config.get('@opentermsarchive/engine.import.mongo.database'));
|
|
61
61
|
|
|
62
|
-
collection = db.collection(config.get('import.mongo.commitsCollection'));
|
|
63
|
-
sourceRepository = new Git({ path: path.resolve(ROOT_PATH, config.get('import.sourcePath')) });
|
|
62
|
+
collection = db.collection(config.get('@opentermsarchive/engine.import.mongo.commitsCollection'));
|
|
63
|
+
sourceRepository = new Git({ path: path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.import.sourcePath')) });
|
|
64
64
|
|
|
65
65
|
await sourceRepository.initialize();
|
|
66
66
|
}
|
|
@@ -36,7 +36,7 @@ export async function initTargetRepo(targetRepoPath) {
|
|
|
36
36
|
}
|
|
37
37
|
await fs.mkdir(targetRepoPath);
|
|
38
38
|
|
|
39
|
-
const targetRepo = await new Git({ path: targetRepoPath, author: config.get('recorder.versions.storage.git.author') });
|
|
39
|
+
const targetRepo = await new Git({ path: targetRepoPath, author: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author') });
|
|
40
40
|
|
|
41
41
|
await targetRepo.initialize();
|
|
42
42
|
|
|
@@ -15,9 +15,9 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
|
15
15
|
|
|
16
16
|
const ROOT_PATH = path.resolve(__dirname, '../../');
|
|
17
17
|
|
|
18
|
-
export const SNAPSHOTS_SOURCE_PATH = path.resolve(ROOT_PATH, config.get('rewrite.snapshotsSourcePath'));
|
|
19
|
-
export const SNAPSHOTS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('recorder.snapshots.storage.git.path'));
|
|
20
|
-
export const VERSIONS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('recorder.versions.storage.git.path'));
|
|
18
|
+
export const SNAPSHOTS_SOURCE_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.rewrite.snapshotsSourcePath'));
|
|
19
|
+
export const SNAPSHOTS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.path'));
|
|
20
|
+
export const VERSIONS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
21
21
|
|
|
22
22
|
const initialize = process.argv.includes('--init');
|
|
23
23
|
|
|
@@ -33,7 +33,7 @@ let recorder;
|
|
|
33
33
|
console.log('Start rewritting history.');
|
|
34
34
|
|
|
35
35
|
await renamer.loadRules();
|
|
36
|
-
const sourceRepo = new Git({ path: SNAPSHOTS_SOURCE_PATH, author: config.get('recorder.snapshots.storage.git.author') });
|
|
36
|
+
const sourceRepo = new Git({ path: SNAPSHOTS_SOURCE_PATH, author: config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.author') });
|
|
37
37
|
|
|
38
38
|
await sourceRepo.initialize();
|
|
39
39
|
|
|
@@ -51,11 +51,11 @@ let recorder;
|
|
|
51
51
|
|
|
52
52
|
recorder = new Recorder({
|
|
53
53
|
versionsRepository: new GitRepository({
|
|
54
|
-
...config.get('recorder.versions.storage.git'),
|
|
54
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
55
55
|
path: VERSIONS_TARGET_PATH,
|
|
56
56
|
}),
|
|
57
57
|
snapshotsRepository: new GitRepository({
|
|
58
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
58
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
59
59
|
path: SNAPSHOTS_TARGET_PATH,
|
|
60
60
|
}),
|
|
61
61
|
});
|
|
@@ -21,9 +21,9 @@ const MARKDOWN_MIME_TYPE = 'text/markdown';
|
|
|
21
21
|
|
|
22
22
|
export const SNAPSHOTS_SOURCE_PATH = path.resolve(
|
|
23
23
|
ROOT_PATH,
|
|
24
|
-
config.get('rewrite.snapshotsSourcePath'),
|
|
24
|
+
config.get('@opentermsarchive/engine.rewrite.snapshotsSourcePath'),
|
|
25
25
|
);
|
|
26
|
-
export const VERSIONS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('recorder.versions.storage.git.path'));
|
|
26
|
+
export const VERSIONS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
27
27
|
|
|
28
28
|
const initialize = process.argv.includes('--init');
|
|
29
29
|
|
|
@@ -42,7 +42,7 @@ let recorder;
|
|
|
42
42
|
|
|
43
43
|
await renamer.loadRules();
|
|
44
44
|
const servicesDeclarations = await services.loadWithHistory();
|
|
45
|
-
const sourceRepo = new Git({ path: SNAPSHOTS_SOURCE_PATH, author: config.get('recorder.snapshots.storage.git.author') });
|
|
45
|
+
const sourceRepo = new Git({ path: SNAPSHOTS_SOURCE_PATH, author: config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.author') });
|
|
46
46
|
|
|
47
47
|
await sourceRepo.initialize();
|
|
48
48
|
|
|
@@ -60,11 +60,11 @@ let recorder;
|
|
|
60
60
|
|
|
61
61
|
recorder = new Recorder({
|
|
62
62
|
versionsRepository: new GitRepository({
|
|
63
|
-
...config.get('recorder.versions.storage.git'),
|
|
63
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
64
64
|
path: VERSIONS_TARGET_PATH,
|
|
65
65
|
}),
|
|
66
66
|
snapshotsRepository: new GitRepository({
|
|
67
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
67
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
68
68
|
path: SNAPSHOTS_SOURCE_PATH,
|
|
69
69
|
}),
|
|
70
70
|
});
|
|
@@ -24,9 +24,9 @@ export { FetchDocumentError } from './errors.js';
|
|
|
24
24
|
export default async function fetch({
|
|
25
25
|
url, executeClientScripts, cssSelectors,
|
|
26
26
|
config: {
|
|
27
|
-
navigationTimeout = config.get('fetcher.navigationTimeout'),
|
|
28
|
-
language = config.get('fetcher.language'),
|
|
29
|
-
waitForElementsTimeout = config.get('fetcher.waitForElementsTimeout'),
|
|
27
|
+
navigationTimeout = config.get('@opentermsarchive/engine.fetcher.navigationTimeout'),
|
|
28
|
+
language = config.get('@opentermsarchive/engine.fetcher.language'),
|
|
29
|
+
waitForElementsTimeout = config.get('@opentermsarchive/engine.fetcher.waitForElementsTimeout'),
|
|
30
30
|
} = {},
|
|
31
31
|
}) {
|
|
32
32
|
try {
|
|
@@ -20,8 +20,8 @@ const { expect } = chai;
|
|
|
20
20
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
21
21
|
|
|
22
22
|
const ROOT_PATH = path.resolve(__dirname, '../../');
|
|
23
|
-
const SNAPSHOTS_PATH = path.resolve(ROOT_PATH, config.get('recorder.snapshots.storage.git.path'));
|
|
24
|
-
const VERSIONS_PATH = path.resolve(ROOT_PATH, config.get('recorder.versions.storage.git.path'));
|
|
23
|
+
const SNAPSHOTS_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.path'));
|
|
24
|
+
const VERSIONS_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
25
25
|
|
|
26
26
|
const MIME_TYPE = 'text/html';
|
|
27
27
|
const FETCH_DATE = new Date('2000-01-02T12:00:00.000Z');
|
|
@@ -55,8 +55,8 @@ describe('Archivist', function () {
|
|
|
55
55
|
gitVersion = new Git({
|
|
56
56
|
path: VERSIONS_PATH,
|
|
57
57
|
author: {
|
|
58
|
-
name: config.get('recorder.versions.storage.git.author.name'),
|
|
59
|
-
email: config.get('recorder.versions.storage.git.author.email'),
|
|
58
|
+
name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
|
|
59
|
+
email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
|
|
60
60
|
},
|
|
61
61
|
});
|
|
62
62
|
await gitVersion.initialize();
|
|
@@ -72,8 +72,8 @@ describe('Archivist', function () {
|
|
|
72
72
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
73
73
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
74
74
|
app = new Archivist({
|
|
75
|
-
recorderConfig: config.get('recorder'),
|
|
76
|
-
fetcherConfig: config.get('fetcher'),
|
|
75
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
76
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
77
77
|
});
|
|
78
78
|
await app.initialize();
|
|
79
79
|
});
|
|
@@ -152,8 +152,8 @@ describe('Archivist', function () {
|
|
|
152
152
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
153
153
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
154
154
|
app = new Archivist({
|
|
155
|
-
recorderConfig: config.get('recorder'),
|
|
156
|
-
fetcherConfig: config.get('fetcher'),
|
|
155
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
156
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
157
157
|
});
|
|
158
158
|
|
|
159
159
|
await app.initialize();
|
|
@@ -212,8 +212,8 @@ describe('Archivist', function () {
|
|
|
212
212
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
213
213
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
214
214
|
app = new Archivist({
|
|
215
|
-
recorderConfig: config.get('recorder'),
|
|
216
|
-
fetcherConfig: config.get('fetcher'),
|
|
215
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
216
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
217
217
|
});
|
|
218
218
|
|
|
219
219
|
await app.initialize();
|
|
@@ -251,8 +251,8 @@ describe('Archivist', function () {
|
|
|
251
251
|
describe('#attach', () => {
|
|
252
252
|
before(async () => {
|
|
253
253
|
app = new Archivist({
|
|
254
|
-
recorderConfig: config.get('recorder'),
|
|
255
|
-
fetcherConfig: config.get('fetcher'),
|
|
254
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
255
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
256
256
|
});
|
|
257
257
|
await app.initialize();
|
|
258
258
|
|
|
@@ -287,8 +287,8 @@ describe('Archivist', function () {
|
|
|
287
287
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
288
288
|
|
|
289
289
|
app = new Archivist({
|
|
290
|
-
recorderConfig: config.get('recorder'),
|
|
291
|
-
fetcherConfig: config.get('fetcher'),
|
|
290
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
291
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
292
292
|
});
|
|
293
293
|
await app.initialize();
|
|
294
294
|
|
|
@@ -338,8 +338,8 @@ describe('Archivist', function () {
|
|
|
338
338
|
|
|
339
339
|
before(async () => {
|
|
340
340
|
app = new Archivist({
|
|
341
|
-
recorderConfig: config.get('recorder'),
|
|
342
|
-
fetcherConfig: config.get('fetcher'),
|
|
341
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
342
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
343
343
|
});
|
|
344
344
|
await app.initialize();
|
|
345
345
|
|
|
@@ -21,7 +21,7 @@ describe('Recorder', () => {
|
|
|
21
21
|
let recorder;
|
|
22
22
|
|
|
23
23
|
before(async () => {
|
|
24
|
-
const options = config.util.cloneDeep(config.recorder);
|
|
24
|
+
const options = config.util.cloneDeep(config.get('@opentermsarchive/engine.recorder'));
|
|
25
25
|
|
|
26
26
|
options.versions.storage.type = repositoryType;
|
|
27
27
|
options.snapshots.storage.type = repositoryType;
|
|
@@ -323,7 +323,7 @@ describe('Recorder', () => {
|
|
|
323
323
|
record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
|
|
324
324
|
});
|
|
325
325
|
|
|
326
|
-
after(() => recorder.versionsRepository.removeAll());
|
|
326
|
+
after(() => recorder.versionsRepository.removeAll());
|
|
327
327
|
|
|
328
328
|
it('records the version with the proper content', async () => {
|
|
329
329
|
expect(await record.content).to.equal(CONTENT);
|
|
@@ -8,7 +8,7 @@ import config from 'config';
|
|
|
8
8
|
import Git from './git.js';
|
|
9
9
|
|
|
10
10
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
11
|
-
const RECORDER_PATH = path.resolve(__dirname, '../../../../../', config.get('recorder.versions.storage.git.path'));
|
|
11
|
+
const RECORDER_PATH = path.resolve(__dirname, '../../../../../', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
12
12
|
|
|
13
13
|
describe('Git', () => {
|
|
14
14
|
const DEFAULT_CONTENT = 'default content';
|
|
@@ -19,8 +19,8 @@ describe('Git', () => {
|
|
|
19
19
|
subject = new Git({
|
|
20
20
|
path: RECORDER_PATH,
|
|
21
21
|
author: {
|
|
22
|
-
name: config.get('recorder.versions.storage.git.author.name'),
|
|
23
|
-
email: config.get('recorder.versions.storage.git.author.email'),
|
|
22
|
+
name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
|
|
23
|
+
email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
|
|
24
24
|
},
|
|
25
25
|
});
|
|
26
26
|
|
|
@@ -18,7 +18,7 @@ const { expect } = chai;
|
|
|
18
18
|
|
|
19
19
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
20
20
|
|
|
21
|
-
const RECORDER_PATH = path.resolve(__dirname, '../../../../..', config.get('recorder.versions.storage.git.path'));
|
|
21
|
+
const RECORDER_PATH = path.resolve(__dirname, '../../../../..', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
22
22
|
|
|
23
23
|
const SERVICE_PROVIDER_ID = 'test_service';
|
|
24
24
|
const TERMS_TYPE = 'Terms of Service';
|
|
@@ -52,15 +52,15 @@ describe('GitRepository', () => {
|
|
|
52
52
|
git = new Git({
|
|
53
53
|
path: RECORDER_PATH,
|
|
54
54
|
author: {
|
|
55
|
-
name: config.get('recorder.versions.storage.git.author.name'),
|
|
56
|
-
email: config.get('recorder.versions.storage.git.author.email'),
|
|
55
|
+
name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
|
|
56
|
+
email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
|
|
57
57
|
},
|
|
58
58
|
});
|
|
59
59
|
|
|
60
60
|
await git.initialize();
|
|
61
61
|
|
|
62
62
|
subject = new GitRepository({
|
|
63
|
-
...config.get('recorder.versions.storage.git'),
|
|
63
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
64
64
|
path: RECORDER_PATH,
|
|
65
65
|
});
|
|
66
66
|
|
|
@@ -266,7 +266,7 @@ describe('GitRepository', () => {
|
|
|
266
266
|
after(() => subject.removeAll());
|
|
267
267
|
|
|
268
268
|
it('stores snapshot ID', () => {
|
|
269
|
-
expect(commit.body).to.include(config.get('recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID));
|
|
269
|
+
expect(commit.body).to.include(config.get('@opentermsarchive/engine.recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID));
|
|
270
270
|
});
|
|
271
271
|
|
|
272
272
|
it('stores the service ID', () => {
|
|
@@ -298,8 +298,8 @@ describe('GitRepository', () => {
|
|
|
298
298
|
after(() => subject.removeAll());
|
|
299
299
|
|
|
300
300
|
it('stores snapshots IDs', () => {
|
|
301
|
-
expect(commit.body).to.include(config.get('recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID_1));
|
|
302
|
-
expect(commit.body).to.include(config.get('recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID_2));
|
|
301
|
+
expect(commit.body).to.include(config.get('@opentermsarchive/engine.recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID_1));
|
|
302
|
+
expect(commit.body).to.include(config.get('@opentermsarchive/engine.recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID_2));
|
|
303
303
|
});
|
|
304
304
|
|
|
305
305
|
it('stores the number of source documents', () => {
|
|
@@ -676,15 +676,15 @@ describe('GitRepository', () => {
|
|
|
676
676
|
git = new Git({
|
|
677
677
|
path: RECORDER_PATH,
|
|
678
678
|
author: {
|
|
679
|
-
name: config.get('recorder.snapshots.storage.git.author.name'),
|
|
680
|
-
email: config.get('recorder.snapshots.storage.git.author.email'),
|
|
679
|
+
name: config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.author.name'),
|
|
680
|
+
email: config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.author.email'),
|
|
681
681
|
},
|
|
682
682
|
});
|
|
683
683
|
|
|
684
684
|
await git.initialize();
|
|
685
685
|
|
|
686
686
|
subject = new GitRepository({
|
|
687
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
687
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
688
688
|
path: RECORDER_PATH,
|
|
689
689
|
});
|
|
690
690
|
|
|
@@ -1257,14 +1257,14 @@ describe('GitRepository', () => {
|
|
|
1257
1257
|
git = new Git({
|
|
1258
1258
|
path: RECORDER_PATH,
|
|
1259
1259
|
author: {
|
|
1260
|
-
name: config.get('recorder.versions.storage.git.author.name'),
|
|
1261
|
-
email: config.get('recorder.versions.storage.git.author.email'),
|
|
1260
|
+
name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
|
|
1261
|
+
email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
|
|
1262
1262
|
},
|
|
1263
1263
|
});
|
|
1264
1264
|
|
|
1265
1265
|
await git.initialize();
|
|
1266
1266
|
subject = new GitRepository({
|
|
1267
|
-
...config.get('recorder.versions.storage.git'),
|
|
1267
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
1268
1268
|
path: RECORDER_PATH,
|
|
1269
1269
|
});
|
|
1270
1270
|
|
|
@@ -15,7 +15,7 @@ import MongoRepository from './index.js';
|
|
|
15
15
|
const { expect } = chai;
|
|
16
16
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
17
17
|
|
|
18
|
-
const { connectionURI } = config.get('recorder.snapshots.storage.mongo');
|
|
18
|
+
const { connectionURI } = config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo');
|
|
19
19
|
const client = new MongoClient(connectionURI);
|
|
20
20
|
|
|
21
21
|
const SERVICE_PROVIDER_ID = 'test_service';
|
|
@@ -41,12 +41,12 @@ describe('MongoRepository', () => {
|
|
|
41
41
|
|
|
42
42
|
context('Version', () => {
|
|
43
43
|
before(async () => {
|
|
44
|
-
subject = new MongoRepository(config.get('recorder.versions.storage.mongo'));
|
|
44
|
+
subject = new MongoRepository(config.get('@opentermsarchive/engine.recorder.versions.storage.mongo'));
|
|
45
45
|
await subject.initialize();
|
|
46
46
|
await client.connect();
|
|
47
|
-
const db = client.db(config.get('recorder.versions.storage.mongo.database'));
|
|
47
|
+
const db = client.db(config.get('@opentermsarchive/engine.recorder.versions.storage.mongo.database'));
|
|
48
48
|
|
|
49
|
-
collection = db.collection(config.get('recorder.versions.storage.mongo.collection'));
|
|
49
|
+
collection = db.collection(config.get('@opentermsarchive/engine.recorder.versions.storage.mongo.collection'));
|
|
50
50
|
});
|
|
51
51
|
|
|
52
52
|
describe('#save', () => {
|
|
@@ -652,12 +652,12 @@ describe('MongoRepository', () => {
|
|
|
652
652
|
|
|
653
653
|
context('Snapshot', () => {
|
|
654
654
|
before(async () => {
|
|
655
|
-
subject = new MongoRepository(config.get('recorder.snapshots.storage.mongo'));
|
|
655
|
+
subject = new MongoRepository(config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo'));
|
|
656
656
|
await subject.initialize();
|
|
657
657
|
await client.connect();
|
|
658
|
-
const db = client.db(config.get('recorder.snapshots.storage.mongo.database'));
|
|
658
|
+
const db = client.db(config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo.database'));
|
|
659
659
|
|
|
660
|
-
collection = db.collection(config.get('recorder.snapshots.storage.mongo.collection'));
|
|
660
|
+
collection = db.collection(config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo.collection'));
|
|
661
661
|
});
|
|
662
662
|
|
|
663
663
|
describe('#save', () => {
|
|
@@ -9,7 +9,7 @@ import SourceDocument from './sourceDocument.js';
|
|
|
9
9
|
import Terms from './terms.js';
|
|
10
10
|
|
|
11
11
|
const fs = fsApi.promises;
|
|
12
|
-
const declarationsPath = path.resolve(process.cwd(), config.get('services.declarationsPath'));
|
|
12
|
+
const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
|
|
13
13
|
|
|
14
14
|
export async function load(servicesIdsToLoad = []) {
|
|
15
15
|
let servicesIds = await getDeclaredServicesIds();
|
|
@@ -11,12 +11,12 @@ const { combine, timestamp, printf, colorize } = winston.format;
|
|
|
11
11
|
|
|
12
12
|
const transports = [new winston.transports.Console()];
|
|
13
13
|
|
|
14
|
-
if (config.get('logger.sendMailOnError')) {
|
|
14
|
+
if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
|
|
15
15
|
transports.push(new winston.transports.Mail({
|
|
16
|
-
to: config.get('logger.sendMailOnError.to'),
|
|
17
|
-
from: config.get('logger.sendMailOnError.from'),
|
|
18
|
-
host: config.get('logger.smtp.host'),
|
|
19
|
-
username: config.get('logger.smtp.username'),
|
|
16
|
+
to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
|
|
17
|
+
from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
|
|
18
|
+
host: config.get('@opentermsarchive/engine.logger.smtp.host'),
|
|
19
|
+
username: config.get('@opentermsarchive/engine.logger.smtp.username'),
|
|
20
20
|
password: process.env.SMTP_PASSWORD,
|
|
21
21
|
ssl: true,
|
|
22
22
|
timeout: 30 * 1000,
|
|
@@ -4,7 +4,7 @@ import request from 'supertest';
|
|
|
4
4
|
|
|
5
5
|
import app from '../server.js';
|
|
6
6
|
|
|
7
|
-
const basePath = config.get('api.basePath');
|
|
7
|
+
const basePath = config.get('@opentermsarchive/engine.collection-api.basePath');
|
|
8
8
|
|
|
9
9
|
describe('Services API', () => {
|
|
10
10
|
describe('GET /services', () => {
|
|
@@ -28,7 +28,7 @@ import { toISODateWithoutMilliseconds } from '../../archivist/utils/date.js';
|
|
|
28
28
|
*/
|
|
29
29
|
const router = express.Router();
|
|
30
30
|
|
|
31
|
-
const versionsRepository = await RepositoryFactory.create(config.get('recorder.versions.storage')).initialize();
|
|
31
|
+
const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage')).initialize();
|
|
32
32
|
|
|
33
33
|
/**
|
|
34
34
|
* @swagger
|
|
@@ -7,7 +7,7 @@ import Version from '../../archivist/recorder/version.js';
|
|
|
7
7
|
import { toISODateWithoutMilliseconds } from '../../archivist/utils/date.js';
|
|
8
8
|
import app from '../server.js';
|
|
9
9
|
|
|
10
|
-
const basePath = config.get('api.basePath');
|
|
10
|
+
const basePath = config.get('@opentermsarchive/engine.collection-api.basePath');
|
|
11
11
|
|
|
12
12
|
const { expect } = chai;
|
|
13
13
|
const request = supertest(app);
|
|
@@ -24,7 +24,7 @@ describe('Versions API', () => {
|
|
|
24
24
|
};
|
|
25
25
|
|
|
26
26
|
before(async () => {
|
|
27
|
-
versionsRepository = RepositoryFactory.create(config.get('recorder.versions.storage'));
|
|
27
|
+
versionsRepository = RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage'));
|
|
28
28
|
|
|
29
29
|
await versionsRepository.initialize();
|
|
30
30
|
|
|
@@ -13,12 +13,12 @@ if (process.env.NODE_ENV !== 'test') {
|
|
|
13
13
|
app.use(loggerMiddleware);
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
const BASE_PATH =
|
|
16
|
+
const BASE_PATH = `/${config.get('@opentermsarchive/engine.collection-api.basePath')}/v1`.replace(/\/\/+/g, '/'); // ensure there are no double slashes
|
|
17
17
|
|
|
18
18
|
app.use(BASE_PATH, apiRouter(BASE_PATH));
|
|
19
19
|
app.use(errorsMiddleware);
|
|
20
20
|
|
|
21
|
-
const port = config.get('api.port');
|
|
21
|
+
const port = config.get('@opentermsarchive/engine.collection-api.port');
|
|
22
22
|
|
|
23
23
|
app.listen(port);
|
|
24
24
|
|
package/src/index.js
CHANGED
|
@@ -8,8 +8,8 @@ import Reporter from './reporter/index.js';
|
|
|
8
8
|
|
|
9
9
|
export default async function track({ services, types, extractOnly, schedule }) {
|
|
10
10
|
const archivist = new Archivist({
|
|
11
|
-
recorderConfig: config.get('recorder'),
|
|
12
|
-
fetcherConfig: config.get('fetcher'),
|
|
11
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
12
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
13
13
|
});
|
|
14
14
|
|
|
15
15
|
archivist.attach(logger);
|
|
@@ -38,20 +38,20 @@ export default async function track({ services, types, extractOnly, schedule })
|
|
|
38
38
|
return;
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
if (process.env.
|
|
41
|
+
if (process.env.OTA_ENGINE_SENDINBLUE_API_KEY) {
|
|
42
42
|
try {
|
|
43
43
|
archivist.attach(new Notifier(archivist.services));
|
|
44
44
|
} catch (error) {
|
|
45
45
|
logger.error('Cannot instantiate the Notifier module; it will be ignored:', error);
|
|
46
46
|
}
|
|
47
47
|
} else {
|
|
48
|
-
logger.warn('Environment variable "
|
|
48
|
+
logger.warn('Environment variable "OTA_ENGINE_SENDINBLUE_API_KEY" was not found; the Notifier module will be ignored');
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
if (process.env.
|
|
51
|
+
if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
|
|
52
52
|
if (config.has('reporter.githubIssues.repositories.declarations')) {
|
|
53
53
|
try {
|
|
54
|
-
const reporter = new Reporter(config.get('reporter'));
|
|
54
|
+
const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
|
|
55
55
|
|
|
56
56
|
await reporter.initialize();
|
|
57
57
|
archivist.attach(reporter);
|
|
@@ -62,7 +62,7 @@ export default async function track({ services, types, extractOnly, schedule })
|
|
|
62
62
|
logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
|
|
63
63
|
}
|
|
64
64
|
} else {
|
|
65
|
-
logger.warn('Environment variable "
|
|
65
|
+
logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored');
|
|
66
66
|
}
|
|
67
67
|
|
|
68
68
|
if (!schedule) {
|
package/src/logger/index.js
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import os from 'os';
|
|
2
2
|
|
|
3
3
|
import config from 'config';
|
|
4
|
-
import dotenv from 'dotenv';
|
|
5
4
|
import winston from 'winston';
|
|
6
5
|
import 'winston-mail';
|
|
7
6
|
|
|
8
|
-
dotenv.config();
|
|
9
7
|
const { combine, timestamp, printf, colorize } = winston.format;
|
|
10
8
|
|
|
11
9
|
const alignedWithColorsAndTime = combine(
|
|
@@ -40,16 +38,16 @@ const logger = winston.createLogger({
|
|
|
40
38
|
rejectionHandlers: transports,
|
|
41
39
|
});
|
|
42
40
|
|
|
43
|
-
if (config.get('logger.sendMailOnError')) {
|
|
44
|
-
if (process.env.
|
|
45
|
-
logger.warn('Environment variable "
|
|
41
|
+
if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
|
|
42
|
+
if (process.env.OTA_ENGINE_SMTP_PASSWORD === undefined) {
|
|
43
|
+
logger.warn('Environment variable "OTA_ENGINE_SMTP_PASSWORD" was not found; log emails cannot be sent');
|
|
46
44
|
} else {
|
|
47
45
|
const mailerOptions = {
|
|
48
|
-
to: config.get('logger.sendMailOnError.to'),
|
|
49
|
-
from: config.get('logger.sendMailOnError.from'),
|
|
50
|
-
host: config.get('logger.smtp.host'),
|
|
51
|
-
username: config.get('logger.smtp.username'),
|
|
52
|
-
password: process.env.
|
|
46
|
+
to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
|
|
47
|
+
from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
|
|
48
|
+
host: config.get('@opentermsarchive/engine.logger.smtp.host'),
|
|
49
|
+
username: config.get('@opentermsarchive/engine.logger.smtp.username'),
|
|
50
|
+
password: process.env.OTA_ENGINE_SMTP_PASSWORD,
|
|
53
51
|
ssl: true,
|
|
54
52
|
timeout: 30 * 1000,
|
|
55
53
|
formatter: args => args[Object.getOwnPropertySymbols(args)[1]], // Returns the full error message, the same visible in the console. It is referenced in the argument object with a Symbol of which we do not have the reference but we know it is the second one.
|
|
@@ -62,7 +60,7 @@ if (config.get('logger.sendMailOnError')) {
|
|
|
62
60
|
subject: `[OTA] Error Report — ${os.hostname()}`,
|
|
63
61
|
}));
|
|
64
62
|
|
|
65
|
-
if (config.get('logger.sendMailOnError.sendWarnings')) {
|
|
63
|
+
if (config.get('@opentermsarchive/engine.logger.sendMailOnError.sendWarnings')) {
|
|
66
64
|
transports.push(new winston.transports.Mail({
|
|
67
65
|
...mailerOptions,
|
|
68
66
|
level: 'warn',
|
package/src/notifier/index.js
CHANGED
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import config from 'config';
|
|
2
|
-
import dotenv from 'dotenv';
|
|
3
2
|
import sendInBlue from 'sib-api-v3-sdk';
|
|
4
3
|
|
|
5
|
-
dotenv.config();
|
|
6
4
|
export default class Notifier {
|
|
7
5
|
constructor(passedServiceProviders) {
|
|
8
6
|
const defaultClient = sendInBlue.ApiClient.instance;
|
|
9
7
|
const authentication = defaultClient.authentications['api-key'];
|
|
10
8
|
|
|
11
|
-
authentication.apiKey = process.env.
|
|
9
|
+
authentication.apiKey = process.env.OTA_ENGINE_SENDINBLUE_API_KEY;
|
|
12
10
|
|
|
13
11
|
this.apiInstance = new sendInBlue.TransactionalEmailsApi();
|
|
14
12
|
this.contactsInstance = new sendInBlue.ContactsApi();
|
|
@@ -35,7 +33,7 @@ export default class Notifier {
|
|
|
35
33
|
|
|
36
34
|
async notifyVersionRecorded(serviceProviderId, termsType, versionId) {
|
|
37
35
|
const sendParams = {
|
|
38
|
-
templateId: config.get('notifier.sendInBlue.updateTemplateId'),
|
|
36
|
+
templateId: config.get('@opentermsarchive/engine.notifier.sendInBlue.updateTemplateId'),
|
|
39
37
|
params: {
|
|
40
38
|
SERVICE_PROVIDER_NAME: this.serviceProviders[serviceProviderId].name,
|
|
41
39
|
DOCUMENT_TYPE: termsType,
|
|
@@ -44,7 +42,7 @@ export default class Notifier {
|
|
|
44
42
|
};
|
|
45
43
|
|
|
46
44
|
const lists = [
|
|
47
|
-
config.get('notifier.sendInBlue.updatesListId'),
|
|
45
|
+
config.get('@opentermsarchive/engine.notifier.sendInBlue.updatesListId'),
|
|
48
46
|
];
|
|
49
47
|
|
|
50
48
|
const notificationListName = `${this.serviceProviders[serviceProviderId].name} - ${termsType} - Update`;
|
package/src/reporter/github.js
CHANGED
|
@@ -16,7 +16,7 @@ export default class GitHub {
|
|
|
16
16
|
constructor(repository) {
|
|
17
17
|
const { version } = require('../../package.json');
|
|
18
18
|
|
|
19
|
-
this.octokit = new Octokit({ auth: process.env.
|
|
19
|
+
this.octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN, userAgent: `opentermsarchive/${version}` });
|
|
20
20
|
|
|
21
21
|
const [ owner, repo ] = repository.split('/');
|
|
22
22
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|