@opentermsarchive/engine 1.2.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/env.js +2 -0
- package/bin/ota-dataset.js +1 -1
- package/bin/ota-serve.js +1 -1
- package/config/ci.json +4 -2
- package/config/default.json +53 -55
- package/config/production.json +21 -19
- package/config/test.json +46 -40
- package/package.json +4 -4
- package/scripts/dataset/assets/README.template.js +2 -2
- package/scripts/dataset/export/index.js +17 -16
- package/scripts/dataset/export/index.test.js +2 -2
- package/scripts/dataset/index.js +1 -1
- package/scripts/dataset/publish/index.js +2 -5
- package/scripts/declarations/lint/index.mocha.js +1 -1
- package/scripts/declarations/validate/index.mocha.js +3 -3
- package/scripts/history/migrate-services.js +8 -8
- package/scripts/history/update-to-full-hash.js +3 -3
- package/scripts/import/index.js +6 -6
- package/scripts/import/loadCommits.js +4 -4
- package/scripts/rewrite/initializer/index.js +1 -1
- package/scripts/rewrite/rewrite-snapshots.js +6 -6
- package/scripts/rewrite/rewrite-versions.js +5 -5
- package/scripts/utils/renamer/index.js +26 -24
- package/scripts/utils/renamer/rules/serviceNames.json +6 -1
- package/src/archivist/fetcher/index.js +3 -3
- package/src/archivist/index.test.js +16 -16
- package/src/archivist/recorder/index.test.js +2 -2
- package/src/archivist/recorder/repositories/git/git.test.js +3 -3
- package/src/archivist/recorder/repositories/git/index.test.js +13 -13
- package/src/archivist/recorder/repositories/mongo/index.test.js +7 -7
- package/src/archivist/services/index.js +1 -1
- package/src/{api → collection-api}/logger.js +5 -5
- package/src/{api → collection-api}/routes/docs.test.js +1 -1
- package/src/{api → collection-api}/routes/services.test.js +1 -1
- package/src/{api → collection-api}/routes/versions.js +1 -1
- package/src/{api → collection-api}/routes/versions.test.js +2 -2
- package/src/{api → collection-api}/server.js +2 -2
- package/src/index.js +7 -7
- package/src/logger/index.js +9 -11
- package/src/notifier/index.js +3 -5
- package/src/reporter/github.js +1 -1
- /package/src/{api → collection-api}/middlewares/errors.js +0 -0
- /package/src/{api → collection-api}/middlewares/logger.js +0 -0
- /package/src/{api → collection-api}/routes/docs.js +0 -0
- /package/src/{api → collection-api}/routes/index.js +0 -0
- /package/src/{api → collection-api}/routes/services.js +0 -0
package/bin/env.js
CHANGED
package/bin/ota-dataset.js
CHANGED
|
@@ -11,7 +11,7 @@ program
|
|
|
11
11
|
.name('ota dataset')
|
|
12
12
|
.description('Export the versions dataset into a ZIP file and optionally publish it to GitHub releases')
|
|
13
13
|
.option('-f, --file <filename>', 'file name of the generated dataset')
|
|
14
|
-
.option('-p, --publish', 'publish dataset to GitHub releases on versions repository. Mandatory authentication to GitHub is provided through the `
|
|
14
|
+
.option('-p, --publish', 'publish dataset to GitHub releases on versions repository. Mandatory authentication to GitHub is provided through the `OTA_ENGINE_GITHUB_TOKEN` environment variable')
|
|
15
15
|
.option('-r, --remove-local-copy', 'remove local copy of dataset after publishing. Works only in combination with --publish option')
|
|
16
16
|
.option('--schedule', 'schedule automatic dataset generation');
|
|
17
17
|
|
package/bin/ota-serve.js
CHANGED
|
@@ -6,4 +6,4 @@ import { fileURLToPath, pathToFileURL } from 'url';
|
|
|
6
6
|
|
|
7
7
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
8
8
|
|
|
9
|
-
await import(pathToFileURL(path.resolve(__dirname, '../src/api/server.js'))); // load asynchronously to ensure env.js is loaded before
|
|
9
|
+
await import(pathToFileURL(path.resolve(__dirname, '../src/collection-api/server.js'))); // load asynchronously to ensure env.js is loaded before
|
package/config/ci.json
CHANGED
package/config/default.json
CHANGED
|
@@ -1,65 +1,63 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
"
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
"
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
|
|
10
|
-
"
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
"
|
|
15
|
-
|
|
2
|
+
"@opentermsarchive/engine": {
|
|
3
|
+
"services": {
|
|
4
|
+
"declarationsPath": "./declarations"
|
|
5
|
+
},
|
|
6
|
+
"recorder": {
|
|
7
|
+
"versions": {
|
|
8
|
+
"storage": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"git": {
|
|
11
|
+
"path": "./data/versions",
|
|
12
|
+
"publish": false,
|
|
13
|
+
"snapshotIdentiferTemplate": "./data/snapshots/%SNAPSHOT_ID",
|
|
14
|
+
"author": {
|
|
15
|
+
"name": "Open Terms Archive Bot",
|
|
16
|
+
"email": "bot@opentermsarchive.org"
|
|
17
|
+
}
|
|
16
18
|
}
|
|
17
19
|
}
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
20
|
+
},
|
|
21
|
+
"snapshots": {
|
|
22
|
+
"storage": {
|
|
23
|
+
"type": "git",
|
|
24
|
+
"git": {
|
|
25
|
+
"path": "./data/snapshots",
|
|
26
|
+
"publish": false,
|
|
27
|
+
"author": {
|
|
28
|
+
"name": "Open Terms Archive Bot",
|
|
29
|
+
"email": "bot@opentermsarchive.org"
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
"mongo": {
|
|
33
|
+
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
34
|
+
"database": "open-terms-archive",
|
|
35
|
+
"collection": "snapshots"
|
|
29
36
|
}
|
|
30
|
-
},
|
|
31
|
-
"mongo": {
|
|
32
|
-
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
33
|
-
"database": "open-terms-archive",
|
|
34
|
-
"collection": "snapshots"
|
|
35
37
|
}
|
|
36
38
|
}
|
|
37
|
-
}
|
|
38
|
-
},
|
|
39
|
-
"fetcher": {
|
|
40
|
-
"waitForElementsTimeout": 10000,
|
|
41
|
-
"navigationTimeout": 30000,
|
|
42
|
-
"language": "en"
|
|
43
|
-
},
|
|
44
|
-
"logger": {
|
|
45
|
-
"smtp": {
|
|
46
|
-
"host": "smtp-relay.sendinblue.com",
|
|
47
|
-
"username": "admin@opentermsarchive.org"
|
|
48
39
|
},
|
|
49
|
-
"
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
40
|
+
"fetcher": {
|
|
41
|
+
"waitForElementsTimeout": 10000,
|
|
42
|
+
"navigationTimeout": 30000,
|
|
43
|
+
"language": "en"
|
|
44
|
+
},
|
|
45
|
+
"logger": {
|
|
46
|
+
"smtp": {
|
|
47
|
+
"host": "smtp-relay.sendinblue.com",
|
|
48
|
+
"username": "admin@opentermsarchive.org"
|
|
49
|
+
},
|
|
50
|
+
"sendMailOnError": false
|
|
51
|
+
},
|
|
52
|
+
"notifier": {
|
|
53
|
+
"sendInBlue": {
|
|
54
|
+
"updatesListId": 850,
|
|
55
|
+
"updateTemplateId": 7
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
"dataset": {
|
|
59
|
+
"title": "sandbox",
|
|
60
|
+
"versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox"
|
|
55
61
|
}
|
|
56
|
-
},
|
|
57
|
-
"dataset": {
|
|
58
|
-
"title": "sandbox",
|
|
59
|
-
"versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox"
|
|
60
|
-
},
|
|
61
|
-
"api": {
|
|
62
|
-
"port": 3000,
|
|
63
|
-
"basePath": "/api"
|
|
64
62
|
}
|
|
65
63
|
}
|
package/config/production.json
CHANGED
|
@@ -1,27 +1,29 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
"
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
|
|
2
|
+
"@opentermsarchive/engine": {
|
|
3
|
+
"recorder": {
|
|
4
|
+
"versions": {
|
|
5
|
+
"storage": {
|
|
6
|
+
"git": {
|
|
7
|
+
"path": "../versions",
|
|
8
|
+
"publish": true
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
},
|
|
12
|
+
"snapshots": {
|
|
13
|
+
"storage": {
|
|
14
|
+
"git": {
|
|
15
|
+
"path": "../snapshots",
|
|
16
|
+
"publish": true
|
|
17
|
+
}
|
|
8
18
|
}
|
|
9
19
|
}
|
|
10
20
|
},
|
|
11
|
-
"
|
|
12
|
-
"
|
|
13
|
-
"
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
}
|
|
21
|
+
"logger": {
|
|
22
|
+
"sendMailOnError": {
|
|
23
|
+
"to": "admin@opentermsarchive.org",
|
|
24
|
+
"from": "noreply@opentermsarchive.org",
|
|
25
|
+
"sendWarnings": false
|
|
17
26
|
}
|
|
18
27
|
}
|
|
19
|
-
},
|
|
20
|
-
"logger": {
|
|
21
|
-
"sendMailOnError": {
|
|
22
|
-
"to": "admin@opentermsarchive.org",
|
|
23
|
-
"from": "noreply@opentermsarchive.org",
|
|
24
|
-
"sendWarnings": false
|
|
25
|
-
}
|
|
26
28
|
}
|
|
27
29
|
}
|
package/config/test.json
CHANGED
|
@@ -1,49 +1,55 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
"
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
"
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
"
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
"
|
|
14
|
-
|
|
2
|
+
"@opentermsarchive/engine": {
|
|
3
|
+
"services": {
|
|
4
|
+
"declarationsPath": "./test/services"
|
|
5
|
+
},
|
|
6
|
+
"recorder": {
|
|
7
|
+
"versions": {
|
|
8
|
+
"storage": {
|
|
9
|
+
"git": {
|
|
10
|
+
"path": "./test/data/versions",
|
|
11
|
+
"publish": false,
|
|
12
|
+
"snapshotIdentiferTemplate": "https://github.com/OpenTermsArchive/sandbox-snapshots/commit/%SNAPSHOT_ID",
|
|
13
|
+
"author": {
|
|
14
|
+
"name": "Open Terms Archive Testing Bot",
|
|
15
|
+
"email": "bot@opentermsarchive.org"
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"mongo": {
|
|
19
|
+
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
20
|
+
"database": "open-terms-archive-test",
|
|
21
|
+
"collection": "versions"
|
|
15
22
|
}
|
|
16
|
-
},
|
|
17
|
-
"mongo": {
|
|
18
|
-
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
19
|
-
"database": "open-terms-archive-test",
|
|
20
|
-
"collection": "versions"
|
|
21
23
|
}
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
24
|
+
},
|
|
25
|
+
"snapshots": {
|
|
26
|
+
"storage": {
|
|
27
|
+
"git": {
|
|
28
|
+
"path": "./test/data/snapshots",
|
|
29
|
+
"publish": false,
|
|
30
|
+
"author": {
|
|
31
|
+
"name": "Open Terms Archive Testing Bot",
|
|
32
|
+
"email": "bot@opentermsarchive.org"
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"mongo": {
|
|
36
|
+
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
37
|
+
"database": "open-terms-archive-test",
|
|
38
|
+
"collection": "snapshots"
|
|
32
39
|
}
|
|
33
|
-
},
|
|
34
|
-
"mongo": {
|
|
35
|
-
"connectionURI": "mongodb://127.0.0.1:27017",
|
|
36
|
-
"database": "open-terms-archive-test",
|
|
37
|
-
"collection": "snapshots"
|
|
38
40
|
}
|
|
39
41
|
}
|
|
42
|
+
},
|
|
43
|
+
"fetcher": {
|
|
44
|
+
"waitForElementsTimeout": 1000
|
|
45
|
+
},
|
|
46
|
+
"dataset": {
|
|
47
|
+
"title": "sandbox",
|
|
48
|
+
"versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox-versions"
|
|
49
|
+
},
|
|
50
|
+
"collection-api": {
|
|
51
|
+
"port": 3000,
|
|
52
|
+
"basePath": "/collection-api"
|
|
40
53
|
}
|
|
41
|
-
},
|
|
42
|
-
"fetcher": {
|
|
43
|
-
"waitForElementsTimeout": 1000
|
|
44
|
-
},
|
|
45
|
-
"dataset": {
|
|
46
|
-
"title": "sandbox",
|
|
47
|
-
"versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox-versions"
|
|
48
54
|
}
|
|
49
55
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opentermsarchive/engine",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "Tracks and makes visible changes to the terms of online services",
|
|
5
5
|
"homepage": "https://opentermsarchive.org",
|
|
6
6
|
"bugs": {
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"declarations:validate:schema": "npm run declarations:validate -- --schema-only",
|
|
41
41
|
"lint": "eslint src test scripts bin",
|
|
42
42
|
"lint:fix": "npm run lint -- --fix",
|
|
43
|
-
"start": "node --max-http-header-size=32768 bin/ota.js track",
|
|
43
|
+
"start": "node -r dotenv/config --max-http-header-size=32768 bin/ota.js track",
|
|
44
44
|
"start:api": "node bin/ota.js serve",
|
|
45
45
|
"start:scheduler": "npm start -- --schedule",
|
|
46
46
|
"test": "cross-env NODE_ENV=test mocha --recursive \"./src/**/*.test.js\" \"./scripts/**/*.test.js\" --exit",
|
|
@@ -72,7 +72,7 @@
|
|
|
72
72
|
"eslint-plugin-import": "^2.29.0",
|
|
73
73
|
"eslint-plugin-json-format": "^2.0.1",
|
|
74
74
|
"eslint-plugin-no-only-tests": "^3.1.0",
|
|
75
|
-
"express": "^4.
|
|
75
|
+
"express": "^4.19.2",
|
|
76
76
|
"express-async-errors": "^3.1.1",
|
|
77
77
|
"fs-extra": "^10.0.0",
|
|
78
78
|
"helmet": "^6.0.1",
|
|
@@ -97,7 +97,7 @@
|
|
|
97
97
|
"sib-api-v3-sdk": "^8.2.1",
|
|
98
98
|
"simple-git": "^3.8.0",
|
|
99
99
|
"swagger-jsdoc": "^6.2.8",
|
|
100
|
-
"swagger-ui-express": "
|
|
100
|
+
"swagger-ui-express": "^5.0.0",
|
|
101
101
|
"winston": "^3.3.3",
|
|
102
102
|
"winston-mail": "^2.0.0"
|
|
103
103
|
},
|
|
@@ -12,7 +12,7 @@ ${body({ servicesCount, firstVersionDate, lastVersionDate })}`;
|
|
|
12
12
|
export function title({ releaseDate }) {
|
|
13
13
|
releaseDate = releaseDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
|
|
14
14
|
|
|
15
|
-
const title = config.get('dataset.title');
|
|
15
|
+
const title = config.get('@opentermsarchive/engine.dataset.title');
|
|
16
16
|
|
|
17
17
|
return `${title} — ${releaseDate} dataset`;
|
|
18
18
|
}
|
|
@@ -21,7 +21,7 @@ export function body({ servicesCount, firstVersionDate, lastVersionDate }) {
|
|
|
21
21
|
firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
|
|
22
22
|
lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
|
|
23
23
|
|
|
24
|
-
const versionsRepositoryURL = config.get('dataset.versionsRepositoryURL');
|
|
24
|
+
const versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL');
|
|
25
25
|
|
|
26
26
|
return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}.
|
|
27
27
|
|
|
@@ -17,7 +17,7 @@ const fs = fsApi.promises;
|
|
|
17
17
|
const ARCHIVE_FORMAT = 'zip'; // for supported formats, see https://www.archiverjs.com/docs/archive-formats
|
|
18
18
|
|
|
19
19
|
export default async function generate({ archivePath, releaseDate }) {
|
|
20
|
-
const versionsRepository = await RepositoryFactory.create(config.get('recorder.versions.storage')).initialize();
|
|
20
|
+
const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage')).initialize();
|
|
21
21
|
|
|
22
22
|
const archive = await initializeArchive(archivePath);
|
|
23
23
|
|
|
@@ -31,27 +31,28 @@ export default async function generate({ archivePath, releaseDate }) {
|
|
|
31
31
|
|
|
32
32
|
for await (const version of versionsRepository.iterate()) {
|
|
33
33
|
const { content, fetchDate } = version;
|
|
34
|
-
const { serviceId, termsType } = renamer.applyRules(version.serviceId, version.termsType);
|
|
35
34
|
|
|
36
|
-
|
|
37
|
-
firstVersionDate
|
|
38
|
-
|
|
35
|
+
for (const { serviceId, termsType } of renamer.applyRules(version.serviceId, version.termsType)) {
|
|
36
|
+
if (firstVersionDate > fetchDate) {
|
|
37
|
+
firstVersionDate = fetchDate;
|
|
38
|
+
}
|
|
39
39
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
40
|
+
if (fetchDate > lastVersionDate) {
|
|
41
|
+
lastVersionDate = fetchDate;
|
|
42
|
+
}
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
services.add(serviceId);
|
|
45
45
|
|
|
46
|
-
|
|
46
|
+
const versionPath = generateVersionPath({ serviceId, termsType, fetchDate });
|
|
47
47
|
|
|
48
|
-
|
|
48
|
+
logger.info({ message: versionPath, counter: index, hash: version.id });
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
50
|
+
archive.stream.append(
|
|
51
|
+
content,
|
|
52
|
+
{ name: `${archive.basename}/${versionPath}` },
|
|
53
|
+
);
|
|
54
|
+
index++;
|
|
55
|
+
}
|
|
55
56
|
}
|
|
56
57
|
|
|
57
58
|
archive.stream.append(
|
|
@@ -48,8 +48,8 @@ describe('Export', () => {
|
|
|
48
48
|
before(async function () {
|
|
49
49
|
this.timeout(10000);
|
|
50
50
|
repository = new GitRepository({
|
|
51
|
-
...config.get('recorder.versions.storage.git'),
|
|
52
|
-
path: path.resolve(__dirname, '../../../', config.get('recorder.versions.storage.git.path')),
|
|
51
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
52
|
+
path: path.resolve(__dirname, '../../../', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path')),
|
|
53
53
|
});
|
|
54
54
|
|
|
55
55
|
await repository.initialize();
|
package/scripts/dataset/index.js
CHANGED
|
@@ -9,7 +9,7 @@ import publishRelease from './publish/index.js';
|
|
|
9
9
|
|
|
10
10
|
export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }) {
|
|
11
11
|
const releaseDate = new Date();
|
|
12
|
-
const archiveName = fileName || `dataset-${config.get('dataset.title')}-${releaseDate.toISOString().replace(/T.*/, '')}`;
|
|
12
|
+
const archiveName = fileName || `dataset-${config.get('@opentermsarchive/engine.dataset.title')}-${releaseDate.toISOString().replace(/T.*/, '')}`;
|
|
13
13
|
const archivePath = `${path.basename(archiveName, '.zip')}.zip`; // allow to pass filename or filename.zip as the archive name and have filename.zip as the result name
|
|
14
14
|
|
|
15
15
|
logger.info('Start exporting dataset…');
|
|
@@ -3,17 +3,14 @@ import path from 'path';
|
|
|
3
3
|
import url from 'url';
|
|
4
4
|
|
|
5
5
|
import config from 'config';
|
|
6
|
-
import dotenv from 'dotenv';
|
|
7
6
|
import { Octokit } from 'octokit';
|
|
8
7
|
|
|
9
8
|
import * as readme from '../assets/README.template.js';
|
|
10
9
|
|
|
11
|
-
dotenv.config();
|
|
12
|
-
|
|
13
10
|
export default async function publish({ archivePath, releaseDate, stats }) {
|
|
14
|
-
const octokit = new Octokit({ auth: process.env.
|
|
11
|
+
const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
|
|
15
12
|
|
|
16
|
-
const [ owner, repo ] = url.parse(config.get('dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
|
|
13
|
+
const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
|
|
17
14
|
|
|
18
15
|
const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
|
|
19
16
|
|
|
@@ -16,7 +16,7 @@ const ESLINT_CONFIG_PATH = path.join(ROOT_PATH, '.eslintrc.yaml');
|
|
|
16
16
|
const eslint = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: false });
|
|
17
17
|
const eslintWithFix = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true });
|
|
18
18
|
|
|
19
|
-
const declarationsPath = path.resolve(process.cwd(), config.get('services.declarationsPath'));
|
|
19
|
+
const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
|
|
20
20
|
const instancePath = path.resolve(declarationsPath, '../');
|
|
21
21
|
|
|
22
22
|
export default async options => {
|
|
@@ -19,7 +19,7 @@ const fs = fsApi.promises;
|
|
|
19
19
|
const MIN_DOC_LENGTH = 100;
|
|
20
20
|
const SLOW_DOCUMENT_THRESHOLD = 10 * 1000; // number of milliseconds after which a document fetch is considered slow
|
|
21
21
|
|
|
22
|
-
const declarationsPath = path.resolve(process.cwd(), config.get('services.declarationsPath'));
|
|
22
|
+
const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
|
|
23
23
|
const instancePath = path.resolve(declarationsPath, '../');
|
|
24
24
|
|
|
25
25
|
export default async options => {
|
|
@@ -114,7 +114,7 @@ export default async options => {
|
|
|
114
114
|
url: location,
|
|
115
115
|
executeClientScripts,
|
|
116
116
|
cssSelectors: sourceDocument.cssSelectors,
|
|
117
|
-
config: config.get('fetcher'),
|
|
117
|
+
config: config.get('@opentermsarchive/engine.fetcher'),
|
|
118
118
|
}));
|
|
119
119
|
});
|
|
120
120
|
|
|
@@ -160,7 +160,7 @@ export default async options => {
|
|
|
160
160
|
url: sourceDocument.location,
|
|
161
161
|
executeClientScripts: sourceDocument.executeClientScripts,
|
|
162
162
|
cssSelectors: sourceDocument.cssSelectors,
|
|
163
|
-
config: config.get('fetcher'),
|
|
163
|
+
config: config.get('@opentermsarchive/engine.fetcher'),
|
|
164
164
|
}));
|
|
165
165
|
const secondFilteredContent = await extract(sourceDocument);
|
|
166
166
|
|
|
@@ -41,22 +41,22 @@ const counters = {
|
|
|
41
41
|
from: {
|
|
42
42
|
snapshots: {
|
|
43
43
|
source: new GitRepository({
|
|
44
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
44
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
45
45
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.snapshots}`),
|
|
46
46
|
}),
|
|
47
47
|
destination: new GitRepository({
|
|
48
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
48
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
49
49
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.snapshots}-migrated`),
|
|
50
50
|
}),
|
|
51
51
|
logger: winston.createLogger({ transports: [ new (winston.transports.File)({ filename: `${__dirname}/logs/${CONFIG.from.snapshots}.log` }), new winston.transports.Console() ], format }),
|
|
52
52
|
},
|
|
53
53
|
versions: {
|
|
54
54
|
source: new GitRepository({
|
|
55
|
-
...config.get('recorder.versions.storage.git'),
|
|
55
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
56
56
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.versions}`),
|
|
57
57
|
}),
|
|
58
58
|
destination: new GitRepository({
|
|
59
|
-
...config.get('recorder.versions.storage.git'),
|
|
59
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
60
60
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.versions}-migrated`),
|
|
61
61
|
prefixMessageToSnapshotId: CONFIG.from.prefixMessageToSnapshotId,
|
|
62
62
|
}),
|
|
@@ -66,22 +66,22 @@ const counters = {
|
|
|
66
66
|
to: {
|
|
67
67
|
snapshots: {
|
|
68
68
|
source: new GitRepository({
|
|
69
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
69
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
70
70
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.snapshots}`),
|
|
71
71
|
}),
|
|
72
72
|
destination: new GitRepository({
|
|
73
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
73
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
74
74
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.snapshots}-migrated`),
|
|
75
75
|
}),
|
|
76
76
|
logger: winston.createLogger({ transports: [ new (winston.transports.File)({ filename: `${__dirname}/logs/${CONFIG.to.snapshots}.log` }), new winston.transports.Console() ], format }),
|
|
77
77
|
},
|
|
78
78
|
versions: {
|
|
79
79
|
source: new GitRepository({
|
|
80
|
-
...config.get('recorder.versions.storage.git'),
|
|
80
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
81
81
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.versions}`),
|
|
82
82
|
}),
|
|
83
83
|
destination: new GitRepository({
|
|
84
|
-
...config.get('recorder.versions.storage.git'),
|
|
84
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
85
85
|
path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.versions}-migrated`),
|
|
86
86
|
prefixMessageToSnapshotId: CONFIG.to.prefixMessageToSnapshotId,
|
|
87
87
|
}),
|
|
@@ -15,18 +15,18 @@ const ROOT_PATH = path.resolve(__dirname, '../../');
|
|
|
15
15
|
console.time('Total time');
|
|
16
16
|
|
|
17
17
|
const versionsRepository = new GitRepository({
|
|
18
|
-
...config.get('recorder.versions.storage.git'),
|
|
18
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
19
19
|
path: path.resolve(ROOT_PATH, './data/france-elections-versions'),
|
|
20
20
|
});
|
|
21
21
|
|
|
22
22
|
const versionsTargetRepository = new GitRepository({
|
|
23
|
-
...config.get('recorder.versions.storage.git'),
|
|
23
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
24
24
|
prefixMessageToSnapshotId: 'This version was recorded after filtering snapshot https://github.com/OpenTermsArchive/france-elections-snapshots/commit/',
|
|
25
25
|
path: path.resolve(ROOT_PATH, './data/france-elections-versions-hash-updated-test'),
|
|
26
26
|
});
|
|
27
27
|
|
|
28
28
|
const snapshotsRepository = new GitRepository({
|
|
29
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
29
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
30
30
|
path: path.resolve(ROOT_PATH, './data/france-elections-snapshots'),
|
|
31
31
|
});
|
|
32
32
|
|
package/scripts/import/index.js
CHANGED
|
@@ -53,15 +53,15 @@ let client;
|
|
|
53
53
|
}());
|
|
54
54
|
|
|
55
55
|
async function initialize() {
|
|
56
|
-
client = new MongoClient(config.get('import.mongo.connectionURI'));
|
|
56
|
+
client = new MongoClient(config.get('@opentermsarchive/engine.import.mongo.connectionURI'));
|
|
57
57
|
|
|
58
58
|
await client.connect();
|
|
59
|
-
const db = client.db(config.get('import.mongo.database'));
|
|
59
|
+
const db = client.db(config.get('@opentermsarchive/engine.import.mongo.database'));
|
|
60
60
|
|
|
61
|
-
snapshotsCollection = db.collection(config.get('import.mongo.snapshotsCollection'));
|
|
62
|
-
commitsCollection = db.collection(config.get('import.mongo.commitsCollection'));
|
|
61
|
+
snapshotsCollection = db.collection(config.get('@opentermsarchive/engine.import.mongo.snapshotsCollection'));
|
|
62
|
+
commitsCollection = db.collection(config.get('@opentermsarchive/engine.import.mongo.commitsCollection'));
|
|
63
63
|
|
|
64
|
-
sourceRepository = new Git({ path: path.resolve(ROOT_PATH, config.get('import.sourcePath')) });
|
|
64
|
+
sourceRepository = new Git({ path: path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.import.sourcePath')) });
|
|
65
65
|
|
|
66
66
|
await sourceRepository.initialize();
|
|
67
67
|
await renamer.loadRules();
|
|
@@ -119,7 +119,7 @@ function queueDrainHandler(totalToTreat) {
|
|
|
119
119
|
|
|
120
120
|
async function getCommitContent({ sha, serviceId, termsType, extension }) {
|
|
121
121
|
const start = performance.now();
|
|
122
|
-
const url = `https://raw.githubusercontent.com/${config.get('import.githubRepository')}/${sha}/${encodeURI(serviceId)}/${encodeURI(termsType)}.${extension}`;
|
|
122
|
+
const url = `https://raw.githubusercontent.com/${config.get('@opentermsarchive/engine.import.githubRepository')}/${sha}/${encodeURI(serviceId)}/${encodeURI(termsType)}.${extension}`;
|
|
123
123
|
const response = await nodeFetch(url);
|
|
124
124
|
const end = performance.now();
|
|
125
125
|
|
|
@@ -54,13 +54,13 @@ let client;
|
|
|
54
54
|
}());
|
|
55
55
|
|
|
56
56
|
async function initialize() {
|
|
57
|
-
client = new MongoClient(config.get('import.mongo.connectionURI'));
|
|
57
|
+
client = new MongoClient(config.get('@opentermsarchive/engine.import.mongo.connectionURI'));
|
|
58
58
|
|
|
59
59
|
await client.connect();
|
|
60
|
-
const db = client.db(config.get('import.mongo.database'));
|
|
60
|
+
const db = client.db(config.get('@opentermsarchive/engine.import.mongo.database'));
|
|
61
61
|
|
|
62
|
-
collection = db.collection(config.get('import.mongo.commitsCollection'));
|
|
63
|
-
sourceRepository = new Git({ path: path.resolve(ROOT_PATH, config.get('import.sourcePath')) });
|
|
62
|
+
collection = db.collection(config.get('@opentermsarchive/engine.import.mongo.commitsCollection'));
|
|
63
|
+
sourceRepository = new Git({ path: path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.import.sourcePath')) });
|
|
64
64
|
|
|
65
65
|
await sourceRepository.initialize();
|
|
66
66
|
}
|
|
@@ -36,7 +36,7 @@ export async function initTargetRepo(targetRepoPath) {
|
|
|
36
36
|
}
|
|
37
37
|
await fs.mkdir(targetRepoPath);
|
|
38
38
|
|
|
39
|
-
const targetRepo = await new Git({ path: targetRepoPath, author: config.get('recorder.versions.storage.git.author') });
|
|
39
|
+
const targetRepo = await new Git({ path: targetRepoPath, author: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author') });
|
|
40
40
|
|
|
41
41
|
await targetRepo.initialize();
|
|
42
42
|
|