@opentermsarchive/engine 1.2.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/env.js +2 -0
- package/bin/ota-dataset.js +1 -1
- package/bin/ota-serve.js +1 -1
- package/config/ci.json +4 -2
- package/config/default.json +53 -55
- package/config/production.json +21 -19
- package/config/test.json +46 -40
- package/package.json +4 -4
- package/scripts/dataset/assets/README.template.js +2 -2
- package/scripts/dataset/export/index.js +17 -16
- package/scripts/dataset/export/index.test.js +2 -2
- package/scripts/dataset/index.js +1 -1
- package/scripts/dataset/publish/index.js +2 -5
- package/scripts/declarations/lint/index.mocha.js +1 -1
- package/scripts/declarations/validate/index.mocha.js +3 -3
- package/scripts/history/migrate-services.js +8 -8
- package/scripts/history/update-to-full-hash.js +3 -3
- package/scripts/import/index.js +6 -6
- package/scripts/import/loadCommits.js +4 -4
- package/scripts/rewrite/initializer/index.js +1 -1
- package/scripts/rewrite/rewrite-snapshots.js +6 -6
- package/scripts/rewrite/rewrite-versions.js +5 -5
- package/scripts/utils/renamer/index.js +26 -24
- package/scripts/utils/renamer/rules/serviceNames.json +6 -1
- package/src/archivist/fetcher/index.js +3 -3
- package/src/archivist/index.test.js +16 -16
- package/src/archivist/recorder/index.test.js +2 -2
- package/src/archivist/recorder/repositories/git/git.test.js +3 -3
- package/src/archivist/recorder/repositories/git/index.test.js +13 -13
- package/src/archivist/recorder/repositories/mongo/index.test.js +7 -7
- package/src/archivist/services/index.js +1 -1
- package/src/{api → collection-api}/logger.js +5 -5
- package/src/{api → collection-api}/routes/docs.test.js +1 -1
- package/src/{api → collection-api}/routes/services.test.js +1 -1
- package/src/{api → collection-api}/routes/versions.js +1 -1
- package/src/{api → collection-api}/routes/versions.test.js +2 -2
- package/src/{api → collection-api}/server.js +2 -2
- package/src/index.js +7 -7
- package/src/logger/index.js +9 -11
- package/src/notifier/index.js +3 -5
- package/src/reporter/github.js +1 -1
- /package/src/{api → collection-api}/middlewares/errors.js +0 -0
- /package/src/{api → collection-api}/middlewares/logger.js +0 -0
- /package/src/{api → collection-api}/routes/docs.js +0 -0
- /package/src/{api → collection-api}/routes/index.js +0 -0
- /package/src/{api → collection-api}/routes/services.js +0 -0
|
@@ -15,9 +15,9 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
|
15
15
|
|
|
16
16
|
const ROOT_PATH = path.resolve(__dirname, '../../');
|
|
17
17
|
|
|
18
|
-
export const SNAPSHOTS_SOURCE_PATH = path.resolve(ROOT_PATH, config.get('rewrite.snapshotsSourcePath'));
|
|
19
|
-
export const SNAPSHOTS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('recorder.snapshots.storage.git.path'));
|
|
20
|
-
export const VERSIONS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('recorder.versions.storage.git.path'));
|
|
18
|
+
export const SNAPSHOTS_SOURCE_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.rewrite.snapshotsSourcePath'));
|
|
19
|
+
export const SNAPSHOTS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.path'));
|
|
20
|
+
export const VERSIONS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
21
21
|
|
|
22
22
|
const initialize = process.argv.includes('--init');
|
|
23
23
|
|
|
@@ -33,7 +33,7 @@ let recorder;
|
|
|
33
33
|
console.log('Start rewritting history.');
|
|
34
34
|
|
|
35
35
|
await renamer.loadRules();
|
|
36
|
-
const sourceRepo = new Git({ path: SNAPSHOTS_SOURCE_PATH, author: config.get('recorder.snapshots.storage.git.author') });
|
|
36
|
+
const sourceRepo = new Git({ path: SNAPSHOTS_SOURCE_PATH, author: config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.author') });
|
|
37
37
|
|
|
38
38
|
await sourceRepo.initialize();
|
|
39
39
|
|
|
@@ -51,11 +51,11 @@ let recorder;
|
|
|
51
51
|
|
|
52
52
|
recorder = new Recorder({
|
|
53
53
|
versionsRepository: new GitRepository({
|
|
54
|
-
...config.get('recorder.versions.storage.git'),
|
|
54
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
55
55
|
path: VERSIONS_TARGET_PATH,
|
|
56
56
|
}),
|
|
57
57
|
snapshotsRepository: new GitRepository({
|
|
58
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
58
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
59
59
|
path: SNAPSHOTS_TARGET_PATH,
|
|
60
60
|
}),
|
|
61
61
|
});
|
|
@@ -21,9 +21,9 @@ const MARKDOWN_MIME_TYPE = 'text/markdown';
|
|
|
21
21
|
|
|
22
22
|
export const SNAPSHOTS_SOURCE_PATH = path.resolve(
|
|
23
23
|
ROOT_PATH,
|
|
24
|
-
config.get('rewrite.snapshotsSourcePath'),
|
|
24
|
+
config.get('@opentermsarchive/engine.rewrite.snapshotsSourcePath'),
|
|
25
25
|
);
|
|
26
|
-
export const VERSIONS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('recorder.versions.storage.git.path'));
|
|
26
|
+
export const VERSIONS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
27
27
|
|
|
28
28
|
const initialize = process.argv.includes('--init');
|
|
29
29
|
|
|
@@ -42,7 +42,7 @@ let recorder;
|
|
|
42
42
|
|
|
43
43
|
await renamer.loadRules();
|
|
44
44
|
const servicesDeclarations = await services.loadWithHistory();
|
|
45
|
-
const sourceRepo = new Git({ path: SNAPSHOTS_SOURCE_PATH, author: config.get('recorder.snapshots.storage.git.author') });
|
|
45
|
+
const sourceRepo = new Git({ path: SNAPSHOTS_SOURCE_PATH, author: config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.author') });
|
|
46
46
|
|
|
47
47
|
await sourceRepo.initialize();
|
|
48
48
|
|
|
@@ -60,11 +60,11 @@ let recorder;
|
|
|
60
60
|
|
|
61
61
|
recorder = new Recorder({
|
|
62
62
|
versionsRepository: new GitRepository({
|
|
63
|
-
...config.get('recorder.versions.storage.git'),
|
|
63
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
64
64
|
path: VERSIONS_TARGET_PATH,
|
|
65
65
|
}),
|
|
66
66
|
snapshotsRepository: new GitRepository({
|
|
67
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
67
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
68
68
|
path: SNAPSHOTS_SOURCE_PATH,
|
|
69
69
|
}),
|
|
70
70
|
});
|
|
@@ -18,28 +18,30 @@ export async function loadRules() {
|
|
|
18
18
|
export function applyRules(serviceId, termsType) {
|
|
19
19
|
const renamedServiceId = renamingRules.serviceNames[serviceId];
|
|
20
20
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
21
|
+
return [].concat(renamedServiceId).map(renamedServiceId => {
|
|
22
|
+
if (renamedServiceId) {
|
|
23
|
+
console.log(`⌙ Rename service "${serviceId}" to "${renamedServiceId}"`);
|
|
24
|
+
serviceId = renamedServiceId;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const renamedTermsType = renamingRules.termsTypes[termsType];
|
|
28
|
+
|
|
29
|
+
if (renamedTermsType) {
|
|
30
|
+
console.log(`⌙ Rename terms type "${termsType}" to "${renamedTermsType}" of "${serviceId}" service`);
|
|
31
|
+
termsType = renamedTermsType;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const renamedServiceTermsType = renamingRules.termsTypesByService[serviceId]
|
|
35
|
+
&& renamingRules.termsTypesByService[serviceId][termsType];
|
|
36
|
+
|
|
37
|
+
if (renamedServiceTermsType) {
|
|
38
|
+
console.log(`⌙ Specific rename terms type "${termsType}" to "${renamedServiceTermsType}" of "${serviceId}" service`);
|
|
39
|
+
termsType = renamedServiceTermsType;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
return {
|
|
43
|
+
serviceId,
|
|
44
|
+
termsType,
|
|
45
|
+
};
|
|
46
|
+
});
|
|
45
47
|
}
|
|
@@ -88,5 +88,10 @@
|
|
|
88
88
|
"XfinityResidentialSubscription": "Xfinity Residential Services",
|
|
89
89
|
"XfinityWebServices": "Xfinity Web Services",
|
|
90
90
|
"Youtube": "YouTube",
|
|
91
|
-
"Verbaudet": "Vertbaudet"
|
|
91
|
+
"Verbaudet": "Vertbaudet",
|
|
92
|
+
"OpenAI": [
|
|
93
|
+
"ChatGPT",
|
|
94
|
+
"DALL·E"
|
|
95
|
+
],
|
|
96
|
+
"Grok": "xAI"
|
|
92
97
|
}
|
|
@@ -24,9 +24,9 @@ export { FetchDocumentError } from './errors.js';
|
|
|
24
24
|
export default async function fetch({
|
|
25
25
|
url, executeClientScripts, cssSelectors,
|
|
26
26
|
config: {
|
|
27
|
-
navigationTimeout = config.get('fetcher.navigationTimeout'),
|
|
28
|
-
language = config.get('fetcher.language'),
|
|
29
|
-
waitForElementsTimeout = config.get('fetcher.waitForElementsTimeout'),
|
|
27
|
+
navigationTimeout = config.get('@opentermsarchive/engine.fetcher.navigationTimeout'),
|
|
28
|
+
language = config.get('@opentermsarchive/engine.fetcher.language'),
|
|
29
|
+
waitForElementsTimeout = config.get('@opentermsarchive/engine.fetcher.waitForElementsTimeout'),
|
|
30
30
|
} = {},
|
|
31
31
|
}) {
|
|
32
32
|
try {
|
|
@@ -20,8 +20,8 @@ const { expect } = chai;
|
|
|
20
20
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
21
21
|
|
|
22
22
|
const ROOT_PATH = path.resolve(__dirname, '../../');
|
|
23
|
-
const SNAPSHOTS_PATH = path.resolve(ROOT_PATH, config.get('recorder.snapshots.storage.git.path'));
|
|
24
|
-
const VERSIONS_PATH = path.resolve(ROOT_PATH, config.get('recorder.versions.storage.git.path'));
|
|
23
|
+
const SNAPSHOTS_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.path'));
|
|
24
|
+
const VERSIONS_PATH = path.resolve(ROOT_PATH, config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
25
25
|
|
|
26
26
|
const MIME_TYPE = 'text/html';
|
|
27
27
|
const FETCH_DATE = new Date('2000-01-02T12:00:00.000Z');
|
|
@@ -55,8 +55,8 @@ describe('Archivist', function () {
|
|
|
55
55
|
gitVersion = new Git({
|
|
56
56
|
path: VERSIONS_PATH,
|
|
57
57
|
author: {
|
|
58
|
-
name: config.get('recorder.versions.storage.git.author.name'),
|
|
59
|
-
email: config.get('recorder.versions.storage.git.author.email'),
|
|
58
|
+
name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
|
|
59
|
+
email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
|
|
60
60
|
},
|
|
61
61
|
});
|
|
62
62
|
await gitVersion.initialize();
|
|
@@ -72,8 +72,8 @@ describe('Archivist', function () {
|
|
|
72
72
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
73
73
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
74
74
|
app = new Archivist({
|
|
75
|
-
recorderConfig: config.get('recorder'),
|
|
76
|
-
fetcherConfig: config.get('fetcher'),
|
|
75
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
76
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
77
77
|
});
|
|
78
78
|
await app.initialize();
|
|
79
79
|
});
|
|
@@ -152,8 +152,8 @@ describe('Archivist', function () {
|
|
|
152
152
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
153
153
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
154
154
|
app = new Archivist({
|
|
155
|
-
recorderConfig: config.get('recorder'),
|
|
156
|
-
fetcherConfig: config.get('fetcher'),
|
|
155
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
156
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
157
157
|
});
|
|
158
158
|
|
|
159
159
|
await app.initialize();
|
|
@@ -212,8 +212,8 @@ describe('Archivist', function () {
|
|
|
212
212
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
213
213
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
214
214
|
app = new Archivist({
|
|
215
|
-
recorderConfig: config.get('recorder'),
|
|
216
|
-
fetcherConfig: config.get('fetcher'),
|
|
215
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
216
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
217
217
|
});
|
|
218
218
|
|
|
219
219
|
await app.initialize();
|
|
@@ -251,8 +251,8 @@ describe('Archivist', function () {
|
|
|
251
251
|
describe('#attach', () => {
|
|
252
252
|
before(async () => {
|
|
253
253
|
app = new Archivist({
|
|
254
|
-
recorderConfig: config.get('recorder'),
|
|
255
|
-
fetcherConfig: config.get('fetcher'),
|
|
254
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
255
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
256
256
|
});
|
|
257
257
|
await app.initialize();
|
|
258
258
|
|
|
@@ -287,8 +287,8 @@ describe('Archivist', function () {
|
|
|
287
287
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
288
288
|
|
|
289
289
|
app = new Archivist({
|
|
290
|
-
recorderConfig: config.get('recorder'),
|
|
291
|
-
fetcherConfig: config.get('fetcher'),
|
|
290
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
291
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
292
292
|
});
|
|
293
293
|
await app.initialize();
|
|
294
294
|
|
|
@@ -338,8 +338,8 @@ describe('Archivist', function () {
|
|
|
338
338
|
|
|
339
339
|
before(async () => {
|
|
340
340
|
app = new Archivist({
|
|
341
|
-
recorderConfig: config.get('recorder'),
|
|
342
|
-
fetcherConfig: config.get('fetcher'),
|
|
341
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
342
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
343
343
|
});
|
|
344
344
|
await app.initialize();
|
|
345
345
|
|
|
@@ -21,7 +21,7 @@ describe('Recorder', () => {
|
|
|
21
21
|
let recorder;
|
|
22
22
|
|
|
23
23
|
before(async () => {
|
|
24
|
-
const options = config.util.cloneDeep(config.recorder);
|
|
24
|
+
const options = config.util.cloneDeep(config.get('@opentermsarchive/engine.recorder'));
|
|
25
25
|
|
|
26
26
|
options.versions.storage.type = repositoryType;
|
|
27
27
|
options.snapshots.storage.type = repositoryType;
|
|
@@ -323,7 +323,7 @@ describe('Recorder', () => {
|
|
|
323
323
|
record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
|
|
324
324
|
});
|
|
325
325
|
|
|
326
|
-
after(() => recorder.versionsRepository.removeAll());
|
|
326
|
+
after(() => recorder.versionsRepository.removeAll());
|
|
327
327
|
|
|
328
328
|
it('records the version with the proper content', async () => {
|
|
329
329
|
expect(await record.content).to.equal(CONTENT);
|
|
@@ -8,7 +8,7 @@ import config from 'config';
|
|
|
8
8
|
import Git from './git.js';
|
|
9
9
|
|
|
10
10
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
11
|
-
const RECORDER_PATH = path.resolve(__dirname, '../../../../../', config.get('recorder.versions.storage.git.path'));
|
|
11
|
+
const RECORDER_PATH = path.resolve(__dirname, '../../../../../', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
12
12
|
|
|
13
13
|
describe('Git', () => {
|
|
14
14
|
const DEFAULT_CONTENT = 'default content';
|
|
@@ -19,8 +19,8 @@ describe('Git', () => {
|
|
|
19
19
|
subject = new Git({
|
|
20
20
|
path: RECORDER_PATH,
|
|
21
21
|
author: {
|
|
22
|
-
name: config.get('recorder.versions.storage.git.author.name'),
|
|
23
|
-
email: config.get('recorder.versions.storage.git.author.email'),
|
|
22
|
+
name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
|
|
23
|
+
email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
|
|
24
24
|
},
|
|
25
25
|
});
|
|
26
26
|
|
|
@@ -18,7 +18,7 @@ const { expect } = chai;
|
|
|
18
18
|
|
|
19
19
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
20
20
|
|
|
21
|
-
const RECORDER_PATH = path.resolve(__dirname, '../../../../..', config.get('recorder.versions.storage.git.path'));
|
|
21
|
+
const RECORDER_PATH = path.resolve(__dirname, '../../../../..', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
22
22
|
|
|
23
23
|
const SERVICE_PROVIDER_ID = 'test_service';
|
|
24
24
|
const TERMS_TYPE = 'Terms of Service';
|
|
@@ -52,15 +52,15 @@ describe('GitRepository', () => {
|
|
|
52
52
|
git = new Git({
|
|
53
53
|
path: RECORDER_PATH,
|
|
54
54
|
author: {
|
|
55
|
-
name: config.get('recorder.versions.storage.git.author.name'),
|
|
56
|
-
email: config.get('recorder.versions.storage.git.author.email'),
|
|
55
|
+
name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
|
|
56
|
+
email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
|
|
57
57
|
},
|
|
58
58
|
});
|
|
59
59
|
|
|
60
60
|
await git.initialize();
|
|
61
61
|
|
|
62
62
|
subject = new GitRepository({
|
|
63
|
-
...config.get('recorder.versions.storage.git'),
|
|
63
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
64
64
|
path: RECORDER_PATH,
|
|
65
65
|
});
|
|
66
66
|
|
|
@@ -266,7 +266,7 @@ describe('GitRepository', () => {
|
|
|
266
266
|
after(() => subject.removeAll());
|
|
267
267
|
|
|
268
268
|
it('stores snapshot ID', () => {
|
|
269
|
-
expect(commit.body).to.include(config.get('recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID));
|
|
269
|
+
expect(commit.body).to.include(config.get('@opentermsarchive/engine.recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID));
|
|
270
270
|
});
|
|
271
271
|
|
|
272
272
|
it('stores the service ID', () => {
|
|
@@ -298,8 +298,8 @@ describe('GitRepository', () => {
|
|
|
298
298
|
after(() => subject.removeAll());
|
|
299
299
|
|
|
300
300
|
it('stores snapshots IDs', () => {
|
|
301
|
-
expect(commit.body).to.include(config.get('recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID_1));
|
|
302
|
-
expect(commit.body).to.include(config.get('recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID_2));
|
|
301
|
+
expect(commit.body).to.include(config.get('@opentermsarchive/engine.recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID_1));
|
|
302
|
+
expect(commit.body).to.include(config.get('@opentermsarchive/engine.recorder.versions.storage.git.snapshotIdentiferTemplate').replace(SNAPSHOT_ID_MARKER, SNAPSHOT_ID_2));
|
|
303
303
|
});
|
|
304
304
|
|
|
305
305
|
it('stores the number of source documents', () => {
|
|
@@ -676,15 +676,15 @@ describe('GitRepository', () => {
|
|
|
676
676
|
git = new Git({
|
|
677
677
|
path: RECORDER_PATH,
|
|
678
678
|
author: {
|
|
679
|
-
name: config.get('recorder.snapshots.storage.git.author.name'),
|
|
680
|
-
email: config.get('recorder.snapshots.storage.git.author.email'),
|
|
679
|
+
name: config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.author.name'),
|
|
680
|
+
email: config.get('@opentermsarchive/engine.recorder.snapshots.storage.git.author.email'),
|
|
681
681
|
},
|
|
682
682
|
});
|
|
683
683
|
|
|
684
684
|
await git.initialize();
|
|
685
685
|
|
|
686
686
|
subject = new GitRepository({
|
|
687
|
-
...config.get('recorder.snapshots.storage.git'),
|
|
687
|
+
...config.get('@opentermsarchive/engine.recorder.snapshots.storage.git'),
|
|
688
688
|
path: RECORDER_PATH,
|
|
689
689
|
});
|
|
690
690
|
|
|
@@ -1257,14 +1257,14 @@ describe('GitRepository', () => {
|
|
|
1257
1257
|
git = new Git({
|
|
1258
1258
|
path: RECORDER_PATH,
|
|
1259
1259
|
author: {
|
|
1260
|
-
name: config.get('recorder.versions.storage.git.author.name'),
|
|
1261
|
-
email: config.get('recorder.versions.storage.git.author.email'),
|
|
1260
|
+
name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
|
|
1261
|
+
email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
|
|
1262
1262
|
},
|
|
1263
1263
|
});
|
|
1264
1264
|
|
|
1265
1265
|
await git.initialize();
|
|
1266
1266
|
subject = new GitRepository({
|
|
1267
|
-
...config.get('recorder.versions.storage.git'),
|
|
1267
|
+
...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
|
|
1268
1268
|
path: RECORDER_PATH,
|
|
1269
1269
|
});
|
|
1270
1270
|
|
|
@@ -15,7 +15,7 @@ import MongoRepository from './index.js';
|
|
|
15
15
|
const { expect } = chai;
|
|
16
16
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
17
17
|
|
|
18
|
-
const { connectionURI } = config.get('recorder.snapshots.storage.mongo');
|
|
18
|
+
const { connectionURI } = config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo');
|
|
19
19
|
const client = new MongoClient(connectionURI);
|
|
20
20
|
|
|
21
21
|
const SERVICE_PROVIDER_ID = 'test_service';
|
|
@@ -41,12 +41,12 @@ describe('MongoRepository', () => {
|
|
|
41
41
|
|
|
42
42
|
context('Version', () => {
|
|
43
43
|
before(async () => {
|
|
44
|
-
subject = new MongoRepository(config.get('recorder.versions.storage.mongo'));
|
|
44
|
+
subject = new MongoRepository(config.get('@opentermsarchive/engine.recorder.versions.storage.mongo'));
|
|
45
45
|
await subject.initialize();
|
|
46
46
|
await client.connect();
|
|
47
|
-
const db = client.db(config.get('recorder.versions.storage.mongo.database'));
|
|
47
|
+
const db = client.db(config.get('@opentermsarchive/engine.recorder.versions.storage.mongo.database'));
|
|
48
48
|
|
|
49
|
-
collection = db.collection(config.get('recorder.versions.storage.mongo.collection'));
|
|
49
|
+
collection = db.collection(config.get('@opentermsarchive/engine.recorder.versions.storage.mongo.collection'));
|
|
50
50
|
});
|
|
51
51
|
|
|
52
52
|
describe('#save', () => {
|
|
@@ -652,12 +652,12 @@ describe('MongoRepository', () => {
|
|
|
652
652
|
|
|
653
653
|
context('Snapshot', () => {
|
|
654
654
|
before(async () => {
|
|
655
|
-
subject = new MongoRepository(config.get('recorder.snapshots.storage.mongo'));
|
|
655
|
+
subject = new MongoRepository(config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo'));
|
|
656
656
|
await subject.initialize();
|
|
657
657
|
await client.connect();
|
|
658
|
-
const db = client.db(config.get('recorder.snapshots.storage.mongo.database'));
|
|
658
|
+
const db = client.db(config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo.database'));
|
|
659
659
|
|
|
660
|
-
collection = db.collection(config.get('recorder.snapshots.storage.mongo.collection'));
|
|
660
|
+
collection = db.collection(config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo.collection'));
|
|
661
661
|
});
|
|
662
662
|
|
|
663
663
|
describe('#save', () => {
|
|
@@ -9,7 +9,7 @@ import SourceDocument from './sourceDocument.js';
|
|
|
9
9
|
import Terms from './terms.js';
|
|
10
10
|
|
|
11
11
|
const fs = fsApi.promises;
|
|
12
|
-
const declarationsPath = path.resolve(process.cwd(), config.get('services.declarationsPath'));
|
|
12
|
+
const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
|
|
13
13
|
|
|
14
14
|
export async function load(servicesIdsToLoad = []) {
|
|
15
15
|
let servicesIds = await getDeclaredServicesIds();
|
|
@@ -11,12 +11,12 @@ const { combine, timestamp, printf, colorize } = winston.format;
|
|
|
11
11
|
|
|
12
12
|
const transports = [new winston.transports.Console()];
|
|
13
13
|
|
|
14
|
-
if (config.get('logger.sendMailOnError')) {
|
|
14
|
+
if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
|
|
15
15
|
transports.push(new winston.transports.Mail({
|
|
16
|
-
to: config.get('logger.sendMailOnError.to'),
|
|
17
|
-
from: config.get('logger.sendMailOnError.from'),
|
|
18
|
-
host: config.get('logger.smtp.host'),
|
|
19
|
-
username: config.get('logger.smtp.username'),
|
|
16
|
+
to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
|
|
17
|
+
from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
|
|
18
|
+
host: config.get('@opentermsarchive/engine.logger.smtp.host'),
|
|
19
|
+
username: config.get('@opentermsarchive/engine.logger.smtp.username'),
|
|
20
20
|
password: process.env.SMTP_PASSWORD,
|
|
21
21
|
ssl: true,
|
|
22
22
|
timeout: 30 * 1000,
|
|
@@ -4,7 +4,7 @@ import request from 'supertest';
|
|
|
4
4
|
|
|
5
5
|
import app from '../server.js';
|
|
6
6
|
|
|
7
|
-
const basePath = config.get('api.basePath');
|
|
7
|
+
const basePath = config.get('@opentermsarchive/engine.collection-api.basePath');
|
|
8
8
|
|
|
9
9
|
describe('Services API', () => {
|
|
10
10
|
describe('GET /services', () => {
|
|
@@ -28,7 +28,7 @@ import { toISODateWithoutMilliseconds } from '../../archivist/utils/date.js';
|
|
|
28
28
|
*/
|
|
29
29
|
const router = express.Router();
|
|
30
30
|
|
|
31
|
-
const versionsRepository = await RepositoryFactory.create(config.get('recorder.versions.storage')).initialize();
|
|
31
|
+
const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage')).initialize();
|
|
32
32
|
|
|
33
33
|
/**
|
|
34
34
|
* @swagger
|
|
@@ -7,7 +7,7 @@ import Version from '../../archivist/recorder/version.js';
|
|
|
7
7
|
import { toISODateWithoutMilliseconds } from '../../archivist/utils/date.js';
|
|
8
8
|
import app from '../server.js';
|
|
9
9
|
|
|
10
|
-
const basePath = config.get('api.basePath');
|
|
10
|
+
const basePath = config.get('@opentermsarchive/engine.collection-api.basePath');
|
|
11
11
|
|
|
12
12
|
const { expect } = chai;
|
|
13
13
|
const request = supertest(app);
|
|
@@ -24,7 +24,7 @@ describe('Versions API', () => {
|
|
|
24
24
|
};
|
|
25
25
|
|
|
26
26
|
before(async () => {
|
|
27
|
-
versionsRepository = RepositoryFactory.create(config.get('recorder.versions.storage'));
|
|
27
|
+
versionsRepository = RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage'));
|
|
28
28
|
|
|
29
29
|
await versionsRepository.initialize();
|
|
30
30
|
|
|
@@ -13,12 +13,12 @@ if (process.env.NODE_ENV !== 'test') {
|
|
|
13
13
|
app.use(loggerMiddleware);
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
const BASE_PATH =
|
|
16
|
+
const BASE_PATH = `/${config.get('@opentermsarchive/engine.collection-api.basePath')}/v1`.replace(/\/\/+/g, '/'); // ensure there are no double slashes
|
|
17
17
|
|
|
18
18
|
app.use(BASE_PATH, apiRouter(BASE_PATH));
|
|
19
19
|
app.use(errorsMiddleware);
|
|
20
20
|
|
|
21
|
-
const port = config.get('api.port');
|
|
21
|
+
const port = config.get('@opentermsarchive/engine.collection-api.port');
|
|
22
22
|
|
|
23
23
|
app.listen(port);
|
|
24
24
|
|
package/src/index.js
CHANGED
|
@@ -8,8 +8,8 @@ import Reporter from './reporter/index.js';
|
|
|
8
8
|
|
|
9
9
|
export default async function track({ services, types, extractOnly, schedule }) {
|
|
10
10
|
const archivist = new Archivist({
|
|
11
|
-
recorderConfig: config.get('recorder'),
|
|
12
|
-
fetcherConfig: config.get('fetcher'),
|
|
11
|
+
recorderConfig: config.get('@opentermsarchive/engine.recorder'),
|
|
12
|
+
fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
|
|
13
13
|
});
|
|
14
14
|
|
|
15
15
|
archivist.attach(logger);
|
|
@@ -38,20 +38,20 @@ export default async function track({ services, types, extractOnly, schedule })
|
|
|
38
38
|
return;
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
if (process.env.
|
|
41
|
+
if (process.env.OTA_ENGINE_SENDINBLUE_API_KEY) {
|
|
42
42
|
try {
|
|
43
43
|
archivist.attach(new Notifier(archivist.services));
|
|
44
44
|
} catch (error) {
|
|
45
45
|
logger.error('Cannot instantiate the Notifier module; it will be ignored:', error);
|
|
46
46
|
}
|
|
47
47
|
} else {
|
|
48
|
-
logger.warn('Environment variable "
|
|
48
|
+
logger.warn('Environment variable "OTA_ENGINE_SENDINBLUE_API_KEY" was not found; the Notifier module will be ignored');
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
if (process.env.
|
|
51
|
+
if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
|
|
52
52
|
if (config.has('reporter.githubIssues.repositories.declarations')) {
|
|
53
53
|
try {
|
|
54
|
-
const reporter = new Reporter(config.get('reporter'));
|
|
54
|
+
const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
|
|
55
55
|
|
|
56
56
|
await reporter.initialize();
|
|
57
57
|
archivist.attach(reporter);
|
|
@@ -62,7 +62,7 @@ export default async function track({ services, types, extractOnly, schedule })
|
|
|
62
62
|
logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
|
|
63
63
|
}
|
|
64
64
|
} else {
|
|
65
|
-
logger.warn('Environment variable "
|
|
65
|
+
logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored');
|
|
66
66
|
}
|
|
67
67
|
|
|
68
68
|
if (!schedule) {
|
package/src/logger/index.js
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import os from 'os';
|
|
2
2
|
|
|
3
3
|
import config from 'config';
|
|
4
|
-
import dotenv from 'dotenv';
|
|
5
4
|
import winston from 'winston';
|
|
6
5
|
import 'winston-mail';
|
|
7
6
|
|
|
8
|
-
dotenv.config();
|
|
9
7
|
const { combine, timestamp, printf, colorize } = winston.format;
|
|
10
8
|
|
|
11
9
|
const alignedWithColorsAndTime = combine(
|
|
@@ -40,16 +38,16 @@ const logger = winston.createLogger({
|
|
|
40
38
|
rejectionHandlers: transports,
|
|
41
39
|
});
|
|
42
40
|
|
|
43
|
-
if (config.get('logger.sendMailOnError')) {
|
|
44
|
-
if (process.env.
|
|
45
|
-
logger.warn('Environment variable "
|
|
41
|
+
if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
|
|
42
|
+
if (process.env.OTA_ENGINE_SMTP_PASSWORD === undefined) {
|
|
43
|
+
logger.warn('Environment variable "OTA_ENGINE_SMTP_PASSWORD" was not found; log emails cannot be sent');
|
|
46
44
|
} else {
|
|
47
45
|
const mailerOptions = {
|
|
48
|
-
to: config.get('logger.sendMailOnError.to'),
|
|
49
|
-
from: config.get('logger.sendMailOnError.from'),
|
|
50
|
-
host: config.get('logger.smtp.host'),
|
|
51
|
-
username: config.get('logger.smtp.username'),
|
|
52
|
-
password: process.env.
|
|
46
|
+
to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
|
|
47
|
+
from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
|
|
48
|
+
host: config.get('@opentermsarchive/engine.logger.smtp.host'),
|
|
49
|
+
username: config.get('@opentermsarchive/engine.logger.smtp.username'),
|
|
50
|
+
password: process.env.OTA_ENGINE_SMTP_PASSWORD,
|
|
53
51
|
ssl: true,
|
|
54
52
|
timeout: 30 * 1000,
|
|
55
53
|
formatter: args => args[Object.getOwnPropertySymbols(args)[1]], // Returns the full error message, the same visible in the console. It is referenced in the argument object with a Symbol of which we do not have the reference but we know it is the second one.
|
|
@@ -62,7 +60,7 @@ if (config.get('logger.sendMailOnError')) {
|
|
|
62
60
|
subject: `[OTA] Error Report — ${os.hostname()}`,
|
|
63
61
|
}));
|
|
64
62
|
|
|
65
|
-
if (config.get('logger.sendMailOnError.sendWarnings')) {
|
|
63
|
+
if (config.get('@opentermsarchive/engine.logger.sendMailOnError.sendWarnings')) {
|
|
66
64
|
transports.push(new winston.transports.Mail({
|
|
67
65
|
...mailerOptions,
|
|
68
66
|
level: 'warn',
|
package/src/notifier/index.js
CHANGED
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import config from 'config';
|
|
2
|
-
import dotenv from 'dotenv';
|
|
3
2
|
import sendInBlue from 'sib-api-v3-sdk';
|
|
4
3
|
|
|
5
|
-
dotenv.config();
|
|
6
4
|
export default class Notifier {
|
|
7
5
|
constructor(passedServiceProviders) {
|
|
8
6
|
const defaultClient = sendInBlue.ApiClient.instance;
|
|
9
7
|
const authentication = defaultClient.authentications['api-key'];
|
|
10
8
|
|
|
11
|
-
authentication.apiKey = process.env.
|
|
9
|
+
authentication.apiKey = process.env.OTA_ENGINE_SENDINBLUE_API_KEY;
|
|
12
10
|
|
|
13
11
|
this.apiInstance = new sendInBlue.TransactionalEmailsApi();
|
|
14
12
|
this.contactsInstance = new sendInBlue.ContactsApi();
|
|
@@ -35,7 +33,7 @@ export default class Notifier {
|
|
|
35
33
|
|
|
36
34
|
async notifyVersionRecorded(serviceProviderId, termsType, versionId) {
|
|
37
35
|
const sendParams = {
|
|
38
|
-
templateId: config.get('notifier.sendInBlue.updateTemplateId'),
|
|
36
|
+
templateId: config.get('@opentermsarchive/engine.notifier.sendInBlue.updateTemplateId'),
|
|
39
37
|
params: {
|
|
40
38
|
SERVICE_PROVIDER_NAME: this.serviceProviders[serviceProviderId].name,
|
|
41
39
|
DOCUMENT_TYPE: termsType,
|
|
@@ -44,7 +42,7 @@ export default class Notifier {
|
|
|
44
42
|
};
|
|
45
43
|
|
|
46
44
|
const lists = [
|
|
47
|
-
config.get('notifier.sendInBlue.updatesListId'),
|
|
45
|
+
config.get('@opentermsarchive/engine.notifier.sendInBlue.updatesListId'),
|
|
48
46
|
];
|
|
49
47
|
|
|
50
48
|
const notificationListName = `${this.serviceProviders[serviceProviderId].name} - ${termsType} - Update`;
|
package/src/reporter/github.js
CHANGED
|
@@ -16,7 +16,7 @@ export default class GitHub {
|
|
|
16
16
|
constructor(repository) {
|
|
17
17
|
const { version } = require('../../package.json');
|
|
18
18
|
|
|
19
|
-
this.octokit = new Octokit({ auth: process.env.
|
|
19
|
+
this.octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN, userAgent: `opentermsarchive/${version}` });
|
|
20
20
|
|
|
21
21
|
const [ owner, repo ] = repository.split('/');
|
|
22
22
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|