@opentermsarchive/engine 0.26.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ota-track.js +3 -3
- package/bin/ota-validate.js +2 -2
- package/bin/ota.js +1 -1
- package/config/default.json +1 -1
- package/package.json +3 -4
- package/scripts/dataset/export/index.js +4 -4
- package/scripts/dataset/export/index.test.js +11 -17
- package/scripts/declarations/lint/index.mocha.js +1 -1
- package/scripts/declarations/utils/index.js +12 -12
- package/scripts/declarations/validate/definitions.js +1 -1
- package/scripts/declarations/validate/index.mocha.js +30 -34
- package/scripts/declarations/validate/service.history.schema.js +11 -11
- package/scripts/declarations/validate/service.schema.js +13 -13
- package/scripts/history/migrate-services.js +4 -4
- package/scripts/history/update-to-full-hash.js +2 -2
- package/scripts/import/index.js +14 -14
- package/scripts/rewrite/rewrite-snapshots.js +3 -3
- package/scripts/rewrite/rewrite-versions.js +14 -14
- package/scripts/utils/renamer/README.md +3 -3
- package/scripts/utils/renamer/index.js +13 -13
- package/src/archivist/errors.js +1 -1
- package/src/archivist/extract/exports.js +3 -0
- package/src/archivist/{filter → extract}/index.js +23 -27
- package/src/archivist/extract/index.test.js +516 -0
- package/src/archivist/index.js +101 -140
- package/src/archivist/index.test.js +178 -166
- package/src/archivist/recorder/index.js +11 -55
- package/src/archivist/recorder/index.test.js +310 -356
- package/src/archivist/recorder/record.js +18 -7
- package/src/archivist/recorder/repositories/git/dataMapper.js +41 -31
- package/src/archivist/recorder/repositories/git/index.js +11 -15
- package/src/archivist/recorder/repositories/git/index.test.js +1058 -463
- package/src/archivist/recorder/repositories/interface.js +8 -6
- package/src/archivist/recorder/repositories/mongo/dataMapper.js +21 -14
- package/src/archivist/recorder/repositories/mongo/index.js +8 -8
- package/src/archivist/recorder/repositories/mongo/index.test.js +898 -479
- package/src/archivist/recorder/snapshot.js +5 -0
- package/src/archivist/recorder/snapshot.test.js +65 -0
- package/src/archivist/recorder/version.js +14 -0
- package/src/archivist/recorder/version.test.js +65 -0
- package/src/archivist/services/index.js +60 -51
- package/src/archivist/services/index.test.js +63 -83
- package/src/archivist/services/service.js +26 -22
- package/src/archivist/services/service.test.js +46 -68
- package/src/archivist/services/{pageDeclaration.js → sourceDocument.js} +11 -9
- package/src/archivist/services/{pageDeclaration.test.js → sourceDocument.test.js} +21 -21
- package/src/archivist/services/terms.js +26 -0
- package/src/archivist/services/{documentDeclaration.test.js → terms.test.js} +15 -15
- package/src/exports.js +2 -2
- package/src/index.js +16 -13
- package/src/logger/index.js +35 -36
- package/src/notifier/index.js +8 -8
- package/src/tracker/index.js +6 -6
- package/src/archivist/filter/exports.js +0 -3
- package/src/archivist/filter/index.test.js +0 -564
- package/src/archivist/recorder/record.test.js +0 -91
- package/src/archivist/services/documentDeclaration.js +0 -26
- /package/scripts/utils/renamer/rules/{documentTypes.json → termsTypes.json} +0 -0
- /package/scripts/utils/renamer/rules/{documentTypesByService.json → termsTypesByService.json} +0 -0
|
@@ -10,7 +10,7 @@ import sinonChai from 'sinon-chai';
|
|
|
10
10
|
|
|
11
11
|
import Git from './recorder/repositories/git/git.js';
|
|
12
12
|
|
|
13
|
-
import Archivist, {
|
|
13
|
+
import Archivist, { EVENTS } from './index.js';
|
|
14
14
|
|
|
15
15
|
const fs = fsApi.promises;
|
|
16
16
|
|
|
@@ -49,7 +49,7 @@ describe('Archivist', function () {
|
|
|
49
49
|
let serviceBSnapshotExpectedContent;
|
|
50
50
|
let serviceBVersionExpectedContent;
|
|
51
51
|
|
|
52
|
-
const
|
|
52
|
+
const services = [ 'service_A', 'service_B' ];
|
|
53
53
|
|
|
54
54
|
before(async () => {
|
|
55
55
|
gitVersion = new Git({
|
|
@@ -67,16 +67,19 @@ describe('Archivist', function () {
|
|
|
67
67
|
serviceBVersionExpectedContent = await fs.readFile(path.resolve(ROOT_PATH, 'test/fixtures/termsFromPDF.md'), { encoding: 'utf8' });
|
|
68
68
|
});
|
|
69
69
|
|
|
70
|
-
describe('#
|
|
70
|
+
describe('#track', () => {
|
|
71
71
|
before(async () => {
|
|
72
72
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
73
73
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
74
|
-
app = new Archivist({
|
|
74
|
+
app = new Archivist({
|
|
75
|
+
recorderConfig: config.get('recorder'),
|
|
76
|
+
fetcherConfig: config.get('fetcher'),
|
|
77
|
+
});
|
|
75
78
|
await app.initialize();
|
|
76
79
|
});
|
|
77
80
|
|
|
78
81
|
context('when everything works fine', () => {
|
|
79
|
-
before(async () => app.
|
|
82
|
+
before(async () => app.track({ services }));
|
|
80
83
|
|
|
81
84
|
after(resetGitRepositories);
|
|
82
85
|
|
|
@@ -105,12 +108,12 @@ describe('Archivist', function () {
|
|
|
105
108
|
});
|
|
106
109
|
});
|
|
107
110
|
|
|
108
|
-
context('when there is an
|
|
111
|
+
context('when there is an operational error with service A', () => {
|
|
109
112
|
before(async () => {
|
|
110
113
|
// as there is no more HTTP request mocks for service A, it should throw an `ENOTFOUND` error which is considered as an expected error in our workflow
|
|
111
114
|
nock.cleanAll();
|
|
112
115
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
113
|
-
await app.
|
|
116
|
+
await app.track({ services });
|
|
114
117
|
});
|
|
115
118
|
|
|
116
119
|
after(resetGitRepositories);
|
|
@@ -135,97 +138,108 @@ describe('Archivist', function () {
|
|
|
135
138
|
expect(resultingTerms).to.equal(serviceBVersionExpectedContent);
|
|
136
139
|
});
|
|
137
140
|
});
|
|
138
|
-
});
|
|
139
141
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
142
|
+
context('extracting only', () => {
|
|
143
|
+
context('when a service’s filter declaration changes', () => {
|
|
144
|
+
context('when everything works fine', () => {
|
|
145
|
+
let originalSnapshotId;
|
|
146
|
+
let firstVersionId;
|
|
147
|
+
let reExtractedVersionId;
|
|
148
|
+
let reExtractedVersionMessageBody;
|
|
149
|
+
let serviceBCommits;
|
|
148
150
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
151
|
+
before(async () => {
|
|
152
|
+
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
153
|
+
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
154
|
+
app = new Archivist({
|
|
155
|
+
recorderConfig: config.get('recorder'),
|
|
156
|
+
fetcherConfig: config.get('fetcher'),
|
|
157
|
+
});
|
|
153
158
|
|
|
154
|
-
|
|
155
|
-
|
|
159
|
+
await app.initialize();
|
|
160
|
+
await app.track({ services });
|
|
156
161
|
|
|
157
|
-
|
|
158
|
-
|
|
162
|
+
({ id: originalSnapshotId } = await app.recorder.snapshotsRepository.findLatest(SERVICE_A_ID, SERVICE_A_TYPE));
|
|
163
|
+
({ id: firstVersionId } = await app.recorder.versionsRepository.findLatest(SERVICE_A_ID, SERVICE_A_TYPE));
|
|
159
164
|
|
|
160
|
-
|
|
165
|
+
serviceBCommits = await gitVersion.log({ file: SERVICE_B_EXPECTED_VERSION_FILE_PATH });
|
|
161
166
|
|
|
162
|
-
|
|
167
|
+
app.services[SERVICE_A_ID].getTerms(SERVICE_A_TYPE).sourceDocuments[0].contentSelectors = 'h1';
|
|
163
168
|
|
|
164
|
-
|
|
169
|
+
await app.track({ services: [ 'service_A', 'service_B' ], extractOnly: true });
|
|
165
170
|
|
|
166
|
-
|
|
171
|
+
const [reExtractedVersionCommit] = await gitVersion.log({ file: SERVICE_A_EXPECTED_VERSION_FILE_PATH });
|
|
167
172
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
173
|
+
reExtractedVersionId = reExtractedVersionCommit.hash;
|
|
174
|
+
reExtractedVersionMessageBody = reExtractedVersionCommit.body;
|
|
175
|
+
});
|
|
171
176
|
|
|
172
|
-
|
|
177
|
+
after(resetGitRepositories);
|
|
173
178
|
|
|
174
|
-
|
|
175
|
-
|
|
179
|
+
it('updates the version of the changed service', async () => {
|
|
180
|
+
const serviceAContent = await fs.readFile(path.resolve(__dirname, SERVICE_A_EXPECTED_VERSION_FILE_PATH), { encoding: 'utf8' });
|
|
176
181
|
|
|
177
|
-
|
|
178
|
-
|
|
182
|
+
expect(serviceAContent).to.equal('Terms of service with UTF-8 \'çhãràčtęrs"\n========================================');
|
|
183
|
+
});
|
|
179
184
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
185
|
+
it('generates a new version id', async () => {
|
|
186
|
+
expect(reExtractedVersionId).to.not.equal(firstVersionId);
|
|
187
|
+
});
|
|
183
188
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
189
|
+
it('mentions the snapshot id in the changelog', async () => {
|
|
190
|
+
expect(reExtractedVersionMessageBody).to.include(originalSnapshotId);
|
|
191
|
+
});
|
|
187
192
|
|
|
188
|
-
|
|
189
|
-
|
|
193
|
+
it('does not change other services', async () => {
|
|
194
|
+
const serviceBVersion = await fs.readFile(path.resolve(__dirname, SERVICE_B_EXPECTED_VERSION_FILE_PATH), { encoding: 'utf8' });
|
|
190
195
|
|
|
191
|
-
|
|
192
|
-
|
|
196
|
+
expect(serviceBVersion).to.equal(serviceBVersionExpectedContent);
|
|
197
|
+
});
|
|
193
198
|
|
|
194
|
-
|
|
195
|
-
|
|
199
|
+
it('does not generate a new id for other services', async () => {
|
|
200
|
+
const serviceBCommitsAfterExtraction = await gitVersion.log({ file: SERVICE_B_EXPECTED_VERSION_FILE_PATH });
|
|
196
201
|
|
|
197
|
-
|
|
202
|
+
expect(serviceBCommitsAfterExtraction.map(commit => commit.hash)).to.deep.equal(serviceBCommits.map(commit => commit.hash));
|
|
203
|
+
});
|
|
198
204
|
});
|
|
199
|
-
});
|
|
200
|
-
|
|
201
|
-
context('when there is an expected error', () => {
|
|
202
|
-
let inaccessibleContentSpy;
|
|
203
|
-
let versionNotChangedSpy;
|
|
204
205
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
206
|
+
context('when there is an operational error with service A', () => {
|
|
207
|
+
let inaccessibleContentSpy;
|
|
208
|
+
let versionNotChangedSpy;
|
|
209
|
+
let versionB;
|
|
209
210
|
|
|
210
|
-
|
|
211
|
-
|
|
211
|
+
before(async () => {
|
|
212
|
+
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
213
|
+
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
214
|
+
app = new Archivist({
|
|
215
|
+
recorderConfig: config.get('recorder'),
|
|
216
|
+
fetcherConfig: config.get('fetcher'),
|
|
217
|
+
});
|
|
212
218
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
219
|
+
await app.initialize();
|
|
220
|
+
await app.track({ services });
|
|
221
|
+
app.services[SERVICE_A_ID].getTerms(SERVICE_A_TYPE).sourceDocuments[0].contentSelectors = 'inexistant-selector';
|
|
222
|
+
inaccessibleContentSpy = sinon.spy();
|
|
223
|
+
versionNotChangedSpy = sinon.spy();
|
|
224
|
+
app.on('inaccessibleContent', inaccessibleContentSpy);
|
|
225
|
+
app.on('versionNotChanged', record => {
|
|
226
|
+
if (record.serviceId == 'service_B') {
|
|
227
|
+
versionB = record;
|
|
228
|
+
}
|
|
229
|
+
versionNotChangedSpy(record);
|
|
230
|
+
});
|
|
231
|
+
await app.track({ services, extractOnly: true });
|
|
232
|
+
});
|
|
220
233
|
|
|
221
|
-
|
|
234
|
+
after(resetGitRepositories);
|
|
222
235
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
236
|
+
it('emits an inaccessibleContent event', async () => {
|
|
237
|
+
expect(inaccessibleContentSpy).to.have.been.called;
|
|
238
|
+
});
|
|
226
239
|
|
|
227
|
-
|
|
228
|
-
|
|
240
|
+
it('still extracts the terms of other services', async () => {
|
|
241
|
+
expect(versionNotChangedSpy).to.have.been.calledWith(versionB);
|
|
242
|
+
});
|
|
229
243
|
});
|
|
230
244
|
});
|
|
231
245
|
});
|
|
@@ -239,7 +253,7 @@ describe('Archivist', function () {
|
|
|
239
253
|
}
|
|
240
254
|
|
|
241
255
|
function emitsOnly(eventNames) {
|
|
242
|
-
|
|
256
|
+
EVENTS.filter(el => eventNames.indexOf(el) < 0).forEach(event => {
|
|
243
257
|
const handlerName = `on${event[0].toUpperCase()}${event.substr(1)}`;
|
|
244
258
|
|
|
245
259
|
it(`emits no "${event}" event`, () => {
|
|
@@ -249,10 +263,13 @@ describe('Archivist', function () {
|
|
|
249
263
|
}
|
|
250
264
|
|
|
251
265
|
before(async () => {
|
|
252
|
-
app = new Archivist({
|
|
266
|
+
app = new Archivist({
|
|
267
|
+
recorderConfig: config.get('recorder'),
|
|
268
|
+
fetcherConfig: config.get('fetcher'),
|
|
269
|
+
});
|
|
253
270
|
await app.initialize();
|
|
254
271
|
|
|
255
|
-
|
|
272
|
+
EVENTS.forEach(event => {
|
|
256
273
|
const handlerName = `on${event[0].toUpperCase()}${event.substr(1)}`;
|
|
257
274
|
|
|
258
275
|
spies[handlerName] = sinon.spy();
|
|
@@ -260,15 +277,24 @@ describe('Archivist', function () {
|
|
|
260
277
|
});
|
|
261
278
|
});
|
|
262
279
|
|
|
263
|
-
describe('#
|
|
280
|
+
describe('#recordSnapshots', () => {
|
|
281
|
+
let terms;
|
|
282
|
+
let snapshot;
|
|
283
|
+
|
|
284
|
+
before(async () => {
|
|
285
|
+
terms = app.services.service_A.getTerms(SERVICE_A_TYPE);
|
|
286
|
+
terms.fetchDate = FETCH_DATE;
|
|
287
|
+
terms.sourceDocuments.forEach(async sourceDocument => {
|
|
288
|
+
sourceDocument.content = serviceASnapshotExpectedContent;
|
|
289
|
+
sourceDocument.mimeType = MIME_TYPE;
|
|
290
|
+
});
|
|
291
|
+
resetSpiesHistory();
|
|
292
|
+
});
|
|
293
|
+
|
|
264
294
|
context('when it is the first record', () => {
|
|
265
|
-
before(async () =>
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
documentType: SERVICE_A_TYPE,
|
|
269
|
-
mimeType: MIME_TYPE,
|
|
270
|
-
fetchDate: FETCH_DATE,
|
|
271
|
-
}));
|
|
295
|
+
before(async () => {
|
|
296
|
+
[snapshot] = await app.recordSnapshots(terms);
|
|
297
|
+
});
|
|
272
298
|
|
|
273
299
|
after(() => {
|
|
274
300
|
resetSpiesHistory();
|
|
@@ -277,7 +303,7 @@ describe('Archivist', function () {
|
|
|
277
303
|
});
|
|
278
304
|
|
|
279
305
|
it('emits "firstSnapshotRecorded" event', async () => {
|
|
280
|
-
expect(spies.onFirstSnapshotRecorded).to.have.been.calledWith(
|
|
306
|
+
expect(spies.onFirstSnapshotRecorded).to.have.been.calledWith(snapshot);
|
|
281
307
|
});
|
|
282
308
|
|
|
283
309
|
emitsOnly(['firstSnapshotRecorded']);
|
|
@@ -285,22 +311,15 @@ describe('Archivist', function () {
|
|
|
285
311
|
|
|
286
312
|
context('when it is not the first record', () => {
|
|
287
313
|
context('when there are changes', () => {
|
|
314
|
+
let changedSnapshot;
|
|
315
|
+
|
|
288
316
|
before(async () => {
|
|
289
|
-
await app.
|
|
290
|
-
content: 'document content',
|
|
291
|
-
serviceId: SERVICE_A_ID,
|
|
292
|
-
documentType: SERVICE_A_TYPE,
|
|
293
|
-
mimeType: MIME_TYPE,
|
|
294
|
-
fetchDate: FETCH_DATE,
|
|
295
|
-
});
|
|
317
|
+
await app.recordSnapshots(terms);
|
|
296
318
|
resetSpiesHistory();
|
|
297
|
-
|
|
298
|
-
content
|
|
299
|
-
serviceId: SERVICE_A_ID,
|
|
300
|
-
documentType: SERVICE_A_TYPE,
|
|
301
|
-
mimeType: MIME_TYPE,
|
|
302
|
-
fetchDate: FETCH_DATE,
|
|
319
|
+
terms.sourceDocuments.forEach(async sourceDocument => {
|
|
320
|
+
sourceDocument.content = serviceBSnapshotExpectedContent;
|
|
303
321
|
});
|
|
322
|
+
[changedSnapshot] = await app.recordSnapshots(terms);
|
|
304
323
|
});
|
|
305
324
|
|
|
306
325
|
after(() => {
|
|
@@ -310,29 +329,19 @@ describe('Archivist', function () {
|
|
|
310
329
|
});
|
|
311
330
|
|
|
312
331
|
it('emits "snapshotRecorded" event', async () => {
|
|
313
|
-
expect(spies.onSnapshotRecorded).to.have.been.calledWith(
|
|
332
|
+
expect(spies.onSnapshotRecorded).to.have.been.calledWith(changedSnapshot);
|
|
314
333
|
});
|
|
315
334
|
|
|
316
335
|
emitsOnly(['snapshotRecorded']);
|
|
317
336
|
});
|
|
318
337
|
|
|
319
338
|
context('when there are no changes', () => {
|
|
339
|
+
let snapshot;
|
|
340
|
+
|
|
320
341
|
before(async () => {
|
|
321
|
-
await app.
|
|
322
|
-
content: 'document content',
|
|
323
|
-
serviceId: SERVICE_A_ID,
|
|
324
|
-
documentType: SERVICE_A_TYPE,
|
|
325
|
-
mimeType: MIME_TYPE,
|
|
326
|
-
fetchDate: FETCH_DATE,
|
|
327
|
-
});
|
|
342
|
+
await app.recordSnapshots(terms);
|
|
328
343
|
resetSpiesHistory();
|
|
329
|
-
await app.
|
|
330
|
-
content: 'document content',
|
|
331
|
-
serviceId: SERVICE_A_ID,
|
|
332
|
-
documentType: SERVICE_A_TYPE,
|
|
333
|
-
mimeType: MIME_TYPE,
|
|
334
|
-
fetchDate: FETCH_DATE,
|
|
335
|
-
});
|
|
344
|
+
[snapshot] = await app.recordSnapshots(terms);
|
|
336
345
|
});
|
|
337
346
|
|
|
338
347
|
after(() => {
|
|
@@ -342,7 +351,7 @@ describe('Archivist', function () {
|
|
|
342
351
|
});
|
|
343
352
|
|
|
344
353
|
it('emits "snapshotNotChanged" event', async () => {
|
|
345
|
-
expect(spies.onSnapshotNotChanged).to.have.been.calledWith(
|
|
354
|
+
expect(spies.onSnapshotNotChanged).to.have.been.calledWith(snapshot);
|
|
346
355
|
});
|
|
347
356
|
|
|
348
357
|
emitsOnly(['snapshotNotChanged']);
|
|
@@ -351,16 +360,23 @@ describe('Archivist', function () {
|
|
|
351
360
|
});
|
|
352
361
|
|
|
353
362
|
describe('#recordVersion', () => {
|
|
363
|
+
let terms;
|
|
364
|
+
let version;
|
|
365
|
+
|
|
366
|
+
before(async () => {
|
|
367
|
+
terms = app.services.service_A.getTerms(SERVICE_A_TYPE);
|
|
368
|
+
terms.fetchDate = FETCH_DATE;
|
|
369
|
+
terms.sourceDocuments.forEach(async sourceDocument => {
|
|
370
|
+
sourceDocument.content = serviceASnapshotExpectedContent;
|
|
371
|
+
sourceDocument.mimeType = MIME_TYPE;
|
|
372
|
+
});
|
|
373
|
+
resetSpiesHistory();
|
|
374
|
+
});
|
|
375
|
+
|
|
354
376
|
context('when it is the first record', () => {
|
|
355
|
-
before(async () =>
|
|
356
|
-
app.recordVersion(
|
|
357
|
-
|
|
358
|
-
snapshotIds: ['sha'],
|
|
359
|
-
mimeType: MIME_TYPE,
|
|
360
|
-
fetchDate: FETCH_DATE,
|
|
361
|
-
serviceId: SERVICE_A_ID,
|
|
362
|
-
documentType: SERVICE_A_TYPE,
|
|
363
|
-
}));
|
|
377
|
+
before(async () => {
|
|
378
|
+
version = await app.recordVersion(terms);
|
|
379
|
+
});
|
|
364
380
|
|
|
365
381
|
after(() => {
|
|
366
382
|
resetSpiesHistory();
|
|
@@ -369,7 +385,7 @@ describe('Archivist', function () {
|
|
|
369
385
|
});
|
|
370
386
|
|
|
371
387
|
it('emits "firstVersionRecorded" event', async () => {
|
|
372
|
-
expect(spies.onFirstVersionRecorded).to.have.been.calledWith(
|
|
388
|
+
expect(spies.onFirstVersionRecorded).to.have.been.calledWith(version);
|
|
373
389
|
});
|
|
374
390
|
|
|
375
391
|
emitsOnly(['firstVersionRecorded']);
|
|
@@ -377,24 +393,15 @@ describe('Archivist', function () {
|
|
|
377
393
|
|
|
378
394
|
context('when it is not the first record', () => {
|
|
379
395
|
context('when there are changes', () => {
|
|
396
|
+
let changedVersion;
|
|
397
|
+
|
|
380
398
|
before(async () => {
|
|
381
|
-
await app.recordVersion(
|
|
382
|
-
content: serviceASnapshotExpectedContent,
|
|
383
|
-
mimeType: MIME_TYPE,
|
|
384
|
-
fetchDate: FETCH_DATE,
|
|
385
|
-
snapshotIds: ['sha'],
|
|
386
|
-
serviceId: SERVICE_A_ID,
|
|
387
|
-
documentType: SERVICE_A_TYPE,
|
|
388
|
-
});
|
|
399
|
+
await app.recordVersion(terms);
|
|
389
400
|
resetSpiesHistory();
|
|
390
|
-
|
|
391
|
-
content
|
|
392
|
-
mimeType: MIME_TYPE,
|
|
393
|
-
fetchDate: FETCH_DATE,
|
|
394
|
-
snapshotIds: ['sha'],
|
|
395
|
-
serviceId: SERVICE_A_ID,
|
|
396
|
-
documentType: SERVICE_A_TYPE,
|
|
401
|
+
terms.sourceDocuments.forEach(async sourceDocument => {
|
|
402
|
+
sourceDocument.content = serviceBSnapshotExpectedContent;
|
|
397
403
|
});
|
|
404
|
+
changedVersion = await app.recordVersion(terms);
|
|
398
405
|
});
|
|
399
406
|
|
|
400
407
|
after(() => {
|
|
@@ -404,31 +411,19 @@ describe('Archivist', function () {
|
|
|
404
411
|
});
|
|
405
412
|
|
|
406
413
|
it('emits "versionRecorded" event', async () => {
|
|
407
|
-
expect(spies.onVersionRecorded).to.have.been.calledWith(
|
|
414
|
+
expect(spies.onVersionRecorded).to.have.been.calledWith(changedVersion);
|
|
408
415
|
});
|
|
409
416
|
|
|
410
417
|
emitsOnly(['versionRecorded']);
|
|
411
418
|
});
|
|
412
419
|
|
|
413
420
|
context('when there are no changes', () => {
|
|
421
|
+
let version;
|
|
422
|
+
|
|
414
423
|
before(async () => {
|
|
415
|
-
await app.recordVersion(
|
|
416
|
-
content: serviceASnapshotExpectedContent,
|
|
417
|
-
snapshotIds: ['sha'],
|
|
418
|
-
mimeType: MIME_TYPE,
|
|
419
|
-
fetchDate: FETCH_DATE,
|
|
420
|
-
serviceId: SERVICE_A_ID,
|
|
421
|
-
documentType: SERVICE_A_TYPE,
|
|
422
|
-
});
|
|
424
|
+
await app.recordVersion(terms);
|
|
423
425
|
resetSpiesHistory();
|
|
424
|
-
await app.recordVersion(
|
|
425
|
-
content: serviceASnapshotExpectedContent,
|
|
426
|
-
snapshotIds: ['sha'],
|
|
427
|
-
mimeType: MIME_TYPE,
|
|
428
|
-
fetchDate: FETCH_DATE,
|
|
429
|
-
serviceId: SERVICE_A_ID,
|
|
430
|
-
documentType: SERVICE_A_TYPE,
|
|
431
|
-
});
|
|
426
|
+
version = await app.recordVersion(terms);
|
|
432
427
|
});
|
|
433
428
|
|
|
434
429
|
after(() => {
|
|
@@ -438,7 +433,7 @@ describe('Archivist', function () {
|
|
|
438
433
|
});
|
|
439
434
|
|
|
440
435
|
it('emits "versionNotChanged" event', async () => {
|
|
441
|
-
expect(spies.onVersionNotChanged).to.have.been.calledWith(
|
|
436
|
+
expect(spies.onVersionNotChanged).to.have.been.calledWith(version);
|
|
442
437
|
});
|
|
443
438
|
|
|
444
439
|
emitsOnly(['versionNotChanged']);
|
|
@@ -451,7 +446,7 @@ describe('Archivist', function () {
|
|
|
451
446
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
452
447
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
453
448
|
|
|
454
|
-
return app.
|
|
449
|
+
return app.track({ services });
|
|
455
450
|
});
|
|
456
451
|
|
|
457
452
|
after(() => {
|
|
@@ -494,14 +489,14 @@ describe('Archivist', function () {
|
|
|
494
489
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
495
490
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
496
491
|
|
|
497
|
-
await app.
|
|
492
|
+
await app.track({ services });
|
|
498
493
|
|
|
499
494
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
500
495
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
501
496
|
|
|
502
497
|
resetSpiesHistory();
|
|
503
498
|
|
|
504
|
-
return app.
|
|
499
|
+
return app.track({ services });
|
|
505
500
|
});
|
|
506
501
|
|
|
507
502
|
after(() => {
|
|
@@ -539,17 +534,34 @@ describe('Archivist', function () {
|
|
|
539
534
|
});
|
|
540
535
|
|
|
541
536
|
context('when a service changed', () => {
|
|
537
|
+
let snapshotA;
|
|
538
|
+
let snapshotB;
|
|
539
|
+
let versionA;
|
|
540
|
+
let versionB;
|
|
541
|
+
|
|
542
542
|
before(async () => {
|
|
543
543
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
544
544
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
545
545
|
|
|
546
|
-
await app.
|
|
546
|
+
await app.track({ services });
|
|
547
547
|
|
|
548
548
|
nock('https://www.servicea.example').get('/tos').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'text/html' });
|
|
549
549
|
nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
|
|
550
550
|
|
|
551
551
|
resetSpiesHistory();
|
|
552
|
-
|
|
552
|
+
app.on('snapshotNotChanged', record => {
|
|
553
|
+
snapshotB = record;
|
|
554
|
+
});
|
|
555
|
+
app.on('snapshotRecorded', record => {
|
|
556
|
+
snapshotA = record;
|
|
557
|
+
});
|
|
558
|
+
app.on('versionNotChanged', record => {
|
|
559
|
+
versionB = record;
|
|
560
|
+
});
|
|
561
|
+
app.on('versionRecorded', record => {
|
|
562
|
+
versionA = record;
|
|
563
|
+
});
|
|
564
|
+
await app.track({ services });
|
|
553
565
|
});
|
|
554
566
|
|
|
555
567
|
after(() => {
|
|
@@ -563,19 +575,19 @@ describe('Archivist', function () {
|
|
|
563
575
|
});
|
|
564
576
|
|
|
565
577
|
it('emits "snapshotNotChanged" event for the service that was not changed', async () => {
|
|
566
|
-
expect(spies.onSnapshotNotChanged).to.have.been.calledOnceWith(
|
|
578
|
+
expect(spies.onSnapshotNotChanged).to.have.been.calledOnceWith(snapshotB);
|
|
567
579
|
});
|
|
568
580
|
|
|
569
581
|
it('emits "snapshotRecorded" event for the service that was changed', async () => {
|
|
570
|
-
expect(spies.onSnapshotRecorded).to.have.been.calledOnceWith(
|
|
582
|
+
expect(spies.onSnapshotRecorded).to.have.been.calledOnceWith(snapshotA);
|
|
571
583
|
});
|
|
572
584
|
|
|
573
585
|
it('emits "versionNotChanged" events for the service that was not changed', async () => {
|
|
574
|
-
expect(spies.onVersionNotChanged).to.have.been.calledOnceWith(
|
|
586
|
+
expect(spies.onVersionNotChanged).to.have.been.calledOnceWith(versionB);
|
|
575
587
|
});
|
|
576
588
|
|
|
577
589
|
it('emits "versionRecorded" event for the service that was changed', async () => {
|
|
578
|
-
expect(spies.onVersionRecorded).to.have.been.calledOnceWith(
|
|
590
|
+
expect(spies.onVersionRecorded).to.have.been.calledOnceWith(versionA);
|
|
579
591
|
});
|
|
580
592
|
|
|
581
593
|
it('emits "snapshotRecorded" events after "versionRecorded" events', async () => {
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import mime from 'mime';
|
|
2
|
-
|
|
3
|
-
import Record from './record.js';
|
|
4
1
|
import RepositoryFactory from './repositories/factory.js';
|
|
2
|
+
import Snapshot from './snapshot.js';
|
|
3
|
+
import Version from './version.js';
|
|
5
4
|
|
|
6
5
|
export default class Recorder {
|
|
7
6
|
constructor(config) {
|
|
@@ -17,61 +16,18 @@ export default class Recorder {
|
|
|
17
16
|
return Promise.all([ this.versionsRepository.finalize(), this.snapshotsRepository.finalize() ]);
|
|
18
17
|
}
|
|
19
18
|
|
|
20
|
-
async getLatestSnapshot(
|
|
21
|
-
return this.snapshotsRepository.findLatest(
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
async recordSnapshot({ serviceId, documentType, pageId, fetchDate, mimeType, content }) {
|
|
25
|
-
if (!serviceId) {
|
|
26
|
-
throw new Error('A service ID is required');
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
if (!documentType) {
|
|
30
|
-
throw new Error('A terms type is required');
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
if (!fetchDate) {
|
|
34
|
-
throw new Error('The fetch date of the snapshot is required to ensure data consistency');
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
if (!content) {
|
|
38
|
-
throw new Error('A document content is required');
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
if (!mimeType) {
|
|
42
|
-
throw new Error('A document mime type is required to ensure data consistency');
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
return this.snapshotsRepository.save(new Record({ serviceId, documentType, pageId, fetchDate, mimeType, content }));
|
|
19
|
+
async getLatestSnapshot(terms, sourceDocumentId) {
|
|
20
|
+
return this.snapshotsRepository.findLatest(terms.service.id, terms.type, terms.hasMultipleSourceDocuments && sourceDocumentId);
|
|
46
21
|
}
|
|
47
22
|
|
|
48
|
-
async
|
|
49
|
-
|
|
50
|
-
throw new Error('A service ID is required');
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
if (!documentType) {
|
|
54
|
-
throw new Error('A terms type is required');
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
if (!snapshotIds?.length) {
|
|
58
|
-
throw new Error(`At least one snapshot ID is required to ensure data consistency for ${serviceId}'s ${documentType}`);
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
if (!fetchDate) {
|
|
62
|
-
throw new Error('The fetch date of the snapshot is required to ensure data consistency');
|
|
63
|
-
}
|
|
23
|
+
async record(record) {
|
|
24
|
+
record.validate();
|
|
64
25
|
|
|
65
|
-
|
|
66
|
-
|
|
26
|
+
switch (record.constructor) { // eslint-disable-line default-case
|
|
27
|
+
case Version:
|
|
28
|
+
return this.versionsRepository.save(record);
|
|
29
|
+
case Snapshot:
|
|
30
|
+
return this.snapshotsRepository.save(record);
|
|
67
31
|
}
|
|
68
|
-
|
|
69
|
-
const mimeType = mime.getType('markdown'); // A version is always in markdown format
|
|
70
|
-
|
|
71
|
-
return this.versionsRepository.save(new Record({ serviceId, documentType, snapshotIds, fetchDate, mimeType, content, isRefilter }));
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
async recordRefilter(params) {
|
|
75
|
-
return this.recordVersion({ isRefilter: true, ...params });
|
|
76
32
|
}
|
|
77
33
|
}
|