@opentermsarchive/engine 16.0.2 → 16.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/config/default.json +13 -0
  2. package/config/test.json +12 -0
  3. package/package.json +1 -1
  4. package/scripts/declarations/validate/index.mocha.js +7 -0
  5. package/scripts/import/index.js +1 -1
  6. package/scripts/import/loadCommits.js +1 -1
  7. package/scripts/rewrite/initializer/index.js +1 -1
  8. package/scripts/rewrite/rewrite-snapshots.js +1 -1
  9. package/scripts/rewrite/rewrite-versions.js +1 -1
  10. package/src/archivist/fetcher/htmlOnlyFetcher.js +8 -0
  11. package/src/archivist/fetcher/index.test.js +12 -0
  12. package/src/archivist/index.js +106 -23
  13. package/src/archivist/index.test.js +659 -9
  14. package/src/archivist/recorder/repositories/git/dataMapper.js +2 -1
  15. package/src/archivist/recorder/repositories/git/index.js +2 -13
  16. package/src/archivist/recorder/repositories/git/index.test.js +23 -1
  17. package/src/archivist/services/index.js +45 -1
  18. package/src/archivist/services/index.test.js +52 -1
  19. package/src/archivist/services/sourceDocument.js +8 -2
  20. package/src/archivist/services/sourceDocument.test.js +37 -0
  21. package/src/archivist/tracking-results/errors.js +5 -0
  22. package/src/archivist/tracking-results/index.js +233 -0
  23. package/src/archivist/tracking-results/index.test.js +406 -0
  24. package/src/archivist/tracking-results/recorder.js +156 -0
  25. package/src/archivist/tracking-results/recorder.test.js +559 -0
  26. package/src/archivist/tracking-results/repository.js +136 -0
  27. package/src/archivist/tracking-results/repository.test.js +763 -0
  28. package/src/archivist/tracking-results/run/dataMapper.js +51 -0
  29. package/src/archivist/tracking-results/run/dataMapper.test.js +168 -0
  30. package/src/archivist/tracking-results/run/index.js +115 -0
  31. package/src/archivist/tracking-results/run/index.test.js +221 -0
  32. package/src/archivist/tracking-results/terms-result/dataMapper.js +200 -0
  33. package/src/archivist/tracking-results/terms-result/dataMapper.test.js +575 -0
  34. package/src/archivist/tracking-results/terms-result/index.js +42 -0
  35. package/src/archivist/tracking-results/terms-result/index.test.js +228 -0
  36. package/src/git/errors.js +1 -0
  37. package/src/{archivist/recorder/repositories/git/git.js → git/index.js} +59 -9
  38. package/src/git/index.test.js +266 -0
  39. package/src/git/pathSegment.js +18 -0
  40. package/src/git/pathSegment.test.js +33 -0
  41. package/src/index.js +5 -4
  42. package/src/reporter/factory.js +2 -2
  43. package/src/reporter/github/index.js +7 -5
  44. package/src/reporter/github/index.test.js +16 -1
  45. package/src/reporter/gitlab/index.js +6 -6
  46. package/src/reporter/gitlab/index.test.js +24 -1
  47. package/src/reporter/index.js +9 -4
  48. package/src/reporter/index.test.js +74 -9
  49. package/src/archivist/recorder/repositories/git/git.test.js +0 -114
  50. /package/src/{archivist/recorder/repositories/git → git}/trailers.js +0 -0
  51. /package/src/{archivist/recorder/repositories/git → git}/trailers.test.js +0 -0
@@ -0,0 +1,763 @@
1
+ import fsApi from 'fs';
2
+ import path from 'path';
3
+ import { fileURLToPath } from 'url';
4
+
5
+ import { expect } from 'chai';
6
+ import sinon from 'sinon';
7
+
8
+ import Git from '../../git/index.js';
9
+
10
+ import TrackingResultsRepository from './repository.js';
11
+ import Run from './run/index.js';
12
+ import { EVENT_TYPES as TERMS_RESULT_EVENT_TYPES } from './terms-result/dataMapper.js';
13
+ import TermsResult, { STATUSES } from './terms-result/index.js';
14
+
15
+ const fs = fsApi.promises;
16
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
17
+
18
+ const REPOSITORY_PATH = path.resolve(__dirname, '../../../test/data/tracking-results');
19
+
20
+ const AUTHOR = {
21
+ name: 'Open Terms Archive Testing Bot',
22
+ email: 'bot@opentermsarchive.org',
23
+ };
24
+
25
+ const SERVICE_ID = 'Facebook';
26
+ const TERMS_TYPE = 'Terms of Service';
27
+ const SERVICE_NAME = 'Facebook';
28
+
29
+ const SOURCE_DOCUMENTS = [{
30
+ id: 'main',
31
+ fetch: 'https://www.facebook.com/legal/terms',
32
+ select: '.content',
33
+ remove: '',
34
+ filter: [],
35
+ executeClientScripts: false,
36
+ mimeType: 'text/html',
37
+ snapshotId: 'def456',
38
+ }];
39
+
40
+ function makeResult({
41
+ status = STATUSES.ok,
42
+ serviceName = SERVICE_NAME,
43
+ sourceDocuments = SOURCE_DOCUMENTS,
44
+ reasons,
45
+ transientError,
46
+ serviceId = SERVICE_ID,
47
+ termsType = TERMS_TYPE,
48
+ date = '2026-01-10T10:30:00Z',
49
+ } = {}) {
50
+ const event = { date, serviceName, sourceDocuments };
51
+
52
+ if (reasons) {
53
+ event.reasons = reasons;
54
+ }
55
+
56
+ if (transientError) {
57
+ event.transientError = transientError;
58
+ }
59
+
60
+ return new TermsResult({ serviceId, termsType, status, event });
61
+ }
62
+
63
+ function makeRun({ runId = 'ota-run-f47ac10b-58cc-4372-a567-0e02b2c3d479' } = {}) {
64
+ return new Run({
65
+ runId,
66
+ collectionId: 'france',
67
+ schedule: '30 */12 * * *',
68
+ startDate: '2026-04-06T10:30:00Z',
69
+ engineVersion: '12.0.0',
70
+ declarationsCommit: 'abc123def456',
71
+ servicesCount: 1,
72
+ termsCount: 1,
73
+ });
74
+ }
75
+
76
+ describe('TrackingResultsRepository', () => {
77
+ let subject;
78
+ let git;
79
+
80
+ before(async function () {
81
+ this.timeout(5000);
82
+
83
+ git = new Git({ path: REPOSITORY_PATH, author: AUTHOR });
84
+ await git.initialize();
85
+
86
+ subject = new TrackingResultsRepository({ path: REPOSITORY_PATH, author: AUTHOR, publish: false });
87
+ await subject.initialize();
88
+ });
89
+
90
+ after(() => subject.removeAll());
91
+
92
+ describe('#saveTermsResult', () => {
93
+ context('when no previous result exists', () => {
94
+ let saved;
95
+ let commit;
96
+
97
+ before(async () => {
98
+ saved = await subject.saveTermsResult(makeResult());
99
+ ([commit] = await git.log());
100
+ });
101
+
102
+ after(() => subject.removeAll());
103
+
104
+ it('returns the commit SHA', () => {
105
+ expect(saved.sha).to.be.a('string').with.length.greaterThan(6);
106
+ });
107
+
108
+ it('returns the FIRST_TRACKING event type', () => {
109
+ expect(saved.eventType).to.equal(TERMS_RESULT_EVENT_TYPES.FIRST_TRACKING);
110
+ });
111
+
112
+ it('creates a commit with the first-record subject', () => {
113
+ expect(commit.message).to.equal('Record first tracking of Facebook Terms of Service');
114
+ });
115
+
116
+ it('persists the file at the expected path', () => {
117
+ expect(fsApi.existsSync(path.join(REPOSITORY_PATH, 'Facebook', 'Terms of Service.json'))).to.be.true;
118
+ });
119
+
120
+ it('persists JSON matching toJSON of the TermsResult', async () => {
121
+ const content = JSON.parse(await fs.readFile(path.join(REPOSITORY_PATH, 'Facebook', 'Terms of Service.json'), 'utf8'));
122
+
123
+ expect(content).to.deep.equal({
124
+ status: 'ok',
125
+ event: {
126
+ date: '2026-01-10T10:30:00Z',
127
+ serviceName: 'Facebook',
128
+ sourceDocuments: SOURCE_DOCUMENTS,
129
+ },
130
+ });
131
+ });
132
+
133
+ it('uses the event date as the commit date', () => {
134
+ expect(new Date(commit.date).toISOString()).to.equal('2026-01-10T10:30:00.000Z');
135
+ });
136
+ });
137
+
138
+ context('when no substantive change', () => {
139
+ let saved;
140
+ let commitsBefore;
141
+
142
+ before(async () => {
143
+ await subject.saveTermsResult(makeResult());
144
+ commitsBefore = (await git.log()).length;
145
+ saved = await subject.saveTermsResult(makeResult({ date: '2026-02-10T10:30:00Z' }));
146
+ });
147
+
148
+ after(() => subject.removeAll());
149
+
150
+ it('returns null sha and null event type', () => {
151
+ expect(saved).to.deep.equal({ sha: null, eventType: null });
152
+ });
153
+
154
+ it('does not create a new commit', async () => {
155
+ expect((await git.log()).length).to.equal(commitsBefore);
156
+ });
157
+ });
158
+
159
+ context('with trailers', () => {
160
+ const RUN_ID = 'ota-run-f47ac10b-58cc-4372-a567-0e02b2c3d479';
161
+ let commit;
162
+
163
+ before(async () => {
164
+ await subject.saveTermsResult(makeResult(), { trailers: { 'x-run-id': RUN_ID } });
165
+ commit = await git.getCommit([]);
166
+ });
167
+
168
+ after(() => subject.removeAll());
169
+
170
+ it('records the trailers on the commit', () => {
171
+ expect(commit.trailers['x-run-id']).to.equal(RUN_ID);
172
+ });
173
+ });
174
+
175
+ context('when the results carry undefined-valued declaration fields', () => { // Regression test: JSON.stringify drops undefined-valued keys at write time, so without normalisation the re-read form would differ from the in-memory one and produce a spurious DECLARATION_UPDATED on every run
176
+ let saved;
177
+ let commitsBefore;
178
+
179
+ const sourceDocumentsWithUndefinedKeys = [{
180
+ id: 'main',
181
+ fetch: 'https://www.facebook.com/legal/terms',
182
+ select: '.content',
183
+ remove: undefined,
184
+ filter: undefined,
185
+ executeClientScripts: false,
186
+ mimeType: 'text/html',
187
+ snapshotId: 'def456',
188
+ }];
189
+
190
+ before(async () => {
191
+ await subject.saveTermsResult(makeResult({ sourceDocuments: sourceDocumentsWithUndefinedKeys }));
192
+ commitsBefore = (await git.log()).length;
193
+ saved = await subject.saveTermsResult(makeResult({ sourceDocuments: sourceDocumentsWithUndefinedKeys, date: '2026-02-10T10:30:00Z' }));
194
+ });
195
+
196
+ after(() => subject.removeAll());
197
+
198
+ it('detects no substantive change on an identical follow-up save', () => {
199
+ expect(saved).to.deep.equal({ sha: null, eventType: null });
200
+ });
201
+
202
+ it('does not create a new commit', async () => {
203
+ expect((await git.log()).length).to.equal(commitsBefore);
204
+ });
205
+ });
206
+
207
+ context('when status transitions from ok to failed', () => {
208
+ let commit;
209
+
210
+ before(async () => {
211
+ await subject.saveTermsResult(makeResult());
212
+ await subject.saveTermsResult(makeResult({ status: STATUSES.failed, reasons: ['[fetch] HTTP 500'] }));
213
+ ([commit] = await git.log());
214
+ });
215
+
216
+ after(() => subject.removeAll());
217
+
218
+ it('uses the failure subject with the derived failure type', () => {
219
+ expect(commit.message).to.equal('Record tracking failure of Facebook Terms of Service (fetch)');
220
+ });
221
+ });
222
+
223
+ context('when status transitions from failed to ok', () => {
224
+ let commit;
225
+
226
+ before(async () => {
227
+ await subject.saveTermsResult(makeResult({ status: STATUSES.failed, reasons: ['[fetch] HTTP 500'] }));
228
+ await subject.saveTermsResult(makeResult());
229
+ ([commit] = await git.log());
230
+ });
231
+
232
+ after(() => subject.removeAll());
233
+
234
+ it('uses the recovery subject', () => {
235
+ expect(commit.message).to.equal('Record tracking recovery of Facebook Terms of Service');
236
+ });
237
+ });
238
+
239
+ context('when a transient error is introduced', () => {
240
+ let commit;
241
+
242
+ before(async () => {
243
+ await subject.saveTermsResult(makeResult());
244
+ await subject.saveTermsResult(makeResult({ transientError: { reasons: ['[fetch] HTTP 503'] } }));
245
+ ([commit] = await git.log());
246
+ });
247
+
248
+ after(() => subject.removeAll());
249
+
250
+ it('uses the transient-error subject', () => {
251
+ expect(commit.message).to.equal('Record transient error of Facebook Terms of Service');
252
+ });
253
+ });
254
+
255
+ context('when a transient error is then resolved', () => {
256
+ let commit;
257
+
258
+ before(async () => {
259
+ await subject.saveTermsResult(makeResult());
260
+ await subject.saveTermsResult(makeResult({ transientError: { reasons: ['[fetch] HTTP 503'] } }));
261
+ await subject.saveTermsResult(makeResult());
262
+ ([commit] = await git.log());
263
+ });
264
+
265
+ after(() => subject.removeAll());
266
+
267
+ it('uses the clear-transient-error subject', () => {
268
+ expect(commit.message).to.equal('Clear transient error of Facebook Terms of Service');
269
+ });
270
+ });
271
+
272
+ context('when the declaration is updated', () => {
273
+ let commit;
274
+
275
+ before(async () => {
276
+ await subject.saveTermsResult(makeResult({ sourceDocuments: [{ ...SOURCE_DOCUMENTS[0], select: '.old' }] }));
277
+ await subject.saveTermsResult(makeResult({ sourceDocuments: [{ ...SOURCE_DOCUMENTS[0], select: '.new' }] }));
278
+ ([commit] = await git.log());
279
+ });
280
+
281
+ after(() => subject.removeAll());
282
+
283
+ it('uses the declaration-update subject', () => {
284
+ expect(commit.message).to.equal('Update tracking declaration of Facebook Terms of Service');
285
+ });
286
+ });
287
+
288
+ context('when only the MIME type changes', () => {
289
+ let commit;
290
+
291
+ before(async () => {
292
+ await subject.saveTermsResult(makeResult({ sourceDocuments: [{ ...SOURCE_DOCUMENTS[0], mimeType: 'text/html' }] }));
293
+ await subject.saveTermsResult(makeResult({ sourceDocuments: [{ ...SOURCE_DOCUMENTS[0], mimeType: 'application/pdf' }] }));
294
+ ([commit] = await git.log());
295
+ });
296
+
297
+ after(() => subject.removeAll());
298
+
299
+ it('uses the mime-type-update subject', () => {
300
+ expect(commit.message).to.equal('Update MIME type of Facebook Terms of Service');
301
+ });
302
+ });
303
+
304
+ context('when only the service name changes', () => {
305
+ let commit;
306
+
307
+ before(async () => {
308
+ await subject.saveTermsResult(makeResult({ serviceName: 'Facebook' }));
309
+ await subject.saveTermsResult(makeResult({ serviceName: 'Facebook Inc.' }));
310
+ ([commit] = await git.log());
311
+ });
312
+
313
+ after(() => subject.removeAll());
314
+
315
+ it('uses the service-name-update subject, keyed by the stable serviceId', () => {
316
+ expect(commit.message).to.equal('Update service name of Facebook Terms of Service');
317
+ });
318
+ });
319
+ });
320
+
321
+ describe('when a commit fails', () => {
322
+ const commitError = new Error('Unable to create index.lock');
323
+
324
+ context('for a terms that already has a committed result', () => {
325
+ let error;
326
+
327
+ before(async () => {
328
+ await subject.saveTermsResult(makeResult());
329
+
330
+ const commitStub = sinon.stub(subject.git, 'commit').rejects(commitError);
331
+
332
+ error = await subject.saveTermsResult(makeResult({ status: STATUSES.failed, reasons: ['[fetch] HTTP 500'] })).catch(error => error);
333
+ commitStub.restore();
334
+ });
335
+
336
+ after(() => subject.removeAll());
337
+
338
+ it('throws an error exposing the cause', () => {
339
+ expect(error.cause).to.equal(commitError);
340
+ });
341
+
342
+ it('keeps the committed result as the latest one', async () => {
343
+ const latest = await subject.findLatestTermsResult(SERVICE_ID, TERMS_TYPE);
344
+
345
+ expect(latest.status).to.equal(STATUSES.ok);
346
+ });
347
+ });
348
+
349
+ context('for a terms that has no committed result yet', () => {
350
+ before(async () => {
351
+ const commitStub = sinon.stub(subject.git, 'commit').rejects(commitError);
352
+
353
+ await subject.saveTermsResult(makeResult()).catch(() => {});
354
+ commitStub.restore();
355
+ });
356
+
357
+ after(() => subject.removeAll());
358
+
359
+ it('leaves no result behind', async () => {
360
+ expect(await subject.findLatestTermsResult(SERVICE_ID, TERMS_TYPE)).to.be.null;
361
+ });
362
+ });
363
+
364
+ context('for the run', () => {
365
+ before(async () => {
366
+ await subject.saveRun(makeRun());
367
+
368
+ const commitStub = sinon.stub(subject.git, 'commit').rejects(commitError);
369
+ const crashed = makeRun();
370
+
371
+ crashed.markCrashed('2026-04-06T11:00:00Z');
372
+ await subject.saveRun(crashed).catch(() => {});
373
+ commitStub.restore();
374
+ });
375
+
376
+ after(() => subject.removeAll());
377
+
378
+ it('keeps the committed run as the latest one', async () => {
379
+ const latest = await subject.findLatestRun();
380
+
381
+ expect(latest.lastRun.status).to.equal('in_progress');
382
+ });
383
+ });
384
+ });
385
+
386
+ describe('#findLatestTermsResult', () => {
387
+ context('when no result exists', () => {
388
+ it('returns null', async () => {
389
+ expect(await subject.findLatestTermsResult(SERVICE_ID, TERMS_TYPE)).to.be.null;
390
+ });
391
+ });
392
+
393
+ context('when a result has been saved', () => {
394
+ let result;
395
+
396
+ before(async () => {
397
+ await subject.saveTermsResult(makeResult());
398
+ result = await subject.findLatestTermsResult(SERVICE_ID, TERMS_TYPE);
399
+ });
400
+
401
+ after(() => subject.removeAll());
402
+
403
+ it('returns the persisted result as a TermsResult', () => {
404
+ expect(result).to.be.instanceOf(TermsResult);
405
+ expect(result.serviceId).to.equal(SERVICE_ID);
406
+ expect(result.termsType).to.equal(TERMS_TYPE);
407
+ expect(result.status).to.equal(STATUSES.ok);
408
+ expect(result.event.serviceName).to.equal(SERVICE_NAME);
409
+ });
410
+ });
411
+ });
412
+
413
+ describe('#saveRun', () => {
414
+ context('with an in-progress run', () => {
415
+ let commit;
416
+
417
+ before(async () => {
418
+ await subject.saveRun(makeRun());
419
+ ([commit] = await git.log());
420
+ });
421
+
422
+ after(() => subject.removeAll());
423
+
424
+ it('creates a commit with the Start run subject', () => {
425
+ expect(commit.message).to.equal('Start run ota-run-f47ac10b');
426
+ });
427
+
428
+ it('persists run.json at the root of the repository', () => {
429
+ expect(fsApi.existsSync(path.join(REPOSITORY_PATH, 'run.json'))).to.be.true;
430
+ });
431
+ });
432
+
433
+ context('called twice in a row with identical content', () => {
434
+ let commitsBefore;
435
+ let commitCount;
436
+ let secondSha;
437
+
438
+ before(async () => {
439
+ await subject.saveRun(makeRun());
440
+ commitsBefore = (await git.log()).length;
441
+ secondSha = await subject.saveRun(makeRun()); // Same content as the previous saveRun
442
+ commitCount = (await git.log()).length;
443
+ });
444
+
445
+ after(() => subject.removeAll());
446
+
447
+ it('attempts the commit but Git creates no new revision when the run.json content is byte-identical', () => {
448
+ // saveRun does not proactively short-circuit like saveTermsResult does; Git's own "nothing to commit" guard is the last line of defense.
449
+ expect(commitCount).to.equal(commitsBefore);
450
+ });
451
+
452
+ it('returns no SHA for the no-op commit attempt', () => {
453
+ expect(secondSha).to.be.undefined;
454
+ });
455
+ });
456
+
457
+ context('with a completed run', () => {
458
+ let commit;
459
+
460
+ before(async () => {
461
+ const run = makeRun();
462
+
463
+ run.markCompleted('2026-04-06T10:42:34Z');
464
+ run.tracked = { ok: 1518, failed: 5 };
465
+ await subject.saveRun(run);
466
+ ([commit] = await git.log());
467
+ });
468
+
469
+ after(() => subject.removeAll());
470
+
471
+ it('creates a commit with counts in the subject', () => {
472
+ expect(commit.message).to.equal('Complete run ota-run-f47ac10b (1518 ok, 5 failed)');
473
+ });
474
+ });
475
+
476
+ context('with a crashed run', () => {
477
+ let commit;
478
+
479
+ before(async () => {
480
+ const run = makeRun();
481
+
482
+ run.markCrashed('2026-04-06T10:42:34Z');
483
+ await subject.saveRun(run);
484
+ ([commit] = await git.log());
485
+ });
486
+
487
+ after(() => subject.removeAll());
488
+
489
+ it('creates a commit with the Finalize crashed run subject', () => {
490
+ expect(commit.message).to.equal('Finalize crashed run ota-run-f47ac10b');
491
+ });
492
+ });
493
+ });
494
+
495
+ describe('#findLatestRun', () => {
496
+ context('when no run.json exists', () => {
497
+ it('returns null', async () => {
498
+ expect(await subject.findLatestRun()).to.be.null;
499
+ });
500
+ });
501
+
502
+ context('after a saveRun', () => {
503
+ let run;
504
+
505
+ before(async () => {
506
+ await subject.saveRun(makeRun());
507
+ run = await subject.findLatestRun();
508
+ });
509
+
510
+ after(() => subject.removeAll());
511
+
512
+ it('returns the persisted run as a Run instance', () => {
513
+ expect(run).to.be.instanceOf(Run);
514
+ expect(run.runId).to.equal('ota-run-f47ac10b-58cc-4372-a567-0e02b2c3d479');
515
+ expect(run.lastRun.status).to.equal('in_progress');
516
+ });
517
+ });
518
+
519
+ context('when run.json is corrupt JSON', () => {
520
+ before(async () => {
521
+ await subject.saveRun(makeRun());
522
+ await fs.writeFile(path.join(REPOSITORY_PATH, 'run.json'), '{ broken json');
523
+ });
524
+
525
+ after(() => subject.removeAll());
526
+
527
+ it('throws an error contextualised with the file path', async () => {
528
+ try {
529
+ await subject.findLatestRun();
530
+ expect.fail('expected findLatestRun to throw');
531
+ } catch (error) {
532
+ expect(error.message).to.match(/Could not parse JSON in/);
533
+ expect(error.message).to.include('run.json');
534
+ }
535
+ });
536
+ });
537
+ });
538
+
539
+ describe('#removeAll', () => {
540
+ it('deletes all persisted files and leaves the repository in a valid empty state', async () => {
541
+ await subject.saveTermsResult(makeResult());
542
+ await subject.removeAll();
543
+ expect(await subject.findLatestTermsResult(SERVICE_ID, TERMS_TYPE)).to.be.null;
544
+ });
545
+
546
+ it('is safe to call on an already-empty repository', async () => {
547
+ await subject.removeAll(); // First call empties it
548
+ let threw = false;
549
+
550
+ try {
551
+ await subject.removeAll(); // Second call on empty repo
552
+ } catch {
553
+ threw = true;
554
+ }
555
+ expect(threw).to.be.false;
556
+ });
557
+ });
558
+
559
+ describe('#initialize', () => {
560
+ it('is idempotent: calling it twice does not throw and leaves the repo usable', async () => {
561
+ await subject.initialize();
562
+ await subject.initialize();
563
+ await subject.saveTermsResult(makeResult());
564
+ expect(await subject.findLatestTermsResult(SERVICE_ID, TERMS_TYPE)).to.not.be.null;
565
+ await subject.removeAll();
566
+ });
567
+ });
568
+
569
+ describe('#finalize', () => {
570
+ context('when publish is false', () => {
571
+ it('does not throw', () => subject.finalize());
572
+ });
573
+ });
574
+
575
+ describe('#findLatestRunCommitSha', () => {
576
+ context('when no run.json exists yet', () => {
577
+ it('returns null', async () => {
578
+ expect(await subject.findLatestRunCommitSha()).to.be.null;
579
+ });
580
+ });
581
+
582
+ context('after a single saveRun', () => {
583
+ let savedSha;
584
+
585
+ before(async () => {
586
+ savedSha = await subject.saveRun(makeRun());
587
+ });
588
+
589
+ after(() => subject.removeAll());
590
+
591
+ it('returns the SHA of that commit', async () => {
592
+ expect(await subject.findLatestRunCommitSha()).to.equal(savedSha);
593
+ });
594
+ });
595
+
596
+ context('after several saveRun calls', () => {
597
+ let lastSha;
598
+
599
+ before(async () => {
600
+ await subject.saveRun(makeRun());
601
+
602
+ const completed = makeRun();
603
+
604
+ completed.markCompleted('2026-04-06T10:42:34Z');
605
+ completed.tracked = { ok: 0, failed: 0 };
606
+ lastSha = await subject.saveRun(completed);
607
+ });
608
+
609
+ after(() => subject.removeAll());
610
+
611
+ it('returns the SHA of the latest commit on run.json', async () => {
612
+ expect(await subject.findLatestRunCommitSha()).to.equal(lastSha);
613
+ });
614
+ });
615
+
616
+ context('when the latest activity is on a per-terms file rather than run.json', () => {
617
+ let runSha;
618
+
619
+ before(async () => {
620
+ runSha = await subject.saveRun(makeRun());
621
+ await subject.saveTermsResult(makeResult({ serviceId: 'Facebook', termsType: 'Terms of Service' }));
622
+ });
623
+
624
+ after(() => subject.removeAll());
625
+
626
+ it('still returns the latest commit on run.json, ignoring intervening per-terms commits', async () => {
627
+ expect(await subject.findLatestRunCommitSha()).to.equal(runSha);
628
+ });
629
+ });
630
+ });
631
+
632
+ describe('#findCommittedTermsResultsSince', () => {
633
+ context('with a null sha argument', () => {
634
+ it('returns an empty array', async () => {
635
+ expect(await subject.findCommittedTermsResultsSince(null)).to.deep.equal([]);
636
+ });
637
+ });
638
+
639
+ context('when no per-terms commits happened after the sha', () => {
640
+ let runSha;
641
+
642
+ before(async () => {
643
+ runSha = await subject.saveRun(makeRun());
644
+ });
645
+
646
+ after(() => subject.removeAll());
647
+
648
+ it('returns an empty array', async () => {
649
+ expect(await subject.findCommittedTermsResultsSince(runSha)).to.deep.equal([]);
650
+ });
651
+ });
652
+
653
+ context('when several distinct terms were committed after the sha', () => {
654
+ let runSha;
655
+ let results;
656
+
657
+ before(async () => {
658
+ runSha = await subject.saveRun(makeRun());
659
+ await subject.saveTermsResult(makeResult({ serviceId: 'Facebook', termsType: 'Terms of Service' }));
660
+ await subject.saveTermsResult(makeResult({ serviceId: 'Google', termsType: 'Privacy Policy' }));
661
+ results = await subject.findCommittedTermsResultsSince(runSha);
662
+ });
663
+
664
+ after(() => subject.removeAll());
665
+
666
+ it('returns each committed terms exactly once, with serviceId and termsType', () => {
667
+ const keys = results.map(r => `${r.serviceId}/${r.termsType}`).sort();
668
+
669
+ expect(keys).to.deep.equal([ 'Facebook/Terms of Service', 'Google/Privacy Policy' ]);
670
+ });
671
+ });
672
+
673
+ context('when the same terms is committed multiple times since the sha', () => {
674
+ let runSha;
675
+ let results;
676
+
677
+ before(async () => {
678
+ runSha = await subject.saveRun(makeRun());
679
+ await subject.saveTermsResult(makeResult({ serviceId: 'Facebook', termsType: 'Terms of Service' }));
680
+ await subject.saveTermsResult(makeResult({ serviceId: 'Facebook', termsType: 'Terms of Service', status: STATUSES.failed, reasons: ['[fetch] HTTP 500'] }));
681
+ results = await subject.findCommittedTermsResultsSince(runSha);
682
+ });
683
+
684
+ after(() => subject.removeAll());
685
+
686
+ it('deduplicates to a single entry per terms', () => {
687
+ expect(results).to.deep.equal([{ serviceId: 'Facebook', termsType: 'Terms of Service' }]);
688
+ });
689
+ });
690
+
691
+ context('when run.json is committed in between', () => {
692
+ let runSha;
693
+ let results;
694
+
695
+ before(async () => {
696
+ runSha = await subject.saveRun(makeRun());
697
+ await subject.saveTermsResult(makeResult({ serviceId: 'Facebook', termsType: 'Terms of Service' }));
698
+
699
+ const completed = makeRun();
700
+
701
+ completed.markCompleted('2026-04-06T10:42:34Z');
702
+ completed.tracked = { ok: 1, failed: 0 };
703
+ await subject.saveRun(completed);
704
+
705
+ results = await subject.findCommittedTermsResultsSince(runSha);
706
+ });
707
+
708
+ after(() => subject.removeAll());
709
+
710
+ it('does not include run.json in the results', () => {
711
+ expect(results).to.deep.equal([{ serviceId: 'Facebook', termsType: 'Terms of Service' }]);
712
+ });
713
+ });
714
+ });
715
+
716
+ describe('crash recovery scenario', () => { // End-to-end test of the primitives composed the way the recorder composes them
717
+ let startedRun;
718
+
719
+ before(async () => {
720
+ startedRun = makeRun({ runId: Run.generateId() });
721
+ await subject.saveRun(startedRun);
722
+ await subject.saveTermsResult(makeResult({ serviceId: 'Facebook', termsType: 'Terms of Service' }));
723
+ await subject.saveTermsResult(makeResult({ serviceId: 'Facebook', termsType: 'Privacy Policy' }));
724
+ // Simulated crash: no run-completion commit. run.json on disk still has status: in_progress.
725
+ });
726
+
727
+ after(() => subject.removeAll());
728
+
729
+ it('exposes the in-progress run via findLatestRun', async () => {
730
+ const found = await subject.findLatestRun();
731
+
732
+ expect(found.lastRun.status).to.equal('in_progress');
733
+ expect(found.runId).to.equal(startedRun.runId);
734
+ });
735
+
736
+ it('lists exactly the terms committed during the partial run', async () => {
737
+ const sha = await subject.findLatestRunCommitSha();
738
+ const committed = await subject.findCommittedTermsResultsSince(sha);
739
+ const keys = committed.map(r => `${r.serviceId}/${r.termsType}`).sort();
740
+
741
+ expect(keys).to.deep.equal([ 'Facebook/Privacy Policy', 'Facebook/Terms of Service' ]);
742
+ });
743
+
744
+ it('can be finalised additively by marking the run as crashed and saving it again', async () => {
745
+ const found = await subject.findLatestRun();
746
+
747
+ found.markCrashed('2026-04-06T11:00:00Z');
748
+ found.coverage = {
749
+ processed: 2,
750
+ skipped: [{ serviceId: 'Google', termsType: 'Terms of Service', reason: 'no outcome recorded before the engine crashed' }],
751
+ };
752
+
753
+ await subject.saveRun(found);
754
+
755
+ const refound = await subject.findLatestRun();
756
+
757
+ expect(refound.lastRun.status).to.equal('crashed');
758
+ expect(refound.lastRun.endDate).to.equal('2026-04-06T11:00:00Z');
759
+ expect(refound.coverage.processed).to.equal(2);
760
+ expect(refound.coverage.skipped).to.deep.equal([{ serviceId: 'Google', termsType: 'Terms of Service', reason: 'no outcome recorded before the engine crashed' }]);
761
+ });
762
+ });
763
+ });