@opentermsarchive/engine 14.1.0 → 15.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/archivist/extract/exposedFilters.js +15 -0
- package/src/archivist/extract/exposedFilters.test.js +49 -1
- package/src/archivist/fetcher/fullDomFetcher.test.js +1 -1
- package/src/archivist/index.test.js +1 -1
- package/src/archivist/recorder/repositories/git/dataMapper.js +4 -0
- package/src/archivist/recorder/repositories/git/git.js +90 -8
- package/src/archivist/recorder/repositories/git/index.js +91 -4
- package/src/archivist/recorder/repositories/git/index.test.js +288 -0
- package/src/archivist/recorder/repositories/interface.js +34 -0
- package/src/archivist/recorder/repositories/mongo/index.js +53 -0
- package/src/archivist/recorder/repositories/mongo/index.test.js +171 -0
- package/src/archivist/services/index.test.js +1 -1
- package/src/collection-api/middlewares/errors.js +1 -1
- package/src/collection-api/routes/index.js +2 -1
- package/src/collection-api/routes/services.js +34 -21
- package/src/collection-api/routes/versions.js +521 -27
- package/src/collection-api/routes/versions.test.js +655 -25
|
@@ -400,6 +400,59 @@ describe('GitRepository', () => {
|
|
|
400
400
|
expect(await subject.findById('inexistantID')).to.equal(null);
|
|
401
401
|
});
|
|
402
402
|
});
|
|
403
|
+
|
|
404
|
+
context('when the requested ID is well formed but absent from the repository', () => {
|
|
405
|
+
it('returns null rather than throwing a "bad object" error', async () => {
|
|
406
|
+
expect(await subject.findById('ecd9407eb26b1bf0613186175ee80edbdeedd47f')).to.equal(null);
|
|
407
|
+
});
|
|
408
|
+
});
|
|
409
|
+
|
|
410
|
+
context('when the requested ID could be interpreted as a git option', () => {
|
|
411
|
+
const INJECTION_PROOF_FILE_PATH = path.resolve(__dirname, 'findById-argument-injection-proof.txt');
|
|
412
|
+
|
|
413
|
+
after(() => fs.rmSync(INJECTION_PROOF_FILE_PATH, { force: true }));
|
|
414
|
+
|
|
415
|
+
it('returns null without letting the ID reach git as an argument', async () => {
|
|
416
|
+
expect(await subject.findById(`--output=${INJECTION_PROOF_FILE_PATH}`)).to.equal(null);
|
|
417
|
+
expect(fs.existsSync(INJECTION_PROOF_FILE_PATH), 'a version ID must never be interpreted as a git option').to.be.false;
|
|
418
|
+
});
|
|
419
|
+
});
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
describe('#findMetadataById', () => {
|
|
423
|
+
let id;
|
|
424
|
+
|
|
425
|
+
before(async () => {
|
|
426
|
+
({ id } = await subject.save(new Version({
|
|
427
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
428
|
+
termsType: TERMS_TYPE,
|
|
429
|
+
content: CONTENT,
|
|
430
|
+
fetchDate: FETCH_DATE,
|
|
431
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
432
|
+
mimeType: HTML_MIME_TYPE,
|
|
433
|
+
metadata: METADATA,
|
|
434
|
+
})));
|
|
435
|
+
});
|
|
436
|
+
|
|
437
|
+
after(() => subject.removeAll());
|
|
438
|
+
|
|
439
|
+
it('returns the record', async () => {
|
|
440
|
+
const record = await subject.findMetadataById(id);
|
|
441
|
+
|
|
442
|
+
expect(record).to.be.an.instanceof(Version);
|
|
443
|
+
expect(record.id).to.include(id);
|
|
444
|
+
});
|
|
445
|
+
|
|
446
|
+
context('when the requested ID could be interpreted as a git option', () => {
|
|
447
|
+
const INJECTION_PROOF_FILE_PATH = path.resolve(__dirname, 'findMetadataById-argument-injection-proof.txt');
|
|
448
|
+
|
|
449
|
+
after(() => fs.rmSync(INJECTION_PROOF_FILE_PATH, { force: true }));
|
|
450
|
+
|
|
451
|
+
it('returns null without letting the ID reach git as an argument', async () => {
|
|
452
|
+
expect(await subject.findMetadataById(`--output=${INJECTION_PROOF_FILE_PATH}`)).to.equal(null);
|
|
453
|
+
expect(fs.existsSync(INJECTION_PROOF_FILE_PATH), 'a version ID must never be interpreted as a git option').to.be.false;
|
|
454
|
+
});
|
|
455
|
+
});
|
|
403
456
|
});
|
|
404
457
|
|
|
405
458
|
describe('#findByDate', () => {
|
|
@@ -485,6 +538,37 @@ describe('GitRepository', () => {
|
|
|
485
538
|
expect(record.metadata).to.deep.equal(METADATA);
|
|
486
539
|
});
|
|
487
540
|
});
|
|
541
|
+
|
|
542
|
+
context('when the service ID is a git argument injection attempt', () => {
|
|
543
|
+
const INJECTION_PROOF_FILE_PATH = path.resolve(__dirname, 'findByDate-argument-injection-proof.*');
|
|
544
|
+
|
|
545
|
+
before(async () => {
|
|
546
|
+
await subject.save(new Version({
|
|
547
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
548
|
+
termsType: TERMS_TYPE,
|
|
549
|
+
content: CONTENT,
|
|
550
|
+
fetchDate: FETCH_DATE,
|
|
551
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
552
|
+
}));
|
|
553
|
+
});
|
|
554
|
+
|
|
555
|
+
after(async () => {
|
|
556
|
+
fs.rmSync(INJECTION_PROOF_FILE_PATH, { force: true });
|
|
557
|
+
await subject.removeAll();
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
it('treats the service ID as a path so it cannot reach git as an option', async () => {
|
|
561
|
+
await subject.findByDate(`--output=${__dirname}`, 'findByDate-argument-injection-proof', FETCH_DATE_LATER);
|
|
562
|
+
|
|
563
|
+
expect(fs.existsSync(INJECTION_PROOF_FILE_PATH), 'a service ID must never be interpreted as a git option').to.be.false;
|
|
564
|
+
});
|
|
565
|
+
});
|
|
566
|
+
|
|
567
|
+
context('when the service ID is a path traversal attempt', () => {
|
|
568
|
+
it('returns null instead of erroring', async () => {
|
|
569
|
+
expect(await subject.findByDate('../../outside', TERMS_TYPE, FETCH_DATE)).to.equal(null);
|
|
570
|
+
});
|
|
571
|
+
});
|
|
488
572
|
});
|
|
489
573
|
|
|
490
574
|
describe('#findAll', () => {
|
|
@@ -645,6 +729,13 @@ describe('GitRepository', () => {
|
|
|
645
729
|
});
|
|
646
730
|
});
|
|
647
731
|
|
|
732
|
+
context('when the service ID or terms type is a path traversal attempt', () => {
|
|
733
|
+
it('returns an empty array instead of erroring', async () => {
|
|
734
|
+
expect(await subject.findByServiceAndTermsType('../../outside', TERMS_TYPE)).to.be.an('array').that.is.empty;
|
|
735
|
+
expect(await subject.findByServiceAndTermsType(SERVICE_PROVIDER_ID, '../../outside')).to.be.an('array').that.is.empty;
|
|
736
|
+
});
|
|
737
|
+
});
|
|
738
|
+
|
|
648
739
|
context('with includeTechnicalUpgrades: false', () => {
|
|
649
740
|
let filteredRecords;
|
|
650
741
|
let technicalUpgradeId;
|
|
@@ -761,6 +852,12 @@ describe('GitRepository', () => {
|
|
|
761
852
|
});
|
|
762
853
|
});
|
|
763
854
|
|
|
855
|
+
context('when the service ID is a path traversal attempt', () => {
|
|
856
|
+
it('returns an empty array instead of erroring', async () => {
|
|
857
|
+
expect(await subject.findByService('../../outside')).to.be.an('array').that.is.empty;
|
|
858
|
+
});
|
|
859
|
+
});
|
|
860
|
+
|
|
764
861
|
context('with includeTechnicalUpgrades: false', () => {
|
|
765
862
|
let filteredRecords;
|
|
766
863
|
let technicalUpgradeId;
|
|
@@ -842,6 +939,12 @@ describe('GitRepository', () => {
|
|
|
842
939
|
});
|
|
843
940
|
});
|
|
844
941
|
|
|
942
|
+
context('when the service ID is a path traversal attempt', () => {
|
|
943
|
+
it('returns zero instead of erroring', async () => {
|
|
944
|
+
expect(await subject.count('../../outside', TERMS_TYPE)).to.equal(0);
|
|
945
|
+
});
|
|
946
|
+
});
|
|
947
|
+
|
|
845
948
|
context('with only serviceId filter', () => {
|
|
846
949
|
it('returns count for all terms types of a service', async () => {
|
|
847
950
|
// Add a version with different terms type
|
|
@@ -860,6 +963,167 @@ describe('GitRepository', () => {
|
|
|
860
963
|
});
|
|
861
964
|
});
|
|
862
965
|
|
|
966
|
+
describe('#getNavigationIds', () => {
|
|
967
|
+
let firstVersion;
|
|
968
|
+
let middleVersion;
|
|
969
|
+
let lastVersion;
|
|
970
|
+
|
|
971
|
+
before(async function () {
|
|
972
|
+
this.timeout(5000);
|
|
973
|
+
|
|
974
|
+
firstVersion = await subject.save(new Version({
|
|
975
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
976
|
+
termsType: TERMS_TYPE,
|
|
977
|
+
content: 'first content',
|
|
978
|
+
fetchDate: FETCH_DATE_EARLIER,
|
|
979
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
980
|
+
}));
|
|
981
|
+
|
|
982
|
+
middleVersion = await subject.save(new Version({
|
|
983
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
984
|
+
termsType: TERMS_TYPE,
|
|
985
|
+
content: 'middle content',
|
|
986
|
+
fetchDate: FETCH_DATE,
|
|
987
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
988
|
+
}));
|
|
989
|
+
|
|
990
|
+
lastVersion = await subject.save(new Version({
|
|
991
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
992
|
+
termsType: TERMS_TYPE,
|
|
993
|
+
content: 'last content',
|
|
994
|
+
fetchDate: FETCH_DATE_LATER,
|
|
995
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
996
|
+
}));
|
|
997
|
+
});
|
|
998
|
+
|
|
999
|
+
after(() => subject.removeAll());
|
|
1000
|
+
|
|
1001
|
+
context('for the oldest version', () => {
|
|
1002
|
+
let navigationIds;
|
|
1003
|
+
|
|
1004
|
+
before(async () => { navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, firstVersion.id); });
|
|
1005
|
+
|
|
1006
|
+
it('has no previous version', () => expect(navigationIds.prev).to.be.null);
|
|
1007
|
+
it('points to the next version', () => expect(navigationIds.next).to.equal(middleVersion.id));
|
|
1008
|
+
it('points to itself as first', () => expect(navigationIds.first).to.equal(firstVersion.id));
|
|
1009
|
+
it('points to the newest version as last', () => expect(navigationIds.last).to.equal(lastVersion.id));
|
|
1010
|
+
});
|
|
1011
|
+
|
|
1012
|
+
context('for a middle version', () => {
|
|
1013
|
+
let navigationIds;
|
|
1014
|
+
|
|
1015
|
+
before(async () => { navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, middleVersion.id); });
|
|
1016
|
+
|
|
1017
|
+
it('points to the previous version', () => expect(navigationIds.prev).to.equal(firstVersion.id));
|
|
1018
|
+
it('points to the next version', () => expect(navigationIds.next).to.equal(lastVersion.id));
|
|
1019
|
+
it('points to the oldest version as first', () => expect(navigationIds.first).to.equal(firstVersion.id));
|
|
1020
|
+
it('points to the newest version as last', () => expect(navigationIds.last).to.equal(lastVersion.id));
|
|
1021
|
+
});
|
|
1022
|
+
|
|
1023
|
+
context('for the newest version', () => {
|
|
1024
|
+
let navigationIds;
|
|
1025
|
+
|
|
1026
|
+
before(async () => { navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, lastVersion.id); });
|
|
1027
|
+
|
|
1028
|
+
it('points to the previous version', () => expect(navigationIds.prev).to.equal(middleVersion.id));
|
|
1029
|
+
it('has no next version', () => expect(navigationIds.next).to.be.null);
|
|
1030
|
+
it('points to itself as last', () => expect(navigationIds.last).to.equal(lastVersion.id));
|
|
1031
|
+
});
|
|
1032
|
+
|
|
1033
|
+
context('when the version does not exist', () => {
|
|
1034
|
+
it('returns only null IDs', async () => {
|
|
1035
|
+
expect(await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, 'ffffffffffffffffffffffffffffffffffffffff')).to.deep.equal({ first: null, prev: null, next: null, last: null });
|
|
1036
|
+
});
|
|
1037
|
+
});
|
|
1038
|
+
|
|
1039
|
+
context('when the service ID is a path traversal attempt', () => {
|
|
1040
|
+
it('returns only null IDs instead of erroring', async () => {
|
|
1041
|
+
expect(await subject.getNavigationIds('../../outside', TERMS_TYPE, firstVersion.id)).to.deep.equal({ first: null, prev: null, next: null, last: null });
|
|
1042
|
+
});
|
|
1043
|
+
});
|
|
1044
|
+
|
|
1045
|
+
context('when a technical upgrade is recorded out of chronological order', () => {
|
|
1046
|
+
// A technical upgrade re-renders an old snapshot: it carries an OLD fetch date but is committed last (topologically recent).
|
|
1047
|
+
// The previous implementation mixed chronological (findPrevious) and topological (findNext) order, so prev/next disagreed here.
|
|
1048
|
+
// The navigation IDs are now derived from a single chronological order, so they stay exact inverses.
|
|
1049
|
+
const TECHNICAL_UPGRADE_DATE = new Date('2000-01-01T09:00:00.000Z'); // between firstVersion (06:00) and middleVersion (12:00)
|
|
1050
|
+
let technicalUpgrade;
|
|
1051
|
+
|
|
1052
|
+
before(async () => {
|
|
1053
|
+
technicalUpgrade = await subject.save(new Version({
|
|
1054
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
1055
|
+
termsType: TERMS_TYPE,
|
|
1056
|
+
content: 'technical upgrade content',
|
|
1057
|
+
fetchDate: TECHNICAL_UPGRADE_DATE,
|
|
1058
|
+
isTechnicalUpgrade: true,
|
|
1059
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
1060
|
+
}));
|
|
1061
|
+
});
|
|
1062
|
+
|
|
1063
|
+
it('orders it by its fetch date, not its commit position', async () => {
|
|
1064
|
+
const navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, technicalUpgrade.id);
|
|
1065
|
+
|
|
1066
|
+
expect(navigationIds.prev).to.equal(firstVersion.id);
|
|
1067
|
+
expect(navigationIds.next).to.equal(middleVersion.id);
|
|
1068
|
+
});
|
|
1069
|
+
|
|
1070
|
+
it('keeps prev and next as exact inverses (round-trips)', async () => {
|
|
1071
|
+
const fromFirst = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, firstVersion.id);
|
|
1072
|
+
|
|
1073
|
+
expect(fromFirst.next).to.equal(technicalUpgrade.id);
|
|
1074
|
+
|
|
1075
|
+
const backFromUpgrade = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, fromFirst.next);
|
|
1076
|
+
|
|
1077
|
+
expect(backFromUpgrade.prev).to.equal(firstVersion.id);
|
|
1078
|
+
});
|
|
1079
|
+
|
|
1080
|
+
it('can exclude technical upgrades from the sequence', async () => {
|
|
1081
|
+
const navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, firstVersion.id, { includeTechnicalUpgrades: false });
|
|
1082
|
+
|
|
1083
|
+
expect(navigationIds.next).to.equal(middleVersion.id);
|
|
1084
|
+
});
|
|
1085
|
+
});
|
|
1086
|
+
});
|
|
1087
|
+
|
|
1088
|
+
describe('#getNavigationIds with versions sharing the same fetch date', () => {
|
|
1089
|
+
// Git stores commit dates at second precision, so distinct versions can share a fetch date.
|
|
1090
|
+
// The record ID tiebreaker must keep the order deterministic so prev/next still round-trip.
|
|
1091
|
+
let ids;
|
|
1092
|
+
|
|
1093
|
+
before(async function () {
|
|
1094
|
+
this.timeout(5000);
|
|
1095
|
+
ids = [];
|
|
1096
|
+
for (const content of [ 'tie A', 'tie B', 'tie C' ]) {
|
|
1097
|
+
ids.push((await subject.save(new Version({ serviceId: SERVICE_PROVIDER_ID, termsType: TERMS_TYPE, content, fetchDate: FETCH_DATE, snapshotIds: [SNAPSHOT_ID] }))).id);
|
|
1098
|
+
}
|
|
1099
|
+
});
|
|
1100
|
+
|
|
1101
|
+
after(() => subject.removeAll());
|
|
1102
|
+
|
|
1103
|
+
it('round-trips next then prev for every version', async () => {
|
|
1104
|
+
for (const id of ids) {
|
|
1105
|
+
const navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, id);
|
|
1106
|
+
|
|
1107
|
+
if (navigationIds.next) {
|
|
1108
|
+
expect((await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, navigationIds.next)).prev).to.equal(id);
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
});
|
|
1112
|
+
|
|
1113
|
+
it('exposes all three versions as a single ordered chain', async () => {
|
|
1114
|
+
const head = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, ids[0]);
|
|
1115
|
+
const walked = new Set([head.first]);
|
|
1116
|
+
let cursor = head.first;
|
|
1117
|
+
|
|
1118
|
+
while (cursor) {
|
|
1119
|
+
walked.add(cursor);
|
|
1120
|
+
cursor = (await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, cursor)).next;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
expect(walked).to.have.lengthOf(ids.length);
|
|
1124
|
+
});
|
|
1125
|
+
});
|
|
1126
|
+
|
|
863
1127
|
describe('#findLatest', () => {
|
|
864
1128
|
context('when there are records for the given service', () => {
|
|
865
1129
|
let lastSnapshotId;
|
|
@@ -920,6 +1184,30 @@ describe('GitRepository', () => {
|
|
|
920
1184
|
expect(latestRecord).to.equal(null);
|
|
921
1185
|
});
|
|
922
1186
|
});
|
|
1187
|
+
|
|
1188
|
+
context('when the service ID could be interpreted as a git option', () => {
|
|
1189
|
+
before(async () => {
|
|
1190
|
+
await subject.save(new Version({
|
|
1191
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
1192
|
+
termsType: TERMS_TYPE,
|
|
1193
|
+
content: CONTENT,
|
|
1194
|
+
fetchDate: FETCH_DATE,
|
|
1195
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
1196
|
+
}));
|
|
1197
|
+
});
|
|
1198
|
+
|
|
1199
|
+
after(() => subject.removeAll());
|
|
1200
|
+
|
|
1201
|
+
it('treats the service ID as a path and returns null instead of erroring', async () => {
|
|
1202
|
+
expect(await subject.findLatest('--not-a-git-option', TERMS_TYPE)).to.equal(null);
|
|
1203
|
+
});
|
|
1204
|
+
});
|
|
1205
|
+
|
|
1206
|
+
context('when the service ID is a path traversal attempt', () => {
|
|
1207
|
+
it('returns null instead of erroring', async () => {
|
|
1208
|
+
expect(await subject.findLatest('../../outside', TERMS_TYPE)).to.equal(null);
|
|
1209
|
+
});
|
|
1210
|
+
});
|
|
923
1211
|
});
|
|
924
1212
|
|
|
925
1213
|
describe('#iterate', () => {
|
|
@@ -69,6 +69,15 @@ class RepositoryInterface {
|
|
|
69
69
|
throw new Error(`#findById method is not implemented in ${this.constructor.name}`);
|
|
70
70
|
}
|
|
71
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Find the metadata of the record that matches the given record ID, without loading its content
|
|
74
|
+
* @param {string} recordId - Record ID of the record to find
|
|
75
|
+
* @returns {Promise<Record>} Promise that will be resolved with the found record (without content) or null if none match the given ID
|
|
76
|
+
*/
|
|
77
|
+
async findMetadataById(recordId) {
|
|
78
|
+
throw new Error(`#findMetadataById method is not implemented in ${this.constructor.name}`);
|
|
79
|
+
}
|
|
80
|
+
|
|
72
81
|
/**
|
|
73
82
|
* Find all records, in descending chronological order (newest first; opposite of #iterate)
|
|
74
83
|
* For performance reasons, the content of the records will not be loaded by default. Use #loadRecordContent to load the content of individual records
|
|
@@ -99,6 +108,20 @@ class RepositoryInterface {
|
|
|
99
108
|
throw new Error(`#findByService method is not implemented in ${this.constructor.name}`);
|
|
100
109
|
}
|
|
101
110
|
|
|
111
|
+
/**
|
|
112
|
+
* Get the IDs locating a version within the history of its terms: the first (oldest) and last (newest) versions, as well as the immediately previous (older) and next (newer) ones
|
|
113
|
+
* These IDs are computed from a single deterministic chronological order (by fetch date, with the record ID as a stable tiebreaker), so navigating prev/next always round-trips
|
|
114
|
+
* @param {string} serviceId - Service ID of the version
|
|
115
|
+
* @param {string} termsType - Terms type of the version
|
|
116
|
+
* @param {string} versionId - ID of the version to locate within its terms history
|
|
117
|
+
* @param {object} [options] - Query options
|
|
118
|
+
* @param {boolean} [options.includeTechnicalUpgrades] - When false, exclude technical upgrade records from the sequence. Default: true
|
|
119
|
+
* @returns {Promise<{first: ?string, prev: ?string, next: ?string, last: ?string}>} Promise resolved with the related version IDs, each null when there is none
|
|
120
|
+
*/
|
|
121
|
+
async getNavigationIds(serviceId, termsType, versionId, options = {}) {
|
|
122
|
+
throw new Error(`#getNavigationIds method is not implemented in ${this.constructor.name}`);
|
|
123
|
+
}
|
|
124
|
+
|
|
102
125
|
/**
|
|
103
126
|
* Find all records for a specific service and terms type, in descending chronological order
|
|
104
127
|
* For performance reasons, the content of the records will not be loaded by default. Use #loadRecordContent to load the content of individual records
|
|
@@ -152,6 +175,17 @@ class RepositoryInterface {
|
|
|
152
175
|
async loadRecordContent(record) {
|
|
153
176
|
throw new Error(`#loadRecordContent method is not implemented in ${this.constructor.name}`);
|
|
154
177
|
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Get the number of lines added and deleted by a record, relative to the previous state of the same terms
|
|
181
|
+
* This is an optional capability: backends that retain diffs (Git) override it, while backends that store full snapshots (MongoDB) inherit this default and report the statistics as unavailable
|
|
182
|
+
* @param {string} recordId - Record ID to get diff stats for
|
|
183
|
+
* @returns {Promise<{additions: ?number, deletions: ?number}>} Promise resolved with the number of added and deleted lines, each null when the backend cannot provide them
|
|
184
|
+
*/
|
|
185
|
+
// eslint-disable-next-line class-methods-use-this, no-unused-vars
|
|
186
|
+
getDiffStats(recordId) {
|
|
187
|
+
return { additions: null, deletions: null };
|
|
188
|
+
}
|
|
155
189
|
}
|
|
156
190
|
|
|
157
191
|
export default RepositoryInterface;
|
|
@@ -88,6 +88,19 @@ export default class MongoRepository extends RepositoryInterface {
|
|
|
88
88
|
return this.#toDomain(mongoDocument);
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
+
async findMetadataById(recordId) {
|
|
92
|
+
if (!ObjectId.isValid(recordId)) {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const document = await this.collection.findOne(
|
|
97
|
+
{ _id: ObjectId.createFromHexString(recordId) },
|
|
98
|
+
{ projection: { content: 0 } },
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
return document ? this.#toDomain(document, { deferContentLoading: true }) : null;
|
|
102
|
+
}
|
|
103
|
+
|
|
91
104
|
async findAll({ limit, offset, includeTechnicalUpgrades = true } = {}) {
|
|
92
105
|
const filter = includeTechnicalUpgrades ? {} : { isTechnicalUpgrade: { $ne: true } };
|
|
93
106
|
let query = this.collection.find(filter).project({ content: 0 }).sort({ fetchDate: -1 });
|
|
@@ -146,6 +159,46 @@ export default class MongoRepository extends RepositoryInterface {
|
|
|
146
159
|
.map(mongoDocument => this.#toDomain(mongoDocument, { deferContentLoading: true })));
|
|
147
160
|
}
|
|
148
161
|
|
|
162
|
+
async getNavigationIds(serviceId, termsType, versionId, { includeTechnicalUpgrades = true } = {}) {
|
|
163
|
+
const empty = { first: null, prev: null, next: null, last: null };
|
|
164
|
+
|
|
165
|
+
if (!ObjectId.isValid(versionId)) {
|
|
166
|
+
return empty;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const _id = ObjectId.createFromHexString(versionId);
|
|
170
|
+
const filter = { serviceId, termsType };
|
|
171
|
+
|
|
172
|
+
if (!includeTechnicalUpgrades) {
|
|
173
|
+
filter.isTechnicalUpgrade = { $ne: true };
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const current = await this.collection.findOne({ ...filter, _id }, { projection: { fetchDate: 1 } });
|
|
177
|
+
|
|
178
|
+
if (!current) {
|
|
179
|
+
return empty;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const { fetchDate } = current;
|
|
183
|
+
const idOnly = { projection: { _id: 1 } };
|
|
184
|
+
|
|
185
|
+
// Deterministic total order (fetchDate, then _id) ascending; prev is the greatest record strictly before the current one, next the least strictly after.
|
|
186
|
+
// Using _id as a tiebreaker makes prev and next well-defined even for versions sharing the same fetch date, so navigation always round-trips.
|
|
187
|
+
const [ oldest, newest, previous, next ] = await Promise.all([
|
|
188
|
+
this.collection.find(filter, idOnly).sort({ fetchDate: 1, _id: 1 }).limit(1).next(),
|
|
189
|
+
this.collection.find(filter, idOnly).sort({ fetchDate: -1, _id: -1 }).limit(1).next(),
|
|
190
|
+
this.collection.find({ ...filter, $or: [{ fetchDate: { $lt: fetchDate } }, { fetchDate, _id: { $lt: _id } }] }, idOnly).sort({ fetchDate: -1, _id: -1 }).limit(1).next(),
|
|
191
|
+
this.collection.find({ ...filter, $or: [{ fetchDate: { $gt: fetchDate } }, { fetchDate, _id: { $gt: _id } }] }, idOnly).sort({ fetchDate: 1, _id: 1 }).limit(1).next(),
|
|
192
|
+
]);
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
first: oldest?._id?.toString() || null,
|
|
196
|
+
last: newest?._id?.toString() || null,
|
|
197
|
+
prev: previous?._id?.toString() || null,
|
|
198
|
+
next: next?._id?.toString() || null,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
149
202
|
count(serviceId, termsType) {
|
|
150
203
|
const filter = {};
|
|
151
204
|
|
|
@@ -988,6 +988,177 @@ describe('MongoRepository', () => {
|
|
|
988
988
|
});
|
|
989
989
|
});
|
|
990
990
|
|
|
991
|
+
describe('#getNavigationIds', () => {
|
|
992
|
+
let firstVersion;
|
|
993
|
+
let middleVersion;
|
|
994
|
+
let lastVersion;
|
|
995
|
+
|
|
996
|
+
before(async () => {
|
|
997
|
+
firstVersion = await subject.save(new Version({
|
|
998
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
999
|
+
termsType: TERMS_TYPE,
|
|
1000
|
+
content: 'first content',
|
|
1001
|
+
fetchDate: FETCH_DATE_EARLIER,
|
|
1002
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
1003
|
+
}));
|
|
1004
|
+
|
|
1005
|
+
middleVersion = await subject.save(new Version({
|
|
1006
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
1007
|
+
termsType: TERMS_TYPE,
|
|
1008
|
+
content: 'middle content',
|
|
1009
|
+
fetchDate: FETCH_DATE,
|
|
1010
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
1011
|
+
}));
|
|
1012
|
+
|
|
1013
|
+
lastVersion = await subject.save(new Version({
|
|
1014
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
1015
|
+
termsType: TERMS_TYPE,
|
|
1016
|
+
content: 'last content',
|
|
1017
|
+
fetchDate: FETCH_DATE_LATER,
|
|
1018
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
1019
|
+
}));
|
|
1020
|
+
});
|
|
1021
|
+
|
|
1022
|
+
after(() => subject.removeAll());
|
|
1023
|
+
|
|
1024
|
+
context('for the oldest version', () => {
|
|
1025
|
+
let navigationIds;
|
|
1026
|
+
|
|
1027
|
+
before(async () => { navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, firstVersion.id); });
|
|
1028
|
+
|
|
1029
|
+
it('has no previous version', () => expect(navigationIds.prev).to.be.null);
|
|
1030
|
+
it('points to the next version', () => expect(navigationIds.next).to.equal(middleVersion.id));
|
|
1031
|
+
it('points to itself as first', () => expect(navigationIds.first).to.equal(firstVersion.id));
|
|
1032
|
+
it('points to the newest version as last', () => expect(navigationIds.last).to.equal(lastVersion.id));
|
|
1033
|
+
});
|
|
1034
|
+
|
|
1035
|
+
context('for a middle version', () => {
|
|
1036
|
+
let navigationIds;
|
|
1037
|
+
|
|
1038
|
+
before(async () => { navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, middleVersion.id); });
|
|
1039
|
+
|
|
1040
|
+
it('points to the previous version', () => expect(navigationIds.prev).to.equal(firstVersion.id));
|
|
1041
|
+
it('points to the next version', () => expect(navigationIds.next).to.equal(lastVersion.id));
|
|
1042
|
+
it('points to the oldest version as first', () => expect(navigationIds.first).to.equal(firstVersion.id));
|
|
1043
|
+
it('points to the newest version as last', () => expect(navigationIds.last).to.equal(lastVersion.id));
|
|
1044
|
+
});
|
|
1045
|
+
|
|
1046
|
+
context('for the newest version', () => {
|
|
1047
|
+
let navigationIds;
|
|
1048
|
+
|
|
1049
|
+
before(async () => { navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, lastVersion.id); });
|
|
1050
|
+
|
|
1051
|
+
it('points to the previous version', () => expect(navigationIds.prev).to.equal(middleVersion.id));
|
|
1052
|
+
it('has no next version', () => expect(navigationIds.next).to.be.null);
|
|
1053
|
+
it('points to itself as last', () => expect(navigationIds.last).to.equal(lastVersion.id));
|
|
1054
|
+
});
|
|
1055
|
+
|
|
1056
|
+
context('when the version does not exist', () => {
|
|
1057
|
+
it('returns only null IDs', async () => {
|
|
1058
|
+
expect(await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, 'ffffffffffffffffffffffff')).to.deep.equal({ first: null, prev: null, next: null, last: null });
|
|
1059
|
+
});
|
|
1060
|
+
});
|
|
1061
|
+
|
|
1062
|
+
context('when a technical upgrade is recorded out of chronological order', () => {
|
|
1063
|
+
// A technical upgrade re-renders an old snapshot: it carries an OLD fetch date but is inserted last.
|
|
1064
|
+
// The navigation IDs are derived from a single chronological order, so prev/next stay exact inverses.
|
|
1065
|
+
const TECHNICAL_UPGRADE_DATE = new Date('2000-01-01T09:00:00.000Z'); // between firstVersion (06:00) and middleVersion (12:00)
|
|
1066
|
+
let technicalUpgrade;
|
|
1067
|
+
|
|
1068
|
+
before(async () => {
|
|
1069
|
+
technicalUpgrade = await subject.save(new Version({
|
|
1070
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
1071
|
+
termsType: TERMS_TYPE,
|
|
1072
|
+
content: 'technical upgrade content',
|
|
1073
|
+
fetchDate: TECHNICAL_UPGRADE_DATE,
|
|
1074
|
+
isTechnicalUpgrade: true,
|
|
1075
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
1076
|
+
}));
|
|
1077
|
+
});
|
|
1078
|
+
|
|
1079
|
+
it('orders it by its fetch date, not its insertion order', async () => {
|
|
1080
|
+
const navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, technicalUpgrade.id);
|
|
1081
|
+
|
|
1082
|
+
expect(navigationIds.prev).to.equal(firstVersion.id);
|
|
1083
|
+
expect(navigationIds.next).to.equal(middleVersion.id);
|
|
1084
|
+
});
|
|
1085
|
+
|
|
1086
|
+
it('keeps prev and next as exact inverses (round-trips)', async () => {
|
|
1087
|
+
const fromFirst = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, firstVersion.id);
|
|
1088
|
+
|
|
1089
|
+
expect(fromFirst.next).to.equal(technicalUpgrade.id);
|
|
1090
|
+
|
|
1091
|
+
const backFromUpgrade = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, fromFirst.next);
|
|
1092
|
+
|
|
1093
|
+
expect(backFromUpgrade.prev).to.equal(firstVersion.id);
|
|
1094
|
+
});
|
|
1095
|
+
|
|
1096
|
+
it('can exclude technical upgrades from the sequence', async () => {
|
|
1097
|
+
const navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, firstVersion.id, { includeTechnicalUpgrades: false });
|
|
1098
|
+
|
|
1099
|
+
expect(navigationIds.next).to.equal(middleVersion.id);
|
|
1100
|
+
});
|
|
1101
|
+
});
|
|
1102
|
+
});
|
|
1103
|
+
|
|
1104
|
+
describe('#getNavigationIds with versions sharing the same fetch date', () => {
|
|
1105
|
+
// Distinct versions can share a fetch date; the _id tiebreaker must keep the order deterministic so prev/next still round-trip.
|
|
1106
|
+
let ids;
|
|
1107
|
+
|
|
1108
|
+
before(async () => {
|
|
1109
|
+
ids = [];
|
|
1110
|
+
for (const content of [ 'tie A', 'tie B', 'tie C' ]) {
|
|
1111
|
+
ids.push((await subject.save(new Version({ serviceId: SERVICE_PROVIDER_ID, termsType: TERMS_TYPE, content, fetchDate: FETCH_DATE, snapshotIds: [SNAPSHOT_ID] }))).id);
|
|
1112
|
+
}
|
|
1113
|
+
});
|
|
1114
|
+
|
|
1115
|
+
after(() => subject.removeAll());
|
|
1116
|
+
|
|
1117
|
+
it('round-trips next then prev for every version', async () => {
|
|
1118
|
+
for (const id of ids) {
|
|
1119
|
+
const navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, id);
|
|
1120
|
+
|
|
1121
|
+
if (navigationIds.next) {
|
|
1122
|
+
expect((await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, navigationIds.next)).prev).to.equal(id);
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
});
|
|
1126
|
+
|
|
1127
|
+
it('exposes all three versions as a single ordered chain', async () => {
|
|
1128
|
+
const head = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, ids[0]);
|
|
1129
|
+
const walked = new Set([head.first]);
|
|
1130
|
+
let cursor = head.first;
|
|
1131
|
+
|
|
1132
|
+
while (cursor) {
|
|
1133
|
+
walked.add(cursor);
|
|
1134
|
+
cursor = (await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, cursor)).next;
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
expect(walked).to.have.lengthOf(ids.length);
|
|
1138
|
+
});
|
|
1139
|
+
});
|
|
1140
|
+
|
|
1141
|
+
describe('#getDiffStats', () => {
|
|
1142
|
+
// Diff statistics are an optional repository capability; MongoDB stores full snapshots rather than diffs, so it inherits the interface default that reports them as unavailable.
|
|
1143
|
+
let versionId;
|
|
1144
|
+
|
|
1145
|
+
before(async () => {
|
|
1146
|
+
({ id: versionId } = await subject.save(new Version({
|
|
1147
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
1148
|
+
termsType: TERMS_TYPE,
|
|
1149
|
+
content: CONTENT,
|
|
1150
|
+
fetchDate: FETCH_DATE,
|
|
1151
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
1152
|
+
})));
|
|
1153
|
+
});
|
|
1154
|
+
|
|
1155
|
+
after(() => subject.removeAll());
|
|
1156
|
+
|
|
1157
|
+
it('reports additions and deletions as unavailable', async () => {
|
|
1158
|
+
expect(await subject.getDiffStats(versionId)).to.deep.equal({ additions: null, deletions: null });
|
|
1159
|
+
});
|
|
1160
|
+
});
|
|
1161
|
+
|
|
991
1162
|
describe('#findLatest', () => {
|
|
992
1163
|
context('when there are records for the given service', () => {
|
|
993
1164
|
let lastSnapshotId;
|
|
@@ -431,7 +431,7 @@ describe('Services', () => {
|
|
|
431
431
|
expect(secondSourceDocument.filters).to.be.an('array');
|
|
432
432
|
expect(secondSourceDocument.filters).to.have.length(2);
|
|
433
433
|
expect(secondSourceDocument.filters[0]).to.be.a('function');
|
|
434
|
-
expect(secondSourceDocument.filters[0].name).to.equal(
|
|
434
|
+
expect(secondSourceDocument.filters[0].name).to.equal(realFilterNames[0]);
|
|
435
435
|
expect(secondSourceDocument.filters[1]).to.be.a('function');
|
|
436
436
|
expect(secondSourceDocument.filters[1].name).to.equal('removePrintButton');
|
|
437
437
|
});
|
|
@@ -2,5 +2,5 @@ import logger from '../logger.js';
|
|
|
2
2
|
|
|
3
3
|
export default function errorsMiddleware(err, req, res, next) {
|
|
4
4
|
logger.error(err.stack);
|
|
5
|
-
res.status(500).json({ error:
|
|
5
|
+
res.status(500).json({ error: 'Internal Server Error' }); // Never echo internal error details: they can expose server internals such as filesystem paths
|
|
6
6
|
}
|
|
@@ -38,6 +38,7 @@ export default async function apiRouter(basePath) {
|
|
|
38
38
|
const collection = await getCollection();
|
|
39
39
|
const versionsStorageConfig = config.get('@opentermsarchive/engine.recorder.versions.storage');
|
|
40
40
|
const versionsRepository = await RepositoryFactory.create(versionsStorageConfig).initialize();
|
|
41
|
+
const snapshotsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.snapshots.storage')).initialize();
|
|
41
42
|
const feedConfig = config.get('@opentermsarchive/engine.collection-api.feed');
|
|
42
43
|
|
|
43
44
|
if (!collection.metadata?.id) {
|
|
@@ -50,7 +51,7 @@ export default async function apiRouter(basePath) {
|
|
|
50
51
|
|
|
51
52
|
router.use(await metadataRouter(collection, services));
|
|
52
53
|
router.use(servicesRouter(services));
|
|
53
|
-
router.use(versionsRouter(versionsRepository));
|
|
54
|
+
router.use(versionsRouter(versionsRepository, snapshotsRepository));
|
|
54
55
|
router.use(feedRouter(services, versionsRepository, versionsStorageConfig.type, feedConfig.limit, feedConfig.versionUrlTemplate));
|
|
55
56
|
|
|
56
57
|
return router;
|