@opentermsarchive/engine 14.1.0 → 15.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/archivist/recorder/repositories/git/dataMapper.js +4 -0
- package/src/archivist/recorder/repositories/git/git.js +90 -8
- package/src/archivist/recorder/repositories/git/index.js +91 -4
- package/src/archivist/recorder/repositories/git/index.test.js +288 -0
- package/src/archivist/recorder/repositories/interface.js +34 -0
- package/src/archivist/recorder/repositories/mongo/index.js +53 -0
- package/src/archivist/recorder/repositories/mongo/index.test.js +171 -0
- package/src/collection-api/middlewares/errors.js +1 -1
- package/src/collection-api/routes/index.js +2 -1
- package/src/collection-api/routes/services.js +34 -21
- package/src/collection-api/routes/versions.js +521 -27
- package/src/collection-api/routes/versions.test.js +655 -25
package/package.json
CHANGED
|
@@ -30,6 +30,10 @@ const MULTIPLE_SOURCE_DOCUMENTS_PREFIX = 'This version was recorded after extrac
|
|
|
30
30
|
|
|
31
31
|
export const COMMIT_MESSAGE_PREFIXES_REGEXP = new RegExp(`^(${Object.values(COMMIT_MESSAGE_PREFIXES).join('|')})`);
|
|
32
32
|
|
|
33
|
+
export function isTechnicalUpgrade(message) {
|
|
34
|
+
return message.startsWith(COMMIT_MESSAGE_PREFIXES.technicalUpgrade) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_refilter);
|
|
35
|
+
}
|
|
36
|
+
|
|
33
37
|
export function toPersistence(record, snapshotIdentiferTemplate) {
|
|
34
38
|
const { serviceId, termsType, documentId, snapshotIds = [], mimeType, metadata } = record;
|
|
35
39
|
|
|
@@ -22,7 +22,7 @@ export default class Git {
|
|
|
22
22
|
|
|
23
23
|
this.git = simpleGit(this.path, {
|
|
24
24
|
trimmed: true,
|
|
25
|
-
maxConcurrentProcesses: 1,
|
|
25
|
+
maxConcurrentProcesses: 1, // Concurrent runs on the same repository race the index and the commit-graph and can corrupt them
|
|
26
26
|
});
|
|
27
27
|
|
|
28
28
|
await this.git.init();
|
|
@@ -106,7 +106,8 @@ export default class Git {
|
|
|
106
106
|
|
|
107
107
|
return commits;
|
|
108
108
|
} catch (error) {
|
|
109
|
-
|
|
109
|
+
// `bad object` is raised for a well-formed but absent object ID; like an unknown revision, it means "no match" rather than a hard failure
|
|
110
|
+
if (/unknown revision or path not in the working tree|does not have any commits yet|bad object/.test(error.message)) {
|
|
110
111
|
return [];
|
|
111
112
|
}
|
|
112
113
|
|
|
@@ -132,15 +133,24 @@ export default class Git {
|
|
|
132
133
|
await fs.rm(path.join(this.path, '.git', 'objects', 'info', 'commit-graph.lock'), { force: true }); // Remove a leftover commit-graph lock from a previous `commit-graph write` that was killed mid-write (e.g. the process was terminated during a deploy or restart). The commit-graph is a disposable cache rebuilt by `writeCommitGraph`, so clearing a stale lock is safe and prevents every subsequent run from failing.
|
|
133
134
|
await this.git.reset('hard');
|
|
134
135
|
|
|
135
|
-
return this.git.clean('f', '-d');
|
|
136
|
+
return this.git.clean('f', '-d'); // Force-remove untracked files (`f`) and untracked directories (`-d`)
|
|
136
137
|
}
|
|
137
138
|
|
|
138
139
|
getFullHash(shortHash) {
|
|
139
|
-
return this.git.show([
|
|
140
|
+
return this.git.show([
|
|
141
|
+
shortHash,
|
|
142
|
+
'--pretty=%H', // Print the full 40-character commit hash
|
|
143
|
+
'-s', // Suppress the diff output, only the formatted hash is wanted
|
|
144
|
+
]);
|
|
140
145
|
}
|
|
141
146
|
|
|
142
147
|
restore(path, commit) {
|
|
143
|
-
return this.git.raw([
|
|
148
|
+
return this.git.raw([
|
|
149
|
+
'restore',
|
|
150
|
+
'-s', commit, // Take the file contents from this specific commit rather than from the index
|
|
151
|
+
'--', // Everything after is a pathspec, not a revision or option
|
|
152
|
+
path,
|
|
153
|
+
]);
|
|
144
154
|
}
|
|
145
155
|
|
|
146
156
|
async destroyHistory() {
|
|
@@ -154,14 +164,86 @@ export default class Git {
|
|
|
154
164
|
}
|
|
155
165
|
|
|
156
166
|
async listFiles(path) {
|
|
157
|
-
return (await this.git.raw([ 'ls-files', path ])).split('\n');
|
|
167
|
+
return (await this.git.raw([ 'ls-files', '--', path ])).split('\n'); // Everything after "--" is a pathspec, not a revision or option
|
|
158
168
|
}
|
|
159
169
|
|
|
160
170
|
async writeCommitGraph() {
|
|
161
|
-
await this.git.raw([
|
|
171
|
+
await this.git.raw([
|
|
172
|
+
'commit-graph',
|
|
173
|
+
'write',
|
|
174
|
+
'--reachable', // Cover every commit reachable from the refs, so the whole history is indexed
|
|
175
|
+
'--changed-paths', // Also store the changed-path Bloom filters that speed up path-limited log/diff
|
|
176
|
+
]);
|
|
162
177
|
}
|
|
163
178
|
|
|
164
179
|
async updateCommitGraph() {
|
|
165
|
-
await this.git.raw([
|
|
180
|
+
await this.git.raw([
|
|
181
|
+
'commit-graph',
|
|
182
|
+
'write',
|
|
183
|
+
'--reachable',
|
|
184
|
+
'--changed-paths',
|
|
185
|
+
'--append', // Extend the existing commit-graph instead of rewriting it in full
|
|
186
|
+
]);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
async listPathRevisions(pathFilter) {
|
|
190
|
+
let output;
|
|
191
|
+
|
|
192
|
+
try {
|
|
193
|
+
// Ordering and technical-upgrade filtering are done by the caller in memory, so `--author-date-order`/`--grep`/`--name-only` are deliberately omitted to keep this walk lean
|
|
194
|
+
output = await this.git.raw([
|
|
195
|
+
'log',
|
|
196
|
+
'--no-merges',
|
|
197
|
+
'--format=%H%x09%at%x09%s', // Tab-separated hash, author date and subject: the minimum needed to order versions and detect technical upgrades, with no diff or message body loaded
|
|
198
|
+
'--', // Everything after is a pathspec, never a revision or an option
|
|
199
|
+
pathFilter,
|
|
200
|
+
]);
|
|
201
|
+
} catch (error) {
|
|
202
|
+
if (/unknown revision or path not in the working tree|does not have any commits yet/.test(error.message)) {
|
|
203
|
+
return [];
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
if (!output) {
|
|
210
|
+
return [];
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
return output.trim().split('\n').filter(Boolean).map(line => {
|
|
214
|
+
const [ hash, timestamp, ...subjectParts ] = line.split('\t');
|
|
215
|
+
|
|
216
|
+
return { hash, timestamp: parseInt(timestamp, 10), subject: subjectParts.join('\t') };
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async getDiffStats(commitHash) {
|
|
221
|
+
const output = await this.git.raw([
|
|
222
|
+
'show',
|
|
223
|
+
'--numstat', // Report added/deleted line counts per file as tab-separated numbers, instead of a textual diff
|
|
224
|
+
'--format=', // Drop the commit header so the output holds only the numstat lines
|
|
225
|
+
commitHash,
|
|
226
|
+
]);
|
|
227
|
+
|
|
228
|
+
let additions = 0;
|
|
229
|
+
let deletions = 0;
|
|
230
|
+
|
|
231
|
+
for (const line of output.trim().split('\n')) {
|
|
232
|
+
if (!line) {
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const [ added, deleted ] = line.split('\t');
|
|
237
|
+
|
|
238
|
+
// Binary files show '-' for additions/deletions
|
|
239
|
+
if (added !== '-') {
|
|
240
|
+
additions += parseInt(added, 10);
|
|
241
|
+
}
|
|
242
|
+
if (deleted !== '-') {
|
|
243
|
+
deletions += parseInt(deleted, 10);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
return { additions, deletions };
|
|
166
248
|
}
|
|
167
249
|
}
|
|
@@ -15,6 +15,25 @@ import Git from './git.js';
|
|
|
15
15
|
|
|
16
16
|
const fs = fsApi.promises;
|
|
17
17
|
|
|
18
|
+
const RECORD_ID_REGEXP = /^[0-9a-f]{7,40}$/i; // Git commit SHA 7 (abbreviated) to 40 (full) hexadecimal characters. Prevent value such as `--output=…` to be parsed as a command-line option
|
|
19
|
+
|
|
20
|
+
const CONTROL_CHARACTERS_REGEXP = /\p{Cc}/u; // Matches any Unicode "control" character: the C0 range (U+0000 to U+001F), DEL (U+007F) and the C1 range (U+0080 to U+009F), i.e. 65 non-printable characters including NUL. The `u` flag is required for the `\p{...}` property escape to be recognised, otherwise the pattern would match the literal text `p{Cc}`. Legitimate service IDs, terms types and document IDs never contain these, and NUL in particular can truncate a value once it reaches git or the filesystem, so any segment holding one is rejected.
|
|
21
|
+
|
|
22
|
+
// Keeps hostile values from reaching git, where a pathspec that resolves outside the repository (such as `../foo/*`) aborts with an error that exposes the repository location.
|
|
23
|
+
function isPlainPathSegment(segment) {
|
|
24
|
+
return segment.length > 0
|
|
25
|
+
&& segment !== '.'
|
|
26
|
+
&& segment !== '..'
|
|
27
|
+
&& !segment.includes('/')
|
|
28
|
+
&& !segment.includes('\\')
|
|
29
|
+
&& !CONTROL_CHARACTERS_REGEXP.test(segment);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function canMatchRecordFilePath(...pathSegments) {
|
|
33
|
+
// A non-string segment means "not provided" (`undefined`, or `false` for an absent document ID) and constrains nothing
|
|
34
|
+
return pathSegments.every(segment => typeof segment !== 'string' || isPlainPathSegment(segment));
|
|
35
|
+
}
|
|
36
|
+
|
|
18
37
|
export default class GitRepository extends RepositoryInterface {
|
|
19
38
|
constructor({ path, author, publish, snapshotIdentiferTemplate }) {
|
|
20
39
|
super();
|
|
@@ -64,6 +83,10 @@ export default class GitRepository extends RepositoryInterface {
|
|
|
64
83
|
}
|
|
65
84
|
|
|
66
85
|
async findLatest(serviceId, termsType, documentId) {
|
|
86
|
+
if (!canMatchRecordFilePath(serviceId, termsType, documentId)) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
|
|
67
90
|
const matchingFilesPaths = await this.git.listFiles(DataMapper.generateFilePath(serviceId, termsType, documentId));
|
|
68
91
|
|
|
69
92
|
if (!matchingFilesPaths.length) {
|
|
@@ -76,35 +99,95 @@ export default class GitRepository extends RepositoryInterface {
|
|
|
76
99
|
}
|
|
77
100
|
|
|
78
101
|
async findByDate(serviceId, termsType, date, documentId) {
|
|
102
|
+
if (!canMatchRecordFilePath(serviceId, termsType, documentId)) {
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
|
|
79
106
|
const filePath = DataMapper.generateFilePath(serviceId, termsType, documentId);
|
|
80
|
-
const commit = await this.git.getCommit([ `--until=${date?.toISOString()}`, filePath ]);
|
|
107
|
+
const commit = await this.git.getCommit([ `--until=${date?.toISOString()}`, '--', filePath ]);
|
|
81
108
|
|
|
82
109
|
return this.#toDomain(commit);
|
|
83
110
|
}
|
|
84
111
|
|
|
85
112
|
async findById(recordId) {
|
|
86
|
-
|
|
113
|
+
if (!RECORD_ID_REGEXP.test(recordId)) {
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const commit = await this.git.getCommit([ '--end-of-options', recordId ]); // `--end-of-options` forces git to treat `recordId` as a revision, never as an option: a second line of defence that keeps the lookup safe from argument injection even if the format guard above is ever relaxed
|
|
87
118
|
|
|
88
119
|
return this.#toDomain(commit);
|
|
89
120
|
}
|
|
90
121
|
|
|
122
|
+
async findMetadataById(recordId) {
|
|
123
|
+
if (!RECORD_ID_REGEXP.test(recordId)) {
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const commit = await this.git.getCommit([ '--end-of-options', recordId ]); // `--end-of-options` forces git to treat `recordId` as a revision, never as an option: a second line of defence that keeps the lookup safe from argument injection even if the format guard above is ever relaxed
|
|
128
|
+
|
|
129
|
+
return this.#toDomain(commit, { deferContentLoading: true });
|
|
130
|
+
}
|
|
131
|
+
|
|
91
132
|
async findAll({ limit, offset, includeTechnicalUpgrades = true } = {}) {
|
|
92
133
|
return Promise.all((await this.#getCommits({ limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
|
|
93
134
|
}
|
|
94
135
|
|
|
136
|
+
async findByServiceAndTermsType(serviceId, termsType, { limit, offset, includeTechnicalUpgrades = true } = {}) {
|
|
137
|
+
if (!canMatchRecordFilePath(serviceId, termsType)) {
|
|
138
|
+
return [];
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const pathPattern = DataMapper.generateFilePath(serviceId, termsType);
|
|
142
|
+
|
|
143
|
+
return Promise.all((await this.#getCommits({ pathFilter: pathPattern, limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
|
|
144
|
+
}
|
|
145
|
+
|
|
95
146
|
async findByService(serviceId, { limit, offset, includeTechnicalUpgrades = true } = {}) {
|
|
147
|
+
if (!canMatchRecordFilePath(serviceId)) {
|
|
148
|
+
return [];
|
|
149
|
+
}
|
|
150
|
+
|
|
96
151
|
const pathPattern = DataMapper.generateFilePath(serviceId);
|
|
97
152
|
|
|
98
153
|
return Promise.all((await this.#getCommits({ pathFilter: pathPattern, limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
|
|
99
154
|
}
|
|
100
155
|
|
|
101
|
-
async
|
|
156
|
+
async getNavigationIds(serviceId, termsType, versionId, { includeTechnicalUpgrades = true } = {}) {
|
|
157
|
+
if (!canMatchRecordFilePath(serviceId, termsType)) {
|
|
158
|
+
return { first: null, prev: null, next: null, last: null };
|
|
159
|
+
}
|
|
160
|
+
|
|
102
161
|
const pathPattern = DataMapper.generateFilePath(serviceId, termsType);
|
|
162
|
+
let revisions = await this.git.listPathRevisions(pathPattern); // single lean walk of the terms history
|
|
103
163
|
|
|
104
|
-
|
|
164
|
+
if (!includeTechnicalUpgrades) {
|
|
165
|
+
revisions = revisions.filter(revision => !DataMapper.isTechnicalUpgrade(revision.subject));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Deterministic total order: most recent first, commit SHA as a stable tiebreaker for versions sharing the same fetch date (git stores second precision).
|
|
169
|
+
// prev/next are then adjacent entries in this single order, so navigation always round-trips, unlike the previous chronological/topological mix.
|
|
170
|
+
revisions.sort((a, b) => b.timestamp - a.timestamp || (a.hash < b.hash ? -1 : 1));
|
|
171
|
+
|
|
172
|
+
const index = revisions.findIndex(revision => revision.hash === versionId);
|
|
173
|
+
|
|
174
|
+
if (index === -1) {
|
|
175
|
+
return { first: null, prev: null, next: null, last: null };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
return {
|
|
179
|
+
last: revisions[0].hash,
|
|
180
|
+
first: revisions[revisions.length - 1].hash,
|
|
181
|
+
next: index > 0 ? revisions[index - 1].hash : null,
|
|
182
|
+
prev: index < revisions.length - 1 ? revisions[index + 1].hash : null,
|
|
183
|
+
};
|
|
105
184
|
}
|
|
106
185
|
|
|
107
186
|
async count(serviceId, termsType) {
|
|
187
|
+
if (!canMatchRecordFilePath(serviceId, termsType)) {
|
|
188
|
+
return 0;
|
|
189
|
+
}
|
|
190
|
+
|
|
108
191
|
const grepOptions = Object.values(DataMapper.COMMIT_MESSAGE_PREFIXES).map(prefix => `--grep=${prefix}`);
|
|
109
192
|
const pathOptions = [];
|
|
110
193
|
|
|
@@ -158,6 +241,10 @@ export default class GitRepository extends RepositoryInterface {
|
|
|
158
241
|
record.content = pdfBuffer;
|
|
159
242
|
}
|
|
160
243
|
|
|
244
|
+
getDiffStats(recordId) {
|
|
245
|
+
return this.git.getDiffStats(recordId);
|
|
246
|
+
}
|
|
247
|
+
|
|
161
248
|
async #getCommits({ pathFilter, reverse = false, limit, offset, includeTechnicalUpgrades = true } = {}) {
|
|
162
249
|
const prefixes = includeTechnicalUpgrades
|
|
163
250
|
? DataMapper.COMMIT_MESSAGE_PREFIXES
|
|
@@ -400,6 +400,59 @@ describe('GitRepository', () => {
|
|
|
400
400
|
expect(await subject.findById('inexistantID')).to.equal(null);
|
|
401
401
|
});
|
|
402
402
|
});
|
|
403
|
+
|
|
404
|
+
context('when the requested ID is well formed but absent from the repository', () => {
|
|
405
|
+
it('returns null rather than throwing a "bad object" error', async () => {
|
|
406
|
+
expect(await subject.findById('ecd9407eb26b1bf0613186175ee80edbdeedd47f')).to.equal(null);
|
|
407
|
+
});
|
|
408
|
+
});
|
|
409
|
+
|
|
410
|
+
context('when the requested ID could be interpreted as a git option', () => {
|
|
411
|
+
const INJECTION_PROOF_FILE_PATH = path.resolve(__dirname, 'findById-argument-injection-proof.txt');
|
|
412
|
+
|
|
413
|
+
after(() => fs.rmSync(INJECTION_PROOF_FILE_PATH, { force: true }));
|
|
414
|
+
|
|
415
|
+
it('returns null without letting the ID reach git as an argument', async () => {
|
|
416
|
+
expect(await subject.findById(`--output=${INJECTION_PROOF_FILE_PATH}`)).to.equal(null);
|
|
417
|
+
expect(fs.existsSync(INJECTION_PROOF_FILE_PATH), 'a version ID must never be interpreted as a git option').to.be.false;
|
|
418
|
+
});
|
|
419
|
+
});
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
describe('#findMetadataById', () => {
|
|
423
|
+
let id;
|
|
424
|
+
|
|
425
|
+
before(async () => {
|
|
426
|
+
({ id } = await subject.save(new Version({
|
|
427
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
428
|
+
termsType: TERMS_TYPE,
|
|
429
|
+
content: CONTENT,
|
|
430
|
+
fetchDate: FETCH_DATE,
|
|
431
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
432
|
+
mimeType: HTML_MIME_TYPE,
|
|
433
|
+
metadata: METADATA,
|
|
434
|
+
})));
|
|
435
|
+
});
|
|
436
|
+
|
|
437
|
+
after(() => subject.removeAll());
|
|
438
|
+
|
|
439
|
+
it('returns the record', async () => {
|
|
440
|
+
const record = await subject.findMetadataById(id);
|
|
441
|
+
|
|
442
|
+
expect(record).to.be.an.instanceof(Version);
|
|
443
|
+
expect(record.id).to.include(id);
|
|
444
|
+
});
|
|
445
|
+
|
|
446
|
+
context('when the requested ID could be interpreted as a git option', () => {
|
|
447
|
+
const INJECTION_PROOF_FILE_PATH = path.resolve(__dirname, 'findMetadataById-argument-injection-proof.txt');
|
|
448
|
+
|
|
449
|
+
after(() => fs.rmSync(INJECTION_PROOF_FILE_PATH, { force: true }));
|
|
450
|
+
|
|
451
|
+
it('returns null without letting the ID reach git as an argument', async () => {
|
|
452
|
+
expect(await subject.findMetadataById(`--output=${INJECTION_PROOF_FILE_PATH}`)).to.equal(null);
|
|
453
|
+
expect(fs.existsSync(INJECTION_PROOF_FILE_PATH), 'a version ID must never be interpreted as a git option').to.be.false;
|
|
454
|
+
});
|
|
455
|
+
});
|
|
403
456
|
});
|
|
404
457
|
|
|
405
458
|
describe('#findByDate', () => {
|
|
@@ -485,6 +538,37 @@ describe('GitRepository', () => {
|
|
|
485
538
|
expect(record.metadata).to.deep.equal(METADATA);
|
|
486
539
|
});
|
|
487
540
|
});
|
|
541
|
+
|
|
542
|
+
context('when the service ID is a git argument injection attempt', () => {
|
|
543
|
+
const INJECTION_PROOF_FILE_PATH = path.resolve(__dirname, 'findByDate-argument-injection-proof.*');
|
|
544
|
+
|
|
545
|
+
before(async () => {
|
|
546
|
+
await subject.save(new Version({
|
|
547
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
548
|
+
termsType: TERMS_TYPE,
|
|
549
|
+
content: CONTENT,
|
|
550
|
+
fetchDate: FETCH_DATE,
|
|
551
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
552
|
+
}));
|
|
553
|
+
});
|
|
554
|
+
|
|
555
|
+
after(async () => {
|
|
556
|
+
fs.rmSync(INJECTION_PROOF_FILE_PATH, { force: true });
|
|
557
|
+
await subject.removeAll();
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
it('treats the service ID as a path so it cannot reach git as an option', async () => {
|
|
561
|
+
await subject.findByDate(`--output=${__dirname}`, 'findByDate-argument-injection-proof', FETCH_DATE_LATER);
|
|
562
|
+
|
|
563
|
+
expect(fs.existsSync(INJECTION_PROOF_FILE_PATH), 'a service ID must never be interpreted as a git option').to.be.false;
|
|
564
|
+
});
|
|
565
|
+
});
|
|
566
|
+
|
|
567
|
+
context('when the service ID is a path traversal attempt', () => {
|
|
568
|
+
it('returns null instead of erroring', async () => {
|
|
569
|
+
expect(await subject.findByDate('../../outside', TERMS_TYPE, FETCH_DATE)).to.equal(null);
|
|
570
|
+
});
|
|
571
|
+
});
|
|
488
572
|
});
|
|
489
573
|
|
|
490
574
|
describe('#findAll', () => {
|
|
@@ -645,6 +729,13 @@ describe('GitRepository', () => {
|
|
|
645
729
|
});
|
|
646
730
|
});
|
|
647
731
|
|
|
732
|
+
context('when the service ID or terms type is a path traversal attempt', () => {
|
|
733
|
+
it('returns an empty array instead of erroring', async () => {
|
|
734
|
+
expect(await subject.findByServiceAndTermsType('../../outside', TERMS_TYPE)).to.be.an('array').that.is.empty;
|
|
735
|
+
expect(await subject.findByServiceAndTermsType(SERVICE_PROVIDER_ID, '../../outside')).to.be.an('array').that.is.empty;
|
|
736
|
+
});
|
|
737
|
+
});
|
|
738
|
+
|
|
648
739
|
context('with includeTechnicalUpgrades: false', () => {
|
|
649
740
|
let filteredRecords;
|
|
650
741
|
let technicalUpgradeId;
|
|
@@ -761,6 +852,12 @@ describe('GitRepository', () => {
|
|
|
761
852
|
});
|
|
762
853
|
});
|
|
763
854
|
|
|
855
|
+
context('when the service ID is a path traversal attempt', () => {
|
|
856
|
+
it('returns an empty array instead of erroring', async () => {
|
|
857
|
+
expect(await subject.findByService('../../outside')).to.be.an('array').that.is.empty;
|
|
858
|
+
});
|
|
859
|
+
});
|
|
860
|
+
|
|
764
861
|
context('with includeTechnicalUpgrades: false', () => {
|
|
765
862
|
let filteredRecords;
|
|
766
863
|
let technicalUpgradeId;
|
|
@@ -842,6 +939,12 @@ describe('GitRepository', () => {
|
|
|
842
939
|
});
|
|
843
940
|
});
|
|
844
941
|
|
|
942
|
+
context('when the service ID is a path traversal attempt', () => {
|
|
943
|
+
it('returns zero instead of erroring', async () => {
|
|
944
|
+
expect(await subject.count('../../outside', TERMS_TYPE)).to.equal(0);
|
|
945
|
+
});
|
|
946
|
+
});
|
|
947
|
+
|
|
845
948
|
context('with only serviceId filter', () => {
|
|
846
949
|
it('returns count for all terms types of a service', async () => {
|
|
847
950
|
// Add a version with different terms type
|
|
@@ -860,6 +963,167 @@ describe('GitRepository', () => {
|
|
|
860
963
|
});
|
|
861
964
|
});
|
|
862
965
|
|
|
966
|
+
describe('#getNavigationIds', () => {
|
|
967
|
+
let firstVersion;
|
|
968
|
+
let middleVersion;
|
|
969
|
+
let lastVersion;
|
|
970
|
+
|
|
971
|
+
before(async function () {
|
|
972
|
+
this.timeout(5000);
|
|
973
|
+
|
|
974
|
+
firstVersion = await subject.save(new Version({
|
|
975
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
976
|
+
termsType: TERMS_TYPE,
|
|
977
|
+
content: 'first content',
|
|
978
|
+
fetchDate: FETCH_DATE_EARLIER,
|
|
979
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
980
|
+
}));
|
|
981
|
+
|
|
982
|
+
middleVersion = await subject.save(new Version({
|
|
983
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
984
|
+
termsType: TERMS_TYPE,
|
|
985
|
+
content: 'middle content',
|
|
986
|
+
fetchDate: FETCH_DATE,
|
|
987
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
988
|
+
}));
|
|
989
|
+
|
|
990
|
+
lastVersion = await subject.save(new Version({
|
|
991
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
992
|
+
termsType: TERMS_TYPE,
|
|
993
|
+
content: 'last content',
|
|
994
|
+
fetchDate: FETCH_DATE_LATER,
|
|
995
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
996
|
+
}));
|
|
997
|
+
});
|
|
998
|
+
|
|
999
|
+
after(() => subject.removeAll());
|
|
1000
|
+
|
|
1001
|
+
context('for the oldest version', () => {
|
|
1002
|
+
let navigationIds;
|
|
1003
|
+
|
|
1004
|
+
before(async () => { navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, firstVersion.id); });
|
|
1005
|
+
|
|
1006
|
+
it('has no previous version', () => expect(navigationIds.prev).to.be.null);
|
|
1007
|
+
it('points to the next version', () => expect(navigationIds.next).to.equal(middleVersion.id));
|
|
1008
|
+
it('points to itself as first', () => expect(navigationIds.first).to.equal(firstVersion.id));
|
|
1009
|
+
it('points to the newest version as last', () => expect(navigationIds.last).to.equal(lastVersion.id));
|
|
1010
|
+
});
|
|
1011
|
+
|
|
1012
|
+
context('for a middle version', () => {
|
|
1013
|
+
let navigationIds;
|
|
1014
|
+
|
|
1015
|
+
before(async () => { navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, middleVersion.id); });
|
|
1016
|
+
|
|
1017
|
+
it('points to the previous version', () => expect(navigationIds.prev).to.equal(firstVersion.id));
|
|
1018
|
+
it('points to the next version', () => expect(navigationIds.next).to.equal(lastVersion.id));
|
|
1019
|
+
it('points to the oldest version as first', () => expect(navigationIds.first).to.equal(firstVersion.id));
|
|
1020
|
+
it('points to the newest version as last', () => expect(navigationIds.last).to.equal(lastVersion.id));
|
|
1021
|
+
});
|
|
1022
|
+
|
|
1023
|
+
context('for the newest version', () => {
|
|
1024
|
+
let navigationIds;
|
|
1025
|
+
|
|
1026
|
+
before(async () => { navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, lastVersion.id); });
|
|
1027
|
+
|
|
1028
|
+
it('points to the previous version', () => expect(navigationIds.prev).to.equal(middleVersion.id));
|
|
1029
|
+
it('has no next version', () => expect(navigationIds.next).to.be.null);
|
|
1030
|
+
it('points to itself as last', () => expect(navigationIds.last).to.equal(lastVersion.id));
|
|
1031
|
+
});
|
|
1032
|
+
|
|
1033
|
+
context('when the version does not exist', () => {
|
|
1034
|
+
it('returns only null IDs', async () => {
|
|
1035
|
+
expect(await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, 'ffffffffffffffffffffffffffffffffffffffff')).to.deep.equal({ first: null, prev: null, next: null, last: null });
|
|
1036
|
+
});
|
|
1037
|
+
});
|
|
1038
|
+
|
|
1039
|
+
context('when the service ID is a path traversal attempt', () => {
|
|
1040
|
+
it('returns only null IDs instead of erroring', async () => {
|
|
1041
|
+
expect(await subject.getNavigationIds('../../outside', TERMS_TYPE, firstVersion.id)).to.deep.equal({ first: null, prev: null, next: null, last: null });
|
|
1042
|
+
});
|
|
1043
|
+
});
|
|
1044
|
+
|
|
1045
|
+
context('when a technical upgrade is recorded out of chronological order', () => {
|
|
1046
|
+
// A technical upgrade re-renders an old snapshot: it carries an OLD fetch date but is committed last (topologically recent).
|
|
1047
|
+
// The previous implementation mixed chronological (findPrevious) and topological (findNext) order, so prev/next disagreed here.
|
|
1048
|
+
// The navigation IDs are now derived from a single chronological order, so they stay exact inverses.
|
|
1049
|
+
const TECHNICAL_UPGRADE_DATE = new Date('2000-01-01T09:00:00.000Z'); // between firstVersion (06:00) and middleVersion (12:00)
|
|
1050
|
+
let technicalUpgrade;
|
|
1051
|
+
|
|
1052
|
+
before(async () => {
|
|
1053
|
+
technicalUpgrade = await subject.save(new Version({
|
|
1054
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
1055
|
+
termsType: TERMS_TYPE,
|
|
1056
|
+
content: 'technical upgrade content',
|
|
1057
|
+
fetchDate: TECHNICAL_UPGRADE_DATE,
|
|
1058
|
+
isTechnicalUpgrade: true,
|
|
1059
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
1060
|
+
}));
|
|
1061
|
+
});
|
|
1062
|
+
|
|
1063
|
+
it('orders it by its fetch date, not its commit position', async () => {
|
|
1064
|
+
const navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, technicalUpgrade.id);
|
|
1065
|
+
|
|
1066
|
+
expect(navigationIds.prev).to.equal(firstVersion.id);
|
|
1067
|
+
expect(navigationIds.next).to.equal(middleVersion.id);
|
|
1068
|
+
});
|
|
1069
|
+
|
|
1070
|
+
it('keeps prev and next as exact inverses (round-trips)', async () => {
|
|
1071
|
+
const fromFirst = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, firstVersion.id);
|
|
1072
|
+
|
|
1073
|
+
expect(fromFirst.next).to.equal(technicalUpgrade.id);
|
|
1074
|
+
|
|
1075
|
+
const backFromUpgrade = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, fromFirst.next);
|
|
1076
|
+
|
|
1077
|
+
expect(backFromUpgrade.prev).to.equal(firstVersion.id);
|
|
1078
|
+
});
|
|
1079
|
+
|
|
1080
|
+
it('can exclude technical upgrades from the sequence', async () => {
|
|
1081
|
+
const navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, firstVersion.id, { includeTechnicalUpgrades: false });
|
|
1082
|
+
|
|
1083
|
+
expect(navigationIds.next).to.equal(middleVersion.id);
|
|
1084
|
+
});
|
|
1085
|
+
});
|
|
1086
|
+
});
|
|
1087
|
+
|
|
1088
|
+
describe('#getNavigationIds with versions sharing the same fetch date', () => {
|
|
1089
|
+
// Git stores commit dates at second precision, so distinct versions can share a fetch date.
|
|
1090
|
+
// The record ID tiebreaker must keep the order deterministic so prev/next still round-trip.
|
|
1091
|
+
let ids;
|
|
1092
|
+
|
|
1093
|
+
before(async function () {
|
|
1094
|
+
this.timeout(5000);
|
|
1095
|
+
ids = [];
|
|
1096
|
+
for (const content of [ 'tie A', 'tie B', 'tie C' ]) {
|
|
1097
|
+
ids.push((await subject.save(new Version({ serviceId: SERVICE_PROVIDER_ID, termsType: TERMS_TYPE, content, fetchDate: FETCH_DATE, snapshotIds: [SNAPSHOT_ID] }))).id);
|
|
1098
|
+
}
|
|
1099
|
+
});
|
|
1100
|
+
|
|
1101
|
+
after(() => subject.removeAll());
|
|
1102
|
+
|
|
1103
|
+
it('round-trips next then prev for every version', async () => {
|
|
1104
|
+
for (const id of ids) {
|
|
1105
|
+
const navigationIds = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, id);
|
|
1106
|
+
|
|
1107
|
+
if (navigationIds.next) {
|
|
1108
|
+
expect((await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, navigationIds.next)).prev).to.equal(id);
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
});
|
|
1112
|
+
|
|
1113
|
+
it('exposes all three versions as a single ordered chain', async () => {
|
|
1114
|
+
const head = await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, ids[0]);
|
|
1115
|
+
const walked = new Set([head.first]);
|
|
1116
|
+
let cursor = head.first;
|
|
1117
|
+
|
|
1118
|
+
while (cursor) {
|
|
1119
|
+
walked.add(cursor);
|
|
1120
|
+
cursor = (await subject.getNavigationIds(SERVICE_PROVIDER_ID, TERMS_TYPE, cursor)).next;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
expect(walked).to.have.lengthOf(ids.length);
|
|
1124
|
+
});
|
|
1125
|
+
});
|
|
1126
|
+
|
|
863
1127
|
describe('#findLatest', () => {
|
|
864
1128
|
context('when there are records for the given service', () => {
|
|
865
1129
|
let lastSnapshotId;
|
|
@@ -920,6 +1184,30 @@ describe('GitRepository', () => {
|
|
|
920
1184
|
expect(latestRecord).to.equal(null);
|
|
921
1185
|
});
|
|
922
1186
|
});
|
|
1187
|
+
|
|
1188
|
+
context('when the service ID could be interpreted as a git option', () => {
|
|
1189
|
+
before(async () => {
|
|
1190
|
+
await subject.save(new Version({
|
|
1191
|
+
serviceId: SERVICE_PROVIDER_ID,
|
|
1192
|
+
termsType: TERMS_TYPE,
|
|
1193
|
+
content: CONTENT,
|
|
1194
|
+
fetchDate: FETCH_DATE,
|
|
1195
|
+
snapshotIds: [SNAPSHOT_ID],
|
|
1196
|
+
}));
|
|
1197
|
+
});
|
|
1198
|
+
|
|
1199
|
+
after(() => subject.removeAll());
|
|
1200
|
+
|
|
1201
|
+
it('treats the service ID as a path and returns null instead of erroring', async () => {
|
|
1202
|
+
expect(await subject.findLatest('--not-a-git-option', TERMS_TYPE)).to.equal(null);
|
|
1203
|
+
});
|
|
1204
|
+
});
|
|
1205
|
+
|
|
1206
|
+
context('when the service ID is a path traversal attempt', () => {
|
|
1207
|
+
it('returns null instead of erroring', async () => {
|
|
1208
|
+
expect(await subject.findLatest('../../outside', TERMS_TYPE)).to.equal(null);
|
|
1209
|
+
});
|
|
1210
|
+
});
|
|
923
1211
|
});
|
|
924
1212
|
|
|
925
1213
|
describe('#iterate', () => {
|