@opentermsarchive/engine 16.0.2 → 16.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/default.json +13 -0
- package/config/test.json +12 -0
- package/package.json +1 -1
- package/scripts/declarations/validate/index.mocha.js +7 -0
- package/scripts/import/index.js +1 -1
- package/scripts/import/loadCommits.js +1 -1
- package/scripts/rewrite/initializer/index.js +1 -1
- package/scripts/rewrite/rewrite-snapshots.js +1 -1
- package/scripts/rewrite/rewrite-versions.js +1 -1
- package/src/archivist/fetcher/htmlOnlyFetcher.js +8 -0
- package/src/archivist/fetcher/index.test.js +12 -0
- package/src/archivist/index.js +106 -23
- package/src/archivist/index.test.js +659 -9
- package/src/archivist/recorder/repositories/git/dataMapper.js +2 -1
- package/src/archivist/recorder/repositories/git/index.js +2 -13
- package/src/archivist/recorder/repositories/git/index.test.js +23 -1
- package/src/archivist/services/index.js +45 -1
- package/src/archivist/services/index.test.js +52 -1
- package/src/archivist/services/sourceDocument.js +8 -2
- package/src/archivist/services/sourceDocument.test.js +37 -0
- package/src/archivist/tracking-results/errors.js +5 -0
- package/src/archivist/tracking-results/index.js +233 -0
- package/src/archivist/tracking-results/index.test.js +406 -0
- package/src/archivist/tracking-results/recorder.js +156 -0
- package/src/archivist/tracking-results/recorder.test.js +559 -0
- package/src/archivist/tracking-results/repository.js +136 -0
- package/src/archivist/tracking-results/repository.test.js +763 -0
- package/src/archivist/tracking-results/run/dataMapper.js +51 -0
- package/src/archivist/tracking-results/run/dataMapper.test.js +168 -0
- package/src/archivist/tracking-results/run/index.js +115 -0
- package/src/archivist/tracking-results/run/index.test.js +221 -0
- package/src/archivist/tracking-results/terms-result/dataMapper.js +200 -0
- package/src/archivist/tracking-results/terms-result/dataMapper.test.js +575 -0
- package/src/archivist/tracking-results/terms-result/index.js +42 -0
- package/src/archivist/tracking-results/terms-result/index.test.js +228 -0
- package/src/git/errors.js +1 -0
- package/src/{archivist/recorder/repositories/git/git.js → git/index.js} +59 -9
- package/src/git/index.test.js +266 -0
- package/src/git/pathSegment.js +18 -0
- package/src/git/pathSegment.test.js +33 -0
- package/src/index.js +5 -4
- package/src/reporter/factory.js +2 -2
- package/src/reporter/github/index.js +7 -5
- package/src/reporter/github/index.test.js +16 -1
- package/src/reporter/gitlab/index.js +6 -6
- package/src/reporter/gitlab/index.test.js +24 -1
- package/src/reporter/index.js +9 -4
- package/src/reporter/index.test.js +74 -9
- package/src/archivist/recorder/repositories/git/git.test.js +0 -114
- /package/src/{archivist/recorder/repositories/git → git}/trailers.js +0 -0
- /package/src/{archivist/recorder/repositories/git → git}/trailers.test.js +0 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import { expect } from 'chai';
|
|
2
|
+
|
|
3
|
+
import TermsResult from './index.js';
|
|
4
|
+
|
|
5
|
+
const VALID_PARAMS = {
|
|
6
|
+
serviceId: 'Google',
|
|
7
|
+
termsType: 'Terms of Service',
|
|
8
|
+
status: 'ok',
|
|
9
|
+
event: {
|
|
10
|
+
date: '2026-01-10T10:30:00Z',
|
|
11
|
+
serviceName: 'Google',
|
|
12
|
+
sourceDocuments: [{
|
|
13
|
+
id: 'terms',
|
|
14
|
+
fetch: 'https://policies.google.com/terms',
|
|
15
|
+
select: '.content',
|
|
16
|
+
remove: '.banner',
|
|
17
|
+
filter: ['removeLinks'],
|
|
18
|
+
executeClientScripts: false,
|
|
19
|
+
mimeType: 'text/html',
|
|
20
|
+
snapshotId: 'def456',
|
|
21
|
+
}],
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
describe('TermsResult', () => {
|
|
26
|
+
let subject;
|
|
27
|
+
|
|
28
|
+
describe('#constructor', () => {
|
|
29
|
+
it('normalises params to their JSON round-trip, dropping undefined-valued keys', () => {
|
|
30
|
+
subject = new TermsResult({
|
|
31
|
+
...VALID_PARAMS,
|
|
32
|
+
event: {
|
|
33
|
+
...VALID_PARAMS.event,
|
|
34
|
+
sourceDocuments: [{
|
|
35
|
+
id: 'terms',
|
|
36
|
+
fetch: 'https://policies.google.com/terms',
|
|
37
|
+
select: '.content',
|
|
38
|
+
remove: undefined,
|
|
39
|
+
filter: undefined,
|
|
40
|
+
}],
|
|
41
|
+
},
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
expect(subject.event.sourceDocuments[0]).to.deep.equal({
|
|
45
|
+
id: 'terms',
|
|
46
|
+
fetch: 'https://policies.google.com/terms',
|
|
47
|
+
select: '.content',
|
|
48
|
+
});
|
|
49
|
+
expect(subject.event.sourceDocuments[0]).to.not.have.property('remove');
|
|
50
|
+
expect(subject.event.sourceDocuments[0]).to.not.have.property('filter');
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe('#validate', () => {
|
|
55
|
+
context('when all required params are present', () => {
|
|
56
|
+
it('does not throw', () => {
|
|
57
|
+
subject = new TermsResult(VALID_PARAMS);
|
|
58
|
+
expect(subject.validate.bind(subject)).to.not.throw();
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
[ 'serviceId', 'termsType', 'status', 'event' ].forEach(requiredParam => {
|
|
63
|
+
describe(`"${requiredParam}"`, () => {
|
|
64
|
+
context('when missing', () => {
|
|
65
|
+
it('throws an error', () => {
|
|
66
|
+
const params = { ...VALID_PARAMS };
|
|
67
|
+
|
|
68
|
+
delete params[requiredParam];
|
|
69
|
+
subject = new TermsResult(params);
|
|
70
|
+
expect(subject.validate.bind(subject)).to.throw(RegExp(requiredParam));
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
context('when null', () => {
|
|
75
|
+
it('throws an error', () => {
|
|
76
|
+
subject = new TermsResult({ ...VALID_PARAMS, [requiredParam]: null });
|
|
77
|
+
expect(subject.validate.bind(subject)).to.throw(RegExp(requiredParam));
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
describe('"status"', () => {
|
|
84
|
+
context('when not a valid value', () => {
|
|
85
|
+
it('throws an error', () => {
|
|
86
|
+
subject = new TermsResult({ ...VALID_PARAMS, status: 'unknown' });
|
|
87
|
+
expect(subject.validate.bind(subject)).to.throw(/status/);
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
[ 'ok', 'failed' ].forEach(validStatus => {
|
|
92
|
+
context(`when set to "${validStatus}"`, () => {
|
|
93
|
+
it('does not throw', () => {
|
|
94
|
+
const params = { ...VALID_PARAMS, status: validStatus };
|
|
95
|
+
|
|
96
|
+
if (validStatus === 'failed') {
|
|
97
|
+
params.event = { ...VALID_PARAMS.event, reasons: ['[fetch] HTTP code 503'] };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
subject = new TermsResult(params);
|
|
101
|
+
expect(subject.validate.bind(subject)).to.not.throw();
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
describe('"event.date"', () => {
|
|
108
|
+
context('when missing', () => {
|
|
109
|
+
it('throws an error', () => {
|
|
110
|
+
subject = new TermsResult({ ...VALID_PARAMS, event: { ...VALID_PARAMS.event, date: undefined } });
|
|
111
|
+
expect(subject.validate.bind(subject)).to.throw(/event\.date/);
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
describe('"event.serviceName"', () => {
|
|
117
|
+
context('when missing', () => {
|
|
118
|
+
it('throws an error', () => {
|
|
119
|
+
subject = new TermsResult({ ...VALID_PARAMS, event: { ...VALID_PARAMS.event, serviceName: undefined } });
|
|
120
|
+
expect(subject.validate.bind(subject)).to.throw(/event\.serviceName/);
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
describe('"event.sourceDocuments"', () => {
|
|
126
|
+
context('when not an array', () => {
|
|
127
|
+
it('throws an error', () => {
|
|
128
|
+
subject = new TermsResult({ ...VALID_PARAMS, event: { ...VALID_PARAMS.event, sourceDocuments: 'not-an-array' } });
|
|
129
|
+
expect(subject.validate.bind(subject)).to.throw(/event\.sourceDocuments/);
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
describe('"event.reasons" when status is "failed"', () => {
|
|
135
|
+
context('when missing', () => {
|
|
136
|
+
it('throws an error', () => {
|
|
137
|
+
subject = new TermsResult({ ...VALID_PARAMS, status: 'failed' });
|
|
138
|
+
expect(subject.validate.bind(subject)).to.throw(/event\.reasons/);
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
context('when empty', () => {
|
|
143
|
+
it('throws an error', () => {
|
|
144
|
+
subject = new TermsResult({ ...VALID_PARAMS, status: 'failed', event: { ...VALID_PARAMS.event, reasons: [] } });
|
|
145
|
+
expect(subject.validate.bind(subject)).to.throw(/event\.reasons/);
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
context('when non-empty', () => {
|
|
150
|
+
it('does not throw', () => {
|
|
151
|
+
subject = new TermsResult({
|
|
152
|
+
...VALID_PARAMS,
|
|
153
|
+
status: 'failed',
|
|
154
|
+
event: { ...VALID_PARAMS.event, reasons: ['[fetch] HTTP code 503'] },
|
|
155
|
+
});
|
|
156
|
+
expect(subject.validate.bind(subject)).to.not.throw();
|
|
157
|
+
});
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
describe('"event.transientError"', () => {
|
|
162
|
+
context('when present but reasons is empty', () => {
|
|
163
|
+
it('throws an error', () => {
|
|
164
|
+
subject = new TermsResult({
|
|
165
|
+
...VALID_PARAMS,
|
|
166
|
+
event: { ...VALID_PARAMS.event, transientError: { reasons: [] } },
|
|
167
|
+
});
|
|
168
|
+
expect(subject.validate.bind(subject)).to.throw(/transientError/);
|
|
169
|
+
});
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
context('when present but reasons is missing (non-object with a truthy value)', () => {
|
|
173
|
+
it('throws an error', () => {
|
|
174
|
+
subject = new TermsResult({
|
|
175
|
+
...VALID_PARAMS,
|
|
176
|
+
event: { ...VALID_PARAMS.event, transientError: 'not-an-object' }, // Scalar truthy value has no .reasons array
|
|
177
|
+
});
|
|
178
|
+
expect(subject.validate.bind(subject)).to.throw(/transientError/);
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
context('when present but reasons is not an array', () => {
|
|
183
|
+
it('throws an error', () => {
|
|
184
|
+
subject = new TermsResult({
|
|
185
|
+
...VALID_PARAMS,
|
|
186
|
+
event: { ...VALID_PARAMS.event, transientError: { reasons: 'a-string' } },
|
|
187
|
+
});
|
|
188
|
+
expect(subject.validate.bind(subject)).to.throw(/transientError/);
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
context('when present with non-empty reasons', () => {
|
|
193
|
+
it('does not throw', () => {
|
|
194
|
+
subject = new TermsResult({
|
|
195
|
+
...VALID_PARAMS,
|
|
196
|
+
event: { ...VALID_PARAMS.event, transientError: { reasons: ['[fetch] HTTP code 503'] } },
|
|
197
|
+
});
|
|
198
|
+
expect(subject.validate.bind(subject)).to.not.throw();
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
describe('"status"', () => {
|
|
204
|
+
context('when the value is the right type but wrong case ("OK" instead of "ok")', () => {
|
|
205
|
+
it('throws an error', () => {
|
|
206
|
+
subject = new TermsResult({ ...VALID_PARAMS, status: 'OK' });
|
|
207
|
+
expect(subject.validate.bind(subject)).to.throw(/status/);
|
|
208
|
+
});
|
|
209
|
+
});
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
describe('"event.sourceDocuments"', () => {
|
|
213
|
+
context('when null', () => {
|
|
214
|
+
it('throws an error', () => {
|
|
215
|
+
subject = new TermsResult({ ...VALID_PARAMS, event: { ...VALID_PARAMS.event, sourceDocuments: null } });
|
|
216
|
+
expect(subject.validate.bind(subject)).to.throw(/event\.sourceDocuments/);
|
|
217
|
+
});
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
context('when an empty array', () => {
|
|
221
|
+
it('does not throw', () => {
|
|
222
|
+
subject = new TermsResult({ ...VALID_PARAMS, event: { ...VALID_PARAMS.event, sourceDocuments: [] } });
|
|
223
|
+
expect(subject.validate.bind(subject)).to.not.throw();
|
|
224
|
+
});
|
|
225
|
+
});
|
|
226
|
+
});
|
|
227
|
+
});
|
|
228
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export class GitObjectNotFoundError extends Error {} // The requested commit, tree or file cannot be resolved in the repository (unknown SHA, shallow clone, rewritten history, file absent from the commit)
|
|
@@ -3,13 +3,42 @@ import path from 'path';
|
|
|
3
3
|
|
|
4
4
|
import simpleGit from 'simple-git';
|
|
5
5
|
|
|
6
|
+
import { GitObjectNotFoundError } from './errors.js';
|
|
6
7
|
import { parseTrailers, formatTrailers } from './trailers.js';
|
|
7
8
|
|
|
9
|
+
export { GitObjectNotFoundError } from './errors.js';
|
|
10
|
+
|
|
8
11
|
process.env.LC_ALL = 'en_GB'; // Ensure git messages will be in English as some errors are handled by analysing the message content
|
|
9
12
|
|
|
10
13
|
const fs = fsApi.promises;
|
|
11
14
|
|
|
15
|
+
const OBJECT_NOT_FOUND_MESSAGES = /bad object|not a tree|invalid object name|unknown revision|does not exist|exists on disk, but not in/i;
|
|
16
|
+
|
|
12
17
|
export default class Git {
|
|
18
|
+
static async getHeadSha(repositoryPath) { // Used by callers that need to capture the current state of a repository without instantiating a full Git wrapper (which would mutate the repo via `init`).
|
|
19
|
+
try {
|
|
20
|
+
const git = simpleGit(repositoryPath, { trimmed: true });
|
|
21
|
+
|
|
22
|
+
return await git.revparse(['HEAD']);
|
|
23
|
+
} catch (error) {
|
|
24
|
+
if (/not a git repository|does not exist|unknown revision|ambiguous argument|does not have any commits/i.test(error.message)) {
|
|
25
|
+
return null; // Not a repository, or an empty one: a legitimate "no commit to reference" answer
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
throw error; // An actual git failure, which callers must not conflate with the absence of a repository
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
static async listFilesAtCommit(repositoryPath, commit) {
|
|
33
|
+
const output = await readObjectAtCommit(repositoryPath, [ 'ls-tree', '--name-only', commit, '--', './' ]); // `repositoryPath` may be a subdirectory of the repository; git resolves the `./` pathspec against its cwd.
|
|
34
|
+
|
|
35
|
+
return output ? output.split('\n') : [];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
static readFileAtCommit(repositoryPath, commit, fileName) { // Returns the content of `fileName` (relative to `repositoryPath`) at the given commit
|
|
39
|
+
return readObjectAtCommit(repositoryPath, [ 'show', `${commit}:./${fileName}` ]); // The `rev:./path` syntax makes git resolve the path against its cwd, which supports repositoryPath being a subdirectory of the repository
|
|
40
|
+
}
|
|
41
|
+
|
|
13
42
|
constructor({ path: repositoryPath, author }) {
|
|
14
43
|
this.path = repositoryPath;
|
|
15
44
|
this.author = author;
|
|
@@ -23,14 +52,21 @@ export default class Git {
|
|
|
23
52
|
this.#connect();
|
|
24
53
|
await this.git.init();
|
|
25
54
|
|
|
55
|
+
const configFile = path.resolve(this.path, '.git', 'config'); // Anchored to an absolute path: git resolves a relative `--file` argument against its own cwd (the repository), not against process.cwd, so a relative repository path would silently point the write at a nonexistent nested location
|
|
56
|
+
|
|
57
|
+
if (!fsApi.existsSync(configFile)) { // Defensive: init should always produce this file; if it does not, refuse to continue rather than risk writing config to an unintended location
|
|
58
|
+
throw new Error(`Git initialisation failed: expected config file at ${configFile} was not created`);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Each setting is written to the explicit config file path rather than via `addConfig` so neither simpleGit nor git itself can walk up to a parent .git and pollute the configuration of an enclosing project (e.g. the engine's own checkout when this.path is `./data/versions`).
|
|
26
62
|
return this.git
|
|
27
|
-
.
|
|
28
|
-
.
|
|
29
|
-
.
|
|
30
|
-
.
|
|
31
|
-
.
|
|
32
|
-
.
|
|
33
|
-
.
|
|
63
|
+
.raw([ 'config', '--file', configFile, 'core.autocrlf', 'false' ])
|
|
64
|
+
.raw([ 'config', '--file', configFile, 'push.default', 'current' ])
|
|
65
|
+
.raw([ 'config', '--file', configFile, 'user.name', this.author.name ])
|
|
66
|
+
.raw([ 'config', '--file', configFile, 'user.email', this.author.email ])
|
|
67
|
+
.raw([ 'config', '--file', configFile, 'core.quotePath', 'false' ]) // Disable Git's encoding of special characters in pathnames. For example, `service·A` will be encoded as `service\302\267A` without this setting, leading to issues. See https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath
|
|
68
|
+
.raw([ 'config', '--file', configFile, 'core.commitGraph', 'true' ]) // Enable `commit-graph` feature for efficient commit data storage, improving performance of operations like `git log`
|
|
69
|
+
.raw([ 'config', '--file', configFile, 'gc.writeCommitGraph', 'false' ]); // Prevent automatic `git gc` from also writing the commit-graph: the engine writes it explicitly (see `writeCommitGraph`/`updateCommitGraph`), and a concurrent gc write races those, which can leave a stale `commit-graph.lock` and make subsequent operations fail
|
|
34
70
|
}
|
|
35
71
|
|
|
36
72
|
open() {
|
|
@@ -52,6 +88,8 @@ export default class Git {
|
|
|
52
88
|
return this.git.add(this.relativePath(filePath));
|
|
53
89
|
}
|
|
54
90
|
|
|
91
|
+
// Not safe to call concurrently: GIT_AUTHOR_DATE / GIT_COMMITTER_DATE are process-wide env vars, so two overlapping calls can stamp each other's commits.
|
|
92
|
+
// simple-git's `maxConcurrentProcesses: 1` serializes child processes but not the env-var mutation that precedes them. Callers must await each commit before issuing the next.
|
|
55
93
|
async commit({ filePath, message, date = new Date(), trailers = {} }) {
|
|
56
94
|
const commitDate = new Date(date).toISOString();
|
|
57
95
|
let summary;
|
|
@@ -63,7 +101,7 @@ export default class Git {
|
|
|
63
101
|
const trailersSection = formatTrailers(trailers);
|
|
64
102
|
const finalMessage = trailersSection ? `${message}\n\n${trailersSection}` : message;
|
|
65
103
|
|
|
66
|
-
summary = await this.git.commit(finalMessage, filePath, ['--no-verify']); // Skip pre-commit and commit-msg hooks, as commits are programmatically managed, to optimize performance
|
|
104
|
+
summary = await this.git.commit(finalMessage, filePath ? this.relativePath(filePath) : [], ['--no-verify']); // Skip pre-commit and commit-msg hooks, as commits are programmatically managed, to optimize performance. The pathspec must be expressed relative to the repository root; passing the absolute or process-cwd-relative path causes git to look for it under the repo's working directory, which fails when the repo's own path components appear in the resolved location. Without a file path, the whole index is committed
|
|
67
105
|
} finally {
|
|
68
106
|
process.env.GIT_AUTHOR_DATE = '';
|
|
69
107
|
process.env.GIT_COMMITTER_DATE = '';
|
|
@@ -88,7 +126,7 @@ export default class Git {
|
|
|
88
126
|
return this.log([
|
|
89
127
|
...reverseOption, // When `reverse` is true, lists commits oldest-first; otherwise the default newest-first applies
|
|
90
128
|
'--author-date-order', // Best-effort author-date ordering: with --max-count, git applies the cap topologically, so the page can miss strictly-newer commits that #getCommits' JS resort cannot recover
|
|
91
|
-
'--no-merges', // Exclude merge commits
|
|
129
|
+
'--no-merges', // Exclude merge commits; records are stored as regular commits, never as merges
|
|
92
130
|
'--name-only', // Append the modified file names below each commit, used by `toDomain` to extract the record's file path
|
|
93
131
|
...skipOption, // Optional `--skip=N`: drop the first N matching commits (pagination offset)
|
|
94
132
|
...maxCountOption, // Optional `--max-count=N`: cap the result to N commits (pagination limit)
|
|
@@ -253,3 +291,15 @@ export default class Git {
|
|
|
253
291
|
return { additions, deletions };
|
|
254
292
|
}
|
|
255
293
|
}
|
|
294
|
+
|
|
295
|
+
async function readObjectAtCommit(repositoryPath, args) {
|
|
296
|
+
try {
|
|
297
|
+
return await simpleGit(repositoryPath, { trimmed: true, config: ['core.quotePath=false'] }).raw(args); // Disable pathname quoting for the same reason Git.initialize sets it on managed repositories: names with special characters (e.g. "service·A") must come back verbatim, and this repository's configuration is not under the engine's control
|
|
298
|
+
} catch (error) {
|
|
299
|
+
if (OBJECT_NOT_FOUND_MESSAGES.test(error.message)) {
|
|
300
|
+
throw new GitObjectNotFoundError(error.message); // Typed so callers can distinguish "this commit or file cannot be resolved, ever" from a transient git failure worth retrying
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
throw error;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import os from 'os';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import { fileURLToPath } from 'url';
|
|
5
|
+
|
|
6
|
+
import { expect } from 'chai';
|
|
7
|
+
import config from 'config';
|
|
8
|
+
|
|
9
|
+
import Git, { GitObjectNotFoundError } from './index.js';
|
|
10
|
+
|
|
11
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
12
|
+
const RECORDER_PATH = path.resolve(__dirname, '../../', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
13
|
+
|
|
14
|
+
describe('Git', () => {
|
|
15
|
+
const DEFAULT_CONTENT = 'default content';
|
|
16
|
+
const DEFAULT_COMMIT_MESSAGE = 'default commit message';
|
|
17
|
+
let subject;
|
|
18
|
+
|
|
19
|
+
before(() => {
|
|
20
|
+
subject = new Git({
|
|
21
|
+
path: RECORDER_PATH,
|
|
22
|
+
author: {
|
|
23
|
+
name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
|
|
24
|
+
email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
return subject.initialize();
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
describe('#commit', () => {
|
|
32
|
+
const expectedFilePath = `${RECORDER_PATH}/test.md`;
|
|
33
|
+
|
|
34
|
+
let commitId;
|
|
35
|
+
|
|
36
|
+
before(async () => {
|
|
37
|
+
await fs.writeFile(expectedFilePath, DEFAULT_CONTENT);
|
|
38
|
+
|
|
39
|
+
await subject.add(expectedFilePath);
|
|
40
|
+
commitId = await subject.commit({ filePath: expectedFilePath, message: DEFAULT_COMMIT_MESSAGE });
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
after(() => subject.destroyHistory());
|
|
44
|
+
|
|
45
|
+
it('returns a full length SHA1 commit ID', () => {
|
|
46
|
+
expect(commitId).to.match(/\b[0-9a-f]{40}\b/);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
context('when stage area is dirty', () => {
|
|
50
|
+
const expectedFileName = 'file-to-commit.md';
|
|
51
|
+
const expectedFilePath = `${RECORDER_PATH}/${expectedFileName}`;
|
|
52
|
+
const unwantedFilePath = `${RECORDER_PATH}/unwanted-file.md`;
|
|
53
|
+
let commit;
|
|
54
|
+
let committedFiles;
|
|
55
|
+
let committedFileName;
|
|
56
|
+
|
|
57
|
+
before(async () => {
|
|
58
|
+
await fs.writeFile(expectedFilePath, DEFAULT_CONTENT);
|
|
59
|
+
await fs.writeFile(unwantedFilePath, DEFAULT_CONTENT);
|
|
60
|
+
|
|
61
|
+
await subject.add(expectedFilePath);
|
|
62
|
+
await subject.add(unwantedFilePath);
|
|
63
|
+
|
|
64
|
+
const commitId = await subject.commit({ filePath: expectedFilePath, message: DEFAULT_COMMIT_MESSAGE });
|
|
65
|
+
|
|
66
|
+
commit = await subject.getCommit([commitId]);
|
|
67
|
+
|
|
68
|
+
if (!commit) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
({ files: committedFiles } = commit.diff);
|
|
73
|
+
([{ file: committedFileName }] = committedFiles);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
after(() => subject.destroyHistory());
|
|
77
|
+
|
|
78
|
+
it('commits the specified file', () => {
|
|
79
|
+
expect(committedFileName).to.equal(expectedFileName);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('commits only one file', () => {
|
|
83
|
+
expect(committedFiles).to.have.lengthOf(1);
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
context('when no file path is given', () => {
|
|
88
|
+
const stagedFileNames = [ 'first-staged-file.md', 'second-staged-file.md' ];
|
|
89
|
+
let committedFileNames;
|
|
90
|
+
|
|
91
|
+
before(async () => {
|
|
92
|
+
for (const fileName of stagedFileNames) {
|
|
93
|
+
const filePath = `${RECORDER_PATH}/${fileName}`;
|
|
94
|
+
|
|
95
|
+
await fs.writeFile(filePath, DEFAULT_CONTENT);
|
|
96
|
+
await subject.add(filePath);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const commitId = await subject.commit({ message: DEFAULT_COMMIT_MESSAGE });
|
|
100
|
+
const commit = await subject.getCommit([commitId]);
|
|
101
|
+
|
|
102
|
+
committedFileNames = commit.diff.files.map(({ file }) => file);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
after(() => subject.destroyHistory());
|
|
106
|
+
|
|
107
|
+
it('commits all the staged files', () => {
|
|
108
|
+
expect(committedFileNames).to.have.members(stagedFileNames);
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
describe('#cleanUp', () => {
|
|
114
|
+
context('when a commit-graph lock has been left behind by an interrupted process', () => {
|
|
115
|
+
const infoDirectoryPath = path.join(RECORDER_PATH, '.git', 'objects', 'info');
|
|
116
|
+
const lockFilePath = path.join(infoDirectoryPath, 'commit-graph.lock');
|
|
117
|
+
|
|
118
|
+
before(async () => {
|
|
119
|
+
const filePath = `${RECORDER_PATH}/file-to-clean.md`;
|
|
120
|
+
|
|
121
|
+
await fs.writeFile(filePath, DEFAULT_CONTENT);
|
|
122
|
+
await subject.add(filePath);
|
|
123
|
+
await subject.commit({ filePath, message: DEFAULT_COMMIT_MESSAGE });
|
|
124
|
+
|
|
125
|
+
await fs.mkdir(infoDirectoryPath, { recursive: true });
|
|
126
|
+
await fs.writeFile(lockFilePath, '');
|
|
127
|
+
|
|
128
|
+
await subject.cleanUp();
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
after(() => subject.destroyHistory());
|
|
132
|
+
|
|
133
|
+
it('removes the stale commit-graph lock', async () => {
|
|
134
|
+
const infoDirectoryContent = await fs.readdir(infoDirectoryPath);
|
|
135
|
+
|
|
136
|
+
expect(infoDirectoryContent).to.not.include('commit-graph.lock');
|
|
137
|
+
});
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
describe('.getHeadSha', () => {
|
|
142
|
+
context('with a directory that does not exist', () => {
|
|
143
|
+
it('returns null', async () => {
|
|
144
|
+
expect(await Git.getHeadSha(path.join(os.tmpdir(), 'ota-nonexistent-directory'))).to.be.null;
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
context('with a directory that is not inside a Git repository', () => {
|
|
149
|
+
let directory;
|
|
150
|
+
|
|
151
|
+
before(async () => {
|
|
152
|
+
directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ota-git-test-')); // Under the OS temp directory, so no enclosing Git repository can be found by walking up
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
after(() => fs.rm(directory, { recursive: true, force: true }));
|
|
156
|
+
|
|
157
|
+
it('returns null', async () => {
|
|
158
|
+
expect(await Git.getHeadSha(directory)).to.be.null;
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
context('with a repository that has no commits yet', () => {
|
|
163
|
+
let directory;
|
|
164
|
+
|
|
165
|
+
before(async () => {
|
|
166
|
+
directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ota-git-test-'));
|
|
167
|
+
await new Git({ path: directory, author: { name: 'Test', email: 'test@example.com' } }).initialize();
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
after(() => fs.rm(directory, { recursive: true, force: true }));
|
|
171
|
+
|
|
172
|
+
it('returns null', async () => {
|
|
173
|
+
expect(await Git.getHeadSha(directory)).to.be.null;
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
context('with a repository that has commits', () => {
|
|
178
|
+
let commitId;
|
|
179
|
+
|
|
180
|
+
before(async () => {
|
|
181
|
+
const filePath = `${RECORDER_PATH}/test.md`;
|
|
182
|
+
|
|
183
|
+
await fs.writeFile(filePath, DEFAULT_CONTENT);
|
|
184
|
+
await subject.add(filePath);
|
|
185
|
+
commitId = await subject.commit({ filePath, message: DEFAULT_COMMIT_MESSAGE });
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
after(() => subject.destroyHistory());
|
|
189
|
+
|
|
190
|
+
it('returns the SHA of HEAD', async () => {
|
|
191
|
+
expect(await Git.getHeadSha(RECORDER_PATH)).to.equal(commitId);
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
describe('reading at a specific commit', () => {
|
|
197
|
+
let repositoryPath;
|
|
198
|
+
let subdirectoryPath;
|
|
199
|
+
let firstCommitSha;
|
|
200
|
+
let secondCommitSha;
|
|
201
|
+
|
|
202
|
+
before(async () => {
|
|
203
|
+
repositoryPath = await fs.mkdtemp(path.join(os.tmpdir(), 'ota-git-test-')); // Under the OS temp directory so the fixture repository is not nested in the engine's own repository
|
|
204
|
+
subdirectoryPath = path.join(repositoryPath, 'declarations');
|
|
205
|
+
await fs.mkdir(subdirectoryPath);
|
|
206
|
+
|
|
207
|
+
const git = new Git({ path: repositoryPath, author: { name: 'Test', email: 'test@example.com' } });
|
|
208
|
+
|
|
209
|
+
await git.initialize();
|
|
210
|
+
|
|
211
|
+
const firstFilePath = path.join(subdirectoryPath, 'Service A.json');
|
|
212
|
+
|
|
213
|
+
await fs.writeFile(firstFilePath, '{ "name": "Service A" }');
|
|
214
|
+
await git.add(firstFilePath);
|
|
215
|
+
firstCommitSha = await git.commit({ filePath: firstFilePath, message: 'Add Service A' });
|
|
216
|
+
|
|
217
|
+
const secondFilePath = path.join(subdirectoryPath, 'Service B.json');
|
|
218
|
+
|
|
219
|
+
await fs.writeFile(secondFilePath, '{ "name": "Service B" }');
|
|
220
|
+
await git.add(secondFilePath);
|
|
221
|
+
secondCommitSha = await git.commit({ filePath: secondFilePath, message: 'Add Service B' });
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
after(() => fs.rm(repositoryPath, { recursive: true, force: true }));
|
|
225
|
+
|
|
226
|
+
describe('.listFilesAtCommit', () => {
|
|
227
|
+
it('lists the files of the subdirectory as they were at the given commit', async () => {
|
|
228
|
+
expect(await Git.listFilesAtCommit(subdirectoryPath, firstCommitSha)).to.deep.equal(['Service A.json']);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
it('reflects later commits when given their SHA', async () => {
|
|
232
|
+
expect(await Git.listFilesAtCommit(subdirectoryPath, secondCommitSha)).to.deep.equal([ 'Service A.json', 'Service B.json' ]);
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
it('throws a GitObjectNotFoundError for an unknown commit', async () => {
|
|
236
|
+
try {
|
|
237
|
+
await Git.listFilesAtCommit(subdirectoryPath, 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef');
|
|
238
|
+
} catch (error) {
|
|
239
|
+
expect(error).to.be.an.instanceOf(GitObjectNotFoundError);
|
|
240
|
+
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
expect.fail('No error was thrown');
|
|
245
|
+
});
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
describe('.readFileAtCommit', () => {
|
|
249
|
+
it('returns the file content as it was at the given commit', async () => {
|
|
250
|
+
expect(await Git.readFileAtCommit(subdirectoryPath, firstCommitSha, 'Service A.json')).to.equal('{ "name": "Service A" }');
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
it('throws a GitObjectNotFoundError for a file absent from the commit', async () => {
|
|
254
|
+
try {
|
|
255
|
+
await Git.readFileAtCommit(subdirectoryPath, firstCommitSha, 'Service B.json');
|
|
256
|
+
} catch (error) {
|
|
257
|
+
expect(error).to.be.an.instanceOf(GitObjectNotFoundError);
|
|
258
|
+
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
expect.fail('No error was thrown');
|
|
263
|
+
});
|
|
264
|
+
});
|
|
265
|
+
});
|
|
266
|
+
});
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
const CONTROL_CHARACTERS_REGEXP = /\p{Cc}/u; // Matches any Unicode "control" character: the C0 range (U+0000 to U+001F), DEL (U+007F) and the C1 range (U+0080 to U+009F), i.e. 65 non-printable characters including NUL. The `u` flag is required for the `\p{...}` property escape to be recognised, otherwise the pattern would match the literal text `p{Cc}`. Legitimate service IDs, terms types and document IDs never contain these, and NUL in particular can truncate a value once it reaches git or the filesystem, so any segment holding one is rejected.
|
|
2
|
+
|
|
3
|
+
const WINDOWS_RESERVED_CHARACTERS_REGEXP = /[:"<>|*?]/; // Forbidden in Windows file names, so a segment holding one cannot exist on every supported platform; the service ID guidelines exclude them for the same reason
|
|
4
|
+
|
|
5
|
+
// Keeps hostile values from reaching git, where a pathspec that resolves outside the repository (such as `../foo/*`) aborts with an error that exposes the repository location.
|
|
6
|
+
export function isPlainPathSegment(segment) {
|
|
7
|
+
return segment.length > 0
|
|
8
|
+
&& segment !== '.'
|
|
9
|
+
&& segment !== '..'
|
|
10
|
+
&& !segment.includes('/')
|
|
11
|
+
&& !segment.includes('\\')
|
|
12
|
+
&& !CONTROL_CHARACTERS_REGEXP.test(segment);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// Accepts only segments that are usable as a file name on every supported platform. As git C-quotes only double-quotes, backslashes and control characters in the paths it prints (see core.quotePath), all rejected here, an accepted segment also never comes back quoted from a git log.
|
|
16
|
+
export function isPortableFileName(segment) {
|
|
17
|
+
return isPlainPathSegment(segment) && !WINDOWS_RESERVED_CHARACTERS_REGEXP.test(segment);
|
|
18
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { expect } from 'chai';
|
|
2
|
+
|
|
3
|
+
import { isPlainPathSegment, isPortableFileName } from './pathSegment.js';
|
|
4
|
+
|
|
5
|
+
describe('PathSegment', () => {
|
|
6
|
+
describe('#isPlainPathSegment', () => {
|
|
7
|
+
[ 'Facebook', 'Terms of Service', 'Facebook v2.0', 'service·A' ].forEach(segment => {
|
|
8
|
+
it(`accepts ${JSON.stringify(segment)}`, () => {
|
|
9
|
+
expect(isPlainPathSegment(segment)).to.be.true;
|
|
10
|
+
});
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
[ '', '.', '..', 'a/b', 'a\\b', 'with\0null', 'line\nbreak' ].forEach(segment => {
|
|
14
|
+
it(`rejects ${JSON.stringify(segment)}`, () => {
|
|
15
|
+
expect(isPlainPathSegment(segment)).to.be.false;
|
|
16
|
+
});
|
|
17
|
+
});
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
describe('#isPortableFileName', () => {
|
|
21
|
+
[ 'Facebook', 'Terms of Service', 'Booking.com', 'Yahoo!', 'service·A' ].forEach(segment => {
|
|
22
|
+
it(`accepts ${JSON.stringify(segment)}`, () => {
|
|
23
|
+
expect(isPortableFileName(segment)).to.be.true;
|
|
24
|
+
});
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
[ 'Service "A"', 're:start', 'a<b', 'a>b', 'a|b', 'a*b', 'a?b', '', '.', '..', 'a/b', 'a\\b', 'with\0null' ].forEach(segment => {
|
|
28
|
+
it(`rejects ${JSON.stringify(segment)}`, () => {
|
|
29
|
+
expect(isPortableFileName(segment)).to.be.false;
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
});
|