@adguard/hostlist-compiler 2.1.0 → 3.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +29 -0
- package/.github/workflows/ci.yml +22 -0
- package/.github/workflows/mirror.yml +24 -0
- package/.github/workflows/prepare-release.yml +26 -0
- package/.github/workflows/publish-release.yml +44 -0
- package/.markdownlint.json +29 -0
- package/.markdownlintignore +1 -0
- package/AGENTS.md +314 -103
- package/CHANGELOG.md +76 -7
- package/DEPLOYMENT.md +171 -0
- package/DEVELOPMENT.md +174 -76
- package/README.md +240 -140
- package/examples/china/README.md +2 -2
- package/examples/energized/README.md +4 -2
- package/examples/sdn/README.md +2 -2
- package/examples/whitelist/README.md +2 -2
- package/package.json +29 -24
- package/src/cli.js +59 -33
- package/src/compile-source.js +4 -3
- package/src/configuration.js +132 -2
- package/src/download.js +27 -0
- package/src/filter.js +2 -2
- package/src/index.d.ts +2 -0
- package/src/index.js +21 -51
- package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
- package/src/schemas/configuration.schema.js +137 -0
- package/src/source-types.js +17 -0
- package/src/transformations/enum.js +49 -0
- package/src/transformations/include.js +1 -1
- package/src/transformations/transform.js +40 -74
- package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
- package/src/transformations/validate-allow-ip.js +1 -1
- package/src/utils.js +1284 -0
- package/src/validation-conflicts.js +106 -0
- package/test/cli-error-output.test.js +305 -0
- package/test/cli.test.js +47 -0
- package/test/compile-source.test.js +167 -7
- package/test/configuration.test.js +438 -0
- package/test/download.test.js +260 -0
- package/test/helpers/http-mock.js +94 -0
- package/test/index.test.js +497 -4
- package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
- package/test/resolve-include.test.js +8 -8
- package/test/transformations/covert-to-ascii.test.js +1 -1
- package/test/transformations/exclude.test.js +2 -2
- package/test/transformations/include.test.js +9 -3
- package/test/transformations/ip-alignment.test.js +1 -1
- package/test/transformations/transform.test.js +66 -1
- package/test/utils.test.js +1559 -0
- package/test/validation-conflicts.test.js +230 -0
- package/bamboo-specs/bamboo.yaml +0 -14
- package/bamboo-specs/build.yaml +0 -92
- package/bamboo-specs/deploy.yaml +0 -38
- package/bamboo-specs/increment.yaml +0 -54
- package/bamboo-specs/permissions.yaml +0 -16
- package/bamboo-specs/test.yaml +0 -56
- package/src/schemas/configuration.schema.json +0 -369
- package/tools/build-txt.js +0 -19
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Validation-conflict rules for transformation lists.
|
|
3
|
+
*
|
|
4
|
+
* This module is the single source of truth for the validation-conflict rules:
|
|
5
|
+
* the runtime checks (used by `transform.js` and `index.js`) and the
|
|
6
|
+
* JSON-schema conflict rules (used by `schemas/configuration.schema.js`) are
|
|
7
|
+
* all derived from `VALIDATION_TRANSFORMATIONS`, which is defined next to the
|
|
8
|
+
* `TRANSFORMATIONS` enum in `transformations/enum.js`, so they cannot drift
|
|
9
|
+
* apart.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const { VALIDATION_TRANSFORMATIONS } = require('./transformations/enum');
|
|
13
|
+
const { stripKnownUrlCredentials } = require('./utils');
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Returns the validation transformations selected in the given list.
|
|
17
|
+
*
|
|
18
|
+
* @param {Array<string>} transformations - a transformations list.
|
|
19
|
+
* @returns {Array<string>} the selected validation transformations.
|
|
20
|
+
*/
|
|
21
|
+
function getSelectedValidationTransformations(transformations) {
|
|
22
|
+
return VALIDATION_TRANSFORMATIONS
|
|
23
|
+
.filter((transformation) => transformations.indexOf(transformation) !== -1);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Throws when multiple validation transformations are selected together.
|
|
28
|
+
* Combining them causes silent data loss: each validator runs on the already-filtered
|
|
29
|
+
* output of the previous one, so allow-modes (AllowIp, AllowPublicSuffix) become ineffective.
|
|
30
|
+
*
|
|
31
|
+
* @param {Array<string>} transformations - configured transformations.
|
|
32
|
+
*/
|
|
33
|
+
function checkIncompatibleValidationTransformations(transformations) {
|
|
34
|
+
const selectedValidations = getSelectedValidationTransformations(transformations);
|
|
35
|
+
|
|
36
|
+
if (selectedValidations.length > 1) {
|
|
37
|
+
throw new Error(`Validation transformations cannot be combined: ${selectedValidations.join(', ')}.`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Checks for conflicting validation transformations across source-level and top-level.
|
|
43
|
+
* Throws if a validation transformation is used at both levels, as this causes silent data loss.
|
|
44
|
+
*
|
|
45
|
+
* @param {*} configuration - compilation configuration.
|
|
46
|
+
*/
|
|
47
|
+
function checkCrossLevelValidationConflicts(configuration) {
|
|
48
|
+
const topLevelTransformations = configuration.transformations || [];
|
|
49
|
+
const topLevelValidations = getSelectedValidationTransformations(topLevelTransformations);
|
|
50
|
+
|
|
51
|
+
if (topLevelValidations.length === 0) {
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Check each source for conflicting validations
|
|
56
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
57
|
+
for (const source of configuration.sources) {
|
|
58
|
+
const sourceTransformations = source.transformations || [];
|
|
59
|
+
const sourceValidations = getSelectedValidationTransformations(sourceTransformations);
|
|
60
|
+
|
|
61
|
+
if (sourceValidations.length > 0) {
|
|
62
|
+
throw new Error(
|
|
63
|
+
'Validation transformations cannot be used at both source and top level. '
|
|
64
|
+
+ `Source "${stripKnownUrlCredentials(source.source)}" uses [${sourceValidations.join(', ')}], `
|
|
65
|
+
+ `but top level uses [${topLevelValidations.join(', ')}]. `
|
|
66
|
+
+ 'This causes the source-level validation to be overridden by the top-level one, '
|
|
67
|
+
+ 'making the source-level validation ineffective.',
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Builds JSON-schema fragments that reject a transformations list containing
|
|
75
|
+
* more than one validation transformation. The fragments are generated from
|
|
76
|
+
* `VALIDATION_TRANSFORMATIONS` so the schema and the runtime checks always
|
|
77
|
+
* stay in sync.
|
|
78
|
+
*
|
|
79
|
+
* @returns {Array<Object>} an array of `not`/`allOf`/`contains` schema fragments.
|
|
80
|
+
*/
|
|
81
|
+
function buildSchemaConflictRules() {
|
|
82
|
+
return VALIDATION_TRANSFORMATIONS.flatMap((first, index) => (
|
|
83
|
+
VALIDATION_TRANSFORMATIONS.slice(index + 1).map((second) => ({
|
|
84
|
+
not: {
|
|
85
|
+
allOf: [
|
|
86
|
+
{
|
|
87
|
+
contains: {
|
|
88
|
+
const: first,
|
|
89
|
+
},
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
contains: {
|
|
93
|
+
const: second,
|
|
94
|
+
},
|
|
95
|
+
},
|
|
96
|
+
],
|
|
97
|
+
},
|
|
98
|
+
}))
|
|
99
|
+
));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
module.exports = {
|
|
103
|
+
checkIncompatibleValidationTransformations,
|
|
104
|
+
checkCrossLevelValidationConflicts,
|
|
105
|
+
buildSchemaConflictRules,
|
|
106
|
+
};
|
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLI error output tests.
|
|
3
|
+
*
|
|
4
|
+
* `cli.test.js` mocks `consola`, so it cannot catch the output that the real
|
|
5
|
+
* reporter produces. This suite uses the real `consola` and captures its
|
|
6
|
+
* `stderr` stream.
|
|
7
|
+
*/
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const consola = require('consola');
|
|
10
|
+
|
|
11
|
+
jest.mock('../src/index', () => jest.fn());
|
|
12
|
+
|
|
13
|
+
describe('CLI error output', () => {
|
|
14
|
+
const originalArgv = process.argv;
|
|
15
|
+
let exitSpy;
|
|
16
|
+
let stderrSpy;
|
|
17
|
+
let stderrOutput;
|
|
18
|
+
|
|
19
|
+
beforeEach(() => {
|
|
20
|
+
jest.resetModules();
|
|
21
|
+
process.argv = ['node', 'cli', '-i', 'https://example.org/hosts.txt', '-o', 'output.txt'];
|
|
22
|
+
stderrOutput = '';
|
|
23
|
+
exitSpy = jest.spyOn(process, 'exit').mockImplementation(() => {});
|
|
24
|
+
stderrSpy = jest.spyOn(consola.stderr, 'write').mockImplementation((chunk) => {
|
|
25
|
+
stderrOutput += chunk.toString();
|
|
26
|
+
return true;
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
afterEach(() => {
|
|
31
|
+
process.argv = originalArgv;
|
|
32
|
+
exitSpy.mockRestore();
|
|
33
|
+
stderrSpy.mockRestore();
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('does not print source credentials in the verbose configuration dump', async () => {
|
|
37
|
+
process.argv = [
|
|
38
|
+
'node',
|
|
39
|
+
'cli',
|
|
40
|
+
'-v',
|
|
41
|
+
'-i',
|
|
42
|
+
'https://user:secret@example.org/hosts.txt',
|
|
43
|
+
'-o',
|
|
44
|
+
'output.txt',
|
|
45
|
+
];
|
|
46
|
+
|
|
47
|
+
let stdoutOutput = '';
|
|
48
|
+
const originalLevel = consola.level;
|
|
49
|
+
const stdoutSpy = jest.spyOn(consola.stdout, 'write').mockImplementation((chunk) => {
|
|
50
|
+
stdoutOutput += chunk.toString();
|
|
51
|
+
return true;
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
try {
|
|
55
|
+
// eslint-disable-next-line global-require
|
|
56
|
+
const compileMock = require('../src/index');
|
|
57
|
+
compileMock.mockRejectedValue(new Error('compile failed'));
|
|
58
|
+
|
|
59
|
+
// eslint-disable-next-line global-require
|
|
60
|
+
require('../src/cli');
|
|
61
|
+
await new Promise((resolve) => { setImmediate(resolve); });
|
|
62
|
+
|
|
63
|
+
expect(stdoutOutput).toContain('https://example.org/hosts.txt');
|
|
64
|
+
expect(stdoutOutput).not.toContain('secret');
|
|
65
|
+
// The download needs the original credentials, so compile must get
|
|
66
|
+
// the unsanitized configuration.
|
|
67
|
+
expect(compileMock).toHaveBeenCalledWith(expect.objectContaining({
|
|
68
|
+
sources: [{ source: 'https://user:secret@example.org/hosts.txt', type: 'hosts' }],
|
|
69
|
+
}));
|
|
70
|
+
} finally {
|
|
71
|
+
stdoutSpy.mockRestore();
|
|
72
|
+
consola.level = originalLevel;
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('does not print credentials from a malformed configuration in the verbose dump', async () => {
|
|
77
|
+
process.argv = [
|
|
78
|
+
'node',
|
|
79
|
+
'cli',
|
|
80
|
+
'-v',
|
|
81
|
+
'-c',
|
|
82
|
+
'config.json',
|
|
83
|
+
'-o',
|
|
84
|
+
'output.txt',
|
|
85
|
+
];
|
|
86
|
+
|
|
87
|
+
const accessSpy = jest.spyOn(fs.promises, 'access').mockResolvedValue();
|
|
88
|
+
const readFileSpy = jest.spyOn(fs.promises, 'readFile').mockResolvedValue(Buffer.from(JSON.stringify({
|
|
89
|
+
name: 'test',
|
|
90
|
+
homepage: "https://o'brien:secret@example.org",
|
|
91
|
+
sources: [
|
|
92
|
+
{
|
|
93
|
+
source: 'test.txt',
|
|
94
|
+
exclusion_sources: ['https://user:secret@example.org/exclusions.txt'],
|
|
95
|
+
},
|
|
96
|
+
],
|
|
97
|
+
})));
|
|
98
|
+
const writeFileSpy = jest.spyOn(fs.promises, 'writeFile').mockResolvedValue();
|
|
99
|
+
|
|
100
|
+
let stdoutOutput = '';
|
|
101
|
+
const originalLevel = consola.level;
|
|
102
|
+
const stdoutSpy = jest.spyOn(consola.stdout, 'write').mockImplementation((chunk) => {
|
|
103
|
+
stdoutOutput += chunk.toString();
|
|
104
|
+
return true;
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
try {
|
|
108
|
+
// eslint-disable-next-line global-require
|
|
109
|
+
const compileMock = require('../src/index');
|
|
110
|
+
compileMock.mockResolvedValue([]);
|
|
111
|
+
|
|
112
|
+
// eslint-disable-next-line global-require
|
|
113
|
+
require('../src/cli');
|
|
114
|
+
await new Promise((resolve) => { setImmediate(resolve); });
|
|
115
|
+
|
|
116
|
+
expect(stdoutOutput).toContain('"exclusion_sources": [');
|
|
117
|
+
expect(stdoutOutput).toContain('https://example.org/exclusions.txt');
|
|
118
|
+
expect(stdoutOutput).toContain('"homepage": "https://example.org"');
|
|
119
|
+
expect(stdoutOutput).not.toContain('secret');
|
|
120
|
+
} finally {
|
|
121
|
+
stdoutSpy.mockRestore();
|
|
122
|
+
accessSpy.mockRestore();
|
|
123
|
+
readFileSpy.mockRestore();
|
|
124
|
+
writeFileSpy.mockRestore();
|
|
125
|
+
consola.level = originalLevel;
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it('does not corrupt the verbose dump for a path-less URL and a later email', async () => {
|
|
130
|
+
process.argv = [
|
|
131
|
+
'node',
|
|
132
|
+
'cli',
|
|
133
|
+
'-v',
|
|
134
|
+
'-c',
|
|
135
|
+
'config.json',
|
|
136
|
+
'-o',
|
|
137
|
+
'output.txt',
|
|
138
|
+
];
|
|
139
|
+
|
|
140
|
+
const accessSpy = jest.spyOn(fs.promises, 'access').mockResolvedValue();
|
|
141
|
+
const readFileSpy = jest.spyOn(fs.promises, 'readFile').mockResolvedValue(Buffer.from(JSON.stringify({
|
|
142
|
+
homepage: 'https://example.org',
|
|
143
|
+
description: 'admin@example.com',
|
|
144
|
+
sources: [{ source: 'test.txt' }],
|
|
145
|
+
})));
|
|
146
|
+
const writeFileSpy = jest.spyOn(fs.promises, 'writeFile').mockResolvedValue();
|
|
147
|
+
|
|
148
|
+
let stdoutOutput = '';
|
|
149
|
+
const originalLevel = consola.level;
|
|
150
|
+
const stdoutSpy = jest.spyOn(consola.stdout, 'write').mockImplementation((chunk) => {
|
|
151
|
+
stdoutOutput += chunk.toString();
|
|
152
|
+
return true;
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
try {
|
|
156
|
+
// eslint-disable-next-line global-require
|
|
157
|
+
const compileMock = require('../src/index');
|
|
158
|
+
compileMock.mockResolvedValue([]);
|
|
159
|
+
|
|
160
|
+
// eslint-disable-next-line global-require
|
|
161
|
+
require('../src/cli');
|
|
162
|
+
await new Promise((resolve) => { setImmediate(resolve); });
|
|
163
|
+
|
|
164
|
+
expect(stdoutOutput).toContain('"homepage": "https://example.org"');
|
|
165
|
+
expect(stdoutOutput).toContain('"description": "admin@example.com"');
|
|
166
|
+
expect(stdoutOutput).toContain('"source": "test.txt"');
|
|
167
|
+
} finally {
|
|
168
|
+
stdoutSpy.mockRestore();
|
|
169
|
+
accessSpy.mockRestore();
|
|
170
|
+
readFileSpy.mockRestore();
|
|
171
|
+
writeFileSpy.mockRestore();
|
|
172
|
+
consola.level = originalLevel;
|
|
173
|
+
}
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it('prints every error in the cause chain', async () => {
|
|
177
|
+
const networkError = new Error('connect ECONNREFUSED 127.0.0.1:443');
|
|
178
|
+
const fetchError = new TypeError('fetch failed');
|
|
179
|
+
fetchError.cause = networkError;
|
|
180
|
+
const error = new Error("Failed to request url 'https://example.org/hosts.txt'");
|
|
181
|
+
error.cause = fetchError;
|
|
182
|
+
|
|
183
|
+
// eslint-disable-next-line global-require
|
|
184
|
+
require('../src/index').mockRejectedValue(error);
|
|
185
|
+
|
|
186
|
+
// Loading the module runs main() with the process.argv prepared above.
|
|
187
|
+
// eslint-disable-next-line global-require
|
|
188
|
+
require('../src/cli');
|
|
189
|
+
// Let main() settle before asserting on the logged errors.
|
|
190
|
+
await new Promise((resolve) => { setImmediate(resolve); });
|
|
191
|
+
|
|
192
|
+
expect(stderrOutput).toContain("Failed to request url 'https://example.org/hosts.txt'");
|
|
193
|
+
expect(stderrOutput).toContain('Caused by: fetch failed');
|
|
194
|
+
expect(stderrOutput).toContain('Caused by: connect ECONNREFUSED 127.0.0.1:443');
|
|
195
|
+
expect(exitSpy).toHaveBeenCalledWith(1);
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it('prints every message from an aggregate cause', async () => {
|
|
199
|
+
const ipv6Error = new Error('connect ECONNREFUSED ::1:443');
|
|
200
|
+
const ipv4Error = new Error('connect ECONNREFUSED 127.0.0.1:443');
|
|
201
|
+
// A dual-stack connection failure has an empty message, so the useful
|
|
202
|
+
// messages are in the `errors` array.
|
|
203
|
+
const aggregateError = new global.AggregateError([ipv6Error, ipv4Error]);
|
|
204
|
+
const fetchError = new TypeError('fetch failed');
|
|
205
|
+
fetchError.cause = aggregateError;
|
|
206
|
+
const error = new Error("Failed to request url 'https://example.org/hosts.txt'");
|
|
207
|
+
error.cause = fetchError;
|
|
208
|
+
|
|
209
|
+
// eslint-disable-next-line global-require
|
|
210
|
+
require('../src/index').mockRejectedValue(error);
|
|
211
|
+
|
|
212
|
+
// eslint-disable-next-line global-require
|
|
213
|
+
require('../src/cli');
|
|
214
|
+
await new Promise((resolve) => { setImmediate(resolve); });
|
|
215
|
+
|
|
216
|
+
expect(stderrOutput).toContain("Failed to request url 'https://example.org/hosts.txt'");
|
|
217
|
+
expect(stderrOutput).toContain('Caused by: fetch failed');
|
|
218
|
+
expect(stderrOutput).toContain('Caused by: connect ECONNREFUSED ::1:443');
|
|
219
|
+
expect(stderrOutput).toContain('Caused by: connect ECONNREFUSED 127.0.0.1:443');
|
|
220
|
+
expect(stderrOutput).not.toContain('Caused by: AggregateError');
|
|
221
|
+
expect(exitSpy).toHaveBeenCalledWith(1);
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
it('does not print credentials from a malformed source URL in an error', async () => {
|
|
225
|
+
const innerError = new TypeError(
|
|
226
|
+
'Request cannot be constructed from a URL that includes credentials: '
|
|
227
|
+
+ 'https:///user:secret@example.org/list.txt',
|
|
228
|
+
);
|
|
229
|
+
const error = new Error("Failed to request url 'http:/user:secret@example.org/list.txt'");
|
|
230
|
+
error.cause = innerError;
|
|
231
|
+
|
|
232
|
+
// eslint-disable-next-line global-require
|
|
233
|
+
require('../src/index').mockRejectedValue(error);
|
|
234
|
+
|
|
235
|
+
// eslint-disable-next-line global-require
|
|
236
|
+
require('../src/cli');
|
|
237
|
+
await new Promise((resolve) => { setImmediate(resolve); });
|
|
238
|
+
|
|
239
|
+
expect(stderrOutput).not.toContain('secret');
|
|
240
|
+
expect(stderrOutput).toContain('https:///example.org/list.txt');
|
|
241
|
+
expect(stderrOutput).toContain('http:/example.org/list.txt');
|
|
242
|
+
expect(exitSpy).toHaveBeenCalledWith(1);
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
it('does not print credentials from a URL object key in the verbose dump', async () => {
|
|
246
|
+
process.argv = [
|
|
247
|
+
'node',
|
|
248
|
+
'cli',
|
|
249
|
+
'-v',
|
|
250
|
+
'-c',
|
|
251
|
+
'config.json',
|
|
252
|
+
'-o',
|
|
253
|
+
'output.txt',
|
|
254
|
+
];
|
|
255
|
+
|
|
256
|
+
const accessSpy = jest.spyOn(fs.promises, 'access').mockResolvedValue();
|
|
257
|
+
const readFileSpy = jest.spyOn(fs.promises, 'readFile').mockResolvedValue(Buffer.from(JSON.stringify({
|
|
258
|
+
name: 'test',
|
|
259
|
+
sources: [{ source: 'test.txt' }],
|
|
260
|
+
'https://user:secret@example.org/key.txt': 'value',
|
|
261
|
+
})));
|
|
262
|
+
const writeFileSpy = jest.spyOn(fs.promises, 'writeFile').mockResolvedValue();
|
|
263
|
+
|
|
264
|
+
let stdoutOutput = '';
|
|
265
|
+
const originalLevel = consola.level;
|
|
266
|
+
const stdoutSpy = jest.spyOn(consola.stdout, 'write').mockImplementation((chunk) => {
|
|
267
|
+
stdoutOutput += chunk.toString();
|
|
268
|
+
return true;
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
try {
|
|
272
|
+
// eslint-disable-next-line global-require
|
|
273
|
+
const compileMock = require('../src/index');
|
|
274
|
+
compileMock.mockResolvedValue([]);
|
|
275
|
+
|
|
276
|
+
// eslint-disable-next-line global-require
|
|
277
|
+
require('../src/cli');
|
|
278
|
+
await new Promise((resolve) => { setImmediate(resolve); });
|
|
279
|
+
|
|
280
|
+
expect(stdoutOutput).toContain('"https://example.org/key.txt"');
|
|
281
|
+
expect(stdoutOutput).not.toContain('secret');
|
|
282
|
+
} finally {
|
|
283
|
+
stdoutSpy.mockRestore();
|
|
284
|
+
accessSpy.mockRestore();
|
|
285
|
+
readFileSpy.mockRestore();
|
|
286
|
+
writeFileSpy.mockRestore();
|
|
287
|
+
consola.level = originalLevel;
|
|
288
|
+
}
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
it('prints a cause that is not an error object', async () => {
|
|
292
|
+
const error = new Error('Failed to request url https://example.org/hosts.txt');
|
|
293
|
+
error.cause = 'connection reset';
|
|
294
|
+
|
|
295
|
+
// eslint-disable-next-line global-require
|
|
296
|
+
require('../src/index').mockRejectedValue(error);
|
|
297
|
+
|
|
298
|
+
// eslint-disable-next-line global-require
|
|
299
|
+
require('../src/cli');
|
|
300
|
+
await new Promise((resolve) => { setImmediate(resolve); });
|
|
301
|
+
|
|
302
|
+
expect(stderrOutput).toContain('Caused by: connection reset');
|
|
303
|
+
expect(exitSpy).toHaveBeenCalledWith(1);
|
|
304
|
+
});
|
|
305
|
+
});
|
package/test/cli.test.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
const consola = require('consola');
|
|
2
|
+
const compile = require('../src/index');
|
|
3
|
+
|
|
4
|
+
jest.mock('consola', () => ({
|
|
5
|
+
error: jest.fn(),
|
|
6
|
+
info: jest.fn(),
|
|
7
|
+
debug: jest.fn(),
|
|
8
|
+
}));
|
|
9
|
+
jest.mock('../src/index', () => jest.fn());
|
|
10
|
+
|
|
11
|
+
describe('CLI', () => {
|
|
12
|
+
const originalArgv = process.argv;
|
|
13
|
+
let exitSpy;
|
|
14
|
+
|
|
15
|
+
beforeEach(() => {
|
|
16
|
+
jest.clearAllMocks();
|
|
17
|
+
process.argv = ['node', 'cli', '-i', 'https://example.org/hosts.txt', '-o', 'output.txt'];
|
|
18
|
+
exitSpy = jest.spyOn(process, 'exit').mockImplementation(() => {});
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
afterEach(() => {
|
|
22
|
+
process.argv = originalArgv;
|
|
23
|
+
exitSpy.mockRestore();
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it('prints the error cause chain when compilation fails', async () => {
|
|
27
|
+
const networkError = new Error('connect ECONNREFUSED 127.0.0.1:80');
|
|
28
|
+
const fetchError = new TypeError('fetch failed');
|
|
29
|
+
fetchError.cause = networkError;
|
|
30
|
+
const error = new Error('Failed to request url https://example.org/hosts.txt');
|
|
31
|
+
error.cause = fetchError;
|
|
32
|
+
compile.mockRejectedValue(error);
|
|
33
|
+
|
|
34
|
+
// Loading the module runs main() with the process.argv prepared above.
|
|
35
|
+
// eslint-disable-next-line global-require
|
|
36
|
+
require('../src/cli');
|
|
37
|
+
// Let main() settle before asserting on the logged errors.
|
|
38
|
+
await new Promise((resolve) => { setImmediate(resolve); });
|
|
39
|
+
|
|
40
|
+
expect(consola.error.mock.calls).toEqual([
|
|
41
|
+
[error],
|
|
42
|
+
['Caused by: fetch failed'],
|
|
43
|
+
['Caused by: connect ECONNREFUSED 127.0.0.1:80'],
|
|
44
|
+
]);
|
|
45
|
+
expect(exitSpy).toHaveBeenCalledWith(1);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
const
|
|
1
|
+
const consola = require('consola');
|
|
2
2
|
const mock = require('mock-fs');
|
|
3
3
|
const path = require('path');
|
|
4
|
+
const mockHttp = require('./helpers/http-mock');
|
|
4
5
|
const compileSource = require('../src/compile-source');
|
|
5
|
-
const { TRANSFORMATIONS } = require('../src/transformations/
|
|
6
|
+
const { TRANSFORMATIONS } = require('../src/transformations/enum');
|
|
6
7
|
|
|
7
8
|
const testDirPath = path.resolve(__dirname, 'test/dir');
|
|
8
9
|
const exclusionsFilePath = path.resolve(testDirPath, 'exclusions.txt');
|
|
@@ -14,7 +15,7 @@ describe('Source compiler', () => {
|
|
|
14
15
|
});
|
|
15
16
|
|
|
16
17
|
it('compile a simple URL source', async () => {
|
|
17
|
-
const scope =
|
|
18
|
+
const scope = mockHttp('https://example.org')
|
|
18
19
|
.get('/filter.txt')
|
|
19
20
|
.reply(200, 'testrule', {
|
|
20
21
|
'Content-Type': 'text/plain',
|
|
@@ -33,6 +34,165 @@ describe('Source compiler', () => {
|
|
|
33
34
|
scope.done();
|
|
34
35
|
});
|
|
35
36
|
|
|
37
|
+
it('downloads a credentialed source and sends Basic authorization', async () => {
|
|
38
|
+
const authorization = `Basic ${Buffer.from('user:secret').toString('base64')}`;
|
|
39
|
+
const scope = mockHttp('https://example.org')
|
|
40
|
+
.get('/filter.txt', { Authorization: authorization })
|
|
41
|
+
.reply(200, 'testrule', {
|
|
42
|
+
'Content-Type': 'text/plain',
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
const source = {
|
|
46
|
+
name: 'test source',
|
|
47
|
+
source: 'https://user:secret@example.org/filter.txt',
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
const rules = await compileSource(source);
|
|
51
|
+
expect(rules).toContain('testrule');
|
|
52
|
+
|
|
53
|
+
scope.done();
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it('does not expose source credentials in logs', async () => {
|
|
57
|
+
const scope = mockHttp('https://example.org')
|
|
58
|
+
.get('/filter.txt')
|
|
59
|
+
.reply(200, 'testrule', {
|
|
60
|
+
'Content-Type': 'text/plain',
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
const source = {
|
|
64
|
+
name: 'test source',
|
|
65
|
+
source: 'https://user:secret@example.org/filter.txt',
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
69
|
+
|
|
70
|
+
try {
|
|
71
|
+
await compileSource(source);
|
|
72
|
+
|
|
73
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
74
|
+
infoMessages.forEach((message) => {
|
|
75
|
+
expect(message).not.toContain('secret');
|
|
76
|
+
});
|
|
77
|
+
expect(infoMessages).toContain('Start compiling https://example.org/filter.txt');
|
|
78
|
+
} finally {
|
|
79
|
+
infoSpy.mockRestore();
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
scope.done();
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it('does not expose credentials with whitespace in logs', async () => {
|
|
86
|
+
const authorization = `Basic ${Buffer.from('user:p s s').toString('base64')}`;
|
|
87
|
+
const scope = mockHttp('https://example.org')
|
|
88
|
+
.get('/filter.txt', { Authorization: authorization })
|
|
89
|
+
.reply(200, 'testrule', {
|
|
90
|
+
'Content-Type': 'text/plain',
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
const source = {
|
|
94
|
+
name: 'test source',
|
|
95
|
+
source: 'https://user:p s s@example.org/filter.txt',
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
99
|
+
|
|
100
|
+
try {
|
|
101
|
+
await compileSource(source);
|
|
102
|
+
|
|
103
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
104
|
+
infoMessages.forEach((message) => {
|
|
105
|
+
expect(message).not.toContain('p s s');
|
|
106
|
+
});
|
|
107
|
+
expect(infoMessages).toContain('Start compiling https://example.org/filter.txt');
|
|
108
|
+
} finally {
|
|
109
|
+
infoSpy.mockRestore();
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
scope.done();
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('does not expose a literal @ password in a path-less source', async () => {
|
|
116
|
+
const authorization = `Basic ${Buffer.from('user:p@s s').toString('base64')}`;
|
|
117
|
+
const scope = mockHttp('https://example.org')
|
|
118
|
+
.get('/', { Authorization: authorization })
|
|
119
|
+
.reply(200, 'testrule', {
|
|
120
|
+
'Content-Type': 'text/plain',
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
const source = {
|
|
124
|
+
name: 'test source',
|
|
125
|
+
source: 'https://user:p@s s@example.org',
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
129
|
+
|
|
130
|
+
try {
|
|
131
|
+
await compileSource(source);
|
|
132
|
+
|
|
133
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
134
|
+
infoMessages.forEach((message) => {
|
|
135
|
+
expect(message).not.toContain('p@s s');
|
|
136
|
+
});
|
|
137
|
+
expect(infoMessages).toContain('Start compiling https://example.org');
|
|
138
|
+
} finally {
|
|
139
|
+
infoSpy.mockRestore();
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
scope.done();
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it('does not expose a quote or space in the username in logs', async () => {
|
|
146
|
+
const authorization = `Basic ${Buffer.from('John Smith:secret').toString('base64')}`;
|
|
147
|
+
const scope = mockHttp('https://example.org')
|
|
148
|
+
.get('/filter.txt', { Authorization: authorization })
|
|
149
|
+
.reply(200, 'testrule', {
|
|
150
|
+
'Content-Type': 'text/plain',
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
const source = {
|
|
154
|
+
name: 'test source',
|
|
155
|
+
source: 'https://John Smith:secret@example.org/filter.txt',
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
159
|
+
|
|
160
|
+
try {
|
|
161
|
+
await compileSource(source);
|
|
162
|
+
|
|
163
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
164
|
+
infoMessages.forEach((message) => {
|
|
165
|
+
expect(message).not.toContain('secret');
|
|
166
|
+
});
|
|
167
|
+
expect(infoMessages).toContain('Start compiling https://example.org/filter.txt');
|
|
168
|
+
} finally {
|
|
169
|
+
infoSpy.mockRestore();
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
scope.done();
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it('does not expose credentials in a prefixed source log', async () => {
|
|
176
|
+
const source = {
|
|
177
|
+
name: 'test source',
|
|
178
|
+
source: ' https://user:secret@example.org/filter.txt',
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
182
|
+
|
|
183
|
+
try {
|
|
184
|
+
await expect(compileSource(source)).rejects.toThrow();
|
|
185
|
+
|
|
186
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
187
|
+
infoMessages.forEach((message) => {
|
|
188
|
+
expect(message).not.toContain('secret');
|
|
189
|
+
});
|
|
190
|
+
expect(infoMessages).toContain('Start compiling https://example.org/filter.txt');
|
|
191
|
+
} finally {
|
|
192
|
+
infoSpy.mockRestore();
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
|
|
36
196
|
it('compile a simple file source', async () => {
|
|
37
197
|
mock({
|
|
38
198
|
'test/dir': {
|
|
@@ -58,7 +218,7 @@ describe('Source compiler', () => {
|
|
|
58
218
|
||invalidrule/test
|
|
59
219
|
@@||rule3
|
|
60
220
|
||rule4`;
|
|
61
|
-
const scope =
|
|
221
|
+
const scope = mockHttp('https://example.org')
|
|
62
222
|
.get('/filter.txt')
|
|
63
223
|
.reply(200, rules, {
|
|
64
224
|
'Content-Type': 'text/plain',
|
|
@@ -106,7 +266,7 @@ describe('Source compiler', () => {
|
|
|
106
266
|
test1.com
|
|
107
267
|
|
|
108
268
|
test2.com`;
|
|
109
|
-
const scope =
|
|
269
|
+
const scope = mockHttp('https://example.org')
|
|
110
270
|
.get('/test-filter.txt')
|
|
111
271
|
.reply(200, testList, {
|
|
112
272
|
'Content-Type': 'text/plain',
|
|
@@ -143,7 +303,7 @@ describe('Source compiler', () => {
|
|
|
143
303
|
||*.ευ^
|
|
144
304
|
||*.ελ^
|
|
145
305
|
||*.հայ^`;
|
|
146
|
-
const scope =
|
|
306
|
+
const scope = mockHttp('https://example.org')
|
|
147
307
|
.get('/test-filter.txt')
|
|
148
308
|
.reply(200, testList, {
|
|
149
309
|
'Content-Type': 'text/plain',
|
|
@@ -193,7 +353,7 @@ describe('Source compiler', () => {
|
|
|
193
353
|
});
|
|
194
354
|
|
|
195
355
|
it('compile the empty external source', async () => {
|
|
196
|
-
const scope =
|
|
356
|
+
const scope = mockHttp('https://example.org')
|
|
197
357
|
.get('/empty.txt')
|
|
198
358
|
.reply(200, '', {
|
|
199
359
|
'Content-Type': 'text/plain',
|