@node-core/utils 7.2.3 → 7.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ncu-ci.js +52 -6
- package/lib/ci/ci_failure_parser.js +132 -117
- package/lib/ci/failure_file_scanner.js +153 -0
- package/lib/ci/resume_ci.js +146 -0
- package/lib/cli.js +7 -1
- package/lib/prepare_security.js +56 -107
- package/lib/request.js +52 -8
- package/lib/security-release/draft.js +32 -0
- package/lib/security-release/preparation.js +168 -0
- package/lib/security-release/security-release.js +32 -2
- package/package.json +1 -1
package/bin/ncu-ci.js
CHANGED
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
import {
|
|
24
24
|
RunPRJob
|
|
25
25
|
} from '../lib/ci/run_ci.js';
|
|
26
|
+
import { ResumePRJob } from '../lib/ci/resume_ci.js';
|
|
26
27
|
import { writeJson, writeFile } from '../lib/file.js';
|
|
27
28
|
import { getMergedConfig } from '../lib/config.js';
|
|
28
29
|
import { runPromise } from '../lib/run.js';
|
|
@@ -134,6 +135,26 @@ const args = yargs(hideBin(process.argv))
|
|
|
134
135
|
},
|
|
135
136
|
handler
|
|
136
137
|
})
|
|
138
|
+
.command({
|
|
139
|
+
command: 'resume <prid>',
|
|
140
|
+
desc: 'Resume the latest CI run for given PR',
|
|
141
|
+
builder: (yargs) => {
|
|
142
|
+
yargs
|
|
143
|
+
.positional('prid', {
|
|
144
|
+
describe: 'ID of the PR or URL to the PR',
|
|
145
|
+
type: 'string'
|
|
146
|
+
})
|
|
147
|
+
.option('owner', {
|
|
148
|
+
default: '',
|
|
149
|
+
describe: 'GitHub repository owner'
|
|
150
|
+
})
|
|
151
|
+
.option('repo', {
|
|
152
|
+
default: '',
|
|
153
|
+
describe: 'GitHub repository name'
|
|
154
|
+
});
|
|
155
|
+
},
|
|
156
|
+
handler
|
|
157
|
+
})
|
|
137
158
|
.command({
|
|
138
159
|
command: 'url <url>',
|
|
139
160
|
desc: 'Automatically detect CI type and show results',
|
|
@@ -278,11 +299,13 @@ class RunPRJobCommand {
|
|
|
278
299
|
return this.argv.prid;
|
|
279
300
|
}
|
|
280
301
|
|
|
281
|
-
|
|
282
|
-
const {
|
|
283
|
-
cli, request, prid, repo, owner
|
|
284
|
-
} = this;
|
|
302
|
+
validate() {
|
|
303
|
+
const { cli, prid, repo, owner } = this;
|
|
285
304
|
let validArgs = true;
|
|
305
|
+
if (!Number.isSafeInteger(prid) || prid <= 0) {
|
|
306
|
+
validArgs = false;
|
|
307
|
+
cli.error('Pull request ID must be a positive integer');
|
|
308
|
+
}
|
|
286
309
|
if (!repo) {
|
|
287
310
|
validArgs = false;
|
|
288
311
|
cli.error('GitHub repository is missing, please set it via ncu-config ' +
|
|
@@ -295,6 +318,13 @@ class RunPRJobCommand {
|
|
|
295
318
|
}
|
|
296
319
|
if (!validArgs) {
|
|
297
320
|
this.cli.setExitCode(1);
|
|
321
|
+
}
|
|
322
|
+
return validArgs;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
async start() {
|
|
326
|
+
const { cli, request, prid, repo, owner } = this;
|
|
327
|
+
if (!this.validate()) {
|
|
298
328
|
return;
|
|
299
329
|
}
|
|
300
330
|
const { certifySafe, checkForDuplicates } = this.argv;
|
|
@@ -308,6 +338,20 @@ class RunPRJobCommand {
|
|
|
308
338
|
}
|
|
309
339
|
}
|
|
310
340
|
|
|
341
|
+
class ResumePRJobCommand extends RunPRJobCommand {
|
|
342
|
+
async start() {
|
|
343
|
+
const { cli, request, prid, repo, owner } = this;
|
|
344
|
+
if (!this.validate()) {
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
const jobRunner = new ResumePRJob(cli, request, owner, repo, prid);
|
|
348
|
+
if (!(await jobRunner.resume())) {
|
|
349
|
+
cli.setExitCode(1);
|
|
350
|
+
process.exitCode = 1;
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
311
355
|
class CICommand {
|
|
312
356
|
constructor(cli, request, argv) {
|
|
313
357
|
this.cli = cli;
|
|
@@ -565,7 +609,8 @@ async function main(command, argv) {
|
|
|
565
609
|
let commandHandler;
|
|
566
610
|
// Prepare queue.
|
|
567
611
|
switch (command) {
|
|
568
|
-
case 'run':
|
|
612
|
+
case 'run':
|
|
613
|
+
case 'resume': {
|
|
569
614
|
const maybeURL = URL.parse(argv.prid);
|
|
570
615
|
if (maybeURL?.host === 'github.com') {
|
|
571
616
|
const [, owner, repo, , prid, , commit_sha] = maybeURL.pathname.split('/');
|
|
@@ -575,7 +620,8 @@ async function main(command, argv) {
|
|
|
575
620
|
argv.prid = prid;
|
|
576
621
|
}
|
|
577
622
|
argv.prid = Number(argv.prid);
|
|
578
|
-
const
|
|
623
|
+
const Command = command === 'run' ? RunPRJobCommand : ResumePRJobCommand;
|
|
624
|
+
const jobRunner = new Command(cli, request, argv);
|
|
579
625
|
return jobRunner.start();
|
|
580
626
|
}
|
|
581
627
|
case 'rate': {
|
|
@@ -132,35 +132,54 @@ function failureMatcher(Failure, patterns, ctx, text) {
|
|
|
132
132
|
return null;
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
+
const fatalPattern = /fatal: .+/g;
|
|
136
|
+
|
|
137
|
+
// Shared by the whole-log parser and the streaming file scanner.
|
|
138
|
+
export const FAILURE_MARKERS = {
|
|
139
|
+
tap: {
|
|
140
|
+
start: /^not ok \d+/,
|
|
141
|
+
end: / {2}\.\.\.\r?\n/,
|
|
142
|
+
todo: '# TODO :'
|
|
143
|
+
},
|
|
144
|
+
cpp: /\[ {2}FAILED {2}\]/,
|
|
145
|
+
git: [{
|
|
146
|
+
start: /Changes not staged for commit:/,
|
|
147
|
+
end: /no changes added to commit/
|
|
148
|
+
}, {
|
|
149
|
+
start: /error: Your local changes to the following files/,
|
|
150
|
+
end: /Failed to merge in the changes./
|
|
151
|
+
}]
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
const tapPattern = new RegExp(
|
|
155
|
+
`${FAILURE_MARKERS.tap.start.source}[\\s\\S]+?${FAILURE_MARKERS.tap.end.source}`, 'mg');
|
|
156
|
+
|
|
135
157
|
// The elements are ranked by priority
|
|
136
158
|
const FAILURE_FILTERS = [{
|
|
137
159
|
// NOTE(mmarchini): infra-related issues should have the highest priority, as
|
|
138
160
|
// they can cause other issues to happen.
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
}
|
|
152
|
-
];
|
|
153
|
-
return failureMatcher(InfraFailure, patterns, ctx, text);
|
|
161
|
+
Failure: InfraFailure,
|
|
162
|
+
patterns: [{
|
|
163
|
+
pattern: /Read-only file system/g,
|
|
164
|
+
context: { index: 0, contextBefore: 1, contextAfter: 0 }
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
pattern: /Device or resource busy/g,
|
|
168
|
+
context: { index: 0, contextBefore: 1, contextAfter: 0 }
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
pattern: /There is not enough space in the file system./g,
|
|
172
|
+
context: { index: 0, contextBefore: 1, contextAfter: 0 }
|
|
154
173
|
}
|
|
174
|
+
],
|
|
155
175
|
}, {
|
|
156
176
|
// TODO: match indentation to avoid skipping context with '...'
|
|
157
177
|
filter(ctx, text) {
|
|
158
|
-
const
|
|
159
|
-
const matches = text.match(pattern);
|
|
178
|
+
const matches = text.match(tapPattern);
|
|
160
179
|
if (!matches) {
|
|
161
180
|
return null;
|
|
162
181
|
}
|
|
163
|
-
const nonFlaky = matches.filter((m) => !m.includes(
|
|
182
|
+
const nonFlaky = matches.filter((m) => !m.includes(FAILURE_MARKERS.tap.todo));
|
|
164
183
|
if (!nonFlaky.length) {
|
|
165
184
|
return null;
|
|
166
185
|
}
|
|
@@ -169,89 +188,71 @@ const FAILURE_FILTERS = [{
|
|
|
169
188
|
);
|
|
170
189
|
}
|
|
171
190
|
}, {
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
return failureMatcher(CCTestFailure, patterns, ctx, text);
|
|
178
|
-
}
|
|
191
|
+
Failure: CCTestFailure,
|
|
192
|
+
patterns: [{
|
|
193
|
+
pattern: new RegExp(`${FAILURE_MARKERS.cpp.source}.+`, 'g'),
|
|
194
|
+
context: { index: 0, contextBefore: 5, contextAfter: 0 }
|
|
195
|
+
}],
|
|
179
196
|
}, {
|
|
180
197
|
// VS compilation error
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
return failureMatcher(BuildFailure, patterns, ctx, text);
|
|
187
|
-
}
|
|
198
|
+
Failure: BuildFailure,
|
|
199
|
+
patterns: [{
|
|
200
|
+
pattern: /error C\d+:/mg,
|
|
201
|
+
context: { index: 0, contextBefore: 0, contextAfter: 5 }
|
|
202
|
+
}],
|
|
188
203
|
}, {
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
return failureMatcher(JenkinsFailure, patterns, ctx, text);
|
|
198
|
-
}
|
|
204
|
+
Failure: JenkinsFailure,
|
|
205
|
+
patterns: [{
|
|
206
|
+
pattern: /java\.io\.IOException.+/g,
|
|
207
|
+
context: { index: -1, contextBefore: 0, contextAfter: 5 }
|
|
208
|
+
}, {
|
|
209
|
+
pattern: /Build timed out/g,
|
|
210
|
+
context: { index: 0, contextBefore: 0, contextAfter: 1 }
|
|
211
|
+
}],
|
|
199
212
|
}, {
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
context: { index: 0, contextBefore: 0, contextAfter: 0 }
|
|
209
|
-
}, {
|
|
210
|
-
pattern: /warning: failed to remove .+/g,
|
|
211
|
-
context: { index: 0, contextBefore: 0, contextAfter: 0 }
|
|
212
|
-
}];
|
|
213
|
-
return failureMatcher(GitFailure, patterns, ctx, text);
|
|
214
|
-
}
|
|
213
|
+
Failure: GitFailure,
|
|
214
|
+
patterns: [...FAILURE_MARKERS.git.map(({ start, end }) => ({
|
|
215
|
+
pattern: new RegExp(`${start.source}[\\s\\S]+${end.source}`, 'mg'),
|
|
216
|
+
context: { index: 0, contextBefore: 0, contextAfter: 0 }
|
|
217
|
+
})), {
|
|
218
|
+
pattern: /warning: failed to remove .+/g,
|
|
219
|
+
context: { index: 0, contextBefore: 0, contextAfter: 0 }
|
|
220
|
+
}],
|
|
215
221
|
}, {
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
return failureMatcher(GitFailure, patterns, ctx, text);
|
|
228
|
-
}
|
|
222
|
+
Failure: GitFailure,
|
|
223
|
+
patterns: [{
|
|
224
|
+
pattern: /ERROR: Error fetching .+/g,
|
|
225
|
+
context: { index: 0, contextBefore: 0, contextAfter: 5 }
|
|
226
|
+
}, {
|
|
227
|
+
pattern: /hudson\.plugins\.git\.GitException+/g,
|
|
228
|
+
context: { index: 0, contextBefore: 0, contextAfter: 5 }
|
|
229
|
+
}, {
|
|
230
|
+
pattern: /Cannot rebase: .+/g,
|
|
231
|
+
context: { index: 0, contextBefore: 0, contextAfter: 1 }
|
|
232
|
+
}],
|
|
229
233
|
}, {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
return failureMatcher(BuildFailure, patterns, ctx, text);
|
|
250
|
-
}
|
|
234
|
+
Failure: BuildFailure,
|
|
235
|
+
patterns: [{
|
|
236
|
+
pattern: /sh: line /g,
|
|
237
|
+
context: { index: 0, contextBefore: 0, contextAfter: 1 }
|
|
238
|
+
}, {
|
|
239
|
+
pattern: /fatal error:/g,
|
|
240
|
+
context: { index: 0, contextBefore: 0, contextAfter: 1 }
|
|
241
|
+
}, {
|
|
242
|
+
pattern: /dtrace: failed to compile script/g,
|
|
243
|
+
context: { index: 0, contextBefore: 0, contextAfter: 1 }
|
|
244
|
+
}, {
|
|
245
|
+
pattern: /ERROR: .+/g,
|
|
246
|
+
// Pick the last one
|
|
247
|
+
context: { index: -1, contextBefore: 0, contextAfter: 5 }
|
|
248
|
+
}, {
|
|
249
|
+
// Pick the first one
|
|
250
|
+
pattern: /Error: .+/g,
|
|
251
|
+
context: { index: 0, contextBefore: 0, contextAfter: 5 }
|
|
252
|
+
}],
|
|
251
253
|
}, {
|
|
252
254
|
filter(ctx, text) {
|
|
253
|
-
const
|
|
254
|
-
const matches = text.match(pattern);
|
|
255
|
+
const matches = text.match(fatalPattern);
|
|
255
256
|
if (!matches) {
|
|
256
257
|
return null;
|
|
257
258
|
}
|
|
@@ -259,30 +260,42 @@ const FAILURE_FILTERS = [{
|
|
|
259
260
|
return [new BuildFailure(ctx, reason)];
|
|
260
261
|
}
|
|
261
262
|
}, {
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
return failureMatcher(BuildFailure, patterns, ctx, text);
|
|
283
|
-
}
|
|
263
|
+
Failure: BuildFailure,
|
|
264
|
+
patterns: [{
|
|
265
|
+
pattern: /FATAL: .+/g,
|
|
266
|
+
context: { index: -1, contextBefore: 0, contextAfter: 5 }
|
|
267
|
+
}, {
|
|
268
|
+
pattern: /make.*: write error/mg,
|
|
269
|
+
context: { index: 0, contextBefore: 0, contextAfter: 3 }
|
|
270
|
+
}, {
|
|
271
|
+
pattern: /error: .+/g,
|
|
272
|
+
context: { index: 0, contextBefore: 0, contextAfter: 5 }
|
|
273
|
+
}, {
|
|
274
|
+
pattern: /Makefile:.+failed/g,
|
|
275
|
+
context: { index: 0, contextBefore: 0, contextAfter: 5 }
|
|
276
|
+
}, {
|
|
277
|
+
pattern: /make.*: .+ Error \d.*/g,
|
|
278
|
+
context: { index: 0, contextBefore: 0, contextAfter: 3 }
|
|
279
|
+
}, {
|
|
280
|
+
pattern: /warning: failed .+/g,
|
|
281
|
+
context: { index: 0, contextBefore: 0, contextAfter: 3 }
|
|
282
|
+
}],
|
|
284
283
|
}];
|
|
285
284
|
|
|
285
|
+
// Share recognition patterns with streaming consumers without exposing the
|
|
286
|
+
// parser's constructors, filter priority, or context-selection machinery.
|
|
287
|
+
export const FAILURE_PATTERNS = {
|
|
288
|
+
infrastructure: FAILURE_FILTERS
|
|
289
|
+
.filter(({ Failure }) => Failure === InfraFailure)
|
|
290
|
+
.flatMap(({ patterns }) => patterns.map(({ pattern }) => pattern)),
|
|
291
|
+
diagnostic: [
|
|
292
|
+
...FAILURE_FILTERS
|
|
293
|
+
.filter(({ Failure }) => Failure && Failure !== InfraFailure && Failure !== CCTestFailure)
|
|
294
|
+
.flatMap(({ patterns }) => patterns.map(({ pattern }) => pattern)),
|
|
295
|
+
fatalPattern
|
|
296
|
+
]
|
|
297
|
+
};
|
|
298
|
+
|
|
286
299
|
export default class CIFailureParser {
|
|
287
300
|
constructor(ctx, text) {
|
|
288
301
|
this.ctx = ctx;
|
|
@@ -291,8 +304,10 @@ export default class CIFailureParser {
|
|
|
291
304
|
|
|
292
305
|
parse() {
|
|
293
306
|
const text = this.text;
|
|
294
|
-
for (const { filter } of FAILURE_FILTERS) {
|
|
295
|
-
const result = filter
|
|
307
|
+
for (const { filter, Failure, patterns } of FAILURE_FILTERS) {
|
|
308
|
+
const result = filter
|
|
309
|
+
? filter(this.ctx, text)
|
|
310
|
+
: failureMatcher(Failure, patterns, this.ctx, text);
|
|
296
311
|
// TODO: we may want to concat certain types of failures
|
|
297
312
|
if (result) {
|
|
298
313
|
return result;
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { StringDecoder } from 'node:string_decoder';
|
|
2
|
+
|
|
3
|
+
import { FAILURE_MARKERS, FAILURE_PATTERNS } from './ci_failure_parser.js';
|
|
4
|
+
|
|
5
|
+
function createMatcher(patterns) {
|
|
6
|
+
// Preserve matching flags without sharing RegExp.lastIndex between scans.
|
|
7
|
+
const matchers = patterns.map(pattern =>
|
|
8
|
+
new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, '')));
|
|
9
|
+
return text => matchers.some(pattern => pattern.test(text));
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
const diagnostic = createMatcher(FAILURE_PATTERNS.diagnostic);
|
|
13
|
+
const infrastructure = createMatcher(FAILURE_PATTERNS.infrastructure);
|
|
14
|
+
const gitStart = createMatcher(FAILURE_MARKERS.git.map(({ start }) => start));
|
|
15
|
+
const gitEnd = createMatcher(FAILURE_MARKERS.git.map(({ end }) => end));
|
|
16
|
+
|
|
17
|
+
function fileAliases(filename) {
|
|
18
|
+
const paths = new Set([filename]);
|
|
19
|
+
if (filename.startsWith('test/')) {
|
|
20
|
+
paths.add(filename.slice(5));
|
|
21
|
+
paths.add(filename.slice(5).replace(/\.(?:js|mjs|cjs|out)$/, ''));
|
|
22
|
+
} else if (filename.startsWith('lib/')) {
|
|
23
|
+
paths.add(filename.slice(4));
|
|
24
|
+
}
|
|
25
|
+
return [...paths];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Emit overlapping, normalized windows rather than buffering whole log lines.
|
|
29
|
+
// Explicit line boundaries keep network chunk boundaries out of the matching rules.
|
|
30
|
+
async function * logWindows(source, overlap) {
|
|
31
|
+
const decoder = new StringDecoder('utf8');
|
|
32
|
+
let tail = '';
|
|
33
|
+
let lineStart = true;
|
|
34
|
+
let afterBackslash = false;
|
|
35
|
+
|
|
36
|
+
function * consume(raw) {
|
|
37
|
+
raw = raw.replace(/\r/g, '');
|
|
38
|
+
let text = raw.replace(/\\+/g, '/');
|
|
39
|
+
if (afterBackslash && raw.startsWith('\\')) text = text.slice(1);
|
|
40
|
+
if (raw) afterBackslash = raw.endsWith('\\');
|
|
41
|
+
for (let offset = 0; offset < text.length;) {
|
|
42
|
+
const newline = text.indexOf('\n', offset);
|
|
43
|
+
const end = Math.min(offset + 8192, newline < 0 ? text.length : newline + 1);
|
|
44
|
+
const window = tail + text.slice(offset, end);
|
|
45
|
+
const lineEnd = end === newline + 1;
|
|
46
|
+
yield { text: window, lineStart, lineEnd };
|
|
47
|
+
if (lineEnd) {
|
|
48
|
+
tail = '';
|
|
49
|
+
lineStart = true;
|
|
50
|
+
} else {
|
|
51
|
+
lineStart &&= window.length <= overlap;
|
|
52
|
+
tail = window.slice(-overlap);
|
|
53
|
+
}
|
|
54
|
+
offset = end;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
for await (const chunk of source) yield * consume(decoder.write(chunk));
|
|
59
|
+
yield * consume(decoder.end());
|
|
60
|
+
if (tail) yield { text: `${tail}\n`, lineStart, lineEnd: true };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Each line is reduced to a filename and failure markers. Neither a giant line
|
|
64
|
+
// nor a giant TAP block needs to survive in memory.
|
|
65
|
+
async function * failureLines(windows, matchFile) {
|
|
66
|
+
let line = {};
|
|
67
|
+
for await (const { text, lineStart, lineEnd } of windows) {
|
|
68
|
+
line.file ??= matchFile(text, lineStart);
|
|
69
|
+
line.diagnostic ||= diagnostic(text);
|
|
70
|
+
line.infrastructure ||= infrastructure(text);
|
|
71
|
+
line.cpp ||= FAILURE_MARKERS.cpp.test(text);
|
|
72
|
+
line.tapStart ||= lineStart && FAILURE_MARKERS.tap.start.test(text);
|
|
73
|
+
line.tapEnd ||= lineEnd && FAILURE_MARKERS.tap.end.test(text);
|
|
74
|
+
line.todo ||= text.includes(FAILURE_MARKERS.tap.todo);
|
|
75
|
+
line.gitStart ||= gitStart(text);
|
|
76
|
+
line.gitEnd ||= gitEnd(text);
|
|
77
|
+
if (lineEnd) {
|
|
78
|
+
yield line;
|
|
79
|
+
line = {};
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async function findFailure(lines) {
|
|
85
|
+
let history = [];
|
|
86
|
+
let followingLines = 0;
|
|
87
|
+
let tap = null;
|
|
88
|
+
let git = null;
|
|
89
|
+
|
|
90
|
+
for await (const line of lines) {
|
|
91
|
+
const precedingLines = history;
|
|
92
|
+
history = [...history, line.file].slice(-5);
|
|
93
|
+
if (line.tapStart) {
|
|
94
|
+
tap = {};
|
|
95
|
+
followingLines = 0;
|
|
96
|
+
}
|
|
97
|
+
if (tap) {
|
|
98
|
+
tap.file ??= line.file;
|
|
99
|
+
tap.todo ||= line.todo;
|
|
100
|
+
// A later TODO can mark this as an expected failure; wait for the ending.
|
|
101
|
+
if (line.tapEnd) {
|
|
102
|
+
if (!tap.todo && tap.file) return tap.file;
|
|
103
|
+
tap = null;
|
|
104
|
+
}
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (line.gitStart) git = {};
|
|
109
|
+
if (git) {
|
|
110
|
+
git.file ??= line.file;
|
|
111
|
+
if (line.gitEnd) {
|
|
112
|
+
if (git.file) return git.file;
|
|
113
|
+
git = null;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (line.infrastructure || line.cpp) {
|
|
117
|
+
const before = line.cpp ? 5 : 1;
|
|
118
|
+
const file = line.file || precedingLines.slice(-before).find(Boolean);
|
|
119
|
+
if (file) return file;
|
|
120
|
+
}
|
|
121
|
+
if (line.diagnostic) followingLines = 6;
|
|
122
|
+
if (followingLines > 0) {
|
|
123
|
+
followingLines--;
|
|
124
|
+
if (line.file) return line.file;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export class FailureFileScanner {
|
|
130
|
+
constructor(filenames) {
|
|
131
|
+
const aliases = [...filenames].map(filename => ({ filename, paths: fileAliases(filename) }));
|
|
132
|
+
this.overlap = Math.max(256,
|
|
133
|
+
...aliases.flatMap(({ paths }) => paths.map(path => path.length + 2)));
|
|
134
|
+
const matchers = aliases.map(({ filename, paths }) => ({
|
|
135
|
+
filename,
|
|
136
|
+
patterns: paths.map(path => {
|
|
137
|
+
const escaped = path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
138
|
+
// Both delimiters must be real: a window boundary is not a path boundary.
|
|
139
|
+
return new RegExp(`[^\\w.-]${escaped}(?=[^\\w./-])`);
|
|
140
|
+
})
|
|
141
|
+
}));
|
|
142
|
+
this.matchFile = (text, lineStart) => {
|
|
143
|
+
// The start of a real log line is also a valid left delimiter.
|
|
144
|
+
if (lineStart) text = `\n${text}`;
|
|
145
|
+
return matchers.find(({ patterns }) =>
|
|
146
|
+
patterns.some(pattern => pattern.test(text)))?.filename;
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
scan(source) {
|
|
151
|
+
return findFailure(failureLines(logWindows(source, this.overlap), this.matchFile));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { JobParser, CI_TYPES_KEYS } from './ci_type_parser.js';
|
|
2
|
+
import { CI_CRUMB_URL } from './run_ci.js';
|
|
3
|
+
import { PRBuild } from './build-types/pr_build.js';
|
|
4
|
+
import { getPrURL } from '../links.js';
|
|
5
|
+
import { debuglog } from '../verbosity.js';
|
|
6
|
+
import { FailureFileScanner } from './failure_file_scanner.js';
|
|
7
|
+
|
|
8
|
+
export class ResumePRJob {
|
|
9
|
+
constructor(cli, request, owner, repo, prid) {
|
|
10
|
+
this.cli = cli;
|
|
11
|
+
this.request = request;
|
|
12
|
+
this.owner = owner;
|
|
13
|
+
this.repo = repo;
|
|
14
|
+
this.prid = prid;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
async checkFailures(jobid) {
|
|
18
|
+
const { cli, request, owner, repo, prid } = this;
|
|
19
|
+
const filenames = new Set();
|
|
20
|
+
for await (const file of request.getPullRequestFiles({ owner, repo, prid })) {
|
|
21
|
+
for (const filename of [file.filename, file.previous_filename].filter(Boolean)) {
|
|
22
|
+
filenames.add(filename);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
const overlaps = new Set();
|
|
26
|
+
const buildRequest = Object.create(request);
|
|
27
|
+
buildRequest.json = async(...args) => {
|
|
28
|
+
const data = await request.json(...args);
|
|
29
|
+
// Inspect failed descendants even when their parent job was aborted.
|
|
30
|
+
return data?.result === 'ABORTED' ? { ...data, result: 'FAILURE' } : data;
|
|
31
|
+
};
|
|
32
|
+
// Reuse the build traversal, but consume console responses directly instead
|
|
33
|
+
// of handing whole logs to its summary parser. Serialize downloads so a
|
|
34
|
+
// match cancels this response and prevents queued logs from being opened.
|
|
35
|
+
let pending = Promise.resolve();
|
|
36
|
+
buildRequest.text = (url) => {
|
|
37
|
+
const scan = pending.then(async() => {
|
|
38
|
+
if (!filenames.size || overlaps.size) return '';
|
|
39
|
+
const scanner = new FailureFileScanner(filenames);
|
|
40
|
+
const match = await scanner.scan(request.stream(url));
|
|
41
|
+
if (match) overlaps.add(match);
|
|
42
|
+
return '';
|
|
43
|
+
});
|
|
44
|
+
pending = scan.catch(debuglog);
|
|
45
|
+
return scan;
|
|
46
|
+
};
|
|
47
|
+
const build = new PRBuild(cli, buildRequest, jobid, Infinity);
|
|
48
|
+
try {
|
|
49
|
+
await build.getResults();
|
|
50
|
+
} catch (err) {
|
|
51
|
+
debuglog(err);
|
|
52
|
+
}
|
|
53
|
+
await pending;
|
|
54
|
+
if (overlaps.size) {
|
|
55
|
+
for (const filename of [...overlaps].sort()) {
|
|
56
|
+
cli.error(filename);
|
|
57
|
+
}
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async resume() {
|
|
64
|
+
const { cli, request, prid } = this;
|
|
65
|
+
let crumb;
|
|
66
|
+
cli.startSpinner('Validating Jenkins credentials');
|
|
67
|
+
try {
|
|
68
|
+
({ crumb } = await request.json(CI_CRUMB_URL));
|
|
69
|
+
if (!crumb) {
|
|
70
|
+
throw new Error('Missing Jenkins crumb');
|
|
71
|
+
}
|
|
72
|
+
} catch (err) {
|
|
73
|
+
debuglog(err);
|
|
74
|
+
cli.stopSpinner('Jenkins credentials invalid', cli.SPINNER_STATUS.FAILED);
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
cli.stopSpinner('Jenkins credentials valid');
|
|
78
|
+
|
|
79
|
+
try {
|
|
80
|
+
cli.startSpinner(`Looking for CI runs for pull request ${prid}`);
|
|
81
|
+
const parser = await JobParser.fromPR(getPrURL(this), cli, request);
|
|
82
|
+
const job = parser.parse().get(CI_TYPES_KEYS.PR);
|
|
83
|
+
if (!job) {
|
|
84
|
+
cli.stopSpinner(`No CI run detected from pull request ${prid}`,
|
|
85
|
+
cli.SPINNER_STATUS.FAILED);
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
cli.stopSpinner(`Found PR CI job ${job.jobid}`);
|
|
89
|
+
|
|
90
|
+
const build = new PRBuild(cli, request, job.jobid, undefined,
|
|
91
|
+
'result,building,actions[parameters[name,value]]');
|
|
92
|
+
const { result, building, actions = [] } = await build.getBuildData();
|
|
93
|
+
if (building || (result !== 'FAILURE' && result !== 'ABORTED')) {
|
|
94
|
+
const status = building ? 'RUNNING' : result ?? 'RUNNING';
|
|
95
|
+
cli.error(`CI job ${job.jobid} is in status ${status}, skipping resume`);
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
const approvedSHAs = new Set(actions.flatMap(action => action.parameters ?? [])
|
|
99
|
+
.filter(parameter => parameter.name === 'COMMIT_SHA_CHECK')
|
|
100
|
+
.map(parameter => parameter.value));
|
|
101
|
+
const [approvedSHA] = approvedSHAs;
|
|
102
|
+
if (approvedSHAs.size !== 1 || typeof approvedSHA !== 'string' || !approvedSHA) {
|
|
103
|
+
cli.error(`Refusing to resume CI job ${job.jobid}: cannot determine its approved commit`);
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
cli.startSpinner('Checking failures against changed PR files');
|
|
108
|
+
if (!(await this.checkFailures(job.jobid))) {
|
|
109
|
+
cli.stopSpinner('Refusing to resume CI: failures reference files changed by this PR',
|
|
110
|
+
cli.SPINNER_STATUS.FAILED);
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
cli.stopSpinner('No changed PR files found in available failure details');
|
|
114
|
+
|
|
115
|
+
// Read HEAD after inspecting failures so the comparison is fresh when resuming.
|
|
116
|
+
const pr = await request.getPullRequest(getPrURL(this));
|
|
117
|
+
if (pr.head?.sha !== approvedSHA) {
|
|
118
|
+
cli.error(`Refusing to resume CI job ${job.jobid}: ` +
|
|
119
|
+
'its approved commit does not match the current PR HEAD');
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
cli.startSpinner(`Resuming PR CI job ${job.jobid}`);
|
|
124
|
+
// Jenkins does not export its resume action in the build API. Let the
|
|
125
|
+
// resume endpoint determine whether the action is available.
|
|
126
|
+
const response = await request.fetch(`${build.jobUrl}resume/`, {
|
|
127
|
+
method: 'POST',
|
|
128
|
+
headers: {
|
|
129
|
+
'Jenkins-Crumb': crumb
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
if (response.status !== 200) {
|
|
133
|
+
cli.stopSpinner(
|
|
134
|
+
`Failed to resume PR CI: ${response.status} ${response.statusText}`,
|
|
135
|
+
cli.SPINNER_STATUS.FAILED);
|
|
136
|
+
return false;
|
|
137
|
+
}
|
|
138
|
+
cli.stopSpinner('PR CI job successfully resumed');
|
|
139
|
+
} catch (err) {
|
|
140
|
+
debuglog(err);
|
|
141
|
+
cli.stopSpinner('Failed to resume CI', cli.SPINNER_STATUS.FAILED);
|
|
142
|
+
return false;
|
|
143
|
+
}
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
}
|
package/lib/cli.js
CHANGED
|
@@ -154,7 +154,13 @@ export default class CLI {
|
|
|
154
154
|
case INFO:
|
|
155
155
|
symbol = info;
|
|
156
156
|
}
|
|
157
|
-
|
|
157
|
+
const text = `${symbol} ${rawText}`;
|
|
158
|
+
if (this.spinner.isSpinning) {
|
|
159
|
+
this.spinner.stop(text);
|
|
160
|
+
} else {
|
|
161
|
+
// A nested operation may already have stopped the shared spinner.
|
|
162
|
+
this.log(text);
|
|
163
|
+
}
|
|
158
164
|
}
|
|
159
165
|
|
|
160
166
|
write(text) {
|
package/lib/prepare_security.js
CHANGED
|
@@ -1,24 +1,40 @@
|
|
|
1
|
-
import fs from 'node:fs';
|
|
2
1
|
import path from 'node:path';
|
|
3
2
|
import auth from './auth.js';
|
|
4
3
|
import Request from './request.js';
|
|
5
4
|
import { parsePRFromURL } from './links.js';
|
|
5
|
+
import {
|
|
6
|
+
assertNewSecurityRelease,
|
|
7
|
+
getSecurityReleaseDraftPath,
|
|
8
|
+
writeSecurityReleaseDraft
|
|
9
|
+
} from './security-release/draft.js';
|
|
10
|
+
import {
|
|
11
|
+
buildIncludedTriagedReport,
|
|
12
|
+
getReportPRURL,
|
|
13
|
+
getMissingReportInformation,
|
|
14
|
+
groupMissingReportInformation,
|
|
15
|
+
listSecurityReleaseCandidates,
|
|
16
|
+
prepareSecurityRelease
|
|
17
|
+
} from './security-release/preparation.js';
|
|
18
|
+
|
|
6
19
|
import {
|
|
7
20
|
NEXT_SECURITY_RELEASE_BRANCH,
|
|
8
|
-
NEXT_SECURITY_RELEASE_FOLDER,
|
|
9
21
|
checkoutOnSecurityReleaseBranch,
|
|
10
22
|
commitAndPushVulnerabilitiesJSON,
|
|
11
23
|
validateDate,
|
|
12
24
|
promptDependencies,
|
|
13
25
|
getSupportedVersions,
|
|
14
26
|
getReportSeverity,
|
|
15
|
-
getSummary,
|
|
16
27
|
pickReport,
|
|
17
28
|
confirmSecurityStep,
|
|
18
|
-
writeSecurityFile,
|
|
19
29
|
SecurityRelease
|
|
20
30
|
} from './security-release/security-release.js';
|
|
21
31
|
|
|
32
|
+
export {
|
|
33
|
+
buildIncludedTriagedReport,
|
|
34
|
+
getMissingReportInformation,
|
|
35
|
+
groupMissingReportInformation
|
|
36
|
+
} from './security-release/preparation.js';
|
|
37
|
+
|
|
22
38
|
function relativeDate(date) {
|
|
23
39
|
const days = Math.floor((Date.now() - date) / (1000 * 60 * 60 * 24));
|
|
24
40
|
if (days < 30) return days === 1 ? '1 day ago' : `${days} days ago`;
|
|
@@ -82,79 +98,11 @@ export function getNextTuesdayReleaseDateChoices(fromDate = new Date(), count =
|
|
|
82
98
|
return choices;
|
|
83
99
|
}
|
|
84
100
|
|
|
85
|
-
function getReportPRURL(report) {
|
|
86
|
-
const customFieldValues = report.relationships.custom_field_values?.data ?? [];
|
|
87
|
-
return customFieldValues[0]?.attributes?.value ?? '';
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
export function buildIncludedTriagedReport(report, options = {}) {
|
|
91
|
-
const {
|
|
92
|
-
affectedVersions = '',
|
|
93
|
-
patchAuthors = [],
|
|
94
|
-
prURL = getReportPRURL(report)
|
|
95
|
-
} = options;
|
|
96
|
-
const {
|
|
97
|
-
id,
|
|
98
|
-
attributes: { title, cve_ids = [] },
|
|
99
|
-
relationships: { reporter }
|
|
100
|
-
} = report;
|
|
101
|
-
const link = `https://hackerone.com/reports/${id}`;
|
|
102
|
-
const summaryContent = getSummary(report);
|
|
103
|
-
|
|
104
|
-
return {
|
|
105
|
-
id,
|
|
106
|
-
title,
|
|
107
|
-
cveIds: cve_ids,
|
|
108
|
-
severity: getReportSeverity(report),
|
|
109
|
-
summary: summaryContent ?? '',
|
|
110
|
-
patchAuthors,
|
|
111
|
-
prURL,
|
|
112
|
-
affectedVersions: affectedVersions
|
|
113
|
-
.split(',')
|
|
114
|
-
.map((v) => v.replace('v', '').trim())
|
|
115
|
-
.filter(Boolean),
|
|
116
|
-
link,
|
|
117
|
-
reporter: reporter?.data?.attributes?.username ?? ''
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
export function getMissingReportInformation(report) {
|
|
122
|
-
const missing = [];
|
|
123
|
-
|
|
124
|
-
if (!report.severity?.rating) missing.push('severity rating');
|
|
125
|
-
if (!report.severity?.cvss_vector_string) missing.push('CVSS vector');
|
|
126
|
-
if (!report.severity?.weakness_id) missing.push('weakness ID');
|
|
127
|
-
if (!report.summary) missing.push('team summary');
|
|
128
|
-
if (!report.prURL) missing.push('PR URL');
|
|
129
|
-
if (!report.patchAuthors?.length) missing.push('patch authors');
|
|
130
|
-
if (!report.affectedVersions?.length) missing.push('affected versions');
|
|
131
|
-
|
|
132
|
-
return missing;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
export function groupMissingReportInformation(reports) {
|
|
136
|
-
const grouped = new Map();
|
|
137
|
-
|
|
138
|
-
for (const report of reports) {
|
|
139
|
-
for (const field of report.missing) {
|
|
140
|
-
const current = grouped.get(field) ?? [];
|
|
141
|
-
current.push(report);
|
|
142
|
-
grouped.set(field, current);
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
return Array.from(grouped.entries())
|
|
147
|
-
.map(([field, fieldReports]) => ({
|
|
148
|
-
field,
|
|
149
|
-
reports: fieldReports
|
|
150
|
-
}))
|
|
151
|
-
.sort((a, b) => b.reports.length - a.reports.length);
|
|
152
|
-
}
|
|
153
|
-
|
|
154
101
|
export default class PrepareSecurityRelease extends SecurityRelease {
|
|
155
102
|
title = 'Next Security Release';
|
|
156
103
|
|
|
157
104
|
async start() {
|
|
105
|
+
assertNewSecurityRelease(process.cwd());
|
|
158
106
|
const credentials = await auth({
|
|
159
107
|
github: true,
|
|
160
108
|
h1: true
|
|
@@ -172,14 +120,15 @@ export default class PrepareSecurityRelease extends SecurityRelease {
|
|
|
172
120
|
const content = await this.buildDescription(releaseDate);
|
|
173
121
|
if (createVulnerabilitiesJSON) {
|
|
174
122
|
const reportSelectionMode = await this.promptReportSelectionMode();
|
|
123
|
+
const candidates = await listSecurityReleaseCandidates(this.req);
|
|
175
124
|
if (reportSelectionMode === 'review') {
|
|
176
125
|
const showTriaged = await this.promptShowTriagedWithoutPR();
|
|
177
126
|
if (showTriaged) {
|
|
178
|
-
excludedReports = await this.showTriagedReportsWithoutPR();
|
|
127
|
+
excludedReports = await this.showTriagedReportsWithoutPR(candidates);
|
|
179
128
|
}
|
|
180
129
|
}
|
|
181
130
|
await this.startVulnerabilitiesJSONCreation(
|
|
182
|
-
releaseDate, content, excludedReports, reportSelectionMode);
|
|
131
|
+
releaseDate, content, excludedReports, reportSelectionMode, candidates);
|
|
183
132
|
}
|
|
184
133
|
|
|
185
134
|
this.cli.ok('Done!');
|
|
@@ -251,19 +200,25 @@ export default class PrepareSecurityRelease extends SecurityRelease {
|
|
|
251
200
|
releaseDate,
|
|
252
201
|
content,
|
|
253
202
|
excludedReports = [],
|
|
254
|
-
reportSelectionMode = 'review'
|
|
203
|
+
reportSelectionMode = 'review',
|
|
204
|
+
candidates
|
|
255
205
|
) {
|
|
256
|
-
|
|
257
|
-
await checkoutOnSecurityReleaseBranch(this.cli, this.repository);
|
|
206
|
+
assertNewSecurityRelease(process.cwd());
|
|
258
207
|
|
|
259
208
|
// choose the reports to include in the security release
|
|
260
209
|
const reports = reportSelectionMode === 'include-all'
|
|
261
|
-
? await this.includeAllTriagedReports(excludedReports)
|
|
262
|
-
: await this.chooseReports(excludedReports);
|
|
210
|
+
? await this.includeAllTriagedReports(excludedReports, candidates)
|
|
211
|
+
: await this.chooseReports(excludedReports, candidates);
|
|
263
212
|
const deps = await this.getDependencyUpdates();
|
|
213
|
+
const { release } = prepareSecurityRelease({ releaseDate, reports, dependencies: deps });
|
|
264
214
|
|
|
265
|
-
//
|
|
266
|
-
|
|
215
|
+
// Prepare all data before changing Git state. An existing branch may contain
|
|
216
|
+
// a draft that was not present on the branch where this command started.
|
|
217
|
+
await checkoutOnSecurityReleaseBranch(this.cli, this.repository);
|
|
218
|
+
assertNewSecurityRelease(process.cwd());
|
|
219
|
+
|
|
220
|
+
const filePath = await this.createVulnerabilitiesJSON(
|
|
221
|
+
release.reports, release.dependencies, release.releaseDate);
|
|
267
222
|
|
|
268
223
|
// review the vulnerabilities.json file
|
|
269
224
|
const review = await this.promptReviewVulnerabilitiesJSON();
|
|
@@ -366,11 +321,11 @@ export default class PrepareSecurityRelease extends SecurityRelease {
|
|
|
366
321
|
{ defaultAnswer: true });
|
|
367
322
|
}
|
|
368
323
|
|
|
369
|
-
async showTriagedReportsWithoutPR() {
|
|
324
|
+
async showTriagedReportsWithoutPR(candidates) {
|
|
370
325
|
this.cli.info('Fetching triaged reports without PR URL...');
|
|
371
|
-
const reports = await this.req
|
|
372
|
-
const reportsWithoutPR = reports.
|
|
373
|
-
(report) => !report
|
|
326
|
+
const reports = candidates ?? await listSecurityReleaseCandidates(this.req);
|
|
327
|
+
const reportsWithoutPR = reports.filter(
|
|
328
|
+
(report) => !getReportPRURL(report)
|
|
374
329
|
);
|
|
375
330
|
if (!reportsWithoutPR.length) {
|
|
376
331
|
this.cli.ok('All triaged reports have a PR URL.');
|
|
@@ -408,12 +363,12 @@ export default class PrepareSecurityRelease extends SecurityRelease {
|
|
|
408
363
|
return template;
|
|
409
364
|
}
|
|
410
365
|
|
|
411
|
-
async chooseReports(excludedReports = []) {
|
|
366
|
+
async chooseReports(excludedReports = [], candidates) {
|
|
412
367
|
this.cli.info('Getting triaged H1 reports...');
|
|
413
|
-
const reports = await this.req
|
|
368
|
+
const reports = candidates ?? await listSecurityReleaseCandidates(this.req);
|
|
414
369
|
const selectedReports = [];
|
|
415
370
|
|
|
416
|
-
for (const report of reports
|
|
371
|
+
for (const report of reports) {
|
|
417
372
|
if (excludedReports.includes(report.id)) continue;
|
|
418
373
|
const rep = await pickReport(report, { cli: this.cli, req: this.req });
|
|
419
374
|
if (!rep) continue;
|
|
@@ -422,14 +377,14 @@ export default class PrepareSecurityRelease extends SecurityRelease {
|
|
|
422
377
|
return selectedReports;
|
|
423
378
|
}
|
|
424
379
|
|
|
425
|
-
async includeAllTriagedReports(excludedReports = []) {
|
|
380
|
+
async includeAllTriagedReports(excludedReports = [], candidates) {
|
|
426
381
|
this.cli.info('Getting triaged H1 reports...');
|
|
427
|
-
const reports = await this.req
|
|
382
|
+
const reports = candidates ?? await listSecurityReleaseCandidates(this.req);
|
|
428
383
|
const supportedVersions = await getSupportedVersions();
|
|
429
384
|
const selectedReports = [];
|
|
430
385
|
const missingInformation = [];
|
|
431
386
|
|
|
432
|
-
for (const report of reports
|
|
387
|
+
for (const report of reports) {
|
|
433
388
|
if (excludedReports.includes(report.id)) continue;
|
|
434
389
|
|
|
435
390
|
const reportData = await this.buildIncludedTriagedReport(
|
|
@@ -497,29 +452,23 @@ export default class PrepareSecurityRelease extends SecurityRelease {
|
|
|
497
452
|
}
|
|
498
453
|
|
|
499
454
|
async createVulnerabilitiesJSON(reports, dependencies, releaseDate) {
|
|
500
|
-
|
|
501
|
-
const
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
dependencies
|
|
505
|
-
}, null, 2) + '\n';
|
|
506
|
-
|
|
507
|
-
const folderPath = path.resolve(NEXT_SECURITY_RELEASE_FOLDER);
|
|
508
|
-
const fullPath = path.join(folderPath, 'vulnerabilities.json');
|
|
455
|
+
const { release } = prepareSecurityRelease({ releaseDate, reports, dependencies });
|
|
456
|
+
const directory = process.cwd();
|
|
457
|
+
const fullPath = getSecurityReleaseDraftPath(directory);
|
|
458
|
+
assertNewSecurityRelease(directory);
|
|
509
459
|
await confirmSecurityStep(
|
|
510
460
|
this.cli,
|
|
511
|
-
`create directory \`${
|
|
461
|
+
`create directory \`${path.dirname(fullPath)}\``,
|
|
512
462
|
'This creates the security release folder if it does not already exist.'
|
|
513
463
|
);
|
|
514
|
-
await
|
|
515
|
-
await writeSecurityFile(
|
|
464
|
+
await confirmSecurityStep(
|
|
516
465
|
this.cli,
|
|
517
|
-
fullPath
|
|
518
|
-
fileContent,
|
|
466
|
+
`write \`${fullPath}\``,
|
|
519
467
|
'This creates vulnerabilities.json for the next security release.'
|
|
520
468
|
);
|
|
469
|
+
this.cli.startSpinner('Creating vulnerabilities.json...');
|
|
470
|
+
writeSecurityReleaseDraft(directory, release);
|
|
521
471
|
this.cli.stopSpinner(`Created ${fullPath}`);
|
|
522
|
-
|
|
523
472
|
return fullPath;
|
|
524
473
|
}
|
|
525
474
|
|
package/lib/request.js
CHANGED
|
@@ -58,6 +58,17 @@ export default class Request {
|
|
|
58
58
|
return res.text();
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
async * stream(url, options = {}) {
|
|
62
|
+
const res = await this.fetch(url, options);
|
|
63
|
+
if (!res.ok) {
|
|
64
|
+
await res.body?.cancel();
|
|
65
|
+
throw new Error(`Unable to stream ${url}: ${res.status} ${res.statusText}`);
|
|
66
|
+
}
|
|
67
|
+
// Undici decompresses HTTP content encodings as it reads. Returning early
|
|
68
|
+
// from this iterator also cancels the response body and its HTTP download.
|
|
69
|
+
yield * res.body;
|
|
70
|
+
}
|
|
71
|
+
|
|
61
72
|
async json(url, options = {}) {
|
|
62
73
|
options.headers = options.headers || {};
|
|
63
74
|
const text = await this.text(url, options);
|
|
@@ -106,7 +117,10 @@ export default class Request {
|
|
|
106
117
|
const url =
|
|
107
118
|
`/repos/${owner}/${repo}/pulls/${prid}/files?per_page=100&page=${page}`;
|
|
108
119
|
const batch = await this.json(url);
|
|
109
|
-
if (!Array.isArray(batch)
|
|
120
|
+
if (!Array.isArray(batch)) {
|
|
121
|
+
throw new Error('Unable to retrieve pull request files');
|
|
122
|
+
}
|
|
123
|
+
if (batch.length === 0) {
|
|
110
124
|
break;
|
|
111
125
|
}
|
|
112
126
|
yield * batch;
|
|
@@ -193,22 +207,52 @@ export default class Request {
|
|
|
193
207
|
}
|
|
194
208
|
|
|
195
209
|
async getTriagedReports() {
|
|
196
|
-
|
|
210
|
+
let url = 'https://api.hackerone.com/v1/reports?filter[program][]=nodejs&filter[state][]=triaged';
|
|
197
211
|
const options = {
|
|
198
212
|
method: 'GET',
|
|
213
|
+
redirect: 'error',
|
|
199
214
|
headers: {
|
|
200
215
|
Authorization: `Basic ${this.credentials.h1}`,
|
|
201
216
|
'User-Agent': 'node-core-utils',
|
|
202
217
|
Accept: 'application/json'
|
|
203
218
|
}
|
|
204
219
|
};
|
|
205
|
-
const
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
220
|
+
const reports = [];
|
|
221
|
+
const reportIds = new Set();
|
|
222
|
+
const pages = new Set();
|
|
223
|
+
let result;
|
|
224
|
+
|
|
225
|
+
while (url) {
|
|
226
|
+
const pageUrl = new URL(url);
|
|
227
|
+
if (pageUrl.origin !== 'https://api.hackerone.com' ||
|
|
228
|
+
pageUrl.pathname !== '/v1/reports' || pageUrl.username || pageUrl.password) {
|
|
229
|
+
throw new Error('Invalid HackerOne reports pagination URL');
|
|
230
|
+
}
|
|
231
|
+
if (pages.has(pageUrl.href)) {
|
|
232
|
+
throw new Error('Repeated HackerOne reports pagination URL');
|
|
233
|
+
}
|
|
234
|
+
pages.add(pageUrl.href);
|
|
235
|
+
|
|
236
|
+
result = await this.json(url, options);
|
|
237
|
+
if (result?.errors?.length) {
|
|
238
|
+
throw new Error(
|
|
239
|
+
`Request to fetch triaged reports failed with: ${JSON.stringify(result.errors)}`
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
if (!Array.isArray(result?.data)) {
|
|
243
|
+
throw new Error('Invalid HackerOne reports response: expected a data array');
|
|
244
|
+
}
|
|
245
|
+
for (const report of result.data) {
|
|
246
|
+
if (reportIds.has(report.id)) continue;
|
|
247
|
+
reportIds.add(report.id);
|
|
248
|
+
reports.push(report);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const next = result.links?.next;
|
|
252
|
+
url = next ? new URL(next, pageUrl).href : null;
|
|
210
253
|
}
|
|
211
|
-
|
|
254
|
+
|
|
255
|
+
return { ...result, data: reports };
|
|
212
256
|
}
|
|
213
257
|
|
|
214
258
|
async getPrograms() {
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
|
|
4
|
+
import { NEXT_SECURITY_RELEASE_FOLDER } from './security-release.js';
|
|
5
|
+
import { prepareSecurityRelease } from './preparation.js';
|
|
6
|
+
|
|
7
|
+
export function getSecurityReleaseDraftPath(directory) {
|
|
8
|
+
if (typeof directory !== 'string' || !directory.trim()) {
|
|
9
|
+
throw new Error('Security release repository directory is required');
|
|
10
|
+
}
|
|
11
|
+
return path.resolve(directory, NEXT_SECURITY_RELEASE_FOLDER, 'vulnerabilities.json');
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function assertNewSecurityRelease(directory) {
|
|
15
|
+
const file = getSecurityReleaseDraftPath(directory);
|
|
16
|
+
if (fs.existsSync(file)) {
|
|
17
|
+
throw new Error(
|
|
18
|
+
`Security release draft already exists: ${file}. ` +
|
|
19
|
+
'Use --sync, --add-report, or --remove-report to update the existing release.'
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Local persistence only. The caller handles review, Git, and publication.
|
|
25
|
+
export function writeSecurityReleaseDraft(directory, draft) {
|
|
26
|
+
const { release } = prepareSecurityRelease(draft);
|
|
27
|
+
const file = getSecurityReleaseDraftPath(directory);
|
|
28
|
+
assertNewSecurityRelease(directory);
|
|
29
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
30
|
+
fs.writeFileSync(file, JSON.stringify(release, null, 2) + '\n', { flag: 'wx' });
|
|
31
|
+
return file;
|
|
32
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { getReportSeverity, getSummary } from './security-release.js';
|
|
2
|
+
|
|
3
|
+
// Request.getTriagedReports rejects failed pages rather than returning partial data.
|
|
4
|
+
// Selection, metadata edits, and all mutations belong to the caller.
|
|
5
|
+
export async function listSecurityReleaseCandidates(request) {
|
|
6
|
+
const { data } = await request.getTriagedReports();
|
|
7
|
+
return data;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export function getReportPRURL(report) {
|
|
11
|
+
const customFieldValues = report.relationships.custom_field_values?.data ?? [];
|
|
12
|
+
return customFieldValues[0]?.attributes?.value ?? '';
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function buildIncludedTriagedReport(report, options = {}) {
|
|
16
|
+
const {
|
|
17
|
+
affectedVersions = '',
|
|
18
|
+
patchAuthors = [],
|
|
19
|
+
prURL = getReportPRURL(report)
|
|
20
|
+
} = options;
|
|
21
|
+
const {
|
|
22
|
+
id,
|
|
23
|
+
attributes: { title, cve_ids = [] },
|
|
24
|
+
relationships: { reporter }
|
|
25
|
+
} = report;
|
|
26
|
+
const link = `https://hackerone.com/reports/${id}`;
|
|
27
|
+
const summaryContent = getSummary(report);
|
|
28
|
+
|
|
29
|
+
return {
|
|
30
|
+
id,
|
|
31
|
+
title,
|
|
32
|
+
cveIds: cve_ids,
|
|
33
|
+
severity: getReportSeverity(report),
|
|
34
|
+
summary: summaryContent ?? '',
|
|
35
|
+
patchAuthors,
|
|
36
|
+
prURL,
|
|
37
|
+
affectedVersions: affectedVersions
|
|
38
|
+
.split(',')
|
|
39
|
+
.map((v) => v.replace('v', '').trim())
|
|
40
|
+
.filter(Boolean),
|
|
41
|
+
link,
|
|
42
|
+
reporter: reporter?.data?.attributes?.username ?? ''
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function getMissingReportInformation(report) {
|
|
47
|
+
const missing = [];
|
|
48
|
+
|
|
49
|
+
if (!report.severity?.rating) missing.push('severity rating');
|
|
50
|
+
if (!report.severity?.cvss_vector_string) missing.push('CVSS vector');
|
|
51
|
+
if (!report.severity?.weakness_id) missing.push('weakness ID');
|
|
52
|
+
if (!report.summary) missing.push('team summary');
|
|
53
|
+
if (!report.prURL) missing.push('PR URL');
|
|
54
|
+
if (!report.patchAuthors?.length) missing.push('patch authors');
|
|
55
|
+
if (!Object.keys(report.affectedVersions ?? {}).length) missing.push('affected versions');
|
|
56
|
+
|
|
57
|
+
return missing;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function groupMissingReportInformation(reports) {
|
|
61
|
+
const grouped = new Map();
|
|
62
|
+
|
|
63
|
+
for (const report of reports) {
|
|
64
|
+
for (const field of report.missing) {
|
|
65
|
+
const current = grouped.get(field) ?? [];
|
|
66
|
+
current.push(report);
|
|
67
|
+
grouped.set(field, current);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return Array.from(grouped.entries())
|
|
72
|
+
.map(([field, fieldReports]) => ({
|
|
73
|
+
field,
|
|
74
|
+
reports: fieldReports
|
|
75
|
+
}))
|
|
76
|
+
.sort((a, b) => b.reports.length - a.reports.length);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function normalizeReleaseDate(value) {
|
|
80
|
+
if (value === 'TBD') return value;
|
|
81
|
+
if (typeof value !== 'string' || !/^\d{4}([/-])\d{2}\1\d{2}$/.test(value)) {
|
|
82
|
+
throw new Error('Release date must be YYYY-MM-DD, YYYY/MM/DD, or TBD');
|
|
83
|
+
}
|
|
84
|
+
const date = value.replaceAll('/', '-');
|
|
85
|
+
const timestamp = Date.parse(`${date}T00:00:00Z`);
|
|
86
|
+
if (!Number.isFinite(timestamp) || new Date(timestamp).toISOString().slice(0, 10) !== date) {
|
|
87
|
+
throw new Error('Invalid release date');
|
|
88
|
+
}
|
|
89
|
+
return date;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function normalizeAffectedVersions(value, fallbackPR = '') {
|
|
93
|
+
if (value == null || value === '') return {};
|
|
94
|
+
if (typeof fallbackPR !== 'string') throw new Error('PR URL must be a string');
|
|
95
|
+
let entries;
|
|
96
|
+
if (typeof value === 'string' || Array.isArray(value)) {
|
|
97
|
+
const lines = typeof value === 'string' ? value.split(',') : value;
|
|
98
|
+
entries = lines.map((line) => [line, fallbackPR]);
|
|
99
|
+
} else if (typeof value === 'object') {
|
|
100
|
+
entries = Object.entries(value);
|
|
101
|
+
} else {
|
|
102
|
+
throw new Error('Affected versions must be release lines or a PR map');
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const result = new Map();
|
|
106
|
+
for (const [line, prURL] of entries) {
|
|
107
|
+
if (typeof line !== 'string') throw new Error('Release line must be a string');
|
|
108
|
+
const trimmed = line.trim();
|
|
109
|
+
const match = /^v?(\d+)(?:\.x)?$/.exec(trimmed);
|
|
110
|
+
const normalized = trimmed === 'main' ? 'main' : match && `${match[1]}.x`;
|
|
111
|
+
if (!normalized) throw new Error(`Invalid release line: ${line}`);
|
|
112
|
+
if (typeof prURL !== 'string') throw new Error(`PR URL for ${normalized} must be a string`);
|
|
113
|
+
if (result.has(normalized) && result.get(normalized) !== prURL) {
|
|
114
|
+
throw new Error(`Conflicting PR URLs for ${normalized}`);
|
|
115
|
+
}
|
|
116
|
+
result.set(normalized, prURL);
|
|
117
|
+
}
|
|
118
|
+
return Object.fromEntries(result);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// This function only builds data. It does not fetch reports, prompt, write files,
|
|
122
|
+
// change branches, commit, push, or create a pull request.
|
|
123
|
+
export function prepareSecurityRelease({ releaseDate, reports, dependencies = {} }) {
|
|
124
|
+
const date = normalizeReleaseDate(releaseDate);
|
|
125
|
+
if (!Array.isArray(reports)) throw new Error('Reports must be an array');
|
|
126
|
+
if (!dependencies || typeof dependencies !== 'object' || Array.isArray(dependencies)) {
|
|
127
|
+
throw new Error('Dependencies must be an object');
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const ids = new Set();
|
|
131
|
+
const selected = reports.map((report) => {
|
|
132
|
+
if (!report || typeof report.id !== 'string' || !/^\d+$/.test(report.id)) {
|
|
133
|
+
throw new Error('Report ID must be a numeric string');
|
|
134
|
+
}
|
|
135
|
+
if (ids.has(report.id)) throw new Error(`Duplicate report ID: ${report.id}`);
|
|
136
|
+
ids.add(report.id);
|
|
137
|
+
return {
|
|
138
|
+
...structuredClone(report),
|
|
139
|
+
// A canonical patch is not evidence of a backport for every affected line.
|
|
140
|
+
affectedVersions: normalizeAffectedVersions(report.affectedVersions)
|
|
141
|
+
};
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
const deps = Object.fromEntries(Object.entries(dependencies).map(([name, updates]) => {
|
|
145
|
+
const normalizeUpdate = (update) => {
|
|
146
|
+
if (!update || typeof update !== 'object' || Array.isArray(update)) {
|
|
147
|
+
throw new Error(`Invalid dependency update: ${name}`);
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
...structuredClone(update),
|
|
151
|
+
affectedVersions: normalizeAffectedVersions(update.affectedVersions, update.prURL ?? '')
|
|
152
|
+
};
|
|
153
|
+
};
|
|
154
|
+
const normalized = Array.isArray(updates)
|
|
155
|
+
? updates.map(normalizeUpdate)
|
|
156
|
+
: normalizeUpdate(updates);
|
|
157
|
+
return [name, normalized];
|
|
158
|
+
}));
|
|
159
|
+
|
|
160
|
+
return {
|
|
161
|
+
release: { releaseDate: date, reports: selected, dependencies: deps },
|
|
162
|
+
missingInformation: selected.flatMap((report) => {
|
|
163
|
+
const missing = getMissingReportInformation(report);
|
|
164
|
+
if (!missing.length) return [];
|
|
165
|
+
return [{ id: report.id, title: report.title, link: report.link, missing }];
|
|
166
|
+
})
|
|
167
|
+
};
|
|
168
|
+
}
|
|
@@ -79,10 +79,24 @@ export async function checkoutOnSecurityReleaseBranch(cli, repository) {
|
|
|
79
79
|
cli.info(`Current branch: ${currentBranch} `);
|
|
80
80
|
|
|
81
81
|
if (currentBranch !== NEXT_SECURITY_RELEASE_BRANCH) {
|
|
82
|
+
const refs = new Set(runSync('git', [
|
|
83
|
+
'for-each-ref', '--format=%(refname)',
|
|
84
|
+
`refs/heads/${NEXT_SECURITY_RELEASE_BRANCH}`,
|
|
85
|
+
`refs/remotes/origin/${NEXT_SECURITY_RELEASE_BRANCH}`
|
|
86
|
+
]).trim().split('\n'));
|
|
87
|
+
let args;
|
|
88
|
+
if (refs.has(`refs/heads/${NEXT_SECURITY_RELEASE_BRANCH}`)) {
|
|
89
|
+
args = ['checkout', NEXT_SECURITY_RELEASE_BRANCH];
|
|
90
|
+
} else if (refs.has(`refs/remotes/origin/${NEXT_SECURITY_RELEASE_BRANCH}`)) {
|
|
91
|
+
args = ['checkout', '-b', NEXT_SECURITY_RELEASE_BRANCH,
|
|
92
|
+
'--track', `origin/${NEXT_SECURITY_RELEASE_BRANCH}`];
|
|
93
|
+
} else {
|
|
94
|
+
args = ['checkout', '-b', NEXT_SECURITY_RELEASE_BRANCH];
|
|
95
|
+
}
|
|
82
96
|
await runSecurityGitCommand(
|
|
83
97
|
cli,
|
|
84
|
-
|
|
85
|
-
`This
|
|
98
|
+
args,
|
|
99
|
+
`This switches to ${NEXT_SECURITY_RELEASE_BRANCH} without resetting an existing branch.`
|
|
86
100
|
);
|
|
87
101
|
cli.ok(`Checkout on branch: ${NEXT_SECURITY_RELEASE_BRANCH} `);
|
|
88
102
|
}
|
|
@@ -95,6 +109,22 @@ export async function commitAndPushVulnerabilitiesJSON(
|
|
|
95
109
|
) {
|
|
96
110
|
await checkRemote(cli, repository);
|
|
97
111
|
|
|
112
|
+
const files = (Array.isArray(filePath) ? filePath : [filePath])
|
|
113
|
+
.map((file) => path.resolve(file));
|
|
114
|
+
const stagedFiles = runSync('git', ['diff', '--cached', '--name-only', '-z'])
|
|
115
|
+
.split('\0').filter(Boolean);
|
|
116
|
+
const unrelated = stagedFiles.filter((file) => {
|
|
117
|
+
const absolute = path.resolve(file);
|
|
118
|
+
return !files.some((target) =>
|
|
119
|
+
absolute === target || absolute.startsWith(`${target}${path.sep}`));
|
|
120
|
+
});
|
|
121
|
+
if (unrelated.length) {
|
|
122
|
+
throw new Error(
|
|
123
|
+
'Unrelated staged changes would be included in the security release commit: ' +
|
|
124
|
+
unrelated.join(', ') + '. Commit or unstage them before continuing.'
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
|
|
98
128
|
if (Array.isArray(filePath)) {
|
|
99
129
|
for (const currentPath of filePath) {
|
|
100
130
|
runSync('git', ['add', currentPath]);
|