@node-core/utils 7.2.2 → 7.3.0

package/lib/ci/resume_ci.js

@@ -0,0 +1,150 @@
+import { JobParser, CI_TYPES_KEYS } from './ci_type_parser.js';
+import { CI_CRUMB_URL } from './run_ci.js';
+import { PRBuild } from './build-types/pr_build.js';
+import { getPrURL } from '../links.js';
+import { debuglog } from '../verbosity.js';
+import { FailureFileScanner } from './failure_file_scanner.js';
+
+export class ResumePRJob {
+  constructor(cli, request, owner, repo, prid) {
+    this.cli = cli;
+    this.request = request;
+    this.owner = owner;
+    this.repo = repo;
+    this.prid = prid;
+  }
+
+  async checkFailures(jobid) {
+    const { cli, request, owner, repo, prid } = this;
+    const filenames = new Set();
+    for await (const file of request.getPullRequestFiles({ owner, repo, prid })) {
+      for (const filename of [file.filename, file.previous_filename].filter(Boolean)) {
+        filenames.add(filename);
+      }
+    }
+    const overlaps = new Set();
+    const buildRequest = Object.create(request);
+    buildRequest.json = async(...args) => {
+      const data = await request.json(...args);
+      // Inspect failed descendants even when their parent job was aborted.
+      return data?.result === 'ABORTED' ? { ...data, result: 'FAILURE' } : data;
+    };
+    // Reuse the build traversal, but consume console responses directly instead
+    // of handing whole logs to its summary parser. Serialize downloads so a
+    // match cancels this response and prevents queued logs from being opened.
+    let pending = Promise.resolve();
+    buildRequest.text = (url) => {
+      const scan = pending.then(async() => {
+        if (!filenames.size || overlaps.size) return '';
+        const scanner = new FailureFileScanner(filenames);
+        const match = await scanner.scan(request.stream(url));
+        if (match) overlaps.add(match);
+        return '';
+      });
+      pending = scan.catch(debuglog);
+      return scan;
+    };
+    const build = new PRBuild(cli, buildRequest, jobid, Infinity);
+    try {
+      await build.getResults();
+    } catch (err) {
+      debuglog(err);
+    }
+    await pending;
+    if (overlaps.size) {
+      for (const filename of [...overlaps].sort()) {
+        cli.error(filename);
+      }
+      return false;
+    }
+    return true;
+  }
+
+  async resume() {
+    const { cli, request, prid } = this;
+    let crumb;
+    cli.startSpinner('Validating Jenkins credentials');
+    try {
+      ({ crumb } = await request.json(CI_CRUMB_URL));
+      if (!crumb) {
+        throw new Error('Missing Jenkins crumb');
+      }
+    } catch (err) {
+      debuglog(err);
+      cli.stopSpinner('Jenkins credentials invalid', cli.SPINNER_STATUS.FAILED);
+      return false;
+    }
+    cli.stopSpinner('Jenkins credentials valid');
+
+    try {
+      cli.startSpinner(`Looking for CI runs for pull request ${prid}`);
+      const parser = await JobParser.fromPR(getPrURL(this), cli, request);
+      const job = parser.parse().get(CI_TYPES_KEYS.PR);
+      if (!job) {
+        cli.stopSpinner(`No CI run detected from pull request ${prid}`,
+          cli.SPINNER_STATUS.FAILED);
+        return false;
+      }
+      cli.stopSpinner(`Found PR CI job ${job.jobid}`);
+
+      const build = new PRBuild(cli, request, job.jobid, undefined,
+        'result,building,actions[_class,parameters[name,value]]');
+      const { result, building, actions = [] } = await build.getBuildData();
+      if (building || (result !== 'FAILURE' && result !== 'ABORTED')) {
+        const status = building ? 'RUNNING' : result ?? 'RUNNING';
+        cli.error(`CI job ${job.jobid} is in status ${status}, skipping resume`);
+        return false;
+      }
+      if (!actions.some(action =>
+        action._class === 'com.tikal.jenkins.plugins.multijob.MultiJobResumeBuild')) {
+        cli.error(`CI job ${job.jobid} is not resumable`);
+        return false;
+      }
+
+      const approvedSHAs = new Set(actions.flatMap(action => action.parameters ?? [])
+        .filter(parameter => parameter.name === 'COMMIT_SHA_CHECK')
+        .map(parameter => parameter.value));
+      const [approvedSHA] = approvedSHAs;
+      if (approvedSHAs.size !== 1 || typeof approvedSHA !== 'string' || !approvedSHA) {
+        cli.error(`Refusing to resume CI job ${job.jobid}: cannot determine its approved commit`);
+        return false;
+      }
+
+      cli.startSpinner('Checking failures against changed PR files');
+      if (!(await this.checkFailures(job.jobid))) {
+        cli.stopSpinner('Refusing to resume CI: failures reference files changed by this PR',
+          cli.SPINNER_STATUS.FAILED);
+        return false;
+      }
+      cli.stopSpinner('No changed PR files found in available failure details');
+
+      // Read HEAD after inspecting failures so the comparison is fresh when resuming.
+      const pr = await request.getPullRequest(getPrURL(this));
+      if (pr.head?.sha !== approvedSHA) {
+        cli.error(`Refusing to resume CI job ${job.jobid}: ` +
+          'its approved commit does not match the current PR HEAD');
+        return false;
+      }
+
+      cli.startSpinner(`Resuming PR CI job ${job.jobid}`);
+      const response = await request.fetch(`${build.jobUrl}resume`, {
+        method: 'POST',
+        headers: {
+          'Jenkins-Crumb': crumb
+        }
+      });
+      if (response.status !== 200) {
+        cli.stopSpinner(
+          `Failed to resume PR CI: ${response.status} ${response.statusText}`,
+          cli.SPINNER_STATUS.FAILED);
+        return false;
+      }
+      cli.stopSpinner('PR CI job successfully resumed');
+    } catch (err) {
+      debuglog(err);
+      cli.stopSpinner('Failed to resume CI', cli.SPINNER_STATUS.FAILED);
+      return false;
+    }
+    return true;
+  }
+}

package/lib/ci/run_ci.js

@@ -94,10 +94,20 @@ export class RunPRJob {
     if (checkForDuplicates) {
       await this.prData.getComments();
       const { jobid, link } = new JobParser(this.prData.comments).parse().get('PR') ?? {};
-      const { actions } = jobid
-        ? (await new PRBuild(cli, request, jobid, undefined, 'actions[parameters[name,value]]')
-            .getBuildData())
-        : {};
+      let actions;
+      try {
+        ({ actions } = jobid
+          ? (await new PRBuild(cli, request, jobid, undefined, 'actions[parameters[name,value]]')
+              .getBuildData())
+          : {});
+      } catch (err) {
+        // Jenkins may no longer have the build (e.g. the record was lost) or
+        // may answer with an HTML error page instead of JSON. Do not let that
+        // block starting a new CI run.
+        cli.stopSpinner(`Could not query existing CI run ${link}: ${err.message}`,
+          cli.SPINNER_STATUS.WARN);
+        cli.warn('Skipping the duplicate CI check');
+      }
       const { parameters } = actions?.find(a => 'parameters' in a) ?? {};
       if (parameters?.find(c => c.name === 'COMMIT_SHA_CHECK')?.value === certifySafe) {
         cli.info('Existing CI run found: ' + link);

package/lib/cli.js

@@ -154,7 +154,13 @@ export default class CLI {
       case INFO:
         symbol = info;
     }
-    this.spinner.stop(`${symbol}  ${rawText}`);
+    const text = `${symbol}  ${rawText}`;
+    if (this.spinner.isSpinning) {
+      this.spinner.stop(text);
+    } else {
+      // A nested operation may already have stopped the shared spinner.
+      this.log(text);
+    }
   }
 
   write(text) {

package/lib/prepare_security.js

@@ -1,24 +1,40 @@
-import fs from 'node:fs';
 import path from 'node:path';
 import auth from './auth.js';
 import Request from './request.js';
 import { parsePRFromURL } from './links.js';
+import {
+  assertNewSecurityRelease,
+  getSecurityReleaseDraftPath,
+  writeSecurityReleaseDraft
+} from './security-release/draft.js';
+import {
+  buildIncludedTriagedReport,
+  getReportPRURL,
+  getMissingReportInformation,
+  groupMissingReportInformation,
+  listSecurityReleaseCandidates,
+  prepareSecurityRelease
+} from './security-release/preparation.js';
+
 import {
   NEXT_SECURITY_RELEASE_BRANCH,
-  NEXT_SECURITY_RELEASE_FOLDER,
   checkoutOnSecurityReleaseBranch,
   commitAndPushVulnerabilitiesJSON,
   validateDate,
   promptDependencies,
   getSupportedVersions,
   getReportSeverity,
-  getSummary,
   pickReport,
   confirmSecurityStep,
-  writeSecurityFile,
   SecurityRelease
 } from './security-release/security-release.js';
 
+export {
+  buildIncludedTriagedReport,
+  getMissingReportInformation,
+  groupMissingReportInformation
+} from './security-release/preparation.js';
+
 function relativeDate(date) {
   const days = Math.floor((Date.now() - date) / (1000 * 60 * 60 * 24));
   if (days < 30) return days === 1 ? '1 day ago' : `${days} days ago`;
@@ -82,79 +98,11 @@ export function getNextTuesdayReleaseDateChoices(fromDate = new Date(), count =
   return choices;
 }
 
-function getReportPRURL(report) {
-  const customFieldValues = report.relationships.custom_field_values?.data ?? [];
-  return customFieldValues[0]?.attributes?.value ?? '';
-}
-
-export function buildIncludedTriagedReport(report, options = {}) {
-  const {
-    affectedVersions = '',
-    patchAuthors = [],
-    prURL = getReportPRURL(report)
-  } = options;
-  const {
-    id,
-    attributes: { title, cve_ids = [] },
-    relationships: { reporter }
-  } = report;
-  const link = `https://hackerone.com/reports/${id}`;
-  const summaryContent = getSummary(report);
-
-  return {
-    id,
-    title,
-    cveIds: cve_ids,
-    severity: getReportSeverity(report),
-    summary: summaryContent ?? '',
-    patchAuthors,
-    prURL,
-    affectedVersions: affectedVersions
-      .split(',')
-      .map((v) => v.replace('v', '').trim())
-      .filter(Boolean),
-    link,
-    reporter: reporter?.data?.attributes?.username ?? ''
-  };
-}
-
-export function getMissingReportInformation(report) {
-  const missing = [];
-
-  if (!report.severity?.rating) missing.push('severity rating');
-  if (!report.severity?.cvss_vector_string) missing.push('CVSS vector');
-  if (!report.severity?.weakness_id) missing.push('weakness ID');
-  if (!report.summary) missing.push('team summary');
-  if (!report.prURL) missing.push('PR URL');
-  if (!report.patchAuthors?.length) missing.push('patch authors');
-  if (!report.affectedVersions?.length) missing.push('affected versions');
-
-  return missing;
-}
-
-export function groupMissingReportInformation(reports) {
-  const grouped = new Map();
-
-  for (const report of reports) {
-    for (const field of report.missing) {
-      const current = grouped.get(field) ?? [];
-      current.push(report);
-      grouped.set(field, current);
-    }
-  }
-
-  return Array.from(grouped.entries())
-    .map(([field, fieldReports]) => ({
-      field,
-      reports: fieldReports
-    }))
-    .sort((a, b) => b.reports.length - a.reports.length);
-}
-
 export default class PrepareSecurityRelease extends SecurityRelease {
   title = 'Next Security Release';
 
   async start() {
+    assertNewSecurityRelease(process.cwd());
     const credentials = await auth({
       github: true,
       h1: true
@@ -172,14 +120,15 @@ export default class PrepareSecurityRelease extends SecurityRelease {
     const content = await this.buildDescription(releaseDate);
     if (createVulnerabilitiesJSON) {
       const reportSelectionMode = await this.promptReportSelectionMode();
+      const candidates = await listSecurityReleaseCandidates(this.req);
       if (reportSelectionMode === 'review') {
         const showTriaged = await this.promptShowTriagedWithoutPR();
         if (showTriaged) {
-          excludedReports = await this.showTriagedReportsWithoutPR();
+          excludedReports = await this.showTriagedReportsWithoutPR(candidates);
         }
       }
       await this.startVulnerabilitiesJSONCreation(
-        releaseDate, content, excludedReports, reportSelectionMode);
+        releaseDate, content, excludedReports, reportSelectionMode, candidates);
     }
 
     this.cli.ok('Done!');
@@ -251,19 +200,25 @@ export default class PrepareSecurityRelease extends SecurityRelease {
     releaseDate,
     content,
     excludedReports = [],
-    reportSelectionMode = 'review'
+    reportSelectionMode = 'review',
+    candidates
   ) {
-    // checkout on the next-security-release branch
-    await checkoutOnSecurityReleaseBranch(this.cli, this.repository);
+    assertNewSecurityRelease(process.cwd());
 
     // choose the reports to include in the security release
     const reports = reportSelectionMode === 'include-all'
-      ? await this.includeAllTriagedReports(excludedReports)
-      : await this.chooseReports(excludedReports);
+      ? await this.includeAllTriagedReports(excludedReports, candidates)
+      : await this.chooseReports(excludedReports, candidates);
     const deps = await this.getDependencyUpdates();
+    const { release } = prepareSecurityRelease({ releaseDate, reports, dependencies: deps });
 
-    // create the vulnerabilities.json file in the security-release repo
-    const filePath = await this.createVulnerabilitiesJSON(reports, deps, releaseDate);
+    // Prepare all data before changing Git state. An existing branch may contain
+    // a draft that was not present on the branch where this command started.
+    await checkoutOnSecurityReleaseBranch(this.cli, this.repository);
+    assertNewSecurityRelease(process.cwd());
+
+    const filePath = await this.createVulnerabilitiesJSON(
+      release.reports, release.dependencies, release.releaseDate);
 
     // review the vulnerabilities.json file
     const review = await this.promptReviewVulnerabilitiesJSON();
@@ -366,11 +321,11 @@ export default class PrepareSecurityRelease extends SecurityRelease {
       { defaultAnswer: true });
   }
 
-  async showTriagedReportsWithoutPR() {
+  async showTriagedReportsWithoutPR(candidates) {
     this.cli.info('Fetching triaged reports without PR URL...');
-    const reports = await this.req.getTriagedReports();
-    const reportsWithoutPR = reports.data.filter(
-      (report) => !report.relationships.custom_field_values.data.length
+    const reports = candidates ?? await listSecurityReleaseCandidates(this.req);
+    const reportsWithoutPR = reports.filter(
+      (report) => !getReportPRURL(report)
     );
     if (!reportsWithoutPR.length) {
       this.cli.ok('All triaged reports have a PR URL.');
@@ -408,12 +363,12 @@ export default class PrepareSecurityRelease extends SecurityRelease {
     return template;
   }
 
-  async chooseReports(excludedReports = []) {
+  async chooseReports(excludedReports = [], candidates) {
     this.cli.info('Getting triaged H1 reports...');
-    const reports = await this.req.getTriagedReports();
+    const reports = candidates ?? await listSecurityReleaseCandidates(this.req);
     const selectedReports = [];
 
-    for (const report of reports.data) {
+    for (const report of reports) {
       if (excludedReports.includes(report.id)) continue;
       const rep = await pickReport(report, { cli: this.cli, req: this.req });
       if (!rep) continue;
@@ -422,14 +377,14 @@ export default class PrepareSecurityRelease extends SecurityRelease {
     return selectedReports;
   }
 
-  async includeAllTriagedReports(excludedReports = []) {
+  async includeAllTriagedReports(excludedReports = [], candidates) {
     this.cli.info('Getting triaged H1 reports...');
-    const reports = await this.req.getTriagedReports();
+    const reports = candidates ?? await listSecurityReleaseCandidates(this.req);
     const supportedVersions = await getSupportedVersions();
     const selectedReports = [];
     const missingInformation = [];
 
-    for (const report of reports.data) {
+    for (const report of reports) {
       if (excludedReports.includes(report.id)) continue;
 
       const reportData = await this.buildIncludedTriagedReport(
@@ -497,29 +452,23 @@ export default class PrepareSecurityRelease extends SecurityRelease {
   }
 
   async createVulnerabilitiesJSON(reports, dependencies, releaseDate) {
-    this.cli.startSpinner('Creating vulnerabilities.json...');
-    const fileContent = JSON.stringify({
-      releaseDate,
-      reports,
-      dependencies
-    }, null, 2) + '\n';
-
-    const folderPath = path.resolve(NEXT_SECURITY_RELEASE_FOLDER);
-    const fullPath = path.join(folderPath, 'vulnerabilities.json');
+    const { release } = prepareSecurityRelease({ releaseDate, reports, dependencies });
+    const directory = process.cwd();
+    const fullPath = getSecurityReleaseDraftPath(directory);
+    assertNewSecurityRelease(directory);
     await confirmSecurityStep(
       this.cli,
-      `create directory \`${folderPath}\``,
+      `create directory \`${path.dirname(fullPath)}\``,
       'This creates the security release folder if it does not already exist.'
     );
-    await fs.promises.mkdir(folderPath, { recursive: true });
-    await writeSecurityFile(
+    await confirmSecurityStep(
       this.cli,
-      fullPath,
-      fileContent,
+      `write \`${fullPath}\``,
       'This creates vulnerabilities.json for the next security release.'
     );
+    this.cli.startSpinner('Creating vulnerabilities.json...');
+    writeSecurityReleaseDraft(directory, release);
     this.cli.stopSpinner(`Created ${fullPath}`);
-
     return fullPath;
   }
 

package/lib/request.js

@@ -58,6 +58,17 @@ export default class Request {
     return res.text();
   }
 
+  async * stream(url, options = {}) {
+    const res = await this.fetch(url, options);
+    if (!res.ok) {
+      await res.body?.cancel();
+      throw new Error(`Unable to stream ${url}: ${res.status} ${res.statusText}`);
+    }
+    // Undici decompresses HTTP content encodings as it reads. Returning early
+    // from this iterator also cancels the response body and its HTTP download.
+    yield * res.body;
+  }
+
   async json(url, options = {}) {
     options.headers = options.headers || {};
     const text = await this.text(url, options);
@@ -106,7 +117,10 @@ export default class Request {
       const url =
         `/repos/${owner}/${repo}/pulls/${prid}/files?per_page=100&page=${page}`;
       const batch = await this.json(url);
-      if (!Array.isArray(batch) || batch.length === 0) {
+      if (!Array.isArray(batch)) {
+        throw new Error('Unable to retrieve pull request files');
+      }
+      if (batch.length === 0) {
         break;
       }
       yield * batch;
@@ -193,22 +207,52 @@ export default class Request {
   }
 
   async getTriagedReports() {
-    const url = 'https://api.hackerone.com/v1/reports?filter[program][]=nodejs&filter[state][]=triaged';
+    let url = 'https://api.hackerone.com/v1/reports?filter[program][]=nodejs&filter[state][]=triaged';
     const options = {
       method: 'GET',
+      redirect: 'error',
       headers: {
         Authorization: `Basic ${this.credentials.h1}`,
         'User-Agent': 'node-core-utils',
         Accept: 'application/json'
       }
     };
-    const data = await this.json(url, options);
-    if (data?.errors) {
-      throw new Error(
-        `Request to fetch triaged reports failed with: ${JSON.stringify(data.errors)}`
-      );
+    const reports = [];
+    const reportIds = new Set();
+    const pages = new Set();
+    let result;
+
+    while (url) {
+      const pageUrl = new URL(url);
+      if (pageUrl.origin !== 'https://api.hackerone.com' ||
+          pageUrl.pathname !== '/v1/reports' || pageUrl.username || pageUrl.password) {
+        throw new Error('Invalid HackerOne reports pagination URL');
+      }
+      if (pages.has(pageUrl.href)) {
+        throw new Error('Repeated HackerOne reports pagination URL');
+      }
+      pages.add(pageUrl.href);
+
+      result = await this.json(url, options);
+      if (result?.errors?.length) {
+        throw new Error(
+          `Request to fetch triaged reports failed with: ${JSON.stringify(result.errors)}`
+        );
+      }
+      if (!Array.isArray(result?.data)) {
+        throw new Error('Invalid HackerOne reports response: expected a data array');
+      }
+      for (const report of result.data) {
+        if (reportIds.has(report.id)) continue;
+        reportIds.add(report.id);
+        reports.push(report);
+      }
+
+      const next = result.links?.next;
+      url = next ? new URL(next, pageUrl).href : null;
     }
-    return data;
+
+    return { ...result, data: reports };
   }
 
   async getPrograms() {

package/lib/security-release/draft.js

@@ -0,0 +1,32 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { NEXT_SECURITY_RELEASE_FOLDER } from './security-release.js';
+import { prepareSecurityRelease } from './preparation.js';
+
+export function getSecurityReleaseDraftPath(directory) {
+  if (typeof directory !== 'string' || !directory.trim()) {
+    throw new Error('Security release repository directory is required');
+  }
+  return path.resolve(directory, NEXT_SECURITY_RELEASE_FOLDER, 'vulnerabilities.json');
+}
+
+export function assertNewSecurityRelease(directory) {
+  const file = getSecurityReleaseDraftPath(directory);
+  if (fs.existsSync(file)) {
+    throw new Error(
+      `Security release draft already exists: ${file}. ` +
+      'Use --sync, --add-report, or --remove-report to update the existing release.'
+    );
+  }
+}
+
+// Local persistence only. The caller handles review, Git, and publication.
+export function writeSecurityReleaseDraft(directory, draft) {
+  const { release } = prepareSecurityRelease(draft);
+  const file = getSecurityReleaseDraftPath(directory);
+  assertNewSecurityRelease(directory);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(release, null, 2) + '\n', { flag: 'wx' });
+  return file;
+}