@node-core/utils 4.0.0

package/lib/ci/failure_aggregator.js

@@ -0,0 +1,152 @@
+import _ from 'lodash';
+import chalk from 'chalk';
+
+import { getMachineUrl, parsePRFromURL } from '../links.js';
+import CIFailureParser from './ci_failure_parser.js';
+import {
+  parseJobFromURL,
+  CI_TYPES
+} from './ci_type_parser.js';
+import {
+  fold,
+  getHighlight,
+  markdownRow
+} from './ci_utils.js';
+
+const { FAILURE_TYPES_NAME } = CIFailureParser;
+
+export class FailureAggregator {
+  constructor(cli, data) {
+    this.cli = cli;
+    this.health = data[0];
+    this.failures = data.slice(1);
+    this.aggregates = null;
+  }
+
+  aggregate() {
+    const failures = this.failures;
+    const groupedByReason = _.chain(failures)
+      .groupBy(getHighlight)
+      .toPairs()
+      .sortBy(0)
+      .value();
+    const data = [];
+    for (const item of groupedByReason) {
+      const [reason, failures] = item;
+      // Uncomment this and redirect stderr away to see matched highlights
+      // console.log('HIGHLIGHT', reason);
+
+      // If multiple sub builds of one PR are failed by the same reason,
+      // we'll only take one of those builds, as that might be a genuine failure
+      const prs = _.chain(failures)
+        .uniqBy('source')
+        .sortBy((f) => parseJobFromURL(f.upstream).jobid)
+        .map((item) => ({ source: item.source, upstream: item.upstream }))
+        .value();
+      const machines = _.uniq(failures.map(f => f.builtOn));
+      data.push({
+        reason, type: failures[0].type, failures, prs, machines
+      });
+    };
+
+    const groupedByType = _.groupBy(data, 'type');
+    for (const type of Object.keys(groupedByType)) {
+      groupedByType[type] =
+        _.sortBy(groupedByType[type], r => 0 - (r.prs.length));
+    }
+    this.aggregates = groupedByType;
+    return groupedByType;
+  }
+
+  formatAsMarkdown() {
+    let { aggregates } = this;
+    if (!aggregates) {
+      aggregates = this.aggregates = this.aggregate();
+    }
+
+    const last = parseJobFromURL(this.failures[0].upstream);
+    const first = parseJobFromURL(
+      this.failures[this.failures.length - 1].upstream
+    );
+    const jobName = CI_TYPES.get(first.type).jobName;
+    let output = 'Failures in ';
+    output += `[${jobName}/${first.jobid}](${first.link}) to `;
+    output += `[${jobName}/${last.jobid}](${last.link}) `;
+    output += 'that failed 2 or more PRs\n';
+    output += '(Generated with `ncu-ci ';
+    output += `${process.argv.slice(2).join(' ')}\`)\n\n`;
+
+    output += this.health.formatAsMarkdown() + '\n';
+
+    const todo = [];
+    for (const type of Object.keys(aggregates)) {
+      if (aggregates[type].length === 0) {
+        continue;
+      }
+      output += `\n### ${FAILURE_TYPES_NAME[type]}\n\n`;
+      for (const item of aggregates[type]) {
+        const { reason, type, prs, failures, machines } = item;
+        if (prs.length < 2) { continue; }
+        todo.push({ count: prs.length, reason });
+        output += markdownRow('Reason', `<code>${reason}</code>`);
+        output += markdownRow('-', ':-');
+        output += markdownRow('Type', type);
+        const source = prs.map(f => f.source);
+        output += markdownRow(
+          'Failed PR', `${source.length} (${source.join(', ')})`
+        );
+        output += markdownRow(
+          'Appeared', machines.map(getMachineUrl).join(', ')
+        );
+        if (prs.length > 1) {
+          output += markdownRow('First CI', `${prs[0].upstream}`);
+        }
+        output += markdownRow('Last CI', `${prs[prs.length - 1].upstream}`);
+        output += '\n';
+        const example = failures[0].reason;
+        output += fold(
+          `<a href="${failures[0].url}">Example</a>`,
+          (example.length > 1024 ? example.slice(0, 1024) + '...' : example)
+        );
+        output += '\n\n-------\n\n';
+      }
+    }
+
+    output += '### Progress\n\n';
+    output += todo.map(
+      ({ count, reason }) => `- [ ] \`${reason}\` (${count})`).join('\n'
+    );
+    return output + '\n';
+  }
+
+  display() {
+    let { cli, aggregates } = this;
+    if (!aggregates) {
+      aggregates = this.aggregates = this.aggregate();
+    }
+
+    for (const type of Object.keys(aggregates)) {
+      cli.separator(type);
+      for (const item of aggregates[type]) {
+        const { reason, type, prs, failures, machines } = item;
+        cli.table('Reason', reason);
+        cli.table('Type', type);
+        const source = prs
+          .map(f => {
+            const parsed = parsePRFromURL(f.source);
+            return parsed ? `#${parsed.prid}` : f.source;
+          });
+        cli.table('Failed PR', `${source.length} (${source.join(', ')})`);
+        cli.table('Appeared', machines.join(', '));
+        if (prs.length > 1) {
+          cli.table('First CI', `${prs[0].upstream}`);
+        }
+        cli.table('Last CI', `${prs[prs.length - 1].upstream}`);
+        cli.log('\n' + chalk.bold('Example: ') + `${failures[0].url}\n`);
+        const example = failures[0].reason;
+        cli.log(example.length > 512 ? example.slice(0, 512) + '...' : example);
+        cli.separator();
+      }
+    }
+  }
+}

package/lib/ci/jenkins_constants.js

@@ -0,0 +1,28 @@
+// com.tikal.jenkins.plugins.multijob.MultiJobBuild
+const BUILD_FIELDS = 'builtOn,buildNumber,jobName,result,url';
+const ACTION_TREE = 'actions[parameters[name,value]]';
+const CHANGE_FIELDS = 'commitId,author[absoluteUrl,fullName],authorEmail,' +
+                      'msg,date';
+const CHANGE_TREE = `changeSet[items[${CHANGE_FIELDS}]]`;
+export const PR_TREE =
+  `result,url,number,${ACTION_TREE},${CHANGE_TREE},builtOn,` +
+  `subBuilds[${BUILD_FIELDS},build[subBuilds[${BUILD_FIELDS}]]]`;
+export const COMMIT_TREE =
+  `result,url,number,${ACTION_TREE},${CHANGE_TREE},builtOn,` +
+  `subBuilds[${BUILD_FIELDS}]`;
+export const CITGM_MAIN_TREE =
+  `result,url,number,${ACTION_TREE},${CHANGE_TREE},builtOn`;
+
+export const FANNED_TREE =
+  `result,url,number,subBuilds[phaseName,${BUILD_FIELDS}]`;
+
+// hudson.tasks.test.MatrixTestResult
+const RESULT_TREE = 'result[suites[cases[name,status]]]';
+export const CITGM_REPORT_TREE =
+`failCount,skipCount,totalCount,childReports[child[url],${RESULT_TREE}]`;
+
+// hudson.matrix.MatrixBuild
+export const BUILD_TREE = 'result,runs[url,number,result],builtOn';
+export const LINTER_TREE = 'result,url,number,builtOn';
+const CAUSE_TREE = 'upstreamBuild,upstreamProject,shortDescription,_class';
+export const RUN_TREE = `actions[causes[${CAUSE_TREE}]],builtOn`;

package/lib/ci/run_ci.js

@@ -0,0 +1,120 @@
+import { FormData } from 'undici';
+
+import {
+  CI_DOMAIN,
+  CI_TYPES,
+  CI_TYPES_KEYS
+} from './ci_type_parser.js';
+import PRData from '../pr_data.js';
+import { debuglog } from '../verbosity.js';
+
+export const CI_CRUMB_URL = `https://${CI_DOMAIN}/crumbIssuer/api/json`;
+const CI_PR_NAME = CI_TYPES.get(CI_TYPES_KEYS.PR).jobName;
+export const CI_PR_URL = `https://${CI_DOMAIN}/job/${CI_PR_NAME}/build`;
+
+const CI_V8_NAME = CI_TYPES.get(CI_TYPES_KEYS.V8).jobName;
+export const CI_V8_URL = `https://${CI_DOMAIN}/job/${CI_V8_NAME}/build`;
+
+export class RunPRJob {
+  constructor(cli, request, owner, repo, prid) {
+    this.cli = cli;
+    this.request = request;
+    this.owner = owner;
+    this.repo = repo;
+    this.prid = prid;
+    this.prData = new PRData({ prid, owner, repo }, cli, request);
+  }
+
+  async getCrumb() {
+    try {
+      const { crumb } = await this.request.json(CI_CRUMB_URL);
+      return crumb;
+    } catch (e) {
+      return false;
+    }
+  }
+
+  get payload() {
+    const payload = new FormData();
+    payload.append('json', JSON.stringify({
+      parameter: [
+        { name: 'CERTIFY_SAFE', value: 'on' },
+        { name: 'TARGET_GITHUB_ORG', value: this.owner },
+        { name: 'TARGET_REPO_NAME', value: this.repo },
+        { name: 'PR_ID', value: this.prid },
+        { name: 'REBASE_ONTO', value: '<pr base branch>' },
+        { name: 'DESCRIPTION_SETTER_DESCRIPTION', value: '' }
+      ]
+    }));
+    return payload;
+  }
+
+  get v8Payload() {
+    const payload = new FormData();
+    payload.append('json', JSON.stringify({
+      parameter: [
+        { name: 'GITHUB_ORG', value: this.owner },
+        { name: 'REPO_NAME', value: this.repo },
+        { name: 'GIT_REMOTE_REF', value: `refs/pull/${this.prid}/head` }
+      ]
+    }));
+    return payload;
+  }
+
+  async start() {
+    const { cli } = this;
+    cli.startSpinner('Validating Jenkins credentials');
+    const crumb = await this.getCrumb();
+
+    if (crumb === false) {
+      cli.stopSpinner('Jenkins credentials invalid',
+        this.cli.SPINNER_STATUS.FAILED);
+      return false;
+    }
+    cli.stopSpinner('Jenkins credentials valid');
+
+    try {
+      cli.startSpinner('Starting PR CI job');
+      const response = await this.request.fetch(CI_PR_URL, {
+        method: 'POST',
+        headers: {
+          'Jenkins-Crumb': crumb
+        },
+        body: this.payload
+      });
+      if (response.status !== 201) {
+        cli.stopSpinner(
+          `Failed to start PR CI: ${response.status} ${response.statusText}`,
+          this.cli.SPINNER_STATUS.FAILED);
+        return false;
+      }
+      cli.stopSpinner('PR CI job successfully started');
+
+      // check if the job need a v8 build and trigger it
+      await this.prData.getPR();
+      const labels = this.prData.pr.labels;
+      if (labels.nodes.map(i => i.name).includes('v8 engine')) {
+        cli.startSpinner('Starting V8 CI job');
+        const response = await this.request.fetch(CI_V8_URL, {
+          method: 'POST',
+          headers: {
+            'Jenkins-Crumb': crumb
+          },
+          body: this.v8Payload
+        });
+        if (response.status !== 201) {
+          cli.stopSpinner(
+            `Failed to start V8 CI: ${response.status} ${response.statusText}`,
+            this.cli.SPINNER_STATUS.FAILED);
+          return false;
+        }
+        cli.stopSpinner('V8 CI job successfully started');
+      }
+    } catch (err) {
+      debuglog(err);
+      cli.stopSpinner('Failed to start CI', this.cli.SPINNER_STATUS.FAILED);
+      return false;
+    }
+    return true;
+  }
+}

package/lib/cli.js

@@ -0,0 +1,192 @@
+import ora from 'ora';
+import chalk from 'chalk';
+import inquirer from 'inquirer';
+
+import { warning, error, info, success } from './figures.js';
+
+const SPINNER_STATUS = {
+  SUCCESS: 'success',
+  FAILED: 'failed',
+  WARN: 'warn',
+  INFO: 'info'
+};
+
+const { SUCCESS, FAILED, WARN, INFO } = SPINNER_STATUS;
+
+const QUESTION_TYPE = {
+  INPUT: 'input',
+  NUMBER: 'number',
+  CONFIRM: 'confirm'
+};
+
+const formatter = new Intl.ListFormat('en', { type: 'disjunction' });
+
+function head(text, length = 11) {
+  return chalk.bold(text.padEnd(length));
+}
+
+export default class CLI {
+  constructor(stream, options = {}) {
+    const spinnerOptions = options.spinner ?? {};
+    this.stream = stream || process.stderr;
+    this.spinner = ora({ stream: this.stream, ...spinnerOptions });
+    this.SPINNER_STATUS = SPINNER_STATUS;
+    this.QUESTION_TYPE = QUESTION_TYPE;
+    this.figureIndent = '   ';
+    this.assumeYes = false;
+  }
+
+  get eolIndent() {
+    return `\n${this.figureIndent}`;
+  }
+
+  setFigureIndent(indent) {
+    this.figureIndent = ' '.repeat(indent);
+  }
+
+  async prompt(question, opts = {
+    defaultAnswer: true,
+    noSeparator: false,
+    questionType: 'confirm'
+  }) {
+    if (!opts.noSeparator) {
+      this.separator();
+    }
+
+    const questionType = opts.questionType || QUESTION_TYPE.CONFIRM;
+    const availableTypes = Object.values(QUESTION_TYPE);
+    if (!availableTypes.includes(questionType)) {
+      throw new Error(
+        `${questionType} must be one of ${formatter.format(availableTypes)}`);
+    }
+
+    const defaultAnswer = (opts.defaultAnswer === undefined) ||
+      opts.defaultAnswer;
+    if (typeof defaultAnswer === 'boolean' &&
+        questionType !== QUESTION_TYPE.CONFIRM) {
+      throw new Error(
+        'defaultAnswer must be provided for non-confirmation prompts');
+    }
+
+    const { isSpinning, text: spinningMessage } = this.spinner;
+
+    if (isSpinning) {
+      this.spinner.stop();
+    }
+
+    if (this.assumeYes) {
+      if (isSpinning) {
+        this.spinner.start(spinningMessage);
+      }
+      return defaultAnswer;
+    }
+
+    const { answer } = await inquirer.prompt([{
+      type: questionType,
+      name: 'answer',
+      message: question,
+      default: defaultAnswer
+    }]);
+
+    if (isSpinning) {
+      this.spinner.start(spinningMessage);
+    }
+
+    return answer;
+  }
+
+  setAssumeYes() {
+    this.assumeYes = true;
+  }
+
+  startSpinner(text) {
+    this.spinner.text = text;
+    this.spinner.start();
+  }
+
+  updateSpinner(text) {
+    this.spinner.text = text;
+  }
+
+  stopSpinner(rawText, status = SUCCESS) {
+    let symbol;
+    switch (status) {
+      case SUCCESS:
+        symbol = success;
+        break;
+      case FAILED:
+        symbol = error;
+        break;
+      case WARN:
+        symbol = warning;
+        break;
+      case INFO:
+        symbol = info;
+    }
+    const text = ' ' + rawText;
+    this.spinner.stopAndPersist({
+      symbol, text
+    });
+  }
+
+  write(text) {
+    this.stream.write(text);
+  }
+
+  log(text) {
+    this.write(text + '\n');
+  }
+
+  table(first, second = '', length = 11) {
+    this.log(head(first, length) + second);
+  }
+
+  separator(text = '', length = 80, sep = '-') {
+    if (!text) {
+      this.log(sep.repeat(length));
+      return;
+    }
+    const rest = (length - text.length - 2);
+    const half = sep.repeat(Math.abs(Math.floor(rest / 2)));
+    if (rest % 2 === 0) {
+      this.log(`${half} ${chalk.bold(text)} ${half}`);
+    } else {
+      this.log(`${half} ${chalk.bold(text)} ${sep}${half}`);
+    }
+  }
+
+  ok(text, options = {}) {
+    const prefix = options.newline ? this.eolIndent : this.figureIndent;
+    this.log(`${prefix}${success}  ${text}`);
+  }
+
+  warn(text, options = {}) {
+    const prefix = options.newline ? this.eolIndent : this.figureIndent;
+    this.log(prefix + chalk.bold(`${warning}  ${text}`));
+  }
+
+  info(text, options = {}) {
+    const prefix = options.newline ? this.eolIndent : this.figureIndent;
+    this.log(`${prefix}${info}  ${text}`);
+  }
+
+  error(obj, options = {}) {
+    const prefix = options.newline ? this.eolIndent : this.figureIndent;
+    if (obj instanceof Error) {
+      this.log(`${prefix}${error}  ${obj.message}`);
+      this.log(obj.stack);
+      if (obj.data) {
+        this.log(JSON.stringify(obj.data, null, 2));
+      }
+    } else {
+      this.log(prefix + chalk.bold(`${error}  ${obj}`));
+    }
+  }
+
+  setExitCode(statusCode) {
+    process.exitCode = statusCode;
+  }
+};
+
+CLI.SPINNER_STATUS = SPINNER_STATUS;
+CLI.QUESTION_TYPE = QUESTION_TYPE;

package/lib/collaborators.js

@@ -0,0 +1,140 @@
+import fs from 'node:fs';
+
+const TSC_TITLE = '#### TSC voting members';
+const TSC_REGULAR_TITLE = '#### TSC regular members';
+const TSCE_TITLE = '#### TSC emeriti members';
+const CL_TITLE = '### Collaborators';
+const CLE_TITLE = '### Collaborator emeriti';
+const CONTACT_RE = /\* +\[(.+?)\]\(.+?\) +-\s+\*\*([^*]+?)\*\* +(?:&lt;|\\<|<<)([^>]+?)(?:&gt;|>)/mg;
+
+const TSC = 'TSC';
+const COLLABORATOR = 'COLLABORATOR';
+
+export class Collaborator {
+  constructor(login, name, email, type) {
+    this.login = login;  // This is not lowercased
+    this.name = name;
+    this.email = email;
+    this.type = type;
+  }
+
+  isActor(actor) {
+    if (!actor || !actor.login) {  // ghost
+      return false;
+    }
+    return actor.login.toLowerCase() === this.login.toLowerCase();
+  }
+
+  isTSC() {
+    return this.type === TSC;
+  }
+
+  getName() {
+    return `${this.name} (@${this.login})`;
+  }
+
+  getContact() {
+    return `${this.name} <${this.email}>`;
+  }
+}
+
+Collaborator.TYPES = {
+  TSC, COLLABORATOR
+};
+
+export async function getCollaborators(cli, request, argv) {
+  const { readme, owner, repo } = argv;
+  let readmeText;
+  if (readme) {
+    cli.updateSpinner(`Reading collaborator contacts from ${readme}`);
+    readmeText = fs.readFileSync(readme, 'utf8');
+  } else {
+    cli.updateSpinner(
+      'Getting collaborator contacts from README of nodejs/node');
+    const url = 'https://raw.githubusercontent.com/nodejs/node/HEAD/README.md';
+    readmeText = await request.text(url);
+  }
+
+  let collaborators;
+  try {
+    collaborators = parseCollaborators(readmeText, cli);
+  } catch (err) {
+    const readmePath = readme || `${owner}/${repo}/README.md`;
+    cli.stopSpinner(`Failed to get collaborator info from ${readmePath}`,
+      cli.SPINNER_STATUS.FAILED);
+    throw err;
+  }
+  return collaborators;
+}
+
+function parseCollaborators(readme, cli) {
+  // This is more or less taken from
+  // https://github.com/rvagg/archived-iojs-tools/blob/main/pr-metadata/pr-metadata.js
+  const collaborators = new Map();
+  let m;
+
+  const tscIndex = readme.toUpperCase().indexOf(TSC_TITLE.toUpperCase());
+  const tscrIndex = readme.toUpperCase().indexOf(TSC_REGULAR_TITLE.toUpperCase());
+  const tsceIndex = readme.toUpperCase().indexOf(TSCE_TITLE.toUpperCase());
+  const clIndex = readme.toUpperCase().indexOf(CL_TITLE.toUpperCase());
+  const cleIndex = readme.toUpperCase().indexOf(CLE_TITLE.toUpperCase());
+
+  if (tscIndex === -1) {
+    throw new Error(`Couldn't find ${TSC_TITLE} in the README`);
+  }
+  if (tscrIndex === -1) {
+    throw new Error(`Couldn't find ${TSC_REGULAR_TITLE} in the README`);
+  }
+  if (tsceIndex === -1) {
+    throw new Error(`Couldn't find ${TSCE_TITLE} in the README`);
+  }
+  if (clIndex === -1) {
+    throw new Error(`Couldn't find ${CL_TITLE} in the README`);
+  }
+  if (cleIndex === -1) {
+    throw new Error(`Couldn't find ${CLE_TITLE} in the README`);
+  }
+
+  if (!(tscIndex < tscrIndex &&
+        tscrIndex < tsceIndex &&
+        tsceIndex < clIndex &&
+        clIndex < cleIndex)) {
+    cli.warn('Contacts in the README is out of order, ' +
+             'analysis could go wrong.', { newline: true });
+  }
+
+  // We also assume that TSC & TSC Emeriti are also listed as collaborators
+  CONTACT_RE.lastIndex = tscIndex;
+  // eslint-disable-next-line no-cond-assign
+  while ((m = CONTACT_RE.exec(readme)) && CONTACT_RE.lastIndex < tscrIndex) {
+    const login = m[1].toLowerCase();
+    const user = new Collaborator(m[1], m[2], m[3], TSC);
+    collaborators.set(login, user);
+  }
+
+  CONTACT_RE.lastIndex = clIndex;
+  // eslint-disable-next-line no-cond-assign
+  while ((m = CONTACT_RE.exec(readme)) &&
+    CONTACT_RE.lastIndex < cleIndex) {
+    const login = m[1].toLowerCase();
+    if (!collaborators.get(login)) {
+      const user = new Collaborator(m[1], m[2], m[3], COLLABORATOR);
+      collaborators.set(login, user);
+    }
+  }
+
+  if (!collaborators.size) {
+    throw new Error('Could not find any collaborators');
+  }
+
+  return collaborators;
+}
+
+/**
+ * @param {Map<string, Collaborator>} collaborators
+ * @param {{login?: string}} user
+ */
+export function isCollaborator(collaborators, user) {
+  return (user && user.login &&  // could be a ghost
+    collaborators.get(user.login.toLowerCase()));
+}

package/lib/config.js

@@ -0,0 +1,72 @@
+import path from 'node:path';
+import os from 'node:os';
+
+import { readJson, writeJson } from './file.js';
+
+export const GLOBAL_CONFIG = Symbol('globalConfig');
+export const PROJECT_CONFIG = Symbol('projectConfig');
+export const LOCAL_CONFIG = Symbol('localConfig');
+
+export function getNcurcPath() {
+  if (process.env.XDG_CONFIG_HOME !== 'undefined' &&
+      process.env.XDG_CONFIG_HOME !== undefined) {
+    return path.join(process.env.XDG_CONFIG_HOME, 'ncurc');
+  } else {
+    return path.join(os.homedir(), '.ncurc');
+  }
+}
+
+export function getMergedConfig(dir, home) {
+  const globalConfig = getConfig(GLOBAL_CONFIG, home);
+  const projectConfig = getConfig(PROJECT_CONFIG, dir);
+  const localConfig = getConfig(LOCAL_CONFIG, dir);
+  return Object.assign(globalConfig, projectConfig, localConfig);
+};
+
+export function getConfig(configType, dir) {
+  const configPath = getConfigPath(configType, dir);
+  try {
+    return readJson(configPath);
+  } catch (cause) {
+    throw new Error('Unable to parse config file ' + configPath, { cause });
+  }
+};
+
+export function getConfigPath(configType, dir) {
+  switch (configType) {
+    case GLOBAL_CONFIG:
+      return getNcurcPath();
+    case PROJECT_CONFIG: {
+      const projectRcPath = path.join(dir || process.cwd(), '.ncurc');
+      return projectRcPath;
+    }
+    case LOCAL_CONFIG: {
+      const ncuDir = getNcuDir(dir);
+      const configPath = path.join(ncuDir, 'config');
+      return configPath;
+    }
+    default:
+      throw Error('Invalid configType');
+  }
+};
+
+export function writeConfig(configType, obj, dir) {
+  writeJson(getConfigPath(configType, dir), obj);
+};
+
+export function updateConfig(configType, obj, dir) {
+  const config = getConfig(configType, dir);
+  const configPath = getConfigPath(configType, dir);
+  writeJson(configPath, Object.assign(config, obj));
+};
+
+export function getHomeDir(home) {
+  if (process.env.XDG_CONFIG_HOME) {
+    return process.env.XDG_CONFIG_HOME;
+  }
+  return home || os.homedir();
+};
+
+export function getNcuDir(dir) {
+  return path.join(dir || process.cwd(), '.ncu');
+};