@node-core/utils 7.2.2 → 7.3.0

package/bin/ncu-ci.js

@@ -23,6 +23,7 @@ import {
 import {
   RunPRJob
 } from '../lib/ci/run_ci.js';
+import { ResumePRJob } from '../lib/ci/resume_ci.js';
 import { writeJson, writeFile } from '../lib/file.js';
 import { getMergedConfig } from '../lib/config.js';
 import { runPromise } from '../lib/run.js';
@@ -134,6 +135,26 @@ const args = yargs(hideBin(process.argv))
     },
     handler
   })
+  .command({
+    command: 'resume <prid>',
+    desc: 'Resume the latest CI run for given PR',
+    builder: (yargs) => {
+      yargs
+        .positional('prid', {
+          describe: 'ID of the PR or URL to the PR',
+          type: 'string'
+        })
+        .option('owner', {
+          default: '',
+          describe: 'GitHub repository owner'
+        })
+        .option('repo', {
+          default: '',
+          describe: 'GitHub repository name'
+        });
+    },
+    handler
+  })
   .command({
     command: 'url <url>',
     desc: 'Automatically detect CI type and show results',
@@ -278,11 +299,13 @@ class RunPRJobCommand {
     return this.argv.prid;
   }
 
-  async start() {
-    const {
-      cli, request, prid, repo, owner
-    } = this;
+  validate() {
+    const { cli, prid, repo, owner } = this;
     let validArgs = true;
+    if (!Number.isSafeInteger(prid) || prid <= 0) {
+      validArgs = false;
+      cli.error('Pull request ID must be a positive integer');
+    }
     if (!repo) {
       validArgs = false;
       cli.error('GitHub repository is missing, please set it via ncu-config ' +
@@ -295,6 +318,13 @@ class RunPRJobCommand {
     }
     if (!validArgs) {
       this.cli.setExitCode(1);
+    }
+    return validArgs;
+  }
+
+  async start() {
+    const { cli, request, prid, repo, owner } = this;
+    if (!this.validate()) {
       return;
     }
     const { certifySafe, checkForDuplicates } = this.argv;
@@ -308,6 +338,20 @@ class RunPRJobCommand {
   }
 }
 
+class ResumePRJobCommand extends RunPRJobCommand {
+  async start() {
+    const { cli, request, prid, repo, owner } = this;
+    if (!this.validate()) {
+      return;
+    }
+    const jobRunner = new ResumePRJob(cli, request, owner, repo, prid);
+    if (!(await jobRunner.resume())) {
+      cli.setExitCode(1);
+      process.exitCode = 1;
+    }
+  }
+}
+
 class CICommand {
   constructor(cli, request, argv) {
     this.cli = cli;
@@ -565,7 +609,8 @@ async function main(command, argv) {
   let commandHandler;
   // Prepare queue.
   switch (command) {
-    case 'run': {
+    case 'run':
+    case 'resume': {
       const maybeURL = URL.parse(argv.prid);
       if (maybeURL?.host === 'github.com') {
         const [, owner, repo, , prid, , commit_sha] = maybeURL.pathname.split('/');
@@ -575,7 +620,8 @@ async function main(command, argv) {
         argv.prid = prid;
       }
       argv.prid = Number(argv.prid);
-      const jobRunner = new RunPRJobCommand(cli, request, argv);
+      const Command = command === 'run' ? RunPRJobCommand : ResumePRJobCommand;
+      const jobRunner = new Command(cli, request, argv);
       return jobRunner.start();
     }
     case 'rate': {

package/lib/ci/ci_failure_parser.js

@@ -132,35 +132,54 @@ function failureMatcher(Failure, patterns, ctx, text) {
   return null;
 }
 
+const fatalPattern = /fatal: .+/g;
+
+// Shared by the whole-log parser and the streaming file scanner.
+export const FAILURE_MARKERS = {
+  tap: {
+    start: /^not ok \d+/,
+    end: / {2}\.\.\.\r?\n/,
+    todo: '# TODO :'
+  },
+  cpp: /\[ {2}FAILED {2}\]/,
+  git: [{
+    start: /Changes not staged for commit:/,
+    end: /no changes added to commit/
+  }, {
+    start: /error: Your local changes to the following files/,
+    end: /Failed to merge in the changes./
+  }]
+};
+
+const tapPattern = new RegExp(
+  `${FAILURE_MARKERS.tap.start.source}[\\s\\S]+?${FAILURE_MARKERS.tap.end.source}`, 'mg');
+
 // The elements are ranked by priority
 const FAILURE_FILTERS = [{
   // NOTE(mmarchini): infra-related issues should have the highest priority, as
   // they can cause other issues to happen.
-  filter(ctx, text) {
-    const patterns = [{
-      pattern: /Read-only file system/g,
-      context: { index: 0, contextBefore: 1, contextAfter: 0 }
-    },
-    {
-      pattern: /Device or resource busy/g,
-      context: { index: 0, contextBefore: 1, contextAfter: 0 }
-    },
-    {
-      pattern: /There is not enough space in the file system./g,
-      context: { index: 0, contextBefore: 1, contextAfter: 0 }
-    }
-    ];
-    return failureMatcher(InfraFailure, patterns, ctx, text);
+  Failure: InfraFailure,
+  patterns: [{
+    pattern: /Read-only file system/g,
+    context: { index: 0, contextBefore: 1, contextAfter: 0 }
+  },
+  {
+    pattern: /Device or resource busy/g,
+    context: { index: 0, contextBefore: 1, contextAfter: 0 }
+  },
+  {
+    pattern: /There is not enough space in the file system./g,
+    context: { index: 0, contextBefore: 1, contextAfter: 0 }
   }
+  ],
 }, {
   // TODO: match indentation to avoid skipping context with '...'
   filter(ctx, text) {
-    const pattern = /^not ok \d+[\s\S]+? {2}\.\.\.\r?\n/mg;
-    const matches = text.match(pattern);
+    const matches = text.match(tapPattern);
     if (!matches) {
       return null;
     }
-    const nonFlaky = matches.filter((m) => !m.includes('# TODO :'));
+    const nonFlaky = matches.filter((m) => !m.includes(FAILURE_MARKERS.tap.todo));
     if (!nonFlaky.length) {
       return null;
     }
@@ -169,89 +188,71 @@ const FAILURE_FILTERS = [{
     );
   }
 }, {
-  filter(ctx, text) {
-    const patterns = [{
-      pattern: /\[ {2}FAILED {2}\].+/g,
-      context: { index: 0, contextBefore: 5, contextAfter: 0 }
-    }];
-    return failureMatcher(CCTestFailure, patterns, ctx, text);
-  }
+  Failure: CCTestFailure,
+  patterns: [{
+    pattern: new RegExp(`${FAILURE_MARKERS.cpp.source}.+`, 'g'),
+    context: { index: 0, contextBefore: 5, contextAfter: 0 }
+  }],
 }, {
   // VS compilation error
-  filter(ctx, text) {
-    const patterns = [{
-      pattern: /error C\d+:/mg,
-      context: { index: 0, contextBefore: 0, contextAfter: 5 }
-    }];
-    return failureMatcher(BuildFailure, patterns, ctx, text);
-  }
+  Failure: BuildFailure,
+  patterns: [{
+    pattern: /error C\d+:/mg,
+    context: { index: 0, contextBefore: 0, contextAfter: 5 }
+  }],
 }, {
-  filter(ctx, text) {
-    const patterns = [{
-      pattern: /java\.io\.IOException.+/g,
-      context: { index: -1, contextBefore: 0, contextAfter: 5 }
-    }, {
-      pattern: /Build timed out/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 1 }
-    }];
-    return failureMatcher(JenkinsFailure, patterns, ctx, text);
-  }
+  Failure: JenkinsFailure,
+  patterns: [{
+    pattern: /java\.io\.IOException.+/g,
+    context: { index: -1, contextBefore: 0, contextAfter: 5 }
+  }, {
+    pattern: /Build timed out/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 1 }
+  }],
 }, {
-  filter(ctx, text) {
-    const patterns = [{
-      pattern:
-        /Changes not staged for commit:[\s\S]+no changes added to commit/mg,
-      context: { index: 0, contextBefore: 0, contextAfter: 0 }
-    }, {
-      pattern:
-        /error: Your local changes to the following files[\s\S]+Failed to merge in the changes./g,
-      context: { index: 0, contextBefore: 0, contextAfter: 0 }
-    }, {
-      pattern: /warning: failed to remove .+/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 0 }
-    }];
-    return failureMatcher(GitFailure, patterns, ctx, text);
-  }
+  Failure: GitFailure,
+  patterns: [...FAILURE_MARKERS.git.map(({ start, end }) => ({
+    pattern: new RegExp(`${start.source}[\\s\\S]+${end.source}`, 'mg'),
+    context: { index: 0, contextBefore: 0, contextAfter: 0 }
+  })), {
+    pattern: /warning: failed to remove .+/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 0 }
+  }],
 }, {
-  filter(ctx, text) {
-    const patterns = [{
-      pattern: /ERROR: Error fetching .+/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 5 }
-    }, {
-      pattern: /hudson\.plugins\.git\.GitException+/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 5 }
-    }, {
-      pattern: /Cannot rebase: .+/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 1 }
-    }];
-    return failureMatcher(GitFailure, patterns, ctx, text);
-  }
+  Failure: GitFailure,
+  patterns: [{
+    pattern: /ERROR: Error fetching .+/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 5 }
+  }, {
+    pattern: /hudson\.plugins\.git\.GitException+/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 5 }
+  }, {
+    pattern: /Cannot rebase: .+/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 1 }
+  }],
 }, {
-  filter(ctx, text) {
-    const patterns = [{
-      pattern: /sh: line /g,
-      context: { index: 0, contextBefore: 0, contextAfter: 1 }
-    }, {
-      pattern: /fatal error:/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 1 }
-    }, {
-      pattern: /dtrace: failed to compile script/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 1 }
-    }, {
-      pattern: /ERROR: .+/g,
-      // Pick the last one
-      context: { index: -1, contextBefore: 0, contextAfter: 5 }
-    }, {
-      // Pick the first one
-      pattern: /Error: .+/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 5 }
-    }];
-    return failureMatcher(BuildFailure, patterns, ctx, text);
-  }
+  Failure: BuildFailure,
+  patterns: [{
+    pattern: /sh: line /g,
+    context: { index: 0, contextBefore: 0, contextAfter: 1 }
+  }, {
+    pattern: /fatal error:/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 1 }
+  }, {
+    pattern: /dtrace: failed to compile script/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 1 }
+  }, {
+    pattern: /ERROR: .+/g,
+    // Pick the last one
+    context: { index: -1, contextBefore: 0, contextAfter: 5 }
+  }, {
+    // Pick the first one
+    pattern: /Error: .+/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 5 }
+  }],
 }, {
   filter(ctx, text) {
-    const pattern = /fatal: .+/g;
-    const matches = text.match(pattern);
+    const matches = text.match(fatalPattern);
     if (!matches) {
       return null;
     }
@@ -259,30 +260,42 @@ const FAILURE_FILTERS = [{
     return [new BuildFailure(ctx, reason)];
   }
 }, {
-  filter(ctx, text) {
-    const patterns = [{
-      pattern: /FATAL: .+/g,
-      context: { index: -1, contextBefore: 0, contextAfter: 5 }
-    }, {
-      pattern: /make.*: write error/mg,
-      context: { index: 0, contextBefore: 0, contextAfter: 3 }
-    }, {
-      pattern: /error: .+/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 5 }
-    }, {
-      pattern: /Makefile:.+failed/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 5 }
-    }, {
-      pattern: /make.*: .+ Error \d.*/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 3 }
-    }, {
-      pattern: /warning: failed .+/g,
-      context: { index: 0, contextBefore: 0, contextAfter: 3 }
-    }];
-    return failureMatcher(BuildFailure, patterns, ctx, text);
-  }
+  Failure: BuildFailure,
+  patterns: [{
+    pattern: /FATAL: .+/g,
+    context: { index: -1, contextBefore: 0, contextAfter: 5 }
+  }, {
+    pattern: /make.*: write error/mg,
+    context: { index: 0, contextBefore: 0, contextAfter: 3 }
+  }, {
+    pattern: /error: .+/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 5 }
+  }, {
+    pattern: /Makefile:.+failed/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 5 }
+  }, {
+    pattern: /make.*: .+ Error \d.*/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 3 }
+  }, {
+    pattern: /warning: failed .+/g,
+    context: { index: 0, contextBefore: 0, contextAfter: 3 }
+  }],
 }];
 
+// Share recognition patterns with streaming consumers without exposing the
+// parser's constructors, filter priority, or context-selection machinery.
+export const FAILURE_PATTERNS = {
+  infrastructure: FAILURE_FILTERS
+    .filter(({ Failure }) => Failure === InfraFailure)
+    .flatMap(({ patterns }) => patterns.map(({ pattern }) => pattern)),
+  diagnostic: [
+    ...FAILURE_FILTERS
+      .filter(({ Failure }) => Failure && Failure !== InfraFailure && Failure !== CCTestFailure)
+      .flatMap(({ patterns }) => patterns.map(({ pattern }) => pattern)),
+    fatalPattern
+  ]
+};
+
 export default class CIFailureParser {
   constructor(ctx, text) {
     this.ctx = ctx;
@@ -291,8 +304,10 @@ export default class CIFailureParser {
 
   parse() {
     const text = this.text;
-    for (const { filter } of FAILURE_FILTERS) {
-      const result = filter(this.ctx, text);
+    for (const { filter, Failure, patterns } of FAILURE_FILTERS) {
+      const result = filter
+        ? filter(this.ctx, text)
+        : failureMatcher(Failure, patterns, this.ctx, text);
       // TODO: we may want to concat certain types of failures
       if (result) {
         return result;

package/lib/ci/failure_file_scanner.js

@@ -0,0 +1,153 @@
+import { StringDecoder } from 'node:string_decoder';
+
+import { FAILURE_MARKERS, FAILURE_PATTERNS } from './ci_failure_parser.js';
+
+function createMatcher(patterns) {
+  // Preserve matching flags without sharing RegExp.lastIndex between scans.
+  const matchers = patterns.map(pattern =>
+    new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, '')));
+  return text => matchers.some(pattern => pattern.test(text));
+}
+
+const diagnostic = createMatcher(FAILURE_PATTERNS.diagnostic);
+const infrastructure = createMatcher(FAILURE_PATTERNS.infrastructure);
+const gitStart = createMatcher(FAILURE_MARKERS.git.map(({ start }) => start));
+const gitEnd = createMatcher(FAILURE_MARKERS.git.map(({ end }) => end));
+
+function fileAliases(filename) {
+  const paths = new Set([filename]);
+  if (filename.startsWith('test/')) {
+    paths.add(filename.slice(5));
+    paths.add(filename.slice(5).replace(/\.(?:js|mjs|cjs|out)$/, ''));
+  } else if (filename.startsWith('lib/')) {
+    paths.add(filename.slice(4));
+  }
+  return [...paths];
+}
+
+// Emit overlapping, normalized windows rather than buffering whole log lines.
+// Explicit line boundaries keep network chunk boundaries out of the matching rules.
+async function * logWindows(source, overlap) {
+  const decoder = new StringDecoder('utf8');
+  let tail = '';
+  let lineStart = true;
+  let afterBackslash = false;
+
+  function * consume(raw) {
+    raw = raw.replace(/\r/g, '');
+    let text = raw.replace(/\\+/g, '/');
+    if (afterBackslash && raw.startsWith('\\')) text = text.slice(1);
+    if (raw) afterBackslash = raw.endsWith('\\');
+    for (let offset = 0; offset < text.length;) {
+      const newline = text.indexOf('\n', offset);
+      const end = Math.min(offset + 8192, newline < 0 ? text.length : newline + 1);
+      const window = tail + text.slice(offset, end);
+      const lineEnd = end === newline + 1;
+      yield { text: window, lineStart, lineEnd };
+      if (lineEnd) {
+        tail = '';
+        lineStart = true;
+      } else {
+        lineStart &&= window.length <= overlap;
+        tail = window.slice(-overlap);
+      }
+      offset = end;
+    }
+  }
+
+  for await (const chunk of source) yield * consume(decoder.write(chunk));
+  yield * consume(decoder.end());
+  if (tail) yield { text: `${tail}\n`, lineStart, lineEnd: true };
+}
+
+// Each line is reduced to a filename and failure markers. Neither a giant line
+// nor a giant TAP block needs to survive in memory.
+async function * failureLines(windows, matchFile) {
+  let line = {};
+  for await (const { text, lineStart, lineEnd } of windows) {
+    line.file ??= matchFile(text, lineStart);
+    line.diagnostic ||= diagnostic(text);
+    line.infrastructure ||= infrastructure(text);
+    line.cpp ||= FAILURE_MARKERS.cpp.test(text);
+    line.tapStart ||= lineStart && FAILURE_MARKERS.tap.start.test(text);
+    line.tapEnd ||= lineEnd && FAILURE_MARKERS.tap.end.test(text);
+    line.todo ||= text.includes(FAILURE_MARKERS.tap.todo);
+    line.gitStart ||= gitStart(text);
+    line.gitEnd ||= gitEnd(text);
+    if (lineEnd) {
+      yield line;
+      line = {};
+    }
+  }
+}
+
+async function findFailure(lines) {
+  let history = [];
+  let followingLines = 0;
+  let tap = null;
+  let git = null;
+
+  for await (const line of lines) {
+    const precedingLines = history;
+    history = [...history, line.file].slice(-5);
+    if (line.tapStart) {
+      tap = {};
+      followingLines = 0;
+    }
+    if (tap) {
+      tap.file ??= line.file;
+      tap.todo ||= line.todo;
+      // A later TODO can mark this as an expected failure; wait for the ending.
+      if (line.tapEnd) {
+        if (!tap.todo && tap.file) return tap.file;
+        tap = null;
+      }
+      continue;
+    }
+
+    if (line.gitStart) git = {};
+    if (git) {
+      git.file ??= line.file;
+      if (line.gitEnd) {
+        if (git.file) return git.file;
+        git = null;
+      }
+    }
+    if (line.infrastructure || line.cpp) {
+      const before = line.cpp ? 5 : 1;
+      const file = line.file || precedingLines.slice(-before).find(Boolean);
+      if (file) return file;
+    }
+    if (line.diagnostic) followingLines = 6;
+    if (followingLines > 0) {
+      followingLines--;
+      if (line.file) return line.file;
+    }
+  }
+}
+
+export class FailureFileScanner {
+  constructor(filenames) {
+    const aliases = [...filenames].map(filename => ({ filename, paths: fileAliases(filename) }));
+    this.overlap = Math.max(256,
+      ...aliases.flatMap(({ paths }) => paths.map(path => path.length + 2)));
+    const matchers = aliases.map(({ filename, paths }) => ({
+      filename,
+      patterns: paths.map(path => {
+        const escaped = path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+        // Both delimiters must be real: a window boundary is not a path boundary.
+        return new RegExp(`[^\\w.-]${escaped}(?=[^\\w./-])`);
+      })
+    }));
+    this.matchFile = (text, lineStart) => {
+      // The start of a real log line is also a valid left delimiter.
+      if (lineStart) text = `\n${text}`;
+      return matchers.find(({ patterns }) =>
+        patterns.some(pattern => pattern.test(text)))?.filename;
+    };
+  }
+
+  scan(source) {
+    return findFailure(failureLines(logWindows(source, this.overlap), this.matchFile));
+  }
+}