create-pmos 0.51.0 → 0.53.0

package/workspace/run-tests.js

@@ -2,7 +2,7 @@
 'use strict';
 /* run-tests.js — run the workspace suites side by side.
  *
- *   node run-tests.js <group> [--jobs=N] [--shard=I/N] [--list] [--verbose]
+ *   node run-tests.js <group> [--jobs=N] [--shard=I/N] [--changed[=REF]] [--list] [--verbose]
  *   node run-tests.js --check
  *
  * Groups come from test-files.json: unit (test.js + modules), modules, panels, ui, browser
@@ -17,8 +17,20 @@
  * file in exactly one shard), and --check fails when a suite on disk is in no list, so a new suite
  * that nobody wired in is a red build rather than a suite that runs nowhere.
  *
+ * --changed is for the LOCAL inner loop only. It narrows the group to the suites a diff against REF
+ * (default origin/main) can reach, and runs the whole group whenever it cannot tell. CI never passes
+ * it: Workspace CI runs every suite on every PR, so a suite --changed leaves out still runs before
+ * the merge. See changedSuites() for the rule.
+ *
  * Output: a suite's own output is buffered and printed whole when it finishes, so lines from two
  * suites never interleave. A passing suite prints its last lines; --verbose prints all of it.
+ *
+ * Fragments (pm-os-absorb Runs 2 to 5). Builders who work at the same time add their suites in a file
+ * of their own, test-lists/<name>.json, never in test-files.json: two slices that each append to the
+ * same JSON array always conflict. A fragment has the keys modules, panels, exclusive and seconds of
+ * test-files.json, plus `helpers` (a -test.js file that is a helper, not a suite, with the reason, as
+ * in ELSEWHERE). This file merges every fragment into the lists before it does anything else, so
+ * --check, --shard and every group see one list. A fragment that is not valid fails every command.
  */
 const fs = require('fs');
 const os = require('os');
@@ -26,7 +38,41 @@ const path = require('path');
 const { spawn } = require('child_process');
 
 const DIR = __dirname;
-const FILES = JSON.parse(fs.readFileSync(path.join(DIR, 'test-files.json'), 'utf8'));
+const FRAG_DIR = path.join(DIR, 'test-lists');
+const FRAG_KEYS = ['_comment', 'modules', 'panels', 'exclusive', 'seconds', 'helpers'];
+const fragProblems = [];
+const fragHelpers = {};
+/** test-files.json with every test-lists/*.json fragment merged in (see the header). Problems go to fragProblems. */
+function loadFiles() {
+  const files = JSON.parse(fs.readFileSync(path.join(DIR, 'test-files.json'), 'utf8'));
+  let names = [];
+  try { names = fs.readdirSync(FRAG_DIR).filter((n) => n.endsWith('.json')).sort(); } catch { /* no fragments yet */ }
+  for (const n of names) {
+    const where = `test-lists/${n}`;
+    if (!/^[a-z0-9][a-z0-9-]*\.json$/.test(n)) fragProblems.push(`${where}: a fragment is named <lowercase-id>.json, for example run2-a.json`);
+    let frag;
+    try { frag = JSON.parse(fs.readFileSync(path.join(FRAG_DIR, n), 'utf8')); } catch (e) { fragProblems.push(`${where} is not valid JSON — ${e.message}`); continue; }
+    if (!frag || typeof frag !== 'object' || Array.isArray(frag)) { fragProblems.push(`${where} is not a JSON object`); continue; }
+    for (const k of Object.keys(frag)) if (!FRAG_KEYS.includes(k)) fragProblems.push(`${where}: unknown key "${k}" (the keys are ${FRAG_KEYS.slice(1).join(', ')})`);
+    for (const k of ['modules', 'panels', 'exclusive']) {
+      if (frag[k] === undefined) continue;
+      if (!Array.isArray(frag[k]) || !frag[k].every((f) => typeof f === 'string')) { fragProblems.push(`${where}: "${k}" is a list of file names`); continue; }
+      files[k] = [...(files[k] || []), ...frag[k]];
+    }
+    for (const [f, v] of Object.entries(frag.seconds || {})) {
+      if (typeof v !== 'number' || !(v > 0)) fragProblems.push(`${where}: seconds of ${f} is a positive number`);
+      else if (files.seconds && f in files.seconds) fragProblems.push(`${where}: seconds of ${f} is already recorded elsewhere`);
+      else files.seconds = { ...(files.seconds || {}), [f]: v };
+    }
+    for (const [f, why] of Object.entries(frag.helpers || {})) {
+      if (typeof why !== 'string' || !why.trim()) fragProblems.push(`${where}: helper ${f} needs a reason`);
+      else if (fragHelpers[f]) fragProblems.push(`${where}: helper ${f} is already named by ${fragHelpers[f].where}`);
+      else fragHelpers[f] = { why, where };
+    }
+  }
+  return files;
+}
+const FILES = loadFiles();
 const GROUPS = {
   unit: [...FILES.unit, ...FILES.modules],
   modules: FILES.modules,
@@ -38,7 +84,14 @@ const GROUPS = {
 // Suites that run elsewhere on purpose, so --check does not count them as unwired.
 const ELSEWHERE = {
   'offline-tarball.test.js': 'npm run test:offline (needs a packed tarball)',
+  // A helper module, not a suite: the Breakdown suites require it (pm-os-absorb Run 1). Its name keeps it out of the
+  // tarball (BUNDLE_EXCLUDES has 'pm-*-test.js'), which is why it ends in -test.js.
+  'pm-breakdown-stub-test.js': 'not a suite — the helpers pm-breakdown-test.js, pm-breakdown-phone-test.js and first-paint-count.test.js require',
 };
+for (const [f, h] of Object.entries(fragHelpers)) {
+  if (ELSEWHERE[f]) fragProblems.push(`${h.where}: helper ${f} is already exempt in run-tests.js`);
+  else ELSEWHERE[f] = `${h.why} (${h.where})`;
+}
 const PER_FILE_TIMEOUT_MS = 10 * 60 * 1000;
 
 const args = process.argv.slice(2);
@@ -65,12 +118,60 @@ function shardOf(list, spec) {
   return out[i - 1];
 }
 
+// Files every page or every suite loads. A change to one can break any suite, so --changed runs
+// the whole group. A changed file no suite can be traced to is treated the same way (see below).
+const WIDE = new Set([
+  'server.js', 'test-run.js', 'run-tests.js', 'test-files.json', 'package.json', 'package-lock.json',
+  'public/pm/app.js', 'public/pm/state.js', 'public/pm/dom.js', 'public/pm/data.js',
+  'public/pm/tokens.css', 'public/pm/fixture.js', 'public/pm/mobile.css', 'lib/find-browser.js',
+]);
+
+/* The suites in `list` that the diff against `ref` can reach. A suite is selected when its own file
+ * changed, when its source names a changed file (`panel-tabs` in pm-tabs-test.js), or when its file
+ * name carries the changed file's stem with any panel-/view-/tab- prefix removed (view-memory.js ->
+ * pm-r29-memory-test.js). It errs toward running MORE: a WIDE file, or a changed workspace file that
+ * no suite can be traced to, selects the whole list. Returns { files, suites, why }. */
+function changedSuites(list, ref) {
+  const git = (...a) => {
+    const r = require('child_process').spawnSync('git', a, { cwd: DIR, encoding: 'utf8' });
+    return r.status === 0 ? r.stdout : null;
+  };
+  const top = (git('rev-parse', '--show-toplevel') || '').trim();
+  if (!top) return { files: [], suites: list, why: 'not a git checkout, so every suite runs' };
+  const base = (git('merge-base', 'HEAD', ref) || '').trim();
+  if (!base) return { files: [], suites: list, why: `no merge base with ${ref} (fetch it first), so every suite runs` };
+  const prefix = path.relative(top, DIR).split(path.sep).join('/') + '/';
+  const names = [
+    git('diff', '--name-only', base), // committed on this branch and uncommitted changes
+    git('ls-files', '--others', '--exclude-standard', '--full-name', DIR),
+  ].join('\n').split('\n').map((f) => f.trim()).filter(Boolean);
+  const files = [...new Set(names)].filter((f) => f.startsWith(prefix)).map((f) => f.slice(prefix.length));
+  if (!files.length) return { files, suites: [], why: `nothing under workspace/ differs from ${ref}` };
+  const wide = files.filter((f) => WIDE.has(f));
+  if (wide.length) return { files, suites: list, why: `${wide.join(', ')} changed, and every suite depends on it` };
+  const source = new Map(list.map((f) => [f, fs.existsSync(path.join(DIR, f)) ? fs.readFileSync(path.join(DIR, f), 'utf8') : '']));
+  const picked = new Set();
+  const untraced = [];
+  for (const f of files) {
+    if (source.has(f)) { picked.add(f); continue; }
+    if (/^(guide|docs)\/|\.md$/.test(f)) continue; // prose: no suite asserts on it except guide.test.js, a unit suite
+    const stem = path.basename(f).replace(/\.[^.]+$/, '');
+    const key = stem.replace(/^(panel|view|tab)-/, '');
+    const hits = list.filter((s) => source.get(s).includes(stem) || s.includes(`-${key}-`) || s.includes(`-${key}.`));
+    if (!hits.length) untraced.push(f);
+    for (const h of hits) picked.add(h);
+  }
+  if (untraced.length) return { files, suites: list, why: `no suite names ${untraced.slice(0, 3).join(', ')}${untraced.length > 3 ? ' …' : ''}, so every suite runs` };
+  return { files, suites: list.filter((f) => picked.has(f)), why: `${files.length} changed file(s) reach these suites` };
+}
+
 function check() {
   const listed = new Set(GROUPS.all);
-  const problems = [];
+  const problems = [...fragProblems];
   const dupes = GROUPS.all.filter((f, k) => GROUPS.all.indexOf(f) !== k);
-  for (const f of new Set(dupes)) problems.push(`${f} is listed more than once in test-files.json`);
-  for (const f of listed) if (!fs.existsSync(path.join(DIR, f))) problems.push(`${f} is listed in test-files.json but does not exist`);
+  for (const f of new Set(dupes)) problems.push(`${f} is listed more than once in test-files.json and test-lists/`);
+  for (const f of listed) if (!fs.existsSync(path.join(DIR, f))) problems.push(`${f} is listed in test-files.json or test-lists/ but does not exist`);
+  for (const f of Object.keys(ELSEWHERE)) if (listed.has(f)) problems.push(`${f} is exempt as a helper but is also listed as a suite`);
   for (const f of FILES.exclusive || []) if (!listed.has(f)) problems.push(`${f} is marked exclusive but is in no test list`);
   const onDisk = fs.readdirSync(DIR).filter((f) => f === 'test.js' || /(\.test\.(js|mjs)|-test\.js)$/.test(f));
   for (const f of onDisk) if (!listed.has(f) && !ELSEWHERE[f]) problems.push(`${f} is a suite on disk that no test list runs. Add it to test-files.json.`);
@@ -79,7 +180,8 @@ function check() {
     if (parts.length !== GROUPS[g].length || new Set(parts).size !== GROUPS[g].length) problems.push(`--shard does not partition "${g}" into ${n}`);
   }
   if (problems.length) { for (const p of problems) console.error(`✗ ${p}`); process.exitCode = 1; return; }
-  console.log(`✓ ${listed.size} suites listed, each once, each on disk; ${onDisk.length - Object.keys(ELSEWHERE).length} suites on disk, all listed; shards partition every group.`);
+  const frags = fs.existsSync(FRAG_DIR) ? fs.readdirSync(FRAG_DIR).filter((n) => n.endsWith('.json')).length : 0;
+  console.log(`✓ ${listed.size} suites listed, each once, each on disk; ${onDisk.filter((f) => !ELSEWHERE[f]).length} suites on disk, all listed; shards partition every group; ${frags} fragment(s) in test-lists/.`);
 }
 
 function runOne(file, verbose) {
@@ -107,6 +209,7 @@ function runOne(file, verbose) {
 
 async function main() {
   if (flag('check')) return check();
+  if (fragProblems.length) { for (const p of fragProblems) console.error(`✗ ${p}`); process.exitCode = 2; return; }
   if (!group || !GROUPS[group]) {
     console.error(`usage: node run-tests.js <${Object.keys(GROUPS).join('|')}> [--jobs=N] [--shard=I/N] [--list] [--verbose]\n       node run-tests.js --check`);
     process.exitCode = 2; return;
@@ -114,6 +217,14 @@ async function main() {
   let list = GROUPS[group];
   const shard = flag('shard');
   list = shard ? shardOf(list, shard) : bySlowest(list);
+  const changed = flag('changed');
+  if (changed) {
+    const ref = changed === true ? 'origin/main' : changed;
+    const pick = changedSuites(list, ref);
+    console.log(`run-tests: --changed against ${ref}: ${pick.why}. ${pick.suites.length} of ${list.length} suites run here; CI runs all ${list.length}.`);
+    list = pick.suites;
+    if (!list.length) return;
+  }
   if (flag('list')) { console.log(list.join('\n')); return; }
   const cpus = typeof os.availableParallelism === 'function' ? os.availableParallelism() : os.cpus().length;
   // A browser suite runs a server, a tmux daemon and a Chromium of its own, and several of them assert