browser-commander 0.26.0 → 0.26.2

package/tests/unit/scripts/check-pipeline-status.test.js

@@ -6,9 +6,17 @@
  * `cancelled` as well - run 24045269874 of this repository is one such run on
  * `main`. scripts/check-pipeline-status.sh is the only thing that looks at
  * that, so these tests pin the readings it has to get right: a failure is
- * always an error, a cancellation off the default branch is a warning, a
- * cancellation on the default branch is an error unless a newer run has already
- * taken over the branch head.
+ * always an error, and a cancellation is an error unless a newer commit has
+ * overtaken the run *and* the cancelled job is one a newer run can cancel
+ * (`cancel-in-progress: true`). Anything else - a job that queues instead, a
+ * run still at the branch head, a value the gate cannot read - is a timeout or
+ * a manual stop that nothing else will report.
+ *
+ * The gate used to warn about every cancellation off the default branch, so a
+ * pull request whose test job hit `timeout-minutes` passed its gate (issue
+ * #128), and it excused a cancelled release writer on a superseded `main` run
+ * although `cancel-in-progress: false` means no newer run can cancel it.
+ *
  * Hosted-runner failures can report an undocumented abandoned result; it and
  * other unexpected/missing results must fail on every branch.
  *
@@ -16,27 +24,65 @@
  */
 
 import assert from 'node:assert/strict';
-import { mkdtempSync, rmSync } from 'node:fs';
+import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { spawnSync } from 'node:child_process';
-import { describe, it } from 'node:test';
+import { after, before, describe, it } from 'node:test';
 
+import { readCancelInProgress } from '../../../../scripts/read-job-cancel-in-progress.mjs';
 import {
   BASH_AVAILABLE,
   REPO_ROOT,
+  readRepoText,
   repoPath,
   runBashScript,
 } from '../../helpers/repo.js';
 
 const SCRIPT = repoPath('scripts/check-pipeline-status.sh');
 
-function runGate(needs, env = {}, options = {}) {
-  return runBashScript(SCRIPT, [], {
-    cwd: options.cwd,
-    env: { NEEDS_JSON: JSON.stringify(needs), ...env },
-  });
-}
+const FIXTURE_WORKFLOW = `name: Fixture
+
+concurrency:
+  group: fixture-\${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04
+    concurrency:
+      group: \${{ github.workflow }}-\${{ github.ref }}-test
+      cancel-in-progress: true # superseded runs stop here
+    steps:
+      - run: echo test
+
+  release:
+    runs-on: ubuntu-24.04
+    concurrency:
+      group: main-writer
+      cancel-in-progress: false
+    steps:
+      - run: echo release
+
+  dynamic:
+    runs-on: ubuntu-24.04
+    concurrency:
+      group: dynamic
+      cancel-in-progress: \${{ github.event_name == 'pull_request' }}
+    steps:
+      - run: echo dynamic
+
+  inherits:
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo inherits
+
+  group-only:
+    runs-on: ubuntu-24.04
+    concurrency: just-a-group
+    steps:
+      - run: echo group
+`;
 
 function results(map) {
   return Object.fromEntries(
@@ -44,11 +90,76 @@ function results(map) {
   );
 }
 
+describe('readCancelInProgress', () => {
+  const policy = readCancelInProgress(FIXTURE_WORKFLOW, [
+    'test',
+    'release',
+    'dynamic',
+    'inherits',
+    'group-only',
+    'absent',
+  ]);
+
+  it('reads a literal job-level value, comments and all', () => {
+    assert.equal(policy.get('test'), 'true');
+    assert.equal(policy.get('release'), 'false');
+  });
+
+  it('reports an expression as unknown instead of guessing', () => {
+    assert.equal(policy.get('dynamic'), 'unknown');
+  });
+
+  it('falls back to the workflow-level group, and to false for a bare group', () => {
+    assert.equal(policy.get('inherits'), 'false');
+    assert.equal(policy.get('group-only'), 'false');
+  });
+
+  it('says when the workflow has no such job', () => {
+    assert.equal(policy.get('absent'), 'missing');
+  });
+
+  it('says when neither the job nor the workflow has a group', () => {
+    const bare = 'name: Bare\non: push\njobs:\n  only:\n    runs-on: x\n';
+    assert.equal(readCancelInProgress(bare, ['only']).get('only'), 'none');
+  });
+});
+
 describe('check-pipeline-status.sh', { skip: !BASH_AVAILABLE }, () => {
+  let directory;
+  let workflowFile;
+
+  before(() => {
+    directory = mkdtempSync(path.join(tmpdir(), 'pipeline-status-'));
+    workflowFile = path.join(directory, 'fixture.yml');
+    writeFileSync(workflowFile, FIXTURE_WORKFLOW);
+  });
+
+  after(() => {
+    rmSync(directory, { force: true, recursive: true });
+  });
+
+  // GITHUB_WORKFLOW_REF is set when this suite itself runs in Actions; blank
+  // it so only what a test passes decides which workflow the gate reads.
+  function runGate(needs, env = {}, options = {}) {
+    return runBashScript(SCRIPT, [], {
+      cwd: options.cwd,
+      env: {
+        GITHUB_WORKFLOW_REF: '',
+        WORKFLOW_FILE: workflowFile,
+        BRANCH_NAME: 'main',
+        RUN_SHA: 'abc',
+        BRANCH_HEAD_SHA: 'abc',
+        NEEDS_JSON: JSON.stringify(needs),
+        ...env,
+      },
+    });
+  }
+
+  const superseded = { BRANCH_HEAD_SHA: 'def' };
+
   it('passes when every job succeeded or was legitimately skipped', () => {
     const { status, output } = runGate(
-      results({ lint: 'success', release: 'skipped' }),
-      { IS_MAIN: 'true', RUN_SHA: 'abc', BRANCH_HEAD_SHA: 'abc' }
+      results({ lint: 'success', release: 'skipped' })
     );
 
     assert.equal(status, 0);
@@ -60,19 +171,15 @@ describe('check-pipeline-status.sh', { skip: !BASH_AVAILABLE }, () => {
 
   it('fails on a failed job and names it', () => {
     const { status, output } = runGate(
-      results({ lint: 'failure', test: 'success' }),
-      { IS_MAIN: 'true', RUN_SHA: 'abc', BRANCH_HEAD_SHA: 'abc' }
+      results({ lint: 'failure', test: 'success' })
     );
 
     assert.equal(status, 1);
     assert.match(output, /::error::Pipeline failed\. Failing jobs: lint/);
   });
 
-  for (const env of [
-    { IS_MAIN: 'false' },
-    { IS_MAIN: 'true', RUN_SHA: 'abc', BRANCH_HEAD_SHA: 'def' },
-  ]) {
-    it(`fails on an abandoned runner job with IS_MAIN=${env.IS_MAIN}`, () => {
+  for (const env of [{}, superseded]) {
+    it(`fails on an abandoned runner job (branch head ${env.BRANCH_HEAD_SHA ?? 'abc'})`, () => {
       // Hosted-runner acquisition failure in run 37362314527 left the macOS
       // check cancelled, but GitHub sent "abandoned" to the needs context.
       const { status, output } = runGate(
@@ -98,58 +205,124 @@ describe('check-pipeline-status.sh', { skip: !BASH_AVAILABLE }, () => {
     });
   }
 
-  it('only warns about a cancellation off the default branch', () => {
+  it('fails on a cancellation in a pull request run that nothing overtook', () => {
+    // The old gate only warned off the default branch, so a pull request whose
+    // test job hit timeout-minutes passed its gate.
     const { status, output } = runGate(results({ test: 'cancelled' }), {
-      IS_MAIN: 'false',
-      RUN_SHA: 'abc',
-      BRANCH_HEAD_SHA: 'abc',
+      BRANCH_NAME: 'issue-128-feature',
     });
 
+    assert.equal(status, 1);
+    assert.match(output, /This run tests abc; issue-128-feature is at abc/);
+    assert.match(
+      output,
+      /test: the run is still the head of issue-128-feature/
+    );
+    assert.match(output, /::error::Pipeline has cancelled jobs: test\./);
+  });
+
+  it('fails on a cancellation at the default branch head', () => {
+    const { status, output } = runGate(results({ test: 'cancelled' }));
+
+    assert.equal(status, 1);
+    assert.match(output, /::error::Pipeline has cancelled jobs: test\./);
+  });
+
+  it('only warns when a newer commit overtook a job that cancels in progress', () => {
+    const { status, output } = runGate(
+      results({ test: 'cancelled' }),
+      superseded
+    );
+
     assert.equal(status, 0);
-    assert.match(output, /::warning::Cancelled jobs: test/);
+    assert.match(output, /This run tests abc; main is at def/);
+    assert.match(output, /test: it sets cancel-in-progress: true/);
+    assert.match(
+      output,
+      /::warning::Cancelled jobs in a superseded run: test\./
+    );
+    assert.doesNotMatch(output, /::error::/);
   });
 
-  it('fails on a cancellation on the default branch head', () => {
+  for (const [job, reason] of [
+    ['release', /it sets cancel-in-progress: false, so a newer run queues/],
+    ['dynamic', /cancel-in-progress is an expression or otherwise unreadable/],
+    ['inherits', /it sets cancel-in-progress: false, so a newer run queues/],
+    ['absent', /declares no job by that name/],
+  ]) {
+    it(`fails on a superseded run's cancelled ${job} job, which no supersede explains`, () => {
+      const { status, output } = runGate(
+        results({ test: 'cancelled', [job]: 'cancelled' }),
+        superseded
+      );
+
+      assert.equal(status, 1);
+      assert.match(output, reason);
+      assert.match(
+        output,
+        /::warning::Cancelled jobs in a superseded run: test\./
+      );
+      assert.match(
+        output,
+        new RegExp(`::error::Pipeline has cancelled jobs: ${job}\\.`)
+      );
+    });
+  }
+
+  it('fails closed when it cannot tell which workflow it is gating', () => {
     const { status, output } = runGate(results({ test: 'cancelled' }), {
-      IS_MAIN: 'true',
-      RUN_SHA: 'abc',
-      BRANCH_HEAD_SHA: 'abc',
+      ...superseded,
+      WORKFLOW_FILE: '',
     });
 
     assert.equal(status, 1);
-    assert.match(output, /::error::Pipeline has cancelled jobs on main: test/);
+    assert.match(output, /could not identify its workflow/);
   });
 
-  it('only warns when a newer commit already superseded the run', () => {
-    const { status, output } = runGate(results({ test: 'cancelled' }), {
-      IS_MAIN: 'true',
-      RUN_SHA: 'abc',
-      BRANCH_HEAD_SHA: 'def',
-    });
+  it('finds its workflow through GITHUB_WORKFLOW_REF, from the repository root', () => {
+    // ci-policy.yml's `check` job cancels in progress; `pipeline-status` reads
+    // it from the checkout whichever directory the step runs in.
+    const { status, output } = runGate(
+      results({ check: 'cancelled' }),
+      {
+        ...superseded,
+        WORKFLOW_FILE: '',
+        GITHUB_WORKFLOW_REF:
+          'link-foundation/browser-commander/.github/workflows/ci-policy.yml@refs/pull/129/merge',
+      },
+      { cwd: directory }
+    );
 
-    assert.equal(status, 0);
-    assert.match(output, /This run tests abc; main is at def/);
-    assert.match(output, /::warning::Cancelled jobs: test/);
+    assert.equal(status, 0, output);
+    assert.match(output, /check: it sets cancel-in-progress: true/);
   });
 
   it('fails rather than guesses when the branch head cannot be resolved', () => {
     // A missed supersede costs one noisy warning; a missed overrun on main
     // costs a silent failure, so the unresolvable case has to be the loud one.
-    const directory = mkdtempSync(path.join(tmpdir(), 'pipeline-status-'));
+    const { status, output } = runGate(
+      results({ test: 'cancelled' }),
+      { BRANCH_HEAD_SHA: '', GIT_REMOTE: 'no-such-remote' },
+      { cwd: directory }
+    );
 
-    try {
-      const { status, output } = runGate(
-        results({ test: 'cancelled' }),
-        { IS_MAIN: 'true', RUN_SHA: 'abc', BRANCH_HEAD_SHA: '' },
-        { cwd: directory }
-      );
+    assert.equal(status, 1);
+    assert.match(output, /assuming it is current/);
+    assert.match(output, /::error::Pipeline has cancelled jobs: test\./);
+  });
 
-      assert.equal(status, 1);
-      assert.match(output, /assuming it is current/);
-      assert.match(output, /::error::Pipeline has cancelled jobs on main/);
-    } finally {
-      rmSync(directory, { force: true, recursive: true });
-    }
+  it('prints the classification trace only when asked', () => {
+    const quiet = runGate(results({ test: 'cancelled' }), superseded);
+    const verbose = runGate(results({ test: 'cancelled' }), {
+      ...superseded,
+      PIPELINE_STATUS_VERBOSE: '1',
+    });
+
+    assert.doesNotMatch(quiet.output, /\[pipeline-status\]/);
+    assert.match(
+      verbose.output,
+      /\[pipeline-status\] cancelled test: cancel-in-progress=true, superseded=yes/
+    );
   });
 
   it('refuses to run without the job results', () => {
@@ -163,3 +336,42 @@ describe('check-pipeline-status.sh', { skip: !BASH_AVAILABLE }, () => {
     assert.match(result.stderr, /NEEDS_JSON is required/);
   });
 });
+
+describe('workflows that run the gate', () => {
+  const workflows = readdirSync(repoPath('.github/workflows'))
+    .filter((name) => name.endsWith('.yml'))
+    .map((name) => ({ name, source: readRepoText('.github/workflows', name) }))
+    .filter(({ source }) => source.includes('check-pipeline-status.sh'));
+
+  it('exist', () => {
+    assert.ok(workflows.length > 0);
+  });
+
+  for (const { name, source } of workflows) {
+    it(`${name} tells the gate which commit and branch the run tests`, () => {
+      // A pull request run's github.sha is a merge commit no branch points at,
+      // and github.ref_name is "129/merge"; the head commit and head branch are
+      // what a newer push moves.
+      assert.match(
+        source,
+        /RUN_SHA: \$\{\{ github\.event\.pull_request\.head\.sha \|\| github\.sha \}\}/
+      );
+      assert.match(
+        source,
+        /BRANCH_NAME: \$\{\{ github\.head_ref \|\| github\.ref_name \}\}/
+      );
+      assert.doesNotMatch(source, /IS_MAIN:/);
+    });
+
+    it(`${name} gives every job a cancel-in-progress the gate can read`, () => {
+      const jobsBlock = source.slice(source.search(/^jobs:\s*$/m));
+      const jobs = [
+        ...jobsBlock.matchAll(/^ {2}([A-Za-z_][\w.-]*):\s*$/gm),
+      ].map((match) => match[1]);
+      const unreadable = [...readCancelInProgress(source, jobs)]
+        .filter(([, value]) => !['true', 'false'].includes(value))
+        .map(([job, value]) => `${job}=${value}`);
+      assert.deepEqual(unreadable, []);
+    });
+  }
+});

package/tests/unit/scripts/check-version-modification.test.js

@@ -1,7 +1,17 @@
 import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
 import { describe, it } from 'node:test';
 
-import { findManualVersionChanges } from '../../../../scripts/check-version-modification.mjs';
+import {
+  checkVersionModification,
+  findManualVersionChanges,
+} from '../../../../scripts/check-version-modification.mjs';
+import { repoPath } from '../../helpers/repo.js';
+
+const SCRIPT = repoPath('scripts/check-version-modification.mjs');
 
 // The real implementation shells out to git; injecting the diff keeps these
 // cases independent of the branch the suite happens to run on.
@@ -68,3 +78,99 @@ describe('check-version-modification', () => {
     assert.deepEqual(findManualVersionChanges('main', diff), []);
   });
 });
+
+describe('check-version-modification when git cannot diff', () => {
+  // Issue #128 (link-foundation/rust-ai-driven-development-pipeline-template
+  // #174): a failed `git diff` used to read as "nothing changed", so a missing
+  // base ref turned the check into a pass.
+  function failingDiff() {
+    throw new Error("fatal: ambiguous argument 'origin/main...HEAD'");
+  }
+
+  it('fails rather than reporting no change', () => {
+    assert.throws(
+      () => findManualVersionChanges('main', failingDiff),
+      /Could not diff js\/package\.json against origin\/main/
+    );
+  });
+
+  it('fetches the base once and retries before giving up', () => {
+    let fetched = 0;
+    const diff = (_baseRef, path) => {
+      if (fetched === 0) {
+        failingDiff();
+      }
+      return path === 'rust/Cargo.toml' ? '+version = "0.9.1"' : '';
+    };
+
+    assert.deepEqual(
+      checkVersionModification('main', {
+        diff,
+        fetchBase: () => {
+          fetched += 1;
+        },
+      }),
+      ['rust/Cargo.toml']
+    );
+    assert.equal(fetched, 1);
+  });
+
+  it('reports the failure when the retry fails too', () => {
+    let fetched = 0;
+
+    assert.throws(
+      () =>
+        checkVersionModification('main', {
+          diff: failingDiff,
+          fetchBase: () => {
+            fetched += 1;
+          },
+        }),
+      /Could not diff/
+    );
+    assert.equal(fetched, 1);
+  });
+
+  it('exits 1 with an error annotation from the command line', () => {
+    const directory = mkdtempSync(path.join(tmpdir(), 'version-check-'));
+
+    try {
+      // A repository with no origin: the base ref cannot exist, and neither
+      // can the fetch that would bring it in.
+      spawnSync('git', ['init', '-q', directory]);
+      spawnSync(
+        'git',
+        [
+          '-C',
+          directory,
+          '-c',
+          'user.name=t',
+          '-c',
+          'user.email=t@example.com',
+          'commit',
+          '-q',
+          '--allow-empty',
+          '-m',
+          'init',
+        ],
+        { encoding: 'utf8' }
+      );
+
+      const result = spawnSync(process.execPath, [SCRIPT], {
+        cwd: directory,
+        encoding: 'utf8',
+        env: {
+          ...process.env,
+          GITHUB_BASE_REF: 'main',
+          GITHUB_HEAD_REF: 'feature',
+        },
+      });
+
+      assert.equal(result.status, 1, result.stdout + result.stderr);
+      assert.match(result.stderr, /::error::Could not diff/);
+      assert.doesNotMatch(result.stdout, /No manual version changes/);
+    } finally {
+      rmSync(directory, { force: true, recursive: true });
+    }
+  });
+});

package/tests/unit/scripts/ci-timeout-budgets.test.js

@@ -199,18 +199,31 @@ describe('CI execution budgets', () => {
       'parity.yml cli CLI and API coverage suites',
       'parity.yml cli Cross-language CLI contract',
       'parity.yml cli Rust CLI build',
+      'parity.yml engine-suites Engine lifecycle suites',
+      'parity.yml fixture-suites React fixture suites',
       'parity.yml parity Fingerprint parity suite',
       'parity.yml parity WebDriver suite',
+      'parity.yml snapshots Python headful real-browser launches',
       'parity.yml snapshots Python native snapshot launches',
+      'parity.yml snapshots Rust launch and connect smokes',
       'parity.yml snapshots Rust native snapshots and parity',
       'parity.yml storage-state Rust storage state transfer',
       'python.yml test pytest suite',
       'rust.yml coverage Rust code coverage',
-      'rust.yml test Rust doc tests',
       'rust.yml test Rust test suite',
     ]);
   });
 
+  it('runs the Rust doc tests once, inside the main test suite', () => {
+    // `cargo test --all-features` already runs them. A second `cargo test
+    // --doc` with the default features rebuilt the crate and, on Windows,
+    // stalled silently until its budget killed it (issue #128).
+    const block = getJobBlock(readWorkflow('rust.yml'), 'test');
+
+    assert.match(block, /cargo test --all-features/);
+    assert.doesNotMatch(block, /cargo test --doc/);
+  });
+
   it('warns at the same share of a budget that this invariant allows', () => {
     const wrapper = readFileSync(
       repoPath('scripts/run-with-budget-warning.sh'),
@@ -313,7 +326,7 @@ describe('pipeline status gate', () => {
 
       assert.match(
         block,
-        /RUN_SHA: \$\{\{ github\.sha \}\}/,
+        /RUN_SHA: \$\{\{ github\.event\.pull_request\.head\.sha \|\| github\.sha \}\}/,
         `${fileName}: ${GATE_JOB} cannot tell a supersede from an overrun without RUN_SHA`
       );
     }