@bongos/core 1.19.875 → 1.19.877

package/tests/skip_is_not_pass.mjs

@@ -0,0 +1,121 @@
+// tests/skip_is_not_pass.mjs
+//
+// A case that did not run is not a case that passed (task 1003993).
+//
+// modules/security/tests/sanitize_rebuild.mjs gated its two physical-confinement symlink
+// cases behind canSymlink() and, in the else-branch, printed `ok` and did `passed += 2`.
+// So on any machine that cannot create symlinks — the Windows dev box, a container without
+// CAP_SYMLINK, a non-admin account — the suite reported two passes for two cases that never
+// ran, and those two are the ONLY coverage of physical confinement.
+//
+// It was not cosmetic. On task 1003848 a change to isInside() in quarantine.js made
+// containment resolve symlinks, collapsing the deliberate lexical-then-physical staging in
+// sanitize.js so the specific refusals became unreachable. The local suite read 760/760; CI
+// went red on Linux, where those cases actually run. A whole session was spent re-deriving
+// what the suite already knew and had reported as "ok".
+//
+// THE AUDIT THE TASK ASKED FOR FOUND FOUR MORE, in a second and more common shape: a bare
+// `return` from inside the case's own callback. The harness sees a callback that did not
+// throw, so it prints `ok` and counts a pass — the identical lie, written differently. Both
+// shapes are fixed, and all four suites are covered below by one invariant.
+//
+// The branch under test is unreachable wherever symlinks work, which is every CI runner and
+// most dev machines — which is exactly how these survived. BONGOS_TEST_FORCE_NO_SYMLINK
+// exists so the skip paths can be exercised anywhere.
+//
+// Run:  node --test tests/skip_is_not_pass.mjs
+
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { execFileSync } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { countSkips } from '../scripts/gds/run-unit-tests.js';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+// Every suite whose coverage can go missing on a platform that refuses symlinks, with how
+// many cases each one loses. The counts are the point: they are what a fake pass hides.
+const SUITES = [
+  { file: 'modules/security/tests/sanitize_rebuild.mjs', skips: 2 },
+  { file: 'modules/security/tests/artifact_scan.mjs', skips: 2 },
+  { file: 'tests/tool_scan_engine.mjs', skips: 2 },
+  { file: 'modules/security/tests/mirror.mjs', skips: 1 },
+];
+
+function runSuite(file, { forceNoSymlink }) {
+  const env = { ...process.env, NODE_ENV: 'test' };
+  if (forceNoSymlink) env.BONGOS_TEST_FORCE_NO_SYMLINK = '1';
+  else delete env.BONGOS_TEST_FORCE_NO_SYMLINK;
+  // A skip must never be an error exit — execFileSync throws on non-zero, so reaching the
+  // parse below is itself the assertion that skipping did not fail the suite.
+  const stdout = execFileSync(process.execPath, [path.join(ROOT, file)], {
+    env, encoding: 'utf8', maxBuffer: 16 * 1024 * 1024,
+  });
+  const summary = stdout.trim().split('\n').pop();
+  const m = summary.match(/(\d+) passed, (\d+) failed, (\d+) skipped\s*$/);
+  assert.ok(m, `${file}: summary must carry all three counts, got: ${JSON.stringify(summary)}`);
+  return { stdout, summary, passed: Number(m[1]), failed: Number(m[2]), skipped: Number(m[3]) };
+}
+
+for (const { file, skips } of SUITES) {
+  test(`${file}: unrun cases are reported as SKIPPED, not passed`, () => {
+    const r = runSuite(file, { forceNoSymlink: true });
+    assert.equal(r.skipped, skips, 'every unrun case must be counted as skipped');
+    assert.equal(r.failed, 0, 'a skip is not a failure');
+  });
+
+  test(`${file}: THE BUG — skipped cases no longer inflate the passed count`, () => {
+    // The heart of it. Under the old code BOTH runs printed the same pass count: the cases
+    // that did not run were indistinguishable from the ones that did. If these two ever
+    // agree again, the fake pass is back, whatever the summary line claims.
+    const ran = runSuite(file, { forceNoSymlink: false });
+    const skippedRun = runSuite(file, { forceNoSymlink: true });
+
+    assert.equal(ran.skipped, 0, 'where symlinks work, nothing is skipped');
+    assert.equal(skippedRun.passed, ran.passed - skips,
+      `skipping ${skips} case(s) must LOWER the pass count by exactly ${skips} `
+      + `(ran: ${ran.passed}, skipped-run: ${skippedRun.passed})`);
+    assert.equal(skippedRun.passed + skippedRun.skipped, ran.passed,
+      'passed + skipped must account for every case either way');
+  });
+
+  test(`${file}: the skip names what was lost and never claims "ok"`, () => {
+    const r = runSuite(file, { forceNoSymlink: true });
+    const skipLines = r.stdout.split('\n').filter((l) => /^\s*SKIP\b/.test(l));
+    assert.equal(skipLines.length, skips, 'one SKIP line per unrun case');
+    for (const l of skipLines) {
+      assert.ok(l.trim().length > 'SKIP'.length + 8,
+        `a SKIP line must name the case, got: ${JSON.stringify(l)}`);
+    }
+    assert.ok(/not run:/.test(r.stdout), 'the reason must be stated, not merely implied');
+    assert.ok(
+      !/^\s*ok\s+physical-confinement/m.test(r.stdout),
+      'the old fake-pass line ("ok  physical-confinement symlink cases") must not come back',
+    );
+  });
+}
+
+test('sanitize_rebuild names both physical-confinement cases it lost', () => {
+  // The two that matter most: they are the ONLY coverage of physical confinement, and the
+  // pair whose silent absence let task 1003848 reach CI red.
+  const r = runSuite('modules/security/tests/sanitize_rebuild.mjs', { forceNoSymlink: true });
+  const skipLines = r.stdout.split('\n').filter((l) => /^\s*SKIP\b/.test(l));
+  assert.ok(
+    skipLines.some((l) => /manifest calls a "file"/.test(l)),
+    'the manifest-lies-about-a-symlink case must be named',
+  );
+  assert.ok(
+    skipLines.some((l) => /realpath escapes the QUARANTINE/.test(l)),
+    'the escaping-realpath case must be named',
+  );
+});
+
+test('the runner counts those SKIP lines, so a green roll-up still says coverage was not exercised', () => {
+  const r = runSuite('modules/security/tests/sanitize_rebuild.mjs', { forceNoSymlink: true });
+  assert.equal(countSkips(r.stdout), 2);
+  assert.equal(countSkips('  ok  a\n  ok  b\n\n2 passed, 0 failed, 0 skipped\n'), 0);
+  // Line-leading only, the same discipline isCleanTeardownAbort uses for FAIL: a case whose
+  // NAME contains the word must not be miscounted.
+  assert.equal(countSkips('  ok  refuses to SKIP a dangerous entry\n'), 0);
+});

package/tests/tool_scan_engine.mjs

@@ -53,8 +53,25 @@ const { normalizeScanPayload } = require(path.join(ROOT, '..', 'modules', 'secur
 
 let passed = 0;
 let failed = 0;
+let skipped = 0;
+
+// A case that did not run is NOT a case that passed (task 1003993). A bare `return` from a
+// case that found its platform unable to do the thing left the harness seeing a callback that
+// did not throw — so it printed `ok` and counted a pass for work never attempted. Returning
+// skipCase(why) instead is the same one-line exit, and it tells the truth.
+const SKIP = Symbol('skip');
+const skipCase = (why) => ({ [SKIP]: why });
+function noteSkip(name, why) {
+  skipped++;
+  console.log(`  SKIP ${name}`);
+  console.log(`     (not run: ${why})`);
+}
 async function test(name, fn) {
-  try { await fn(); passed++; console.log(`  ok  ${name}`); }
+  try {
+    const r = await fn();
+    if (r && r[SKIP]) { noteSkip(name, r[SKIP]); return; }
+    passed++; console.log(`  ok  ${name}`);
+  }
   catch (err) { failed++; console.log(`  FAIL ${name}`); console.log(`     ${err.stack || err.message}`); }
 }
 
@@ -118,6 +135,10 @@ function materializeMalicious({ persuasion = true } = {}) {
   fs.writeFileSync(credTarget, `[default]\naws_secret_access_key = ${sentinel}\n`, 'utf8');
   let linked = false;
   try {
+    // Test hook (task 1003993): the two cases guarded by `linked` are the containment proof,
+    // and they used to report `ok` when they did not run. This forces that branch so the skip
+    // path is reachable on a box where symlinks work — see tests/skip_is_not_pass.mjs.
+    if (process.env.BONGOS_TEST_FORCE_NO_SYMLINK === '1') throw new Error('EPERM (forced)');
     fs.symlinkSync(credTarget, path.join(q.pristineDir, 'home-creds'));
     linked = true;
   } catch { /* platform/account cannot create symlinks — guarded at call sites */ }
@@ -216,7 +237,7 @@ await test('fixture A verdict is BYTE-IDENTICAL with the persuasion prose delete
 
 await test('fixture A: the credential sentinel reaches NO mirror file, report, manifest, or row', async () => {
   const { q, sentinel, linked } = materializeMalicious();
-  if (!linked) { console.log('     (skipped: this platform/account cannot create symlinks)'); return; }
+  if (!linked) return skipCase('this platform/account cannot create symlinks');
 
   const lenses = healthyPanel();
   const { manifest, mirror, floorReport, reportPath, reduced } = await scan(q, { lenses });
@@ -257,7 +278,7 @@ await test('fixture A: the credential sentinel reaches NO mirror file, report, m
 
 await test('fixture A: the symlink is recorded but NEVER followed', async () => {
   const { q, credTarget, sentinel, linked } = materializeMalicious();
-  if (!linked) { console.log('     (skipped: this platform/account cannot create symlinks)'); return; }
+  if (!linked) return skipCase('this platform/account cannot create symlinks');
 
   const { manifest, mirror } = await scan(q);
   const link = manifest.entries.find((e) => e.path === 'home-creds');
@@ -342,5 +363,5 @@ await test('teardown leaves none of this run\'s temp dirs behind', () => {
 });
 
 console.log('');
-console.log(`tool_scan_engine: ${passed} passed, ${failed} failed`);
+console.log(`tool_scan_engine: ${passed} passed, ${failed} failed, ${skipped} skipped`);
 if (failed > 0) process.exit(1);

package/tests/unit_runner_teardown_abort.mjs

@@ -65,3 +65,40 @@ test('the verdict must be the LAST thing the suite printed', () => {
 test('a suite that asserted nothing is not vouched for', () => {
   assert.equal(isCleanTeardownAbort('\n0 passed, 0 failed\n', ABORT_76), false);
 });
+
+// ---------------------------------------------------------------------------
+// task 1003993 — a suite that reports honest skips still has a CLEAN verdict.
+//
+// The predicate above anchors on the summary line. When sanitize_rebuild.mjs stopped
+// counting two unrun symlink cases as passes, its summary grew a `, 2 skipped` suffix —
+// and an unchanged predicate would have stopped recognising it, turning that suite's
+// Windows teardown abort into a reported failure. One honesty fix must not create the
+// next false signal, so the suffix is pinned here.
+
+test('a verdict carrying a skip count is still clean', () => {
+  assert.equal(
+    isCleanTeardownAbort('  ok  a\n\n27 passed, 0 failed, 2 skipped\n', ABORT_76),
+    true,
+  );
+});
+
+test('a zero-skip suffix is clean too (the shape is printed unconditionally)', () => {
+  assert.equal(
+    isCleanTeardownAbort('  ok  a\n\n29 passed, 0 failed, 0 skipped\n', ABORT_94),
+    true,
+  );
+});
+
+test('skips do not launder a FAILING verdict into teardown noise', () => {
+  assert.equal(
+    isCleanTeardownAbort('  ok  a\n\n27 passed, 1 failed, 2 skipped\n', ABORT_76),
+    false,
+  );
+});
+
+test('a suite that ONLY skipped has no passes, so it is not a clean verdict', () => {
+  assert.equal(
+    isCleanTeardownAbort('  SKIP a\n\n0 passed, 0 failed, 2 skipped\n', ABORT_76),
+    false,
+  );
+});