jules-orchestrator-kit 0.60.0 → 0.64.0

package/src/ops/test-collection.mjs

@@ -0,0 +1,149 @@
+/**
+ * How many tests the runner actually collected, read out of its own output.
+ *
+ * The gate's oracle is one number: the exit code of the verification command.
+ * That number cannot distinguish "every test passed" from "there were no
+ * tests". Several runners report the second case as success, by design:
+ *
+ *   go test ./...        →  "?  example.com/app  [no test files]", exit 0
+ *   jest --passWithNoTests →  "No tests found, exiting with code 0"
+ *   npm test --workspaces →  exit 0 when the changed package has no suite
+ *   pytest --exitfirst on a path that matches nothing, in some configurations
+ *
+ * So a repository could invert a function, add an untested one, and collect
+ * five green phases — verified against nothing. `verify.required: false` is
+ * the switch for a repository that genuinely has no oracle; silently passing
+ * is not.
+ *
+ * The parsing is deliberately one-sided. A count is only returned when the
+ * runner stated one in a form recognised here; an unrecognised runner yields
+ * `null`, and null is not a failure. Failing on "I could not tell" would break
+ * every runner not on this list, which is most of them.
+ */
+
+/**
+ * Patterns that state a test count, per runner family.
+ *
+ * Each entry captures a single number. The first pattern that matches wins,
+ * so the more specific summaries come first.
+ */
+const COUNT_PATTERNS = [
+  // node:test — spec reporter ("ℹ tests 940") and tap ("# tests 940")
+  { name: "node:test", re: /^[^\n]*?(?:ℹ|#)\s*tests\s+(\d+)\s*$/m },
+  // pytest — "collected 12 items", "12 passed", "no tests ran in 0.01s"
+  { name: "pytest", re: /^\s*collected\s+(\d+)\s+items?/m },
+  { name: "pytest", re: /=+\s*(\d+)\s+passed/m },
+  // cargo — "running 7 tests"
+  { name: "cargo", re: /^\s*running\s+(\d+)\s+tests?\s*$/m },
+  // jest / vitest — "Tests:       12 passed, 12 total"
+  { name: "jest", re: /^\s*Tests:\s+.*?(\d+)\s+total\s*$/m },
+  // mocha — "12 passing"
+  { name: "mocha", re: /^\s*(\d+)\s+passing/m },
+  // Maven / Surefire — "Tests run: 12, Failures: 0"
+  { name: "surefire", re: /\bTests run:\s*(\d+)/i },
+  // PHPUnit — "OK (12 tests, 30 assertions)"
+  { name: "phpunit", re: /\bOK\s*\((\d+)\s+tests?/i },
+  // RSpec / ExUnit — "12 examples, 0 failures" / "12 tests, 0 failures"
+  { name: "rspec", re: /^\s*(\d+)\s+examples?,\s*\d+\s+failures?/m },
+  { name: "exunit", re: /^\s*(\d+)\s+tests?,\s*\d+\s+failures?/m },
+  // dotnet test — "Total tests: 12" / "Passed!  - Failed: 0, Passed: 12"
+  { name: "dotnet", re: /\bTotal(?:\s+tests)?:\s*(\d+)/i },
+  // swift test / XCTest — "Executed 12 tests"
+  { name: "xctest", re: /\bExecuted\s+(\d+)\s+tests?/i },
+];
+
+/** Per-test lines, which `go test` only prints under -v. */
+const GO_PER_TEST = /^\s*--- (?:PASS|FAIL|SKIP):/gm;
+
+/** Phrases that state, in so many words, that nothing was collected. */
+const EXPLICIT_ZERO = [
+  { name: "pytest", re: /\bno tests ran\b/i },
+  { name: "pytest", re: /^\s*collected\s+0\s+items?/m },
+  { name: "jest", re: /\bNo tests found\b/i },
+  { name: "vitest", re: /\bNo test files found\b/i },
+  { name: "mocha", re: /^\s*0\s+passing/m },
+  { name: "cargo", re: /^\s*running\s+0\s+tests?\s*$/m },
+  { name: "phpunit", re: /\bNo tests executed!/i },
+  { name: "gradle", re: /^>\s*Task\s+:\S*test\S*\s+NO-SOURCE\s*$/mi },
+  { name: "ctest", re: /\bNo tests were found\b/i },
+  { name: "flutter", re: /\bNo tests ran\.?/i },
+];
+
+/** Go prints this per package that has no test files at all. */
+const GO_NO_TEST_FILES = /\[no test files\]/;
+/** Any sign that a Go package did run tests. */
+const GO_RAN_SOMETHING = /^(?:ok|FAIL|---\s+(?:PASS|FAIL|SKIP)):?\s/m;
+
+/**
+ * Read a collected-test count out of a runner's output.
+ *
+ * @param {string} [stdout]
+ * @param {string} [stderr]
+ * @returns {{ count: number|null, runner: string|null }}
+ *   `count` is null when no recognised runner stated one — which is not a
+ *   finding, only an absence of evidence.
+ */
+export function parseCollectedTests(stdout = "", stderr = "") {
+  const text = `${stdout || ""}\n${stderr || ""}`;
+  if (!text.trim()) return { count: null, runner: null };
+
+  for (const rule of EXPLICIT_ZERO) {
+    if (rule.re.test(text)) return { count: 0, runner: rule.name };
+  }
+
+  // Go states absence per package rather than as a count, so it needs its own
+  // pass before the generic patterns.
+  if (GO_NO_TEST_FILES.test(text) || GO_RAN_SOMETHING.test(text)) {
+    // Only a run where *no* package did anything is a zero: a monorepo where
+    // one package has no tests and three do is a normal, healthy repository.
+    if (!GO_RAN_SOMETHING.test(text)) return { count: 0, runner: "go" };
+    // Something ran. `--- PASS:` lines are per-test but appear only under -v,
+    // so their absence means the count was not stated — not that it was zero.
+    // Reporting zero here would have failed every ordinary `go test ./...`.
+    const perTest = text.match(GO_PER_TEST);
+    return { count: perTest && perTest.length > 0 ? perTest.length : null, runner: "go" };
+  }
+
+  for (const rule of COUNT_PATTERNS) {
+    const m = rule.re.exec(text);
+    if (!m) continue;
+    const n = Number(m[1]);
+    if (Number.isFinite(n)) return { count: n, runner: rule.name };
+  }
+
+  return { count: null, runner: null };
+}
+
+/**
+ * Decide whether a passing verification command actually verified anything.
+ *
+ * @param {object} testResult - the test stage's result ({ ok, stdout, stderr, command })
+ * @param {object} [opts]
+ * @param {number} [opts.minTests=1] - the floor, from `verify.minTests`.
+ * @returns {{ ok: boolean, count: number|null, runner: string|null, reason: string|null }}
+ */
+export function checkCollectionFloor(testResult, opts = {}) {
+  const minTests = Number.isFinite(opts.minTests) ? opts.minTests : 1;
+  if (minTests <= 0) return { ok: true, count: null, runner: null, reason: null };
+  // Only a *passing* command can lie about this. A failing one already fails.
+  if (!testResult || testResult.ok !== true) {
+    return { ok: true, count: null, runner: null, reason: null };
+  }
+
+  const { count, runner } = parseCollectedTests(testResult.stdout, testResult.stderr);
+  if (count === null || count >= minTests) {
+    return { ok: true, count, runner, reason: null };
+  }
+
+  return {
+    ok: false,
+    count,
+    runner,
+    reason:
+      `The verification command exited 0 without running any tests` +
+      (runner ? ` (${runner} reported ${count})` : "") +
+      `, so this change was approved against nothing. ` +
+      `Point verify.test at a suite that covers this repository, lower the floor with verify.minTests, ` +
+      `or — if this repository intentionally uses only the scope and secret phases — set verify.required: false.`,
+  };
+}

package/src/security.mjs

@@ -1155,14 +1155,42 @@ function locateFindingLine(lines, type, file = null) {
 // (`expect(\n  formatInvoice(bill)\n).toBe(…`) still recognises the chain.
 // The bound is a guess: an argument list longer than 240 characters is
 // rarer than a missed chain.
+// The dialect list is not decoration. `assertEqual` was recognised only
+// because `\.?` made the dot optional and the `i` flag let `Equal` match
+// `equal`; `assertEquals`, one letter longer, fell out of the pattern and
+// took JUnit, PHPUnit, Minitest, RSpec and XCTest with it. The weak forms
+// — assertTrue, assertNotNull, XCTAssertTrue — are deliberately absent:
+// they state no expected value, so their arrival in place of one of these
+// is a weakening, which is a finding of its own.
 const SPECIFIC_ASSERTION = new RegExp(
   [
     "\\bassert(?:\\.strict)?\\.?(?:strictEqual|deepStrictEqual|deepEqual|notStrictEqual|notDeepStrictEqual|equal|notEqual|match|doesNotMatch|throws|rejects|doesNotThrow)\\s*\\(",
     "\\bexpect\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:toBe|toEqual|toStrictEqual|toMatch|toMatchObject|toContain|toHaveBeenCalledWith|toThrow|toHaveLength|toBeCloseTo)\\s*\\(",
     "\\bassert\\.(?:equals|deepEquals|include|lengthOf)\\s*\\(",
     "assert_eq!|assert_ne!",
+    // The bare comparison form. `assert add(1, 2) == 3` is how pytest is
+    // actually written, and Rust's `assert!(a == b)` and Elixir's
+    // `assert f(x) == 3` follow it; none of them name a comparison
+    // function, so a list of function names could never reach them.
+    // Equality only. `assert!(x != 0)` names no expected value — it is the
+    // weaker claim you arrive at by giving one up, and counting it as
+    // specific would make the downgrade from `assert_eq!(x, 5)` invisible to
+    // the weakening check.
+    "\\bassert\\s+[^\\n]*(?:===|==)(?!=)",
+    "\\bassert!\\s*\\([^\\n]*==(?!=)",
     "\\bt\\.(?:Errorf|Fatalf)\\s*\\(",
     "\\brequire\\.(?:Equal|NotEqual|Len|Contains|Error|NoError)\\s*\\(",
+    // Python unittest, stated rather than inherited from the optional dot.
+    "\\bassert(?:Equal|NotEqual|AlmostEqual|NotAlmostEqual|Regex|NotRegex|Raises|In|NotIn|Is|IsNot|ListEqual|DictEqual|SetEqual|TupleEqual|CountEqual|Greater|Less|GreaterEqual|LessEqual)\\s*\\(",
+    // JUnit / TestNG / PHPUnit
+    "\\bassert(?:Equals|NotEquals|Same|NotSame|ArrayEquals|IterableEquals|LinesMatch|Count|StringContainsString|StringEqualsFile|InstanceOf|Contains|Throws)\\s*\\(",
+    "\\bassertThat\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:isEqualTo|isSameAs|contains|containsExactly|hasSize|isCloseTo|matches)\\s*\\(",
+    // Minitest
+    "\\b(?:assert|refute)_(?:equal|includes|match|nil|same|in_delta|in_epsilon|raises|empty|operator|predicate)\\b",
+    // RSpec
+    "\\bexpect\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:to|not_to|to_not)\\s+(?:eq|eql|equal|be|be_within|match|include|contain_exactly|match_array|have_attributes|raise_error|start_with|end_with)\\b",
+    // XCTest
+    "\\bXCTAssert(?:Equal|NotEqual|EqualWithAccuracy|Identical|NotIdentical|GreaterThan|LessThan|GreaterThanOrEqual|LessThanOrEqual|ThrowsError|NoThrow)\\s*\\(",
   ].join("|"),
   "i"
 );
@@ -1203,6 +1231,15 @@ const TEST_LANG_BY_EXT = new Map([
   [".py", "python"], [".pyi", "python"],
   [".go", "go"],
   [".rs", "rust"],
+  // Approximations, chosen for comment and continuation syntax rather than
+  // for kinship: the C-like family reads correctly under the `js` scanner,
+  // and Ruby under the `python` one because both end a comment at `#` and a
+  // statement at the newline. Naming them beats falling through to `js` by
+  // default, which is how a `#` comment came to be read as code.
+  [".java", "js"], [".kt", "js"], [".kts", "js"], [".scala", "js"], [".groovy", "js"],
+  [".swift", "js"], [".cs", "js"], [".php", "js"], [".c", "js"], [".cc", "js"],
+  [".cpp", "js"], [".h", "js"], [".hpp", "js"], [".m", "js"], [".sol", "js"],
+  [".rb", "python"],
 ]);
 
 function langForTestFile(file) {
@@ -1540,6 +1577,21 @@ function stripComments(text, lang) {
 const CONTINUATION_START = /^[)\],.]/;
 const CONTINUATION_OP_START = /^[+\-*/%<>=&|^:]/;
 
+// A comment is not a continuation, however much it looks like one.
+//
+// `//` begins with a division sign and `--` with a minus, so both matched
+// CONTINUATION_OP_START and folded the following comment line into the
+// statement above it. The cost was a false accusation on a virtuous act:
+// adding an assertion next to a `// ...` line made the new assertion absorb
+// the comment, stop matching its unchanged twin, and get reported as a
+// rewritten expectation. Python was unaffected only because `#` is not an
+// operator — which is why the same fixture passed in pytest and failed in
+// Jest, and why it survived every suite written against the pytest layout.
+//
+// A comment inside an open delimiter still joins: `cur.delta > 0` decides
+// that before this test is ever reached.
+const COMMENT_LINE_START = /^(?:\/\/|\/\*|#|--)/;
+
 // A scanner miscount (an unbalanced delimiter inside a regex literal is the
 // usual cause) must not be able to merge a whole file into one statement,
 // which would pair *any* literal change anywhere in the file.
@@ -1578,14 +1630,15 @@ function assembleStatements(sliceLines, lang) {
 
   for (const L of sliceLines) {
     const trimmed = L.text.replace(/^\s+/, "");
+    const startsComment = COMMENT_LINE_START.test(trimmed);
     const joins =
       cur !== null &&
       (cur.delta > 0 ||
         cur.state.str !== null ||
         cur.state.block > 0 ||
         cur.trailingBackslash ||
-        CONTINUATION_START.test(trimmed) ||
-        CONTINUATION_OP_START.test(trimmed));
+        (!startsComment &&
+          (CONTINUATION_START.test(trimmed) || CONTINUATION_OP_START.test(trimmed))));
 
     if (
       joins &&
@@ -1648,7 +1701,17 @@ function splitAssertionArgs(clean, lang) {
   const m = SPECIFIC_ASSERTION.exec(clean);
   if (!m) return null;
 
-  let i = m.index + m[0].length; // just past the opening paren
+  // Not every branch of SPECIFIC_ASSERTION ends at an opening paren:
+  // `assert_eq!`, `assert_equal` and RSpec's `.to eq` all match a bare name.
+  // Starting the walk one character early made every argument boundary wrong,
+  // so a reworded message read as a rewritten value.
+  let i = m.index + m[0].length;
+  if (clean[i - 1] !== "(") {
+    let j = i;
+    while (j < clean.length && /\s/.test(clean[j])) j++;
+    if (clean[j] !== "(") return null;
+    i = j + 1;
+  }
   let depth = 1;
   let quote = null;
   let triple = false;
@@ -1740,7 +1803,45 @@ function messageArgIndices(args) {
  * every time somebody improved the wording of a failure. Firing on that is
  * how an operator learns to pass the override without reading it.
  */
+/**
+ * Split a statement at a trailing `, "message"` written outside the call.
+ *
+ * RSpec puts the message there — `expect(x).to eq(3), "explain"` — and so do
+ * Ruby and Elixir assertions generally. An argument-position check can never
+ * see it, so rewording one read as a rewritten expectation.
+ */
+function splitTrailingMessage(clean) {
+  let depth = 0;
+  let quote = null;
+  let lastComma = -1;
+  for (let i = 0; i < clean.length; i++) {
+    const c = clean[i];
+    if (quote !== null) {
+      if (c === "\\") { i += 1; continue; }
+      if (c === quote) quote = null;
+      continue;
+    }
+    if (c === '"' || c === "'" || c === "`") { quote = c; continue; }
+    if (c === "(" || c === "[" || c === "{") depth++;
+    else if (c === ")" || c === "]" || c === "}") depth--;
+    else if (c === "," && depth === 0) lastComma = i;
+  }
+  if (lastComma === -1) return { head: clean, msg: null };
+  const tail = clean.slice(lastComma + 1).trim();
+  if (!isPureStringLiteral(tail)) return { head: clean, msg: null };
+  return { head: clean.slice(0, lastComma), msg: tail };
+}
+
 function differsOnlyInMessage(cleanRemoved, cleanAdded, lang) {
+  const ta = splitTrailingMessage(cleanRemoved);
+  const tb = splitTrailingMessage(cleanAdded);
+  if (
+    (ta.msg !== null || tb.msg !== null) &&
+    ta.head.replace(/\s+/g, "") === tb.head.replace(/\s+/g, "")
+  ) {
+    return true;
+  }
+
   const a = splitAssertionArgs(cleanRemoved, lang);
   const b = splitAssertionArgs(cleanAdded, lang);
   if (!a || !b || a.length !== b.length || a.length === 0) return false;
@@ -1988,12 +2089,26 @@ export function resolveAllowedTamperKinds(options = {}) {
  * @param {string} diffOrText - Unified git diff
  * @param {Object} [options]
  * @param {boolean} [options.allowTestModifications=false]
- * @returns {{ ok: boolean, violations: Array<{ file: string, type: string, line?: number, reason: string }> }}
+ * @returns {{ ok: boolean, violations: Array<object>, inputsSeen: number, filesSeen: number,
+ *            assertionsSeen: number, unreadable: Array<{file: string, count: number, samples: string[]}>,
+ *            status: "PASS"|"FAIL"|"UNREADABLE"|"NOT_APPLICABLE" }}
+ *   `status` distinguishes "checked and clean" from "nothing was checked";
+ *   `ok: true` alone cannot, and that ambiguity is the defect class this
+ *   field exists to make visible.
  */
 export function checkTestTampering(diffOrText = "", options = {}) {
-  if (!diffOrText || typeof diffOrText !== "string") return { ok: true, violations: [] };
+  if (!diffOrText || typeof diffOrText !== "string") {
+    return { ok: true, violations: [], inputsSeen: 0, status: "NOT_APPLICABLE", reason: "empty diff" };
+  }
   const allowed = resolveAllowedTamperKinds(options);
-  if (allowed.all) return { ok: true, violations: [] };
+  if (allowed.all) {
+    return { ok: true, violations: [], inputsSeen: 0, status: "NOT_APPLICABLE", reason: "all kinds allowed" };
+  }
+
+  // Which predicate decides what this guard even looks at. Injectable so the
+  // meta-check can mutate it: a canary that still passes when the predicate is
+  // replaced by `() => false` was never requiring this guard to activate.
+  const isTestPath_ = typeof options.isTestPath === "function" ? options.isTestPath : isTestPath;
 
   const violations = [];
   const lines = diffOrText.split("\n");
@@ -2002,7 +2117,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
   let currentOldLineNo = null;
   let currentNewLineNo = null;
 
-  const isTestFile = isTestPath;
+  const isTestFile = isTestPath_;
 
   const SKIP_INJECTIONS = [
     { pattern: /\b(?:it|test|describe|context)\.skip\s*\(/i, desc: "Injected test skip (.skip())" },
@@ -2012,9 +2127,24 @@ export function checkTestTampering(diffOrText = "", options = {}) {
     { pattern: /#\[ignore\]/i, desc: "Injected Rust test ignore attribute (#[ignore])" },
     { pattern: /\/\/\s*@ts-ignore/i, desc: "Injected TypeScript ignore comment (// @ts-ignore)" },
     { pattern: /\/\/\s*@ts-nocheck/i, desc: "Injected TypeScript nocheck directive (// @ts-nocheck)" },
+    // A suite that never runs cannot fail, and `@Disabled` is exactly as
+    // effective at arranging that as `it.skip` is.
+    { pattern: /@Disabled\b/, desc: "Injected JUnit 5 disable annotation (@Disabled)" },
+    { pattern: /@Ignore\b/, desc: "Injected JUnit 4 / TestNG ignore annotation (@Ignore)" },
+    { pattern: /@Test\s*\([^)]*enabled\s*=\s*false/i, desc: "Injected TestNG disabled test (enabled = false)" },
+    { pattern: /@unittest\.skip/i, desc: "Injected unittest skip decorator (@unittest.skip)" },
+    { pattern: /\bmarkTest(?:Skipped|Incomplete)\s*\(/i, desc: "Injected PHPUnit skip (markTestSkipped())" },
+    { pattern: /\bXCTSkip(?:If|Unless|IfNot)?\s*\(/, desc: "Injected XCTest skip (XCTSkip())" },
+    { pattern: /\b(?:xit|xdescribe|xcontext|xspecify)\b\s*["\x27]/i, desc: "Injected RSpec disabled example (xit)" },
+    { pattern: /,\s*skip:\s*(?:true|["\x27])/i, desc: "Injected RSpec skip metadata (skip:)" },
+    { pattern: /^\s*(?:skip|pending)\s*(?:["\x27(]|$)/i, desc: "Injected Minitest/RSpec skip statement" },
   ];
 
-  const COMMENTED_ASSERTION = /^\+\s*(?:\/\/|\/\*)\s*(?:expect\(|assert\.|assert\(|t\.expect|t\.assert)/i;
+  // `#` and `--` belong here for the same reason the dialects belong in
+  // ASSERTION_PATTERN: a Ruby or Python assertion commented out is exactly
+  // as gone as a JavaScript one, and was previously not looked for.
+  const COMMENTED_ASSERTION =
+    /^\+\s*(?:\/\/|\/\*|#|--)\s*(?:expect\s*\(|assert(?!ion|ing|ed\b|s\b)[a-zA-Z0-9_$]*\s*[.(]|assert\s|refute_|XCTAssert|t\.expect|t\.assert)/i;
 
   const VACUOUS_ASSERTIONS = [
     { pattern: /\bassert(?:\.ok)?\s*\(\s*true\s*(?:,[^)]*)?\)/i, desc: "Vacuous truth assertion (assert.ok(true))" },
@@ -2025,11 +2155,44 @@ export function checkTestTampering(diffOrText = "", options = {}) {
     { pattern: /\bexpect\s*\(\s*false\s*\)\s*\.toBeFalsy\s*\(/i, desc: "Vacuous falsity expectation (expect(false).toBeFalsy())" },
     { pattern: /\bassert\.(?:isTrue|isOk)\s*\(\s*true\s*(?:,[^)]*)?\)/i, desc: "Vacuous truth assertion (assert.isTrue(true))" },
     { pattern: /\bassert\.(?:isFalse|isNotOk)\s*\(\s*false\s*(?:,[^)]*)?\)/i, desc: "Vacuous falsity assertion (assert.isFalse(false))" },
+    { pattern: /\b(?:XCT)?assertTrue\s*\(\s*true\s*[,)]/i, desc: "Vacuous truth assertion (assertTrue(true))" },
+    { pattern: /\b(?:XCT)?assertFalse\s*\(\s*false\s*[,)]/i, desc: "Vacuous falsity assertion (assertFalse(false))" },
+    { pattern: /\b(?:assertEquals|assertSame|XCTAssertEqual)\s*\(\s*([^,]+?)\s*,\s*\1\s*[,)]/i, desc: "Vacuous identity assertion (assertEquals(X, X))" },
+    { pattern: /\bassert_equal\s*\(?\s*([^,]+?)\s*,\s*\1\s*\)?\s*$/i, desc: "Vacuous identity assertion (assert_equal X, X)" },
+    { pattern: /\bexpect\s*\(\s*true\s*\)\s*\.to\s+be(?:\s+true)?\b/i, desc: "Vacuous truth expectation (expect(true).to be true)" },
   ];
 
-  const ASSERTION_PATTERN = /(?:\b(?:assert(?:\.[a-zA-Z0-9_$]+)?|expect|t\.(?:assert|expect|is|equal|true|false|Errorf|Fatalf)|require\.[a-zA-Z0-9_$]+)\b|assert!|assert_eq!|assert_ne!)/i;
+  // Broad on purpose: this is the denominator, not the verdict. A word
+  // boundary immediately after `assert` never falls in `assertEquals`,
+  // `assert_equal` or `XCTAssertEqual`, so five ecosystems contributed no
+  // assertions to count at all and a gutted JUnit suite was arithmetically
+  // indistinguishable from an untouched one. The lookahead keeps prose and
+  // identifiers — `assertion`, `asserts`, `asserted` — out of the count.
+  const ASSERTION_PATTERN =
+    /(?:\b(?:assert(?!ion|ing|ed\b|s\b)[a-zA-Z0-9_$]*(?:\.[a-zA-Z0-9_$]+)?|refute[a-zA-Z0-9_$]*|XCTAssert[a-zA-Z0-9_$]*|XCTFail|expect|t\.(?:assert|expect|is|equal|true|false|Errorf|Fatalf)|require\.[a-zA-Z0-9_$]+)\b|assert!|assert_eq!|assert_ne!)/i;
+  // The loose net. Not a verdict and never a block — its only job is to
+  // notice that a line was plainly an assertion in *some* dialect that
+  // ASSERTION_PATTERN did not recognise. Without it, adding the seventh
+  // ecosystem is indistinguishable from having covered it all along: the
+  // guard returns the same clean PASS either way. This is the denominator
+  // for the denominator.
+  // Deliberately not call-shaped. Haskell's `x `shouldBe` 3` is an
+  // assertion with no parentheses anywhere near it, and a net that only
+  // catches `name(` reports the same confident PASS on it as on a clean
+  // Node suite. `require` and `check` are absent on purpose: in a
+  // CommonJS test file `require("./calc")` is an import, not a claim.
+  const ASSERTION_SHAPED =
+    /\b(?:assert(?!ion|ing|ed\b|s\b)|expect(?!ed\b|ation)|refute)[a-zA-Z0-9_$]*\b|`\s*should[a-zA-Z0-9_$]*\s*`|\b(?:should|must|verify|ensure|confirm)[a-zA-Z0-9_$]*\s*[(!]|\.\s*(?:should|to|to_not|not_to|must)\b|\bBOOST_[A-Z_]+\s*\(|\b[A-Z]+_(?:EQ|NE|TRUE|FALSE|THAT)\s*\(/;
   const isCommentLine = (str) => /^\s*(?:\/\/|\/\*|\*|#|--|;)/.test(str);
 
+  /** Book-keeping only: what this run looked at, before deciding anything. */
+  const countExamined = (stats, text) => {
+    if (!text.trim() || isCommentLine(text)) return;
+    stats.examined++;
+    if (ASSERTION_PATTERN.test(text)) stats.recognised++;
+    else if (ASSERTION_SHAPED.test(text) && stats.unreadable.length < 5) stats.unreadable.push(text.trim().slice(0, 120));
+  };
+
   const fileAssertions = new Map();
   let pendingHunk = false;
 
@@ -2065,7 +2228,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
     }
 
     if (!fileAssertions.has(currentFile)) {
-      fileAssertions.set(currentFile, { removed: [], added: 0, addedTexts: [], removedSpecific: [], addedSpecific: 0, hunks: [] });
+      fileAssertions.set(currentFile, { removed: [], added: 0, addedTexts: [], removedSpecific: [], addedSpecific: 0, hunks: [], examined: 0, recognised: 0, unreadable: [] });
     }
     const fileStats = fileAssertions.get(currentFile);
     if (pendingHunk) {
@@ -2077,6 +2240,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
     if (line.startsWith("-") && !line.startsWith("---")) {
       const deletedText = line.slice(1);
       if (hunk) hunk.lines.push({ kind: "-", text: deletedText, oldNo: currentOldLineNo, newNo: null });
+      countExamined(fileStats, deletedText);
       if (!isCommentLine(deletedText) && ASSERTION_PATTERN.test(deletedText)) {
         fileStats.removed.push({ line: currentOldLineNo, text: deletedText });
         if (isSpecificAssertion(deletedText)) {
@@ -2087,6 +2251,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
     } else if (line.startsWith("+") && !line.startsWith("+++")) {
       const addedText = line.slice(1);
       if (hunk) hunk.lines.push({ kind: "+", text: addedText, oldNo: null, newNo: currentNewLineNo });
+      countExamined(fileStats, addedText);
       let isVacuous = false;
 
       // Check skip injections
@@ -2207,9 +2372,50 @@ export function checkTestTampering(diffOrText = "", options = {}) {
       ? violations
       : violations.filter((v) => !allowed.kinds.has(TAMPER_KINDS.get(v.type)));
 
+  // What was examined, not only what was found.
+  //
+  // `ok: true` from a guard that looked at nothing is byte-identical to
+  // `ok: true` from a guard that looked at everything and approved it. That
+  // ambiguity is how a substring bug in the file classifier switched this
+  // entire guard off for the standard pytest, Rust and RSpec layouts while
+  // every signal stayed green.
+  //
+  // Counting *files* was not enough. A JUnit diff that rewrote an expected
+  // value produced `inputsSeen: 1` and a clean PASS while not one assertion
+  // in it had been recognised — the same ambiguity, one level down, inside
+  // the mechanism built to remove it. So the denominator is now the thing
+  // the rules actually consume: lines examined, and of those, assertions
+  // understood. `UNREADABLE` is the state that has no business being silent
+  // — assertion-shaped lines were present and none of them parsed, which
+  // means this repository speaks a dialect the guard does not.
+  let examined = 0;
+  let assertionsSeen = 0;
+  const unreadable = [];
+  for (const [file, stats] of fileAssertions.entries()) {
+    examined += stats.examined;
+    assertionsSeen += stats.recognised;
+    if (stats.unreadable.length > 0) {
+      unreadable.push({ file, count: stats.unreadable.length, samples: stats.unreadable.slice(0, 3) });
+    }
+  }
+
+  const status =
+    reported.length > 0
+      ? "FAIL"
+      : assertionsSeen === 0 && unreadable.length > 0
+        ? "UNREADABLE"
+        : examined > 0
+          ? "PASS"
+          : "NOT_APPLICABLE";
+
   return {
     ok: reported.length === 0,
     violations: reported,
+    inputsSeen: examined,
+    filesSeen: fileAssertions.size,
+    assertionsSeen,
+    unreadable,
+    status,
   };
 }
 

package/src/stack-detector.mjs

@@ -39,7 +39,65 @@ export function pytestCmd(env = process.env) {
 }
 
 /**
- * Generate a zero-dependency smoke test file using node:test for untested JS/Generic repos.
+ * Does this Makefile declare a `test` target?
+ *
+ * Read rather than assumed: the presence of the file says nothing about
+ * whether `make test` will run.
+ */
+function makefileHasTestTarget(root) {
+  try {
+    const text = readFileSync(join(root, "Makefile"), "utf-8");
+    return /^\.PHONY:.*\btest\b/m.test(text) || /^test\s*:/m.test(text);
+  } catch (_) {
+    return false;
+  }
+}
+
+/**
+ * A declared test script that runs no tests and exits 0.
+ *
+ * This is the single most dangerous input the gate can receive, because every
+ * downstream check reads "a command ran and passed". `bootstrapZeroTestRepo`
+ * called `"test": "echo 'no tests yet' && exit 0"` an
+ * EXISTING_VERIFICATION_ORACLE — it asked whether the field was set, never
+ * what was in it.
+ *
+ * npm's own default (`echo "Error: no test specified" && exit 1`) is not a
+ * placeholder by this definition, and correctly so: it exits non-zero, which
+ * fails loudly rather than certifying nothing.
+ */
+export function isPlaceholderTestScript(cmd) {
+  if (typeof cmd !== "string") return false;
+  const trimmed = cmd.trim();
+  if (!trimmed) return true;
+  // Drop the announcements; what matters is what the shell is left doing.
+  const remainder = trimmed
+    .split(/&&|;/)
+    .map((part) => part.trim())
+    .filter((part) => part && !/^(?:echo|printf|:)\b/.test(part));
+  if (remainder.length === 0) return true;
+  return remainder.every((part) => /^(?:exit\s+0|true|:)$/.test(part));
+}
+
+/**
+ * Generate the fallback verification oracle for a JS/generic repo with no tests.
+ *
+ * What this used to write could not fail:
+ *
+ *   assert.ok(fs.existsSync(process.cwd()));
+ *   assert.ok(fs.readdirSync(process.cwd()).length > 0);
+ *
+ * Both hold for every repository and every change, so the generated "oracle"
+ * was green against arbitrary broken code — and worse, it *silenced* the
+ * `missingOracle` guard in engine.mjs, which fires only when no command ran at
+ * all. A repository that honestly had no oracle was converted into one that
+ * claimed to have one. That is the tool writing its own blindness to disk.
+ *
+ * The other stacks already get a real static gate at this point — `tsc
+ * --noEmit`, `cargo check`, `go vet`, `compileall` — each of which fails on a
+ * real class of defect. This is the JavaScript equivalent: every source file
+ * must parse. It proves the code compiles, not that it works, and the caller
+ * says so; but a syntax error fails it, which is one more than before.
  */
 export function generateSmokeTestScript(root = process.cwd()) {
   const agentDir = join(root, ".agent");
@@ -48,15 +106,45 @@ export function generateSmokeTestScript(root = process.cwd()) {
   } catch (_) {}
 
   const smokePath = join(agentDir, "smoke.test.mjs");
-  const content = `// Auto-generated zero-dependency smoke test gate (.agent/smoke.test.mjs)
+  const content = `// Auto-generated zero-dependency parse gate (.agent/smoke.test.mjs)
+//
+// Written by \`agentctl bootstrap\` for a repository that had no test suite.
+// It proves that every source file still parses. It does NOT prove the code
+// is correct — replace it with real tests as soon as there are any.
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import fs from "node:fs";
+import { readdirSync, statSync } from "node:fs";
+import { join, extname } from "node:path";
+import { spawnSync } from "node:child_process";
+
+const SKIP = new Set([".git", "node_modules", "vendor", "dist", "build", "coverage", ".venv", "venv", ".next", ".agent"]);
+const SOURCE = new Set([".js", ".mjs", ".cjs"]);
+
+function sources(dir, acc = [], depth = 0) {
+  if (depth > 8) return acc;
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (entry.name.startsWith(".") && entry.name !== ".agent") continue;
+    if (SKIP.has(entry.name)) continue;
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) sources(full, acc, depth + 1);
+    else if (SOURCE.has(extname(entry.name))) acc.push(full);
+  }
+  return acc;
+}
+
+test("every source file parses", () => {
+  const files = sources(process.cwd());
 
-test("Zero-Test Bootstrapped Smoke Verification", () => {
-  assert.ok(fs.existsSync(process.cwd()), "Working directory exists and is accessible");
-  const entries = fs.readdirSync(process.cwd());
-  assert.ok(entries.length > 0, "Repository contains files");
+  // A gate with nothing to check is not a passing gate. Reporting success
+  // over an empty file list is exactly the vacuous oracle this replaced.
+  assert.ok(files.length > 0, "No JavaScript sources found to verify — this is not an oracle. Set verify.test in .agent/config.yml.");
+
+  const broken = [];
+  for (const file of files) {
+    const res = spawnSync(process.execPath, ["--check", file], { encoding: "utf-8" });
+    if (res.status !== 0) broken.push(\`\${file}: \${(res.stderr || "").trim().split("\\n")[0]}\`);
+  }
+  assert.deepEqual(broken, [], \`\${broken.length} file(s) failed to parse\`);
 });
 `;
   writeFileSync(smokePath, content, "utf-8");
@@ -173,7 +261,15 @@ export function detectPolyglotStack(projectRoot = process.cwd()) {
   if (existsSync(join(projectRoot, "Package.swift"))) {
     return { ...container, stack: "swift", testCmd: "swift test", buildCmd: "swift build", triggerFile: "Package.swift" };
   }
-  if (existsSync(join(projectRoot, "app.json")) || existsSync(join(projectRoot, "react-native.config.js"))) {
+  // `app.json` is not a manifest, it is an Expo/React Native *configuration*
+  // file, and the name is generic enough that unrelated projects use it. Its
+  // test command is `npm test`, so without a package.json beside it the
+  // detector was claiming a Node stack for a repository that has no Node in
+  // it: `Cargo.toml` + `app.json` was measured as `react-native` / `npm test`.
+  if (
+    (existsSync(join(projectRoot, "app.json")) && existsSync(join(projectRoot, "package.json"))) ||
+    existsSync(join(projectRoot, "react-native.config.js"))
+  ) {
     const triggerFile = existsSync(join(projectRoot, "app.json")) ? "app.json" : "react-native.config.js";
     return { ...container, stack: "react-native", testCmd: "npm test", buildCmd: "npx react-native bundle --platform android --dev false --entry-file index.js --bundle-output android/main.jsbundle", triggerFile };
   }
@@ -188,7 +284,14 @@ export function detectPolyglotStack(projectRoot = process.cwd()) {
   if (existsSync(join(projectRoot, "go.mod"))) {
     return { ...container, stack: "go", testCmd: "go test ./...", buildCmd: "go build ./...", triggerFile: "go.mod" };
   }
-  if (existsSync(join(projectRoot, "Makefile"))) {
+  // A Makefile is only an oracle if it declares the target we are about to
+  // run. `make test` on a Makefile with only a `build:` target exits 2 with
+  // "No rule to make target 'test'" — measured on a repository whose
+  // package.json declared a perfectly good `vitest run`, because the Makefile
+  // was checked first and the presence of the *file* was the whole test. A
+  // hard red on day one is how a user learns the gate is broken and turns it
+  // off, so the file must earn the claim.
+  if (existsSync(join(projectRoot, "Makefile")) && makefileHasTestTarget(projectRoot)) {
     return { ...container, stack: "make", testCmd: "make test", buildCmd: "make build", triggerFile: "Makefile" };
   }
 
@@ -715,7 +818,7 @@ export function bootstrapZeroTestRepo(root = process.cwd(), options = {}) {
   if (existsSync(join(root, "package.json"))) {
     try {
       const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf-8"));
-      hasPkgTest = Boolean(pkg?.scripts?.test);
+      hasPkgTest = Boolean(pkg?.scripts?.test) && !isPlaceholderTestScript(pkg.scripts.test);
     } catch (_) {}
   }