jules-orchestrator-kit 0.65.0 → 0.66.0

package/README.md

@@ -207,7 +207,7 @@ To maximize PR merge rates, dispatch tasks according to deterministic boundaries
 * **Fail-Closed Security & Secret Redaction:** Evaluates explicit Deny rules before Allow rules against canonicalized, case-folded paths. Redacts high-entropy keys and base64-encoded credentials (such as Kubernetes `Secret` manifests).
 * **Complexity & Cost Router:** Zero-dependency heuristic classifier (`src/router.mjs`) routing mechanical tasks to lightweight models while reserving primary models for complex refactors, with a `node --check` syntax-verification gate that transparently escalates a FAST-tier result to the primary provider if it left broken JS on disk.
 * **Terminal UI & Diagnostic Matrix (`agentctl doctor`):** Interactive terminal dashboard, task sidecar manager, and automated transactional self-repair.
-* **Verified Test Suite:** Tested with **1109 unit tests across 154 suites passing in < 15.0s**.
+* **Verified Test Suite:** Tested with **1133 unit tests across 163 suites passing in < 15.0s**.
 
 <br/>
 

package/bin/agentctl.mjs

@@ -429,6 +429,9 @@ async function main() {
           if (p.violations) {
             p.violations.forEach((v) => console.log(`     - Violation: ${v.file} (Rule: ${v.rule})`));
           }
+          if (p.unverified) {
+            console.log(`     - Unverified: ${p.unverified}`);
+          }
           if (p.setup) {
             console.log(`     - Setup: accepted ${p.setup.length} gate scaffold file(s) this repository did not have yet`);
             p.setup.forEach((f) => console.log(`         ${f}`));
@@ -2667,13 +2670,21 @@ async function main() {
       const taskId = values.task || "unknown";
       const agent = values.agent || "jules";
 
+      const { CONFIRM_AFTER } = await import("../src/memory.mjs");
       const res = harvestFailure(root, { exitCode, logPath, diffText, taskId, agent });
       if (values.json) {
         console.log(JSON.stringify(res, null, 2));
       } else {
         if (res.status === "HARVESTED") {
-          console.log(`🌾 Harvested failure candidate: ${res.candidate.trigger}`);
-          console.log(`   Solution: ${res.candidate.solution}`);
+          // No "Solution:" line any more, because there is no solution: the
+          // repair loop exhausting its budget is the definition of not having
+          // found one. It used to print a fixed sentence here and store it as
+          // if it were a remedy.
+          console.log(`🌾 Recorded failure observation: ${res.candidate.trigger}`);
+          console.log(`   Seen ${res.occurrences}× — ${res.confirmed ? "stated as a rule" : `not injected into prompts until it recurs ${CONFIRM_AFTER}×`}`);
+          if (!res.confirmed) {
+            console.log(`   Record an actual fix for it with: agentctl learning add "<trigger>" "<solution>"`);
+          }
         } else {
           console.log(`⚠️ Harvest rejected: ${res.reason}`);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jules-orchestrator-kit",
-  "version": "0.65.0",
+  "version": "0.66.0",
   "description": "Zero-dependency safety gatekeeper, test oracle generator, and multi-agent coordination protocol for autonomous coding agents — Google Jules, Claude Code, Codex and Gemini CLI.",
   "repository": {
     "type": "git",

package/scripts/guard-reach-check.mjs

@@ -49,6 +49,7 @@ import { parseCollectedTests } from "../src/ops/test-collection.mjs";
 import {
   TEST_PATH_CASES,
   TAMPER_CANARIES,
+  MULTILINE_CANARIES,
   PREDICATE_MUTANTS,
   EMPTY_RUN_CANARIES,
   COUNTED_RUN_CANARIES,
@@ -68,6 +69,9 @@ import {
 function canaryDiff(c) {
   const ctx = c.context || "// context";
   const lines = [`--- a/${c.file}`, `+++ b/${c.file}`, "@@ -1,20 +1,20 @@", ` ${ctx}`];
+  // Unchanged lines the change sits inside. An assertion whose keyword is on
+  // one of these is the shape the line-level denominator could not see.
+  for (const l of c.lead || []) lines.push(` ${l}`);
   for (const l of c.removed) lines.push(`-${l}`);
   for (const l of c.added) lines.push(`+${l}`);
   lines.push(` ${ctx}`);
@@ -130,7 +134,7 @@ const canaryResults = new Map();
   const silent = [];
   const noDenominator = [];
   const noAssertions = [];
-  for (const c of TAMPER_CANARIES) {
+  for (const c of [...TAMPER_CANARIES, ...MULTILINE_CANARIES]) {
     const res = checkTestTampering(canaryDiff(c));
     const hit = (res.violations || []).some((v) => v.type === c.expect);
     canaryResults.set(c.id, hit);
@@ -144,7 +148,7 @@ const canaryResults = new Map();
       noAssertions.push(`${c.id} (${res.assertionsSeen} assertions parsed)`);
     }
   }
-  add("canaries: every tamper rule still fires", silent.length === 0, silent.length ? silent.join("; ") : `${TAMPER_CANARIES.length} canaries red as required`);
+  add("canaries: every tamper rule still fires", silent.length === 0, silent.length ? silent.join("; ") : `${TAMPER_CANARIES.length + MULTILINE_CANARIES.length} canaries red as required`);
   add("canaries: every finding carries a denominator", noDenominator.length === 0, noDenominator.length ? noDenominator.join(", ") : "inputsSeen > 0 on every hit");
   add("canaries: assertion rules parsed an assertion", noAssertions.length === 0, noAssertions.length ? noAssertions.join(", ") : "assertionsSeen > 0 on every assertion finding");
 }
@@ -186,7 +190,7 @@ const canaryResults = new Map();
   const survivors = [];
   for (const mutant of PREDICATE_MUTANTS) {
     let killed = false;
-    for (const c of TAMPER_CANARIES) {
+    for (const c of [...TAMPER_CANARIES, ...MULTILINE_CANARIES]) {
       // Only canaries the healthy predicate catches can kill a mutant.
       if (!canaryResults.get(c.id)) continue;
       const res = checkTestTampering(canaryDiff(c), { isTestPath: mutant.fn });

package/src/engine.mjs

@@ -692,6 +692,13 @@ export async function gate(opts = {}) {
       postTestHash,
       tamperDetected: testTampered,
     },
+    // Verified by exit code alone. Not a failure, and not an approval either:
+    // an unrecognised runner passes on purpose, but the operator has to be
+    // able to tell that apart from a suite the gate actually counted.
+    ...(collectionFloor.unverified ? { unverified: collectionFloor.note } : {}),
+    ...(collectionFloor.count !== null && collectionFloor.count !== undefined
+      ? { testsCollected: collectionFloor.count, runner: collectionFloor.runner }
+      : {}),
   });
   phases.push({
     phase: "evidence",

package/src/guard-policy.mjs

@@ -382,6 +382,15 @@ export const INNOCENT_EDITS = [
     added: ['const calc = require("../src/calc");'],
     why: "`require(` is an import, not a claim — the loose net must not read it as one",
   },
+  {
+    id: "reword-comment-inside-assertion/node",
+    file: "test/calc.test.js",
+    context: "// arithmetic",
+    lead: ["  assert.equal(", "    add(1, 2),"],
+    removed: ["    3, // the answer", "  );"],
+    added: ["    3, // the correct answer", "  );"],
+    why: "line comments were never stripped — `copyCode(i)` left `pending` at the comment and the copy after the loop put it back",
+  },
   {
     id: "rename-helper/node",
     file: "test/calc.test.js",
@@ -510,3 +519,35 @@ export const COUNTED_RUN_CANARIES = [
     why: "Go's own `ok  <package>` line, which a bare `^ok\\s` confused with TAP's `ok 1 - name`",
   },
 ];
+
+/**
+ * Changes that sit inside an assertion whose keyword never appears in the diff.
+ *
+ * Counting `+`/`-` lines answered zero here, and nothing among the changed
+ * lines looked assertion-shaped either, so the guard reported a clean PASS on
+ * a five-element expected list rewritten to one element to match broken
+ * output. Measured on a real repository: five green phases, `APPROVED`.
+ *
+ * The statement machinery that pairs rewrites already assembles context lines
+ * together with changed ones. The denominator has to use it.
+ */
+export const MULTILINE_CANARIES = [
+  {
+    id: "expectation/python-multiline",
+    file: "tests/test_headers.py",
+    context: "# headers",
+    lead: ["    def test_parse(self):", "        self.assertEqual(", "            _parse_http_header(x),"],
+    removed: ['            [("text/xml", {}),', '             ("text/plain", {}),', '             ("more", {})])'],
+    added: ['            [("text/xml", {"tampered": True})])'],
+    expect: "ASSERTION_EXPECTATION_CHANGED",
+  },
+  {
+    id: "expectation/js-multiline",
+    file: "test/headers.test.js",
+    context: "// headers",
+    lead: ["  assert.deepStrictEqual(", "    parse(input),"],
+    removed: ["    [{ type: \"a\" }, { type: \"b\" }, { type: \"c\" }]", "  );"],
+    added: ["    [{ type: \"a\" }]", "  );"],
+    expect: "ASSERTION_EXPECTATION_CHANGED",
+  },
+];

package/src/ops/test-collection.mjs

@@ -148,7 +148,24 @@ export function checkCollectionFloor(testResult, opts = {}) {
   }
 
   const { count, runner } = parseCollectedTests(testResult.stdout, testResult.stderr);
-  if (count === null || count >= minTests) {
+
+  // Deliberately one-sided: only a *stated* zero fails, because failing on
+  // "I could not tell" would break every runner not on the list. But passing
+  // and saying nothing makes `echo "all tests passed"` indistinguishable from
+  // a real suite, so the absence of evidence is reported as an absence.
+  if (count === null) {
+    return {
+      ok: true,
+      count: null,
+      runner: null,
+      reason: null,
+      unverified: true,
+      note:
+        `The verification command exited 0, but no recognised test runner stated how many tests it ran, ` +
+        `so the gate cannot tell a full suite from a command that ran nothing. Verified by exit code alone.`,
+    };
+  }
+  if (count >= minTests) {
     return { ok: true, count, runner, reason: null };
   }
 

package/src/security.mjs

@@ -1523,8 +1523,13 @@ function stripComments(text, lang) {
         continue;
       }
 
-      if (c === "/" && c2 === "/") { copyCode(i); break; }
-      if (lang === "python" && c === "#") { copyCode(i); break; }
+      // `pending = n` is the whole fix. `copyCode(i)` copies the code up to
+      // the comment and leaves `pending` sitting at its start; the
+      // `copyCode(n)` after this loop then copied the comment straight back
+      // in, so no line comment has ever been stripped. Block comments were,
+      // which is why `/* … */` behaved and `// …` did not.
+      if (c === "/" && c2 === "/") { copyCode(i); pending = n; break; }
+      if (lang === "python" && c === "#") { copyCode(i); pending = n; break; }
       if (c === "/" && c2 === "*") {
         copyCode(i);
         state.block = 1;
@@ -1929,6 +1934,9 @@ function detectExpectationRewrites(file, hunks, stats, violations) {
     }
 
     const pairs = [];
+    const pairedOld = new Set();
+    const pairedNew = new Set();
+    const cancelled = new Set();
     for (const [shape, olds] of oldByShape) {
       const news = newByShape.get(shape) || [];
 
@@ -1946,20 +1954,69 @@ function detectExpectationRewrites(file, hunks, stats, violations) {
       const survivingOld = [];
       for (const o of olds) {
         const twin = survivingNew.findIndex((n) => n.canon === o.canon);
-        if (twin === -1) survivingOld.push(o);
-        else survivingNew.splice(twin, 1);
+        if (twin === -1) {
+          survivingOld.push(o);
+        } else {
+          // Present unchanged on both sides: this assertion did not change,
+          // and the argument-level pass below must not be allowed to pair it
+          // with something else and call that a rewrite.
+          cancelled.add(o);
+          cancelled.add(survivingNew[twin]);
+          survivingNew.splice(twin, 1);
+        }
       }
 
       const k = Math.min(survivingOld.length, survivingNew.length);
       for (let t = 0; t < k; t++) {
         const r = survivingOld[t];
         const a = survivingNew[t];
+        // Whatever this pass decides about a pair — reported, or deliberately
+        // let go as a reorder or a reworded message — is the decision. The
+        // argument-level pass below exists only for statements whose shapes
+        // differ so much that they never met in a bucket here; letting it
+        // re-open a case that was already judged turned a comment edit into a
+        // rewritten expectation.
+        pairedOld.add(r);
+        pairedNew.add(a);
         if (r.canon === a.canon) continue;
         if (differsOnlyInMessage(r.clean, a.clean, lang)) continue;
         pairs.push({ r: r.s, a: a.s });
       }
     }
 
+    // Same assertion, same subject, different expected value.
+    //
+    // Shape pairing compares the statement with its literals blanked, so it
+    // only ever matched assertions whose structure survived the edit. Shrink
+    // a five-element expected list to one element to match broken output and
+    // the two images land in different shape buckets, never pair, and the
+    // rewrite is not reported at all — measured on a real repository, where
+    // it collected five green phases.
+    //
+    // The arguments are the better witness here: when both sides call the
+    // same assertion with the same number of arguments and the *subject*
+    // argument is untouched, what changed is what the test expects of it.
+    for (const r of oldCands) {
+      if (pairedOld.has(r) || cancelled.has(r)) continue;
+      const ra = splitAssertionArgs(r.clean, lang);
+      if (!ra || ra.length < 2) continue;
+      for (const a of newCands) {
+        if (pairedNew.has(a) || cancelled.has(a)) continue;
+        const aa = splitAssertionArgs(a.clean, lang);
+        if (!aa || aa.length !== ra.length) continue;
+        const same = (i) => ra[i].replace(/\s+/g, "") === aa[i].replace(/\s+/g, "");
+        // The subject has to be the same expression, or these are two
+        // different assertions that merely resemble each other.
+        if (!same(0)) continue;
+        if (ra.every((_, i) => same(i))) continue;
+        if (differsOnlyInMessage(r.clean, a.clean, lang)) continue;
+        pairs.push({ r: r.s, a: a.s });
+        pairedOld.add(r);
+        pairedNew.add(a);
+        break;
+      }
+    }
+
     // Zero-context hunk: each image is a single fragment and the assertion
     // keyword may sit outside the hunk entirely. The fragment pair is taken
     // only when both sides normalize to the same shape *and* that shape
@@ -2388,21 +2445,57 @@ export function checkTestTampering(diffOrText = "", options = {}) {
   // understood. `UNREADABLE` is the state that has no business being silent
   // — assertion-shaped lines were present and none of them parsed, which
   // means this repository speaks a dialect the guard does not.
+  // An assertion is a statement, not a line.
+  //
+  // Counting `+`/`-` lines missed the commonest shape in every language with
+  // multi-line calls: `self.assertEqual(` sits on an unchanged context line
+  // and only its argument lines are edited. Nothing among the changed lines
+  // matched an assertion pattern, nothing looked assertion-shaped either, so
+  // the guard reported `assertionsSeen: 0` and — because no line looked
+  // suspicious — a clean PASS. A five-element expected list rewritten to one
+  // element to match broken output sailed through five green phases.
+  //
+  // The statement machinery that already exists for pairing knows better:
+  // it assembles context lines together with changed ones. Ask it.
+  for (const [file, stats] of fileAssertions.entries()) {
+    const lang = langForTestFile(file);
+    let touched = 0;
+    for (const hunk of stats.hunks) {
+      const oldSlice = [];
+      const newSlice = [];
+      for (const L of hunk.lines) {
+        if (L.kind !== "+") oldSlice.push(L);
+        if (L.kind !== "-") newSlice.push(L);
+      }
+      for (const stmts of [assembleStatements(oldSlice, lang), assembleStatements(newSlice, lang)]) {
+        for (const st of stmts) {
+          const changed = (st.removedLines?.length || 0) + (st.addedLines?.length || 0) > 0;
+          if (changed && ASSERTION_PATTERN.test(stripComments(st.text, lang))) touched++;
+        }
+      }
+    }
+    stats.statementAssertions = touched;
+  }
+
   let examined = 0;
   let assertionsSeen = 0;
   const unreadable = [];
   for (const [file, stats] of fileAssertions.entries()) {
     examined += stats.examined;
-    assertionsSeen += stats.recognised;
+    assertionsSeen += Math.max(stats.recognised, stats.statementAssertions || 0);
     if (stats.unreadable.length > 0) {
       unreadable.push({ file, count: stats.unreadable.length, samples: stats.unreadable.slice(0, 3) });
     }
   }
 
+  // Changed lines inside a test file, none of them recognisable as part of an
+  // assertion, is not the same as "checked and clean" — it is the state where
+  // this guard has nothing to say. Saying nothing and saying "approved" have
+  // to look different, which is the whole reason `status` exists.
   const status =
     reported.length > 0
       ? "FAIL"
-      : assertionsSeen === 0 && unreadable.length > 0
+      : assertionsSeen === 0 && (unreadable.length > 0 || examined > 0)
         ? "UNREADABLE"
         : examined > 0
           ? "PASS"
@@ -2480,6 +2573,26 @@ export function scanDiff(diffTextStr = "", options = {}) {
     }
   }
 
+  // The gate calls `scanDiff`, not `assertTestIntegrity` — so wiring the
+  // dialect warning into the latter meant it reached nobody. The guard
+  // computed `UNREADABLE`, and the operator was shown an unblemished pass.
+  // A boundary that is not reported is not a boundary.
+  if (tamperingRes.status === "UNREADABLE") {
+    const where = (tamperingRes.unreadable || []).map((u) => u.file);
+    const sample = tamperingRes.unreadable?.[0]?.samples?.[0];
+    findings.push({
+      severity: "MEDIUM",
+      type: "TEST_DIALECT_UNREADABLE",
+      file: where[0] ?? null,
+      line: null,
+      description:
+        `Test Tamper Guard: changed ${tamperingRes.inputsSeen} line(s) in ${tamperingRes.filesSeen} test file(s) ` +
+        `and recognised no assertion among them${sample ? ` (e.g. ${JSON.stringify(sample)})` : ""}. ` +
+        `This change was NOT checked for tampering. That is not a failure — an unlisted assertion library is ` +
+        `normal — but it is not an approval either, so it is reported rather than passed silently.`,
+    });
+  }
+
   return {
     ok: secretsOk && edgeRes.ok && crossPkgRes.ok && tamperingRes.ok,
     findings,