jules-orchestrator-kit 0.22.7 → 0.22.8

package/bin/agentctl.mjs

@@ -14,7 +14,7 @@ const command = args[0];
 
 function printHelp() {
   console.log(`
-🚀 agentctl v0.22.7 — Universal Agent Orchestrator & Safety Gatekeeper
+🚀 agentctl v0.22.8 — Universal Agent Orchestrator & Safety Gatekeeper
 
 Usage: agentctl <command> [options]
 
@@ -44,7 +44,7 @@ async function main() {
   }
 
   if (command === "version" || command === "--version" || command === "-v") {
-    console.log("agentctl v0.22.7");
+    console.log("agentctl v0.22.8");
     process.exit(0);
   }
 
@@ -226,7 +226,7 @@ async function main() {
     }
 
     case "doctor": {
-      console.log(`\n🔍 agentctl System Diagnostics (v0.22.7)`);
+      console.log(`\n🔍 agentctl System Diagnostics (v0.22.8)`);
       console.log(`--------------------------------------------------`);
       console.log(`  Project Root     : ${root}`);
       console.log(`  Config File      : ${config._file || "None (Using defaults)"}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jules-orchestrator-kit",
-  "version": "0.22.7",
+  "version": "0.22.8",
   "description": "Orchestration kit for running Google Jules autonomous agents.",
   "repository": {
     "type": "git",

package/src/engine.mjs

@@ -4,10 +4,13 @@ import { changedFiles, diffBytes, diffText, showFromOrigin, runCmd } from "./git
 import { createProvider } from "./provider.mjs";
 import { withBudget, appendLedger, getQueueDir, ensureDir } from "./state.mjs";
 import { sanitizeUntrustedData, buildAgentEnvelope } from "./prompt-guard.mjs";
+import { recordVerifyRun, readVerifyRuns, flakyVerdict } from "./flaky-ledger.mjs";
 import { readdirSync, readFileSync, renameSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import { createHash } from "node:crypto";
 
+export { recordVerifyRun, readVerifyRuns, flakyVerdict };
+
 /**
  * Validates whether a file in .agent/jules-queue/ is a task file.
  * Filters out README.md and matches TASK-*.md or valid envelope front-matter.
@@ -176,9 +179,25 @@ export async function gate(opts = {}) {
     NODE_OPTIONS: guardNodeOptions,
   };
 
+  let flakyVerdictResult = null;
+
   if (testCmd) {
+    const startTime = Date.now();
     const res = runCmd(testCmd, { cwd: root, ignoreError: true, env: testEnv });
+    const durationMs = Date.now() - startTime;
     testResult = { ok: res.status === 0, status: res.status, stdout: res.stdout, stderr: res.stderr, command: testCmd };
+
+    const fingerprint = !testResult.ok ? fingerprintFailureState(testResult, root) : null;
+    recordVerifyRun(root, testCmd, testResult.ok, fingerprint, durationMs);
+
+    if (!testResult.ok) {
+      const runs = readVerifyRuns(root, testCmd);
+      flakyVerdictResult = flakyVerdict(runs);
+      if (flakyVerdictResult.verdict === "QUARANTINED") {
+        phases.push({ phase: "verify", ok: false, testResult, buildResult, flakyVerdict: flakyVerdictResult });
+        return { ok: false, code: 8, phases, flakyVerdict: flakyVerdictResult };
+      }
+    }
   }
 
   if (testResult.ok && buildCmd) {

package/src/flaky-ledger.mjs

@@ -0,0 +1,173 @@
+import { readFileSync, existsSync, appendFileSync } from "node:fs";
+import { join } from "node:path";
+import { getStateDir, ensureDir } from "./state.mjs";
+
+/**
+ * Calculates Wilson score interval for binomial proportion.
+ */
+export function wilsonScoreInterval(k, n, z = 1.96) {
+  if (n === 0) return { lower: 0, upper: 0 };
+  const p = k / n;
+  const z2 = z * z;
+  const denominator = 1 + z2 / n;
+  const center = (p + z2 / (2 * n)) / denominator;
+  const stdErr = Math.sqrt((p * (1 - p) + z2 / (4 * n)) / n);
+  const margin = (z * stdErr) / denominator;
+  const lower = Math.max(0, center - margin);
+  const upper = Math.min(1, center + margin);
+  return { lower, upper };
+}
+
+/**
+ * Computes oscillation rate (fraction of state transitions between consecutive runs).
+ */
+export function computeOscillation(runs) {
+  if (!runs || runs.length <= 1) return 0;
+  let transitions = 0;
+  for (let i = 0; i < runs.length - 1; i++) {
+    const current = typeof runs[i] === "boolean" ? runs[i] : Boolean(runs[i] && runs[i].pass);
+    const next = typeof runs[i + 1] === "boolean" ? runs[i + 1] : Boolean(runs[i + 1] && runs[i + 1].pass);
+    if (current !== next) {
+      transitions++;
+    }
+  }
+  return transitions / (runs.length - 1);
+}
+
+/**
+ * Appends a test outcome record to .agent/state/flaky.jsonl.
+ */
+export function recordVerifyRun(root, testCmd, pass, fingerprint = null, durationMs = 0) {
+  const stateDir = getStateDir(root);
+  ensureDir(stateDir);
+  const filePath = join(stateDir, "flaky.jsonl");
+
+  const entry = {
+    timestamp: new Date().toISOString(),
+    testCmd: testCmd || "",
+    pass: Boolean(pass),
+    fingerprint: fingerprint || null,
+    durationMs: Number(durationMs) || 0,
+  };
+
+  appendFileSync(filePath, JSON.stringify(entry) + "\n", "utf-8");
+  return entry;
+}
+
+/**
+ * Reads test run records from .agent/state/flaky.jsonl.
+ */
+export function readVerifyRuns(root, testCmd = null) {
+  const stateDir = getStateDir(root);
+  const filePath = join(stateDir, "flaky.jsonl");
+
+  if (!existsSync(filePath)) {
+    return [];
+  }
+
+  try {
+    const raw = readFileSync(filePath, "utf-8");
+    const lines = raw.split("\n").filter((l) => l.trim().length > 0);
+    const runs = [];
+    for (const line of lines) {
+      try {
+        const parsed = JSON.parse(line);
+        if (!testCmd || parsed.testCmd === testCmd) {
+          runs.push(parsed);
+        }
+      } catch (_) {}
+    }
+    return runs;
+  } catch (_) {
+    return [];
+  }
+}
+
+export const getVerifyRuns = readVerifyRuns;
+
+/**
+ * Evaluates flaky test verdict using a sliding window of the last n <= 10 runs.
+ */
+export function flakyVerdict(runs = []) {
+  const normalized = (runs || []).map((r) => {
+    if (typeof r === "boolean") return { pass: r };
+    if (typeof r === "number") return { pass: r === 1 };
+    return { pass: Boolean(r && r.pass) };
+  });
+
+  const window = normalized.slice(-10);
+  const n = window.length;
+  const fails = window.filter((r) => !r.pass).length;
+  const passes = n - fails;
+
+  if (n === 0) {
+    return {
+      verdict: "INSUFFICIENT_DATA",
+      allowRepair: true,
+      n: 0,
+      fails: 0,
+      passes: 0,
+      oscillation: 0,
+    };
+  }
+
+  if (fails === 0) {
+    return {
+      verdict: "HEALTHY",
+      allowRepair: true,
+      n,
+      fails,
+      passes,
+      oscillation: computeOscillation(window),
+    };
+  }
+
+  const trailing3Fail = n >= 3 && window.slice(-3).every((r) => !r.pass);
+  if (fails === n || trailing3Fail) {
+    return {
+      verdict: "REPAIRABLE_REGRESSION",
+      allowRepair: true,
+      n,
+      fails,
+      passes,
+      oscillation: computeOscillation(window),
+    };
+  }
+
+  if (n < 6) {
+    return {
+      verdict: "INSUFFICIENT_DATA",
+      allowRepair: true,
+      n,
+      fails,
+      passes,
+      oscillation: computeOscillation(window),
+    };
+  }
+
+  const oscillation = computeOscillation(window);
+  const wilson = wilsonScoreInterval(fails, n, 1.96);
+  const interiorWilson = wilson.lower > 0 && wilson.upper < 1;
+
+  if (oscillation >= 0.4 && interiorWilson) {
+    return {
+      verdict: "QUARANTINED",
+      allowRepair: false,
+      n,
+      fails,
+      passes,
+      oscillation,
+      wilson,
+    };
+  }
+
+  return {
+    verdict: "REPAIRABLE_REGRESSION",
+    allowRepair: true,
+    n,
+    fails,
+    passes,
+    oscillation,
+    wilson,
+  };
+}

package/src/mcp.mjs

@@ -7,7 +7,7 @@ import { reapOrphanedIntents, reapStaleMutexDirs } from "./journal.mjs";
 
 export const MCP_SERVER_INFO = {
   name: "jules-orchestrator-kit",
-  version: "0.22.7",
+  version: "0.22.8",
 };
 
 export const MAX_MCP_FRAME_SIZE = 4 * 1024 * 1024; // 4 MB memory safety ceiling