marketing-mindset 1.8.0 → 1.9.0

package/README.md

@@ -114,3 +114,26 @@ npx marketing-mindset mde --base 0.03 --per-arm 5000 --json
 (0.05, 0.01, 0.1) are the same knobs as `sample`. When no lift is readable at that volume the command
 says so instead of printing a number, and names the three ways out: more volume, a higher base rate, or a
 cheaper measurable unit (for cold email the reply rate, floor ~1,500–2,000 sends per variant).
+
+## `verdict` — was the difference real, and was the volume ever enough
+
+Reading a finished test is a different job from planning one, and the two honest failure modes are opposite:
+a difference that is inside the noise, and a difference that is real but was never readable at that volume.
+`verdict` answers both from two counts.
+
+```bash
+npx marketing-mindset verdict --a 120/4000 --b 168/4000
+# control 120/4,000 = 3.00% · variant 168/4,000 = 4.20%
+# difference: 1.2pp (40.0% relative) · interval 0.44pp .. 1.96pp
+# separated from noise: true · volume clears the floor: true
+# verdict: readable: the interval excludes zero and the volume clears the 4,994 the observed lift requires ...
+
+npx marketing-mindset verdict --a 120/4000 --b 168/4000 --json
+npx marketing-mindset verdict --a-conv 120 --a-n 4000 --b-conv 168 --b-n 4000
+```
+
+Each arm is `conversions/sample`. The command prints a Wilson interval per arm, a Newcombe interval for the
+difference, and decides: not readable (the interval includes zero), a lead rather than a verdict (separated,
+but below the per-arm volume the observed lift would need), or readable. `--alpha` takes the same three
+values as `sample` (0.05, 0.01, 0.1) and the floor is computed with the same formula, so the answer stays
+consistent with the plan that produced the test.

package/bin/cli.js

@@ -4,6 +4,7 @@
  *   marketing-mindset sample         required per-arm sample size for a two-proportion test
  *   marketing-mindset kill           the kill rule for a running test
  *   marketing-mindset mde            smallest relative lift a fixed per-arm volume can read
+ *   marketing-mindset verdict        read a finished test: interval, floor, honest verdict
  *   marketing-mindset sections       list every section of SKILL.md
  *   marketing-mindset read <text>    print one section by fuzzy title match
  *   marketing-mindset install        copy the skill into your agent's skills directory
@@ -438,6 +439,77 @@ function runMde(opts) {
   return 0;
 }
 
+function wilsonInterval(conv, n, alpha) {
+  const z = ZA[String(alpha)] || ZA["0.05"];
+  const p = conv / n;
+  const d = 1 + (z * z) / n;
+  const centre = (p + (z * z) / (2 * n)) / d;
+  const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / d;
+  return { low: Math.max(0, centre - half), high: Math.min(1, centre + half) };
+}
+
+function parseArm(v) {
+  const s = String(v === undefined ? "" : v).trim();
+  const m = s.match(/^(\d+)\s*\/\s*(\d+)$/);
+  if (!m) { return null; }
+  return { conv: Number(m[1]), n: Number(m[2]) };
+}
+
+function r6(x) { return Math.round(x * 1e6) / 1e6; }
+
+function runVerdict(opts) {
+  const alpha = Number(opts.alpha !== undefined ? opts.alpha : 0.05);
+  let aRaw = opts.a;
+  if (aRaw === undefined && opts["a-conv"] !== undefined && opts["a-n"] !== undefined) {
+    aRaw = opts["a-conv"] + "/" + opts["a-n"];
+  }
+  let bRaw = opts.b;
+  if (bRaw === undefined && opts["b-conv"] !== undefined && opts["b-n"] !== undefined) {
+    bRaw = opts["b-conv"] + "/" + opts["b-n"];
+  }
+  const problems = [];
+  const a = parseArm(aRaw), b = parseArm(bRaw);
+  if (!a) { problems.push("control arm: pass --a 120/4000 (conversions/sample)"); }
+  if (!b) { problems.push("variant arm: pass --b 168/4000 (conversions/sample)"); }
+  if (a && (a.n <= 0 || a.conv > a.n)) { problems.push("control arm must be conversions <= sample and a positive sample"); }
+  if (b && (b.n <= 0 || b.conv > b.n)) { problems.push("variant arm must be conversions <= sample and a positive sample"); }
+  if (!ZA[String(alpha)]) { problems.push("alpha must be 0.05, 0.01 or 0.1"); }
+  if (problems.length) { console.error("verdict: " + problems.join("; ")); return 1; }
+
+  const pa = a.conv / a.n, pb = b.conv / b.n;
+  const ia = wilsonInterval(a.conv, a.n, alpha), ib = wilsonInterval(b.conv, b.n, alpha);
+  const diff = pb - pa;
+  const lo = diff - Math.sqrt(Math.pow(pa - ia.low, 2) + Math.pow(ib.high - pb, 2));
+  const hi = diff + Math.sqrt(Math.pow(ia.high - pa, 2) + Math.pow(pb - ib.low, 2));
+  const decisive = lo > 0 || hi < 0;
+  const rel = pa > 0 ? diff / pa : null;
+  const floor = rel !== null && rel !== 0 ? sampleSize(pa, Math.abs(rel), 0.8, alpha) : null;
+  const underpowered = floor !== null && b.n < floor;
+  let verdict;
+  if (!decisive) {
+    verdict = "not readable: the " + ((1 - alpha) * 100) + "% interval for the difference includes zero, so the point estimate is noise dressed as a result. The honest moves are more volume, a higher base rate, or a bigger difference - not a rebuild.";
+  } else if (underpowered) {
+    verdict = "a lead, not a verdict: the interval excludes zero, but " + fmtNum(b.n) + " per arm is below the " + fmtNum(floor) + " the observed lift would need. Keep it running to the floor before anything is rebuilt on it.";
+  } else {
+    verdict = "readable: the interval excludes zero and the volume clears the " + fmtNum(floor) + " the observed lift requires - this is the number a decision can be taken on.";
+  }
+  const out = {
+    control: { conversions: a.conv, n: a.n, rate: r6(pa), wilson_low: r6(ia.low), wilson_high: r6(ia.high) },
+    variant: { conversions: b.conv, n: b.n, rate: r6(pb), wilson_low: r6(ib.low), wilson_high: r6(ib.high) },
+    absolute_lift_pp: r6(diff * 100),
+    relative_lift: rel === null ? null : r6(rel),
+    difference_interval: [r6(lo), r6(hi)],
+    decisive, floor_per_arm: floor, underpowered, alpha, verdict
+  };
+  if (opts.json) { console.log(JSON.stringify(out, null, 2)); } else {
+    console.log("control " + a.conv + "/" + fmtNum(a.n) + " = " + (pa * 100).toFixed(2) + "% · variant " + b.conv + "/" + fmtNum(b.n) + " = " + (pb * 100).toFixed(2) + "%");
+    console.log("difference: " + out.absolute_lift_pp + "pp" + (rel === null ? "" : " (" + (rel * 100).toFixed(1) + "% relative)") + " · interval " + (lo * 100).toFixed(2) + "pp .. " + (hi * 100).toFixed(2) + "pp");
+    console.log("separated from noise: " + decisive + " · volume clears the floor: " + !underpowered);
+    console.log("verdict: " + verdict);
+  }
+  return 0;
+}
+
 function fmtNum(n) {
   return String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
 }
@@ -568,6 +640,8 @@ if (cmd === "sample") {
       ? "keep running — below the required volume, a verdict now is noise"
       : "read it — volume is sufficient; kill if conversions are at or under the floor"
   }, null, 2));
+} else if (cmd === "verdict") {
+  process.exit(runVerdict(flags(rest)));
 } else if (cmd === "aeo") {
   process.exit(runAeo(flags(rest)));
 } else if (cmd === "brief") {
@@ -610,6 +684,8 @@ Usage:
       --base 0.03 --lift 0.2 --cpl 4 [--daily 500]
   npx marketing-mindset plan            dated plan: weekly volume, gates, first honest read
       --base 0.03 --lift 0.2 --daily 500 [--weeks 4] [--start 2026-09-21]
+  npx marketing-mindset verdict         was the difference real, and was the volume ever enough?
+      --a 120/4000 --b 168/4000 [--alpha 0.05] [--json]
   npx marketing-mindset mde             what a volume you can afford can actually read
       --base 0.03 --per-arm 5000 [--power 0.8] [--alpha 0.05] [--json]
   npx marketing-mindset warmup          day-by-day sending ramp with stop rules

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "marketing-mindset",
-  "version": "1.8.0",
-  "description": "A marketer's operating mindset as an installable agent skill and CLI: test-size floors, kill rules, the lift a volume can actually read, agent-readiness checklist, marketing-engineer job description. npx marketing-mindset sample|kill|mde|budget|brief|plan|warmup|aeo|jd|sections|read|install.",
+  "version": "1.9.0",
+  "description": "A marketer's operating mindset as an installable agent skill and CLI: test-size floors, kill rules, honest verdicts on finished tests, the lift a volume can read, agent-readiness checklist, marketing-engineer job description. npx marketing-mindset sample|kill|verdict|mde|budget|brief|plan|warmup|aeo|jd|sections|read|install.",
   "keywords": [
     "agent-skill",
     "skill",
@@ -13,6 +13,7 @@
     "llm",
     "prompt",
     "sample-size",
+    "ab-test-verdict",
     "minimum-detectable-effect",
     "job-description",
     "cold-email",