@mcptoolshop/promo-kit 0.1.0

package/scripts/gen-decision-drift.mjs

@@ -0,0 +1,253 @@
+#!/usr/bin/env node
+
+/**
+ * Decision Drift Detector
+ *
+ * Compares current vs previous promo-decisions.json to detect
+ * week-over-week drift in promotion decisions — new entrants,
+ * exits, score deltas, and action changes.
+ *
+ * Usage:
+ *   node scripts/gen-decision-drift.mjs [--dry-run]
+ *
+ * Reads:
+ *   site/src/data/promo-decisions.json
+ *   site/src/data/decision-drift-snapshot.json
+ *   site/src/data/governance.json
+ *
+ * Writes:
+ *   site/src/data/decision-drift.json
+ *   site/public/lab/decisions/decision-drift.md
+ *   site/src/data/decision-drift-snapshot.json (updated snapshot)
+ */
+
+import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { resolve, join } from "node:path";
+import { getConfig, getRoot } from "./lib/config.mjs";
+
+const ROOT = getRoot();
+const config = getConfig();
+const DATA_DIR = join(ROOT, config.paths.dataDir);
+const DECISIONS_DIR = join(ROOT, config.paths.publicDir, "lab", "decisions");
+
+// ── Helpers ─────────────────────────────────────────────────
+
+function safeParseJson(filePath, fallback = null) {
+  try {
+    return JSON.parse(readFileSync(filePath, "utf8"));
+  } catch {
+    return fallback;
+  }
+}
+
+// ── Core ────────────────────────────────────────────────────
+
+/**
+ * Build drift report comparing previous and current promo decisions.
+ *
+ * @param {object|null} previous - Previous promo-decisions object (null on first run)
+ * @param {object|null} current  - Current promo-decisions object
+ * @returns {{ entrants: string[], exits: string[], scoreDeltas: Array, reasonChanges: Array, summary: object }}
+ */
+export function buildDrift(previous, current) {
+  const prevDecisions = previous?.decisions ?? [];
+  const currDecisions = current?.decisions ?? [];
+
+  const prevMap = new Map();
+  for (const d of prevDecisions) {
+    if (d && d.slug) prevMap.set(d.slug, d);
+  }
+
+  const currMap = new Map();
+  for (const d of currDecisions) {
+    if (d && d.slug) currMap.set(d.slug, d);
+  }
+
+  const prevSlugs = new Set(prevMap.keys());
+  const currSlugs = new Set(currMap.keys());
+
+  // Entrants: in current but not in previous
+  const entrants = [...currSlugs].filter((s) => !prevSlugs.has(s));
+
+  // Exits: in previous but not in current
+  const exits = [...prevSlugs].filter((s) => !currSlugs.has(s));
+
+  // Common slugs: compute score deltas and action changes
+  const commonSlugs = [...currSlugs].filter((s) => prevSlugs.has(s));
+
+  const scoreDeltas = [];
+  const reasonChanges = [];
+
+  for (const slug of commonSlugs) {
+    const prev = prevMap.get(slug);
+    const curr = currMap.get(slug);
+
+    const prevScore = prev.score ?? 0;
+    const currScore = curr.score ?? 0;
+    const delta = currScore - prevScore;
+
+    scoreDeltas.push({ slug, prevScore, currScore, delta });
+
+    if (prev.action !== curr.action) {
+      reasonChanges.push({ slug, prevAction: prev.action, currAction: curr.action });
+    }
+  }
+
+  // Summary
+  const deltasWithChange = scoreDeltas.filter((d) => d.delta !== 0).length;
+  const actionOnlyChanges = reasonChanges.filter((rc) => {
+    const sd = scoreDeltas.find((d) => d.slug === rc.slug);
+    return sd && sd.delta === 0;
+  }).length;
+  const totalChanged = entrants.length + exits.length + deltasWithChange + actionOnlyChanges;
+  const totalStable = commonSlugs.length - deltasWithChange - actionOnlyChanges;
+
+  return {
+    entrants,
+    exits,
+    scoreDeltas,
+    reasonChanges,
+    summary: { totalChanged, totalStable },
+  };
+}
+
+// ── Markdown generator ──────────────────────────────────────
+
+/**
+ * Build a markdown summary of decision drift.
+ *
+ * @param {object} drift - Output from buildDrift()
+ * @returns {string}
+ */
+function generateDriftMd(drift) {
+  const { entrants, exits, scoreDeltas, reasonChanges, summary } = drift;
+  const lines = [];
+
+  lines.push("# Decision Drift Report");
+  lines.push("");
+  lines.push(`*Generated: ${new Date().toISOString().slice(0, 10)}*`);
+  lines.push("");
+
+  // Entrants
+  lines.push("## Entrants");
+  lines.push("");
+  if (entrants.length > 0) {
+    for (const slug of entrants) {
+      lines.push(`- ${slug}`);
+    }
+  } else {
+    lines.push("No new entrants.");
+  }
+  lines.push("");
+
+  // Exits
+  lines.push("## Exits");
+  lines.push("");
+  if (exits.length > 0) {
+    for (const slug of exits) {
+      lines.push(`- ${slug}`);
+    }
+  } else {
+    lines.push("No exits.");
+  }
+  lines.push("");
+
+  // Score Deltas
+  lines.push("## Score Deltas");
+  lines.push("");
+  if (scoreDeltas.length > 0) {
+    lines.push("| Slug | Prev Score | Curr Score | Delta |");
+    lines.push("|------|-----------|-----------|-------|");
+    for (const d of scoreDeltas) {
+      const sign = d.delta > 0 ? "+" : "";
+      lines.push(`| ${d.slug} | ${d.prevScore} | ${d.currScore} | ${sign}${d.delta} |`);
+    }
+  } else {
+    lines.push("No common slugs to compare.");
+  }
+  lines.push("");
+
+  // Action Changes
+  lines.push("## Action Changes");
+  lines.push("");
+  if (reasonChanges.length > 0) {
+    for (const rc of reasonChanges) {
+      lines.push(`- **${rc.slug}**: ${rc.prevAction} \u2192 ${rc.currAction}`);
+    }
+  } else {
+    lines.push("No action changes.");
+  }
+  lines.push("");
+
+  // Summary
+  lines.push("## Summary");
+  lines.push("");
+  lines.push(`- **Total changed:** ${summary.totalChanged}`);
+  lines.push(`- **Total stable:** ${summary.totalStable}`);
+  lines.push("");
+
+  return lines.join("\n");
+}
+
+// ── Pipeline ─────────────────────────────────────────────────
+
+/**
+ * Generate decision drift artifacts.
+ * @param {{ dataDir?: string, decisionsDir?: string, dryRun?: boolean }} opts
+ */
+export function generateDecisionDrift(opts = {}) {
+  const {
+    dataDir = DATA_DIR,
+    decisionsDir = DECISIONS_DIR,
+    dryRun = false,
+  } = opts;
+
+  const current = safeParseJson(join(dataDir, "promo-decisions.json"), { decisions: [] });
+  const previous = safeParseJson(join(dataDir, "decision-drift-snapshot.json"), null);
+  const governance = safeParseJson(join(dataDir, "governance.json"), {});
+
+  const drift = buildDrift(previous, current);
+
+  const output = {
+    generatedAt: new Date().toISOString(),
+    ...drift,
+  };
+
+  if (dryRun) {
+    console.log("  [dry-run] Decision drift computed.");
+    console.log(`    Entrants: ${drift.entrants.length}, Exits: ${drift.exits.length}`);
+    console.log(`    Score deltas: ${drift.scoreDeltas.length}, Action changes: ${drift.reasonChanges.length}`);
+    return output;
+  }
+
+  // Write drift JSON
+  const driftPath = join(dataDir, "decision-drift.json");
+  writeFileSync(driftPath, JSON.stringify(output, null, 2) + "\n", "utf8");
+
+  // Write drift markdown
+  mkdirSync(decisionsDir, { recursive: true });
+  const md = generateDriftMd(drift);
+  writeFileSync(join(decisionsDir, "decision-drift.md"), md + "\n", "utf8");
+
+  // Update snapshot (unless frozen)
+  if (governance.decisionsFrozen !== true) {
+    writeFileSync(
+      join(dataDir, "decision-drift-snapshot.json"),
+      JSON.stringify(current, null, 2) + "\n",
+      "utf8",
+    );
+  }
+
+  console.log(`  Decision drift written → ${driftPath}`);
+  return output;
+}
+
+// ── Entry point ──────────────────────────────────────────────
+
+const isMain = process.argv[1] && resolve(process.argv[1]).endsWith("gen-decision-drift.mjs");
+if (isMain) {
+  const dryRun = process.argv.includes("--dry-run");
+  console.log("Generating decision drift...");
+  if (dryRun) console.log("  Mode: DRY RUN");
+  generateDecisionDrift({ dryRun });
+}

package/scripts/gen-experiment-decisions.mjs

@@ -0,0 +1,282 @@
+#!/usr/bin/env node
+
+/**
+ * Experiment Decisions Generator
+ *
+ * Reads experiments.json, feedback-summary.json, and governance.json,
+ * evaluates active experiments for winner/loser/insufficient-data status,
+ * writes a separate decisions file. Does NOT modify experiments.json.
+ *
+ * Usage:
+ *   node scripts/gen-experiment-decisions.mjs [--dry-run]
+ *
+ * Reads:
+ *   site/src/data/experiments.json
+ *   site/src/data/feedback-summary.json
+ *   site/src/data/governance.json
+ *
+ * Writes:
+ *   site/src/data/experiment-decisions.json
+ *   site/public/lab/decisions/experiment-decisions.md
+ */
+
+import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { resolve, join } from "node:path";
+import { getConfig, getRoot } from "./lib/config.mjs";
+
+const ROOT = getRoot();
+const config = getConfig();
+const DATA_DIR = join(ROOT, config.paths.dataDir);
+const PUBLIC_DIR = join(ROOT, config.paths.publicDir, "lab", "decisions");
+
+// -- Helpers ---------------------------------------------------------
+
+function safeParseJson(filePath, fallback = null) {
+  try {
+    return JSON.parse(readFileSync(filePath, "utf8"));
+  } catch {
+    return fallback;
+  }
+}
+
+// -- Core ------------------------------------------------------------
+
+/**
+ * Evaluate active experiments against feedback data and governance thresholds.
+ *
+ * @param {{ schemaVersion?: number, experiments: Array<{ id: string, name: string, status: string, control: { key: string }, variant: { key: string } }> }} experiments
+ * @param {{ perExperiment: Record<string, Record<string, { sent: number, opened: number, replied: number, ignored: number, bounced: number }>> }} feedbackSummary
+ * @param {{ minExperimentDataThreshold: number }} governance
+ * @returns {{
+ *   evaluations: Array<{
+ *     experimentId: string,
+ *     name: string,
+ *     status: "needs-more-data"|"winner-found"|"no-decision",
+ *     controlEntries: number,
+ *     variantEntries: number,
+ *     controlReplyRate: number,
+ *     variantReplyRate: number,
+ *     winnerKey: string|null,
+ *     recommendation: string
+ *   }>,
+ *   warnings: string[]
+ * }}
+ */
+export function evaluateExperiments(experiments, feedbackSummary, governance) {
+  const evaluations = [];
+  const warnings = [];
+
+  const threshold = governance.minExperimentDataThreshold || 10;
+  const perExp = feedbackSummary.perExperiment || {};
+  const allExperiments = experiments.experiments || [];
+
+  // Filter to active experiments only (skip draft and concluded)
+  const active = allExperiments.filter((exp) => exp.status === "active");
+
+  if (active.length === 0) {
+    warnings.push("No active experiments found");
+  }
+
+  for (const exp of active) {
+    const expData = perExp[exp.id];
+
+    // No feedback data at all for this experiment
+    if (!expData) {
+      evaluations.push({
+        experimentId: exp.id,
+        name: exp.name,
+        status: "needs-more-data",
+        controlEntries: 0,
+        variantEntries: 0,
+        controlReplyRate: 0,
+        variantReplyRate: 0,
+        winnerKey: null,
+        recommendation: `No feedback data yet for experiment ${exp.id}`,
+      });
+      continue;
+    }
+
+    const controlKey = exp.control.key;
+    const variantKey = exp.variant.key;
+
+    const controlCounts = expData[controlKey] || { sent: 0, opened: 0, replied: 0, ignored: 0, bounced: 0 };
+    const variantCounts = expData[variantKey] || { sent: 0, opened: 0, replied: 0, ignored: 0, bounced: 0 };
+
+    // Compute entry counts per arm: total = sum of all outcome fields
+    const controlEntries = controlCounts.sent + controlCounts.opened + controlCounts.replied + controlCounts.ignored + controlCounts.bounced;
+    const variantEntries = variantCounts.sent + variantCounts.opened + variantCounts.replied + variantCounts.ignored + variantCounts.bounced;
+
+    // Compute reply rates
+    const controlReplyRate = controlCounts.replied / Math.max(controlEntries, 1);
+    const variantReplyRate = variantCounts.replied / Math.max(variantEntries, 1);
+
+    // Insufficient data check
+    if (controlEntries < threshold || variantEntries < threshold) {
+      evaluations.push({
+        experimentId: exp.id,
+        name: exp.name,
+        status: "needs-more-data",
+        controlEntries,
+        variantEntries,
+        controlReplyRate: Math.round(controlReplyRate * 10000) / 10000,
+        variantReplyRate: Math.round(variantReplyRate * 10000) / 10000,
+        winnerKey: null,
+        recommendation: `Insufficient data: ${controlEntries} control, ${variantEntries} variant (threshold: ${threshold})`,
+      });
+      continue;
+    }
+
+    // Winner detection: variant outperforms control at 2x+
+    if (variantReplyRate > 0 && variantReplyRate / Math.max(controlReplyRate, 0.001) > 2) {
+      const ratio = Math.round(variantReplyRate / Math.max(controlReplyRate, 0.001) * 10) / 10;
+      evaluations.push({
+        experimentId: exp.id,
+        name: exp.name,
+        status: "winner-found",
+        controlEntries,
+        variantEntries,
+        controlReplyRate: Math.round(controlReplyRate * 10000) / 10000,
+        variantReplyRate: Math.round(variantReplyRate * 10000) / 10000,
+        winnerKey: variantKey,
+        recommendation: `Variant '${variantKey}' outperforms control at ${ratio}x reply rate`,
+      });
+      continue;
+    }
+
+    // Winner detection: control outperforms variant at 2x+
+    if (controlReplyRate > 0 && controlReplyRate / Math.max(variantReplyRate, 0.001) > 2) {
+      const ratio = Math.round(controlReplyRate / Math.max(variantReplyRate, 0.001) * 10) / 10;
+      evaluations.push({
+        experimentId: exp.id,
+        name: exp.name,
+        status: "winner-found",
+        controlEntries,
+        variantEntries,
+        controlReplyRate: Math.round(controlReplyRate * 10000) / 10000,
+        variantReplyRate: Math.round(variantReplyRate * 10000) / 10000,
+        winnerKey: controlKey,
+        recommendation: `Control '${controlKey}' outperforms variant at ${ratio}x reply rate`,
+      });
+      continue;
+    }
+
+    // No clear winner -- keep collecting
+    evaluations.push({
+      experimentId: exp.id,
+      name: exp.name,
+      status: "no-decision",
+      controlEntries,
+      variantEntries,
+      controlReplyRate: Math.round(controlReplyRate * 10000) / 10000,
+      variantReplyRate: Math.round(variantReplyRate * 10000) / 10000,
+      winnerKey: null,
+      recommendation: `Performance is similar (control: ${Math.round(controlReplyRate * 10000) / 10000}, variant: ${Math.round(variantReplyRate * 10000) / 10000}). Keep collecting data.`,
+    });
+  }
+
+  return { evaluations, warnings };
+}
+
+/**
+ * Generate markdown report from experiment evaluations.
+ *
+ * @param {{ evaluations: Array<object>, warnings: string[] }} result
+ * @returns {string} Markdown string
+ */
+export function generateDecisionsMd(result) {
+  const lines = [];
+
+  lines.push("# Experiment Decisions Report");
+  lines.push("");
+  lines.push(`*Generated: ${new Date().toISOString().slice(0, 10)}*`);
+  lines.push("");
+
+  if (result.evaluations.length === 0) {
+    lines.push("No active experiments to evaluate.");
+    lines.push("");
+  } else {
+    lines.push("## Evaluations");
+    lines.push("");
+    lines.push("| Experiment | Status | Control (n) | Variant (n) | Control Rate | Variant Rate | Winner | Recommendation |");
+    lines.push("|------------|--------|-------------|-------------|--------------|--------------|--------|----------------|");
+
+    for (const ev of result.evaluations) {
+      const winner = ev.winnerKey || "--";
+      lines.push(`| ${ev.name} | ${ev.status} | ${ev.controlEntries} | ${ev.variantEntries} | ${ev.controlReplyRate} | ${ev.variantReplyRate} | ${winner} | ${ev.recommendation} |`);
+    }
+    lines.push("");
+  }
+
+  if (result.warnings.length > 0) {
+    lines.push("## Warnings");
+    lines.push("");
+    for (const w of result.warnings) {
+      lines.push(`- ${w}`);
+    }
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}
+
+// -- Pipeline --------------------------------------------------------
+
+/**
+ * Full pipeline: load data, evaluate experiments, write outputs.
+ *
+ * @param {{ dataDir?: string, publicDir?: string, dryRun?: boolean }} opts
+ * @returns {{ evaluationCount: number, outputPath: string }}
+ */
+export function generateExperimentDecisions(opts = {}) {
+  const { dataDir = DATA_DIR, publicDir = PUBLIC_DIR, dryRun = false } = opts;
+
+  const experiments = safeParseJson(join(dataDir, "experiments.json"), { experiments: [] });
+  const feedbackSummary = safeParseJson(join(dataDir, "feedback-summary.json"), { perExperiment: {} });
+  const governance = safeParseJson(join(dataDir, "governance.json"), { minExperimentDataThreshold: 10 });
+
+  // Freeze check — preserve existing evaluations when frozen
+  const outputPath = join(dataDir, "experiment-decisions.json");
+  if (governance.experimentsFrozen === true) {
+    console.log("  Experiments frozen (governance.experimentsFrozen=true). Skipping regeneration.");
+    const existing = safeParseJson(outputPath, { evaluations: [] });
+    return { evaluationCount: (existing.evaluations || []).length, outputPath, frozen: true };
+  }
+
+  const result = evaluateExperiments(experiments, feedbackSummary, governance);
+
+  if (dryRun) {
+    console.log(`  [dry-run] Would write ${result.evaluations.length} evaluations`);
+    for (const ev of result.evaluations) {
+      console.log(`  [dry-run]   ${ev.experimentId}: ${ev.status} -- ${ev.recommendation}`);
+    }
+    return { evaluationCount: result.evaluations.length, outputPath };
+  }
+
+  // Write experiment-decisions.json
+  const jsonOut = {
+    generatedAt: new Date().toISOString(),
+    ...result,
+  };
+  writeFileSync(outputPath, JSON.stringify(jsonOut, null, 2) + "\n", "utf8");
+  console.log(`  Wrote experiment-decisions.json (${result.evaluations.length} evaluations)`);
+
+  // Write markdown report
+  mkdirSync(publicDir, { recursive: true });
+  const md = generateDecisionsMd(result);
+  writeFileSync(join(publicDir, "experiment-decisions.md"), md, "utf8");
+  console.log(`  Wrote experiment-decisions.md`);
+
+  return { evaluationCount: result.evaluations.length, outputPath };
+}
+
+// -- Entry point -----------------------------------------------------
+
+const isMain = process.argv[1] && resolve(process.argv[1]).endsWith("gen-experiment-decisions.mjs");
+
+if (isMain) {
+  const dryRun = process.argv.includes("--dry-run");
+  console.log("Evaluating experiments...");
+  if (dryRun) console.log("  Mode: DRY RUN");
+  const result = generateExperimentDecisions({ dryRun });
+  console.log(`  Evaluations: ${result.evaluationCount}`);
+}