@mcptoolshop/promo-kit 0.1.0

package/scripts/gen-recommendations.mjs

@@ -0,0 +1,409 @@
+#!/usr/bin/env node
+/**
+ * gen-recommendations.mjs
+ *
+ * Advisory recommendation engine. Analyzes telemetry rollups, queue health,
+ * governance data, and lint reports to surface actionable insights.
+ *
+ * All recommendations are advisory — humans decide, the system proposes.
+ *
+ * Usage:
+ *   node scripts/gen-recommendations.mjs [--dry-run]
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { getConfig, getRoot } from "./lib/config.mjs";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = getRoot();
+const config = getConfig();
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url));
+
+// ── Valid category enum ─────────────────────────────────────
+
+export const VALID_CATEGORIES = [
+  "re-feature",
+  "improve-proof",
+  "stuck-submission",
+  "experiment-graduation",
+  "lint-promotion",
+];
+
+export const VALID_PRIORITIES = ["high", "medium", "low"];
+
+// ── Signal Computation (pure exports) ───────────────────────
+
+/**
+ * Compute Proof Engagement Score per tool from telemetry rollup.
+ * ProofEngagement = click_evidence_link + copy_proof_bullets
+ * @param {Record<string, Record<string, number>>} bySlug - rollup.bySlug
+ * @returns {Record<string, number>} slug → score
+ */
+export function computeProofEngagementBySlug(bySlug) {
+  if (!bySlug || typeof bySlug !== "object") return {};
+  const scores = {};
+  for (const [slug, counts] of Object.entries(bySlug)) {
+    const clicks = counts.click_evidence_link || 0;
+    const bullets = counts.copy_proof_bullets || 0;
+    const score = clicks + bullets;
+    if (score > 0) scores[slug] = score;
+  }
+  return scores;
+}
+
+/**
+ * Compute Submission Friction Score per submission.
+ * Friction = lint warning count + needs-info penalty (2) + stuck penalty (1 if >7d)
+ * @param {object[]} submissions - submissions.json .submissions array
+ * @param {Record<string, object>} lintReports - slug → { errors, warnings }
+ * @param {{ now?: Date }} opts
+ * @returns {Record<string, number>} slug → friction score
+ */
+export function computeSubmissionFrictionBySlug(submissions, lintReports = {}, opts = {}) {
+  if (!Array.isArray(submissions)) return {};
+  const now = opts.now || new Date();
+  const scores = {};
+
+  for (const sub of submissions) {
+    if (!sub.slug) continue;
+    const lint = lintReports[sub.slug] || {};
+    const warningCount = (lint.warnings || []).length;
+    const needsInfoPenalty = sub.status === "needs-info" ? 2 : 0;
+
+    let stuckPenalty = 0;
+    if ((sub.status === "pending" || sub.status === "needs-info") && sub.submittedAt) {
+      const days = Math.floor((now.getTime() - new Date(sub.submittedAt).getTime()) / (1000 * 60 * 60 * 24));
+      if (days > 7) stuckPenalty = 1;
+    }
+
+    const score = warningCount + needsInfoPenalty + stuckPenalty;
+    if (score > 0) scores[sub.slug] = score;
+  }
+
+  return scores;
+}
+
+// ── Category Finders ────────────────────────────────────────
+
+/**
+ * Find tools with high proof engagement that aren't currently featured.
+ * @param {object} signals
+ * @param {object} overrides
+ * @returns {object[]}
+ */
+function findHighTrustTools(signals, overrides = {}) {
+  const results = [];
+  const engagement = signals.proofEngagementBySlug || {};
+
+  const sorted = Object.entries(engagement)
+    .sort((a, b) => b[1] - a[1])
+    .filter(([slug]) => !overrides[slug]?.featured);
+
+  for (const [slug, score] of sorted.slice(0, 5)) {
+    if (score > 5) {
+      results.push({
+        priority: "high",
+        category: "re-feature",
+        slug,
+        title: `Re-feature ${slug}`,
+        insight: `High proof engagement score (${score}) — users actively checking evidence`,
+        action: `Consider adding "${slug}" to featured collection or promo queue`,
+        evidence: { proofEngagementScore: score, currentlyFeatured: false },
+      });
+    }
+  }
+
+  return results.slice(0, 3);
+}
+
+/**
+ * Find tools with high install copies but low proof engagement.
+ * @param {object} signals
+ * @param {object} rollup
+ * @returns {object[]}
+ */
+function findLowProofEngagementTools(signals, rollup = {}) {
+  const results = [];
+  const bySlug = rollup.bySlug || {};
+  const engagement = signals.proofEngagementBySlug || {};
+
+  for (const [slug, counts] of Object.entries(bySlug)) {
+    const installs = counts.copy_install || 0;
+    const proofScore = engagement[slug] || 0;
+
+    if (installs > 5 && proofScore < 2) {
+      results.push({
+        priority: "medium",
+        category: "improve-proof",
+        slug,
+        title: `Improve proof for ${slug}`,
+        insight: `${installs} install copies but only ${proofScore} proof interactions`,
+        action: `Review publicProof quality — add demo, benchmark, or better evidence links`,
+        evidence: { installCopies: installs, proofEngagementScore: proofScore },
+      });
+    }
+  }
+
+  return results
+    .sort((a, b) => b.evidence.installCopies - a.evidence.installCopies)
+    .slice(0, 3);
+}
+
+/**
+ * Find submissions with high friction scores.
+ * @param {object} signals
+ * @param {object} submissions
+ * @returns {object[]}
+ */
+function findHighFrictionSubmissions(signals, submissions = {}) {
+  const results = [];
+  const subs = submissions.submissions || [];
+  const friction = signals.submissionFrictionBySlug || {};
+
+  for (const sub of subs) {
+    const score = friction[sub.slug] || 0;
+    if (score >= 3 && (sub.status === "pending" || sub.status === "needs-info")) {
+      results.push({
+        priority: "high",
+        category: "stuck-submission",
+        slug: sub.slug,
+        title: `Review stuck submission: ${sub.slug}`,
+        insight: `Friction score ${score} — status: ${sub.status}`,
+        action: `Provide guidance to submitter or escalate for manual review`,
+        evidence: { frictionScore: score, status: sub.status, submittedAt: sub.submittedAt },
+      });
+    }
+  }
+
+  return results
+    .sort((a, b) => b.evidence.frictionScore - a.evidence.frictionScore)
+    .slice(0, 3);
+}
+
+/**
+ * Find experiments that have found a winner and are ready to graduate.
+ * @param {object} experimentDecisions
+ * @returns {object[]}
+ */
+function findReadyExperiments(experimentDecisions = {}) {
+  const results = [];
+  const evaluations = experimentDecisions.evaluations || [];
+
+  for (const ev of evaluations) {
+    if (ev.status === "winner-found") {
+      results.push({
+        priority: "medium",
+        category: "experiment-graduation",
+        slug: ev.experimentId,
+        title: `Graduate experiment: ${ev.experimentId}`,
+        insight: `Winner found — variant "${ev.winnerKey}" outperformed`,
+        action: `Apply winning variant to overrides.json and conclude experiment`,
+        evidence: {
+          experimentId: ev.experimentId,
+          winnerKey: ev.winnerKey,
+          recommendation: ev.recommendation || "",
+        },
+      });
+    }
+  }
+
+  return results.slice(0, 3);
+}
+
+/**
+ * Analyze lint failure patterns from queue health data.
+ * @param {object} queueHealth
+ * @returns {{ warningsToElevate: object[], docsToRewrite: object[] }}
+ */
+function analyzeLintPatterns(queueHealth = {}) {
+  const topFailures = queueHealth.topLintFailures || [];
+  const warningsToElevate = [];
+  const docsToRewrite = [];
+
+  for (const failure of topFailures) {
+    if (failure.count > 3) {
+      warningsToElevate.push({
+        warning: failure.reason,
+        count: failure.count,
+        suggestion: `Promote to error — ${failure.count} occurrences indicate systemic gap`,
+      });
+    }
+  }
+
+  // Cluster needs-info patterns — look for repeated lint warnings as signals
+  // that the submission docs need rewriting
+  const docTopics = {};
+  for (const failure of topFailures) {
+    const reason = failure.reason.toLowerCase();
+    if (reason.includes("install")) docTopics["install command"] = (docTopics["install command"] || 0) + failure.count;
+    if (reason.includes("quickstart")) docTopics["quickstart"] = (docTopics["quickstart"] || 0) + failure.count;
+    if (reason.includes("proof")) docTopics["proof links"] = (docTopics["proof links"] || 0) + failure.count;
+    if (reason.includes("pitch")) docTopics["pitch format"] = (docTopics["pitch format"] || 0) + failure.count;
+  }
+
+  for (const [topic, count] of Object.entries(docTopics)) {
+    if (count > 3) {
+      docsToRewrite.push({
+        topic,
+        occurrences: count,
+        suggestion: `Rewrite submission guide section on "${topic}" — ${count} failures suggest unclear docs`,
+      });
+    }
+  }
+
+  return { warningsToElevate, docsToRewrite };
+}
+
+/**
+ * Build lint-promotion recommendations from lint insights.
+ * @param {{ warningsToElevate: object[], docsToRewrite: object[] }} lintInsights
+ * @returns {object[]}
+ */
+function buildLintRecommendations(lintInsights) {
+  const results = [];
+
+  for (const item of (lintInsights.warningsToElevate || []).slice(0, 3)) {
+    results.push({
+      priority: "low",
+      category: "lint-promotion",
+      slug: item.warning,
+      title: `Promote lint warning: "${item.warning}"`,
+      insight: `Appears ${item.count} times — consistent failure pattern`,
+      action: item.suggestion,
+      evidence: { warning: item.warning, count: item.count },
+    });
+  }
+
+  return results;
+}
+
+// ── Main Orchestrator ───────────────────────────────────────
+
+/**
+ * Build all recommendations from input data.
+ * @param {object} inputs
+ * @param {{ maxRecommendations?: number }} opts
+ * @returns {object}
+ */
+export function buildRecommendations(inputs, opts = {}) {
+  const { maxRecommendations = 20 } = opts;
+
+  // Compute signals
+  const trustByWeek = inputs.rollup?.metrics?.trustInteractionScoreByWeek || {};
+  const proofEngagementBySlug = computeProofEngagementBySlug(inputs.rollup?.bySlug || {});
+  const submissionFrictionBySlug = computeSubmissionFrictionBySlug(
+    inputs.submissions?.submissions || [],
+    inputs.lintReports || {},
+    { now: opts.now },
+  );
+
+  const signals = { trustByWeek, proofEngagementBySlug, submissionFrictionBySlug };
+
+  // Gather recommendations from all categories
+  const recommendations = [];
+
+  recommendations.push(...findHighTrustTools(signals, inputs.overrides || {}));
+  recommendations.push(...findLowProofEngagementTools(signals, inputs.rollup || {}));
+  recommendations.push(...findHighFrictionSubmissions(signals, inputs.submissions || {}));
+  recommendations.push(...findReadyExperiments(inputs.experimentDecisions || {}));
+
+  // Lint patterns analysis
+  const lintInsights = analyzeLintPatterns(inputs.queueHealth || {});
+  recommendations.push(...buildLintRecommendations(lintInsights));
+
+  // Sort by priority (high first)
+  const priorityOrder = { high: 0, medium: 1, low: 2 };
+  recommendations.sort((a, b) => (priorityOrder[a.priority] ?? 9) - (priorityOrder[b.priority] ?? 9));
+
+  return {
+    generatedAt: new Date().toISOString(),
+    signals,
+    recommendations: recommendations.slice(0, maxRecommendations),
+    guardrails: {
+      dailyEventCaps: { max: 500 },
+      suspiciousSources: [],
+      multiEventCorroboration: true,
+    },
+    lintInsights,
+  };
+}
+
+// ── Pipeline ────────────────────────────────────────────────
+
+function loadJsonSafe(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Generate recommendations from data files.
+ * @param {object} opts
+ */
+export function genRecommendations(opts = {}) {
+  const {
+    rollupPath = path.join(ROOT, config.paths.dataDir, "telemetry", "rollup.json"),
+    queueHealthPath = path.join(ROOT, config.paths.dataDir, "queue-health.json"),
+    worthyPath = path.join(ROOT, config.paths.dataDir, "worthy.json"),
+    overridesPath = path.join(ROOT, config.paths.dataDir, "overrides.json"),
+    submissionsPath = path.join(ROOT, config.paths.dataDir, "submissions.json"),
+    experimentsPath = path.join(ROOT, config.paths.dataDir, "experiments.json"),
+    experimentDecisionsPath = path.join(ROOT, config.paths.dataDir, "experiment-decisions.json"),
+    lintDir = path.join(ROOT, config.paths.dataDir, "lint-reports"),
+    outputPath = path.join(ROOT, config.paths.dataDir, "recommendations.json"),
+    dryRun = false,
+  } = opts;
+
+  console.log("Generating recommendations...");
+  console.log(`  Mode: ${dryRun ? "DRY RUN" : "LIVE"}`);
+
+  // Load all inputs (fail-soft)
+  const inputs = {
+    rollup: loadJsonSafe(rollupPath),
+    queueHealth: loadJsonSafe(queueHealthPath),
+    worthy: loadJsonSafe(worthyPath),
+    overrides: loadJsonSafe(overridesPath),
+    submissions: loadJsonSafe(submissionsPath),
+    experiments: loadJsonSafe(experimentsPath),
+    experimentDecisions: loadJsonSafe(experimentDecisionsPath),
+    lintReports: {},
+  };
+
+  // Load lint reports if directory exists
+  try {
+    if (fs.existsSync(lintDir)) {
+      const files = fs.readdirSync(lintDir).filter((f) => f.endsWith(".json"));
+      for (const file of files) {
+        const slug = file.replace(/\.json$/, "");
+        const report = loadJsonSafe(path.join(lintDir, file));
+        if (report) inputs.lintReports[slug] = report;
+      }
+    }
+  } catch { /* fail soft */ }
+
+  const result = buildRecommendations(inputs);
+
+  if (dryRun) {
+    console.log(`  [dry-run] Recommendation generation complete.`);
+  } else {
+    fs.writeFileSync(outputPath, JSON.stringify(result, null, 2) + "\n");
+    console.log(`  Wrote ${outputPath}`);
+  }
+
+  console.log(`    Recommendations: ${result.recommendations.length}`);
+  console.log(`    Signals: ${Object.keys(result.signals.proofEngagementBySlug).length} tools with engagement`);
+  console.log(`    Lint insights: ${result.lintInsights.warningsToElevate.length} warnings to elevate`);
+
+  return result;
+}
+
+// ── CLI ─────────────────────────────────────────────────────
+
+if (isMain) {
+  const dryRun = process.argv.includes("--dry-run");
+  genRecommendations({ dryRun });
+}

package/scripts/gen-telemetry-aggregate.mjs

@@ -0,0 +1,266 @@
+#!/usr/bin/env node
+
+/**
+ * Telemetry Aggregator
+ *
+ * Reads raw event JSONL files and produces aggregated rollup + daily JSON.
+ *
+ * Usage:
+ *   node scripts/gen-telemetry-aggregate.mjs [--dry-run]
+ *
+ * Reads:
+ *   site/src/data/telemetry/events/*.jsonl
+ *
+ * Writes:
+ *   site/src/data/telemetry/rollup.json
+ *   site/src/data/telemetry/daily/YYYY-MM-DD.json
+ */
+
+import { readFileSync, readdirSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
+import { resolve, join } from "node:path";
+import { getConfig, getRoot } from "./lib/config.mjs";
+
+const ROOT = getRoot();
+const config = getConfig();
+
+// ── Valid event types ─────────────────────────────────────────
+
+export const VALID_EVENT_TYPES = [
+  "copy_proof_link", "copy_bundle", "copy_verify_cmd",
+  "copy_install", "copy_proof_bullets", "copy_claim",
+  "click_evidence_link", "click_receipt_link", "click_submit_link",
+];
+
+// ── Parsing ───────────────────────────────────────────────────
+
+/**
+ * Parse a JSONL string into an array of events.
+ * Skips malformed lines silently.
+ * @param {string} content
+ * @returns {object[]}
+ */
+export function parseEventsFile(content) {
+  if (!content || !content.trim()) return [];
+  const events = [];
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const evt = JSON.parse(trimmed);
+      if (evt && evt.type && evt.timestamp) {
+        events.push(evt);
+      }
+    } catch {
+      // skip malformed lines
+    }
+  }
+  return events;
+}
+
+// ── Aggregation ───────────────────────────────────────────────
+
+/**
+ * Aggregate events into counts by type, slug, and week.
+ * Includes anti-gaming guardrails: per-day caps and spike detection.
+ * @param {object[]} events
+ * @param {{ enableCaps?: boolean, dailyCapPerType?: number, spikeThreshold?: number }} opts
+ * @returns {{ generatedAt: string, totalEvents: number, byType: object, bySlug: object, byWeek: object, metrics: object, guardrails: object }}
+ */
+export function aggregateEvents(events, opts = {}) {
+  const { enableCaps = true, dailyCapPerType = 50, spikeThreshold = 300 } = opts;
+
+  // Phase 1: Apply per-day caps
+  const dailyTypeCounts = {};
+  const cappedEvents = [];
+  const guardrails = {
+    totalEventsProcessed: events.length,
+    eventsCapped: 0,
+    suspiciousDays: [],
+  };
+
+  for (const evt of events) {
+    if (!evt.timestamp || !evt.type) {
+      cappedEvents.push(evt);
+      continue;
+    }
+
+    const day = evt.timestamp.slice(0, 10);
+    const key = `${day}:${evt.type}`;
+
+    if (enableCaps) {
+      const count = dailyTypeCounts[key] || 0;
+      if (count >= dailyCapPerType) {
+        guardrails.eventsCapped++;
+        continue;
+      }
+      dailyTypeCounts[key] = count + 1;
+    }
+
+    cappedEvents.push(evt);
+  }
+
+  // Phase 2: Spike detection
+  const dailyTotals = {};
+  for (const evt of cappedEvents) {
+    if (evt.timestamp) {
+      const day = evt.timestamp.slice(0, 10);
+      dailyTotals[day] = (dailyTotals[day] || 0) + 1;
+    }
+  }
+  for (const [day, count] of Object.entries(dailyTotals)) {
+    if (count > spikeThreshold) {
+      guardrails.suspiciousDays.push({ day, count });
+    }
+  }
+
+  // Phase 3: Standard aggregation on capped events
+  const byType = {};
+  const bySlug = {};
+  const byWeek = {};
+
+  for (const evt of cappedEvents) {
+    // By type
+    byType[evt.type] = (byType[evt.type] || 0) + 1;
+
+    // By slug
+    const slug = evt.payload?.slug;
+    if (slug) {
+      if (!bySlug[slug]) bySlug[slug] = {};
+      bySlug[slug][evt.type] = (bySlug[slug][evt.type] || 0) + 1;
+    }
+
+    // By week
+    const week = evt.payload?.week;
+    if (week) {
+      if (!byWeek[week]) byWeek[week] = {};
+      byWeek[week][evt.type] = (byWeek[week][evt.type] || 0) + 1;
+    }
+  }
+
+  const metrics = computeMetrics(byType, byWeek);
+
+  return {
+    generatedAt: new Date().toISOString(),
+    totalEvents: cappedEvents.length,
+    byType,
+    bySlug,
+    byWeek,
+    metrics,
+    guardrails,
+  };
+}
+
+/**
+ * Compute derived metrics from aggregated type counts.
+ * @param {object} byType
+ * @param {object} byWeek
+ * @returns {object}
+ */
+export function computeMetrics(byType, byWeek = {}) {
+  const copyProofLink = byType.copy_proof_link || 0;
+  const copyBundle = byType.copy_bundle || 0;
+  const copyVerifyCmd = byType.copy_verify_cmd || 0;
+  const totalVerifyActions = copyProofLink + copyBundle + copyVerifyCmd;
+  const verificationRate = totalVerifyActions > 0 ? copyBundle / totalVerifyActions : 0;
+
+  const totalProofActions =
+    (byType.copy_proof_bullets || 0) +
+    (byType.copy_claim || 0) +
+    (byType.click_evidence_link || 0);
+
+  const submissionClicks = byType.click_submit_link || 0;
+
+  // Trust Interaction Score per week
+  const trustInteractionScoreByWeek = {};
+  for (const [week, counts] of Object.entries(byWeek)) {
+    trustInteractionScoreByWeek[week] =
+      (counts.copy_bundle || 0) +
+      (counts.copy_verify_cmd || 0) +
+      (counts.click_receipt_link || 0);
+  }
+
+  return {
+    verificationRate: Math.round(verificationRate * 10000) / 10000,
+    totalVerifyActions,
+    totalProofActions,
+    submissionClicks,
+    trustInteractionScoreByWeek,
+  };
+}
+
+// ── Pipeline ──────────────────────────────────────────────────
+
+/**
+ * Read all JSONL event files, aggregate, write output.
+ * @param {{ eventsDir?: string, outputPath?: string, dailyDir?: string, dryRun?: boolean }} opts
+ */
+export function genTelemetryAggregate(opts = {}) {
+  const {
+    eventsDir = join(ROOT, config.paths.dataDir, "telemetry", "events"),
+    outputPath = join(ROOT, config.paths.dataDir, "telemetry", "rollup.json"),
+    dailyDir = join(ROOT, config.paths.dataDir, "telemetry", "daily"),
+    dryRun = false,
+  } = opts;
+
+  // Collect all events
+  const allEvents = [];
+  const eventsByDay = {};
+
+  if (existsSync(eventsDir)) {
+    const files = readdirSync(eventsDir).filter((f) => f.endsWith(".jsonl"));
+    for (const file of files) {
+      const content = readFileSync(join(eventsDir, file), "utf8");
+      const events = parseEventsFile(content);
+      allEvents.push(...events);
+    }
+  }
+
+  // Group by day for daily rollups
+  for (const evt of allEvents) {
+    const day = evt.timestamp.slice(0, 10); // YYYY-MM-DD
+    if (!eventsByDay[day]) eventsByDay[day] = [];
+    eventsByDay[day].push(evt);
+  }
+
+  // Aggregate all
+  const rollup = aggregateEvents(allEvents);
+
+  if (dryRun) {
+    console.log(`  [dry-run] Telemetry aggregation complete.`);
+    console.log(`    Total events: ${allEvents.length}`);
+    console.log(`    Days with data: ${Object.keys(eventsByDay).length}`);
+    return { rollup, dailyCount: Object.keys(eventsByDay).length };
+  }
+
+  // Write rollup
+  writeFileSync(outputPath, JSON.stringify(rollup, null, 2) + "\n", "utf8");
+
+  // Write daily rollups
+  mkdirSync(dailyDir, { recursive: true });
+  for (const [day, events] of Object.entries(eventsByDay)) {
+    const daily = aggregateEvents(events);
+    writeFileSync(
+      join(dailyDir, `${day}.json`),
+      JSON.stringify(daily, null, 2) + "\n",
+      "utf8",
+    );
+  }
+
+  return { rollup, dailyCount: Object.keys(eventsByDay).length };
+}
+
+// ── Entry point ───────────────────────────────────────────────
+
+const isMain = process.argv[1] && resolve(process.argv[1]).endsWith("gen-telemetry-aggregate.mjs");
+if (isMain) {
+  const dryRun = process.argv.includes("--dry-run");
+  console.log("Aggregating telemetry events...");
+  if (dryRun) console.log("  Mode: DRY RUN");
+
+  const result = genTelemetryAggregate({ dryRun });
+  if (!dryRun) {
+    console.log(`  Total events: ${result.rollup.totalEvents}`);
+    console.log(`  Daily rollups: ${result.dailyCount}`);
+    console.log(`  Verification rate: ${(result.rollup.metrics.verificationRate * 100).toFixed(1)}%`);
+  }
+}