@eide/foir-cli 0.16.0 → 0.17.0

package/dist/cli.js

@@ -2083,6 +2083,7 @@ function createSegmentsMethods(client) {
 
 // src/lib/rpc/experiments.ts
 import { create as create7 } from "@bufbuild/protobuf";
+import { timestampFromDate } from "@bufbuild/protobuf/wkt";
 import {
   ExperimentStatus,
   CreateExperimentRequestSchema,
@@ -2096,6 +2097,7 @@ import {
   ResumeExperimentRequestSchema,
   EndExperimentRequestSchema,
   GetExperimentStatsRequestSchema,
+  ExperimentMetricSpecSchema,
   ForceAssignExperimentRequestSchema,
   RemoveExperimentAssignmentRequestSchema,
   ApplyExperimentWinnerRequestSchema,
@@ -2105,6 +2107,7 @@ import {
   ExperimentGoalSchema,
   ExperimentFunnelStepSchema
 } from "@eide/foir-proto-ts/experiments/v1/experiments_pb";
+import { MetricAggregator as MetricAggregator2 } from "@eide/foir-proto-ts/experiments/v1/experiments_pb";
 import { ExperimentStatus as ExperimentStatus2 } from "@eide/foir-proto-ts/experiments/v1/experiments_pb";
 var STATUS_TO_PROTO = {
   draft: ExperimentStatus.DRAFT,
@@ -2138,9 +2141,22 @@ function createExperimentsMethods(client) {
         })
       );
     },
-    async getExperimentStats(experimentId) {
+    async getExperimentStats(experimentId, params = {}) {
       const resp = await client.getExperimentStats(
-        create7(GetExperimentStatsRequestSchema, { experimentId })
+        create7(GetExperimentStatsRequestSchema, {
+          experimentId,
+          from: params.from ? timestampFromDate(params.from) : void 0,
+          to: params.to ? timestampFromDate(params.to) : void 0,
+          metrics: params.metrics?.map(
+            (m) => create7(ExperimentMetricSpecSchema, {
+              name: m.name,
+              path: m.path,
+              aggregator: m.aggregator,
+              goalKey: m.goalKey
+            })
+          ) ?? [],
+          minSampleSize: params.minSampleSize
+        })
       );
       return resp.stats ?? null;
     },
@@ -7240,7 +7256,8 @@ function registerSegmentsCommands(program2, globalOpts) {
 import {
   ExperimentSchema,
   ExperimentStatsSchema,
-  ExperimentAssignmentSchema
+  ExperimentAssignmentSchema,
+  MetricAggregator as MetricAggregator3
 } from "@eide/foir-proto-ts/experiments/v1/experiments_pb";
 function registerExperimentsCommands(program2, globalOpts) {
   const experiments = program2.command("experiments").description("Manage experiments");
@@ -7380,13 +7397,37 @@ function registerExperimentsCommands(program2, globalOpts) {
         success("Experiment ended");
     })
   );
-  experiments.command("stats <id>").description("Get experiment statistics").action(
-    withErrorHandler(globalOpts, async (id) => {
-      const opts = globalOpts();
-      const client = await createPlatformClient(opts);
-      const result = await client.experiments.getExperimentStats(id);
-      formatOutputProto(ExperimentStatsSchema, result, opts);
-    })
+  experiments.command("stats <id>").description(
+    "Decision-grade experiment statistics: per-variant assignments, per-goal conversion + significance, and optional metric aggregations."
+  ).option("--from <iso8601>", "Lower bound on converted_at (defaults to experiment start)").option("--to <iso8601>", "Upper bound on converted_at (defaults to now)").option(
+    "--metric <spec>",
+    "Metric aggregation in name=path:aggregator[:goal_key] form (e.g. cart=cartValue:sum:checkout_complete). Repeatable.",
+    collectMetric,
+    []
+  ).option("--min-sample-size <n>", "Per-arm sample-size gate for is_significant (default 100)", parseInt).action(
+    withErrorHandler(
+      globalOpts,
+      async (id, cmdOpts) => {
+        const opts = globalOpts();
+        const client = await createPlatformClient(opts);
+        const metrics = cmdOpts.metric.map(parseMetricSpec);
+        const result = await client.experiments.getExperimentStats(id, {
+          from: cmdOpts.from ? new Date(cmdOpts.from) : void 0,
+          to: cmdOpts.to ? new Date(cmdOpts.to) : void 0,
+          metrics,
+          minSampleSize: cmdOpts.minSampleSize
+        });
+        if (opts.json || opts.jsonl) {
+          formatOutputProto(ExperimentStatsSchema, result, opts);
+          return;
+        }
+        if (!result) {
+          console.log("No stats \u2014 experiment not found or no data in window.");
+          return;
+        }
+        renderStatsTable(result);
+      }
+    )
   );
   experiments.command("force-assign <experimentId>").description(
     "Force a known identity into a specific variant (admin override)."
@@ -7487,6 +7528,206 @@ function registerExperimentsCommands(program2, globalOpts) {
     )
   );
 }
+function collectMetric(value, accumulator) {
+  return [...accumulator, value];
+}
+function parseMetricSpec(raw) {
+  const eq = raw.indexOf("=");
+  if (eq <= 0) {
+    throw new Error(
+      `--metric ${JSON.stringify(raw)}: expected name=path:aggregator[:goal_key]`
+    );
+  }
+  const name = raw.slice(0, eq);
+  const rest = raw.slice(eq + 1);
+  const parts = rest.split(":");
+  if (parts.length < 2 || parts.length > 3) {
+    throw new Error(
+      `--metric ${JSON.stringify(raw)}: expected path:aggregator[:goal_key] after =`
+    );
+  }
+  const path3 = parts[0];
+  const aggStr = parts[1];
+  const goalKey = parts[2];
+  if (!path3 || !aggStr) {
+    throw new Error(
+      `--metric ${JSON.stringify(raw)}: expected path:aggregator[:goal_key] after =`
+    );
+  }
+  return {
+    name,
+    path: path3,
+    aggregator: parseAggregator(aggStr),
+    goalKey: goalKey || void 0
+  };
+}
+function parseAggregator(s) {
+  switch (s.toLowerCase()) {
+    case "sum":
+      return MetricAggregator3.SUM;
+    case "avg":
+    case "mean":
+      return MetricAggregator3.AVG;
+    case "count":
+      return MetricAggregator3.COUNT;
+    case "min":
+      return MetricAggregator3.MIN;
+    case "max":
+      return MetricAggregator3.MAX;
+    case "p50":
+    case "median":
+      return MetricAggregator3.P50;
+    case "p95":
+      return MetricAggregator3.P95;
+  }
+  throw new Error(
+    `unknown aggregator ${JSON.stringify(s)}: expected one of sum, avg, count, min, max, p50, p95`
+  );
+}
+function renderStatsTable(s) {
+  const winFrom = s.windowFrom?.seconds ? new Date(Number(s.windowFrom.seconds) * 1e3).toISOString().slice(0, 19) : "?";
+  const winTo = s.windowTo?.seconds ? new Date(Number(s.windowTo.seconds) * 1e3).toISOString().slice(0, 19) : "?";
+  console.log("");
+  console.log(`Experiment ${s.experimentKey}  (id ${s.experimentId})`);
+  console.log(
+    `Window ${winFrom} \u2192 ${winTo}    grain: ${s.dailyGrain || "day"}    total assignments: ${s.totalAssignments}`
+  );
+  console.log("");
+  console.log("Variants");
+  console.log(
+    "  " + padCols(["variant", "control", "assignments", "share"], [16, 8, 12, 8])
+  );
+  console.log("  " + "-".repeat(48));
+  for (const v of s.variants) {
+    const share = s.totalAssignments > 0 ? (v.assignmentCount / s.totalAssignments * 100).toFixed(1) + "%" : "-";
+    console.log(
+      "  " + padCols(
+        [v.variantKey, v.isControl ? "\u2713" : "", String(v.assignmentCount), share],
+        [16, 8, 12, 8]
+      )
+    );
+  }
+  console.log("");
+  const familySize = familySizeOf(s);
+  for (const goal of s.goals) {
+    console.log(`Goal: ${goal.goalKey}${goal.isPrimary ? "  (primary)" : ""}`);
+    console.log(
+      "  " + padCols(
+        ["variant", "count", "unique", "rate", "lift", "p (raw)", "p (corr)", "sig"],
+        [16, 8, 8, 9, 9, 10, 10, 5]
+      )
+    );
+    console.log("  " + "-".repeat(75));
+    for (const v of s.variants) {
+      const cell = v.goalStats.find((g) => g.goalKey === goal.goalKey);
+      if (!cell) continue;
+      const rate = (cell.conversionRate * 100).toFixed(2) + "%";
+      if (v.isControl) {
+        console.log(
+          "  " + padCols(
+            [v.variantKey, String(cell.conversionEventCount), String(cell.uniqueConverters), rate, "-", "-", "-", "-"],
+            [16, 8, 8, 9, 9, 10, 10, 5]
+          )
+        );
+        continue;
+      }
+      const sig = cell.significance;
+      const rawP = sig?.pValue ?? 1;
+      const corrP = Math.min(1, rawP * familySize);
+      const lift = sig ? (sig.relativeLift >= 0 ? "+" : "") + (sig.relativeLift * 100).toFixed(1) + "%" : "-";
+      const sigFlag = corrP < 0.05 && sig?.isSignificant ? "\u2713" : "";
+      console.log(
+        "  " + padCols(
+          [
+            v.variantKey,
+            String(cell.conversionEventCount),
+            String(cell.uniqueConverters),
+            rate,
+            lift,
+            rawP.toFixed(4),
+            corrP.toFixed(4),
+            sigFlag
+          ],
+          [16, 8, 8, 9, 9, 10, 10, 5]
+        )
+      );
+    }
+    console.log("");
+  }
+  if (s.metricStats.length > 0) {
+    const byMetric = /* @__PURE__ */ new Map();
+    for (const m of s.metricStats) {
+      const list = byMetric.get(m.metricName) ?? [];
+      list.push(m);
+      byMetric.set(m.metricName, list);
+    }
+    for (const [name, rows] of byMetric) {
+      console.log(`Metric: ${name}`);
+      console.log(
+        "  " + padCols(
+          ["variant", "value", "n", "skipped", "lift", "p", "sig"],
+          [16, 12, 8, 8, 9, 8, 5]
+        )
+      );
+      console.log("  " + "-".repeat(70));
+      for (const m of rows) {
+        const sig = m.significance;
+        const lift = sig ? (sig.relativeLift >= 0 ? "+" : "") + (sig.relativeLift * 100).toFixed(1) + "%" : "-";
+        const p = sig ? sig.pValue.toFixed(4) : "-";
+        const sigFlag = sig?.isSignificant ? "\u2713" : "";
+        console.log(
+          "  " + padCols(
+            [
+              m.variantKey,
+              m.value.toFixed(2),
+              String(m.sampleSize),
+              String(m.skippedRowCount),
+              lift,
+              p,
+              sigFlag
+            ],
+            [16, 12, 8, 8, 9, 8, 5]
+          )
+        );
+      }
+      console.log("");
+    }
+  }
+  const winners = winnersOnPrimary(s, familySize);
+  if (winners.length > 0) {
+    console.log(`Significant on primary goal (Bonferroni-corrected): ${winners.join(", ")}`);
+  } else {
+    console.log("No variant has reached significance on the primary goal.");
+  }
+}
+function familySizeOf(s) {
+  const variantCount = s.variants.filter((v) => !v.isControl).length;
+  const goalCount = s.goals.length;
+  return Math.max(1, variantCount * goalCount);
+}
+function winnersOnPrimary(s, familySize) {
+  const primary = s.goals.find((g) => g.isPrimary);
+  if (!primary) return [];
+  const out = [];
+  for (const v of s.variants) {
+    if (v.isControl) continue;
+    const cell = v.goalStats.find((g) => g.goalKey === primary.goalKey);
+    if (!cell?.significance) continue;
+    const corrP = Math.min(1, cell.significance.pValue * familySize);
+    if (corrP < 0.05 && cell.significance.isSignificant) {
+      out.push(v.variantKey);
+    }
+  }
+  return out;
+}
+function padCols(values, widths) {
+  return values.map((v, i) => {
+    const w = widths[i] ?? 12;
+    const s = v ?? "";
+    if (s.length >= w) return s.slice(0, w);
+    return s + " ".repeat(w - s.length);
+  }).join(" ");
+}
 
 // src/commands/schedules.ts
 import { CronScheduleSchema } from "@eide/foir-proto-ts/schedules/v1/schedules_pb";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eide/foir-cli",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Universal platform CLI for Foir platform",
   "type": "module",
   "publishConfig": {
@@ -50,7 +50,7 @@
     "@bufbuild/protovalidate": "^1.1.1",
     "@connectrpc/connect": "^2.0.0",
     "@connectrpc/connect-node": "^2.0.0",
-    "@eide/foir-proto-ts": "^0.44.0",
+    "@eide/foir-proto-ts": "^0.45.0",
     "chalk": "^5.3.0",
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",