@absmartly/cli 1.3.0 → 1.4.0

package/README.md

@@ -317,6 +317,14 @@ abs experiments alerts dismiss 456
 abs experiments recommendations list 123
 abs experiments recommendations dismiss 456
 
+# Analyze experiment design + signals + benchmarks (returns JSON by default)
+abs experiments analyze 123                              # full structured analysis (JSON)
+abs experiments analyze checkout_redesign                # by name
+abs experiments analyze 123 -o table                     # flat one-row summary
+abs experiments analyze 123 -o yaml                      # full analysis as YAML
+abs experiments analyze 123 | jq .recommendation         # pluck a single section
+abs experiments analyze 123 | jq '[.heuristic_output[] | select(.fired)]'
+
 # Access control
 abs experiments access list-users 123
 abs experiments access grant-user 123 --user 1 --role 2
@@ -384,6 +392,249 @@ abs experiments estimate-participants --unit-type user_id --application absmartl
 abs experiments estimate-participants --unit-type user_id -o json
 ```
 
+#### Analyze experiment
+
+`abs experiments analyze <id>` produces a single structured document that combines what you'd otherwise piece together from `get`, `metrics results`, `alerts list`, `recommendations list`, and `list --type`. It's designed to be consumed by an LLM-based analyzer or a human reviewer; the JSON shape is stable.
+
+Default output is **JSON** (the `-o table` global default is overridden when `-o` is not explicitly set, since the data is deeply nested). `-o yaml` works the same way. `-o table`/`-o markdown`/`-o plain` switch to a flat one-row summary suitable for tabular display.
+
+The output has nine top-level sections:
+
+| Section | What it contains |
+|---|---|
+| `experiment` | Headline facts: `name`, `state`, `hypothesis`, `primary_metric_name`, `participant_count`, `leading_variant_*`, `current_recommended_action`, `report_note`. |
+| `alerts` | Active alerts (`{id, type, dismissed}`) — `srm`, `audience_mismatch`, `cleanup_needed`, etc. |
+| `recommendation` | The single best deterministic recommendation (`{theme, title, details}`) picked from the heuristic rule set, or `null`. Warnings outrank successes. |
+| `metric_signals` | Per-metric / per-variant rows from `metrics_snapshot` with `percent_change`, `p_value`, CI bounds, and a derived `status`: `improves` / `contradicts` / `flat` / `inconclusive`. |
+| `related_experiments` | Up to 24 recent same-type experiments. For peers that share the primary metric, `leading_variant_impact_percent` is fetched and compared. |
+| `analysis_confidence` | `high` / `medium` / `low` plus 5 boolean factors and human-readable reasons. Captures *whether there's enough signal to analyze*. |
+| `design_readout` | Whether the experiment design fits the question — summary line, parameter pass-through, and a benchmark from related experiments when available. |
+| `source_signals` | Audit trail mapping each derived field to its API source path (e.g. `experiment.metrics_snapshot.rows[*].percent_change`). Useful for verifying AI analyses. |
+| `heuristic_output` | All 9 heuristic rules with `{rule, fired, theme, title, details, evidence}`. The starting point for an AI analysis when one is available. |
+
+**Example output** (illustrative — a running experiment with a metric snapshot, two comparable peers, a primary-metric win, and a contradicting guardrail):
+
+```json
+{
+  "experiment": {
+    "id": 18234,
+    "name": "checkout_redesign",
+    "type": "test",
+    "state": "running",
+    "hypothesis": "Single-page checkout reduces drop-off by surfacing all required fields earlier.",
+    "primary_metric_name": "Checkout Completion Rate",
+    "unit_type_name": "user_id",
+    "participant_count": 102450,
+    "leading_variant_name": "treatment",
+    "leading_variant_impact_percent": 3.8,
+    "leading_variant_confidence": 0.987,
+    "current_recommended_action": "review",
+    "report_note": "Treatment is winning on primary metric; awaiting eng review of latency regression."
+  },
+  "alerts": [
+    { "id": 1042, "type": "sample_size_reached", "dismissed": false }
+  ],
+  "recommendation": {
+    "theme": "warning",
+    "title": "Review guardrail regressions before rolling out treatment.",
+    "details": "At least one guardrail metric is moving in the wrong direction, so the apparent win on the primary metric should not be treated as rollout-ready yet."
+  },
+  "metric_signals": [
+    {
+      "metric_id": 14, "metric_name": "Checkout Completion Rate", "metric_type": "primary",
+      "variant_id": 1, "variant_name": "treatment",
+      "percent_change": 3.8, "p_value": 0.013, "ci_low": 1.6, "ci_high": 6.0,
+      "status": "improves"
+    },
+    {
+      "metric_id": 18, "metric_name": "P95 Page Latency", "metric_type": "guardrail",
+      "variant_id": 1, "variant_name": "treatment",
+      "percent_change": 4.2, "p_value": 0.022, "ci_low": 0.9, "ci_high": 7.5,
+      "status": "contradicts"
+    },
+    {
+      "metric_id": 21, "metric_name": "Revenue per Visitor", "metric_type": "secondary",
+      "variant_id": 1, "variant_name": "treatment",
+      "percent_change": 0.4, "p_value": 0.61, "ci_low": -1.1, "ci_high": 1.9,
+      "status": "flat"
+    }
+  ],
+  "related_experiments": [
+    {
+      "id": 17988, "name": "checkout_steps_v2", "state": "stopped",
+      "started_at": "2025-09-04T12:00:00Z", "stopped_at": "2025-10-02T16:00:00Z",
+      "primary_metric_id": 14, "primary_metric_name": "Checkout Completion Rate",
+      "leading_variant_impact_percent": 2.1
+    },
+    {
+      "id": 18012, "name": "guest_checkout_default", "state": "stopped",
+      "started_at": "2025-10-15T09:00:00Z", "stopped_at": "2025-11-12T09:00:00Z",
+      "primary_metric_id": 14, "primary_metric_name": "Checkout Completion Rate",
+      "leading_variant_impact_percent": 4.6
+    },
+    {
+      "id": 18103, "name": "trust_badges_above_fold", "state": "running",
+      "started_at": "2025-12-01T10:00:00Z", "stopped_at": null,
+      "primary_metric_id": 14, "primary_metric_name": "Checkout Completion Rate",
+      "leading_variant_impact_percent": null
+    }
+  ],
+  "analysis_confidence": {
+    "level": "high",
+    "reasons": [],
+    "factors": {
+      "sample_size_reached": true,
+      "hypothesis_present": true,
+      "primary_metric_present": true,
+      "guardrails_present": true,
+      "no_blocking_alerts": true
+    }
+  },
+  "design_readout": {
+    "summary": "Designed to detect effects in the expected range; comparable experiments moved this metric by ~3.35%.",
+    "notes": [],
+    "parameters": {
+      "analysis_type": "group_sequential",
+      "required_alpha": 0.1,
+      "required_power": 0.8,
+      "minimum_detectable_effect": 2.5,
+      "baseline_primary_metric_mean": 0.382,
+      "baseline_participants_per_day": 7400,
+      "percentage_of_traffic": 100
+    },
+    "benchmark": {
+      "observed_impacts": [2.1, 4.6],
+      "median_abs_impact": 3.35
+    }
+  },
+  "source_signals": [
+    { "covers": "experiment.participant_count",
+      "source": "experiment.metrics_snapshot.rows[*].cum_unit_count" },
+    { "covers": "experiment.leading_variant_impact_percent",
+      "source": "experiment.metrics_snapshot.rows[*].percent_change" },
+    { "covers": "alerts",
+      "source": "experiment.alerts[*].type" },
+    { "covers": "experiment.current_recommended_action",
+      "source": "experiment.recommended_action.recommendation" },
+    { "covers": "experiment.report_note",
+      "source": "experiment.experiment_report.experiment_note.note" },
+    { "covers": "related_experiments[17988].leading_variant_impact_percent",
+      "source": "experiments/17988/metrics/main.percent_change" },
+    { "covers": "related_experiments[18012].leading_variant_impact_percent",
+      "source": "experiments/18012/metrics/main.percent_change" }
+  ],
+  "heuristic_output": [
+    { "rule": "blocking_alert", "fired": false, "theme": "warning",
+      "title": "Investigate the experiment before making a rollout decision.",
+      "details": "Active health checks indicate that the current results may be misleading. Resolve the data-quality or assignment issue before acting on the outcome.",
+      "evidence": { "alert_types": [] } },
+    { "rule": "cleanup_needed", "fired": false, "theme": "warning",
+      "title": "Clean up stale assignments before proceeding.",
+      "details": "The experiment has cleanup_needed signals; old data may skew the analysis.",
+      "evidence": { "alert_ids": [] } },
+    { "rule": "guardrail_contradicts", "fired": true, "theme": "warning",
+      "title": "Review guardrail regressions before rolling out treatment.",
+      "details": "At least one guardrail metric is moving in the wrong direction, so the apparent win on the primary metric should not be treated as rollout-ready yet.",
+      "evidence": { "metrics": ["P95 Page Latency"] } },
+    { "rule": "primary_metric_significant_loss", "fired": false, "theme": "warning",
+      "title": "Primary metric regressed at significance.",
+      "details": "The leading variant moves the primary metric in the wrong direction with p-value below alpha. Do not roll out.",
+      "evidence": { "variant": "treatment", "impact_percent": 3.8, "p_value": 0.013 } },
+    { "rule": "primary_metric_significant_win", "fired": true, "theme": "success",
+      "title": "Primary metric improved with treatment.",
+      "details": "The leading variant beat baseline on the primary metric with p-value below alpha.",
+      "evidence": { "variant": "treatment", "impact_percent": 3.8, "p_value": 0.013 } },
+    { "rule": "sample_size_not_reached", "fired": false, "theme": "info",
+      "title": "Sample size not reached.",
+      "details": "Group-sequential analysis has not yet hit its planned sample size; treat results as interim.",
+      "evidence": {} },
+    { "rule": "hypothesis_missing", "fired": false, "theme": "info",
+      "title": "No hypothesis recorded for this experiment.",
+      "details": "Without a written hypothesis it is hard to judge whether the result answers the original question.",
+      "evidence": {} },
+    { "rule": "no_recommendation_overdue", "fired": false, "theme": "info",
+      "title": "Experiment has been running long without a recommended action.",
+      "details": "Consider whether to stop, full-on, or iterate; running indefinitely is rarely the best option.",
+      "evidence": { "days_running": 18 } },
+    { "rule": "snapshot_unavailable", "fired": false, "theme": "info",
+      "title": "No metric snapshot is available yet.",
+      "details": "The previewer may not have processed this experiment; analysis is limited to design parameters.",
+      "evidence": {} }
+  ]
+}
+```
+
+Notice how the sections reinforce each other in this example:
+
+- `metric_signals` shows the primary metric improving (`status: improves`) AND a guardrail contradicting (`status: contradicts`).
+- That guardrail status drives `heuristic_output[guardrail_contradicts].fired = true` (theme `warning`).
+- Both `guardrail_contradicts` (warning) and `primary_metric_significant_win` (success) fired. `recommendation` is the first fired warning, so the headline becomes "Review guardrail regressions…" — even though the primary metric is winning.
+- `analysis_confidence` is `high`: all five factors are true (sample size reached, hypothesis present, primary metric present, guardrails present, no blocking alerts).
+- `design_readout.benchmark.median_abs_impact` (3.35%) is computed from the two comparable peers; the summary phrasing "designed to detect effects in the expected range" follows because the median is ≥ 1.5× the MDE (2.5%).
+- `source_signals` records one entry per derived field plus per peer-fetch — the audit trail an LLM analyst can cite.
+
+**Example — distilled view via jq:**
+
+```bash
+$ abs experiments analyze 18234 \
+    | jq '{name: .experiment.name,
+           confidence: .analysis_confidence.level,
+           recommendation: .recommendation.title,
+           fired: [.heuristic_output[] | select(.fired) | .rule]}'
+```
+```json
+{
+  "name": "checkout_redesign",
+  "confidence": "high",
+  "recommendation": "Review guardrail regressions before rolling out treatment.",
+  "fired": [
+    "guardrail_contradicts",
+    "primary_metric_significant_win"
+  ]
+}
+```
+
+Useful jq slices:
+
+```bash
+# Headline only
+abs experiments analyze 123 \
+  | jq '{state: .experiment.state, confidence: .analysis_confidence.level, recommendation}'
+
+# Just the rules that fired
+abs experiments analyze 123 \
+  | jq '[.heuristic_output[] | select(.fired) | {rule, theme, title}]'
+
+# Audit trail of where each derived field came from
+abs experiments analyze 123 | jq .source_signals
+
+# Compare two experiments at a glance
+for id in 123 456; do
+  abs experiments analyze "$id" \
+    | jq --arg id "$id" '{id: $id, name: .experiment.name,
+                          confidence: .analysis_confidence.level,
+                          fired: [.heuristic_output[] | select(.fired) | .rule]}'
+done
+```
+
+The 9 heuristic rules, in evaluation order:
+
+| Rule | Theme | Fires when |
+|---|---|---|
+| `blocking_alert` | warning | active alert in `{srm, audience_mismatch, assignment_conflict, experiments_interact}` |
+| `cleanup_needed` | warning | active `cleanup_needed` alert |
+| `guardrail_contradicts` | warning | any guardrail metric_signal status is `contradicts` |
+| `primary_metric_significant_loss` | warning | leading-variant primary `p_value < alpha` and `impact_percent < 0` |
+| `primary_metric_significant_win` | success | leading-variant primary `p_value < alpha` and `impact_percent > 0` |
+| `sample_size_not_reached` | info | running, group_sequential, no `sample_size_reached` alert |
+| `hypothesis_missing` | info | `experiment.hypothesis` empty/null |
+| `no_recommendation_overdue` | info | running ≥ 21 days without a recommended action |
+| `snapshot_unavailable` | info | `metrics_snapshot` not yet computed by the previewer |
+
+`recommendation` is the first fired `warning` rule, falling back to the first fired `success`, else `null`. When no human/AI analyst is available, `heuristic_output` is the deterministic baseline analysis.
+
+`analysis_confidence.level` is computed from five boolean factors (`sample_size_reached`, `hypothesis_present`, `primary_metric_present`, `guardrails_present`, `no_blocking_alerts`): `low` when any blocking alert is active OR ≥2 factors are missing, `medium` when exactly one factor is missing, `high` when all five hold.
+
 #### Custom field options
 
 After running `abs experiments refresh-fields`, custom fields from your ABsmartly instance appear as native CLI options:

package/dist/commands/experiments/analyze.d.ts

@@ -0,0 +1,3 @@
+import { Command } from 'commander';
+export declare const analyzeCommand: Command;
+//# sourceMappingURL=analyze.d.ts.map

package/dist/commands/experiments/analyze.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../../src/commands/experiments/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoBpC,eAAO,MAAM,cAAc,SAmBxB,CAAC"}

package/dist/commands/experiments/analyze.js

@@ -0,0 +1,33 @@
+import { Command } from 'commander';
+import { getAPIClientFromOptions, getGlobalOptions, printFormatted, withErrorHandling, } from '../../lib/utils/api-helper.js';
+import { parseExperimentIdOrName } from './resolve-id.js';
+import { analyzeExperiment } from '../../core/experiments/analyze/index.js';
+function isOutputExplicit(cmd) {
+    let current = cmd;
+    while (current) {
+        const src = current.getOptionValueSource('output');
+        if (src === 'cli' || src === 'env')
+            return true;
+        current = current.parent ?? undefined;
+    }
+    return false;
+}
+export const analyzeCommand = new Command('analyze')
+    .description('Analyze experiment design, signals, alerts, and related-experiment benchmarks')
+    .argument('<id>', 'experiment ID or name', parseExperimentIdOrName)
+    .action(withErrorHandling(async (nameOrId) => {
+    const globalOptions = getGlobalOptions(analyzeCommand);
+    if (!isOutputExplicit(analyzeCommand))
+        globalOptions.output = 'json';
+    const client = await getAPIClientFromOptions(globalOptions);
+    const id = await client.resolveExperimentId(nameOrId);
+    const result = await analyzeExperiment(client, { experimentId: id });
+    const useFull = globalOptions.output === 'json' || globalOptions.output === 'yaml';
+    if (useFull) {
+        printFormatted(result.data, globalOptions);
+    }
+    else {
+        printFormatted(result.detail, globalOptions);
+    }
+}));
+//# sourceMappingURL=analyze.js.map

package/dist/commands/experiments/analyze.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../../src/commands/experiments/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EACL,uBAAuB,EACvB,gBAAgB,EAChB,cAAc,EACd,iBAAiB,GAClB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,yCAAyC,CAAC;AAE5E,SAAS,gBAAgB,CAAC,GAAY;IACpC,IAAI,OAAO,GAAwB,GAAG,CAAC;IACvC,OAAO,OAAO,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,KAAK;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC,SAAS,CAAC;KACjD,WAAW,CAAC,+EAA+E,CAAC;KAC5F,QAAQ,CAAC,MAAM,EAAE,uBAAuB,EAAE,uBAAuB,CAAC;KAClE,MAAM,CACL,iBAAiB,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;IAC3C,MAAM,aAAa,GAAG,gBAAgB,CAAC,cAAc,CAAC,CAAC;IACvD,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC;QAAE,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC;IACrE,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,aAAa,CAAC,CAAC;IAC5D,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAEtD,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;IAErE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,KAAK,MAAM,IAAI,aAAa,CAAC,MAAM,KAAK,MAAM,CAAC;IACnF,IAAI,OAAO,EAAE,CAAC;QACZ,cAAc,CAAC,MAAM,CAAC,IAA0C,EAAE,aAAa,CAAC,CAAC;IACnF,CAAC;SAAM,CAAC;QACN,cAAc,CAAC,MAAM,CAAC,MAAO,EAAE,aAAa,CAAC,CAAC;IAChD,CAAC;AACH,CAAC,CAAC,CACH,CAAC"}

package/dist/commands/experiments/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/experiments/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkEpC,eAAO,MAAM,kBAAkB,SAKuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/experiments/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoEpC,eAAO,MAAM,kBAAkB,SAKuB,CAAC"}

package/dist/commands/experiments/index.js

@@ -1,6 +1,7 @@
 import { Command } from 'commander';
 import { listCommand } from './list.js';
 import { getCommand } from './get.js';
+import { analyzeCommand } from './analyze.js';
 import { searchCommand } from './search.js';
 import { startCommand } from './start.js';
 import { stopCommand } from './stop.js';
@@ -32,6 +33,7 @@ import { estimateParticipantsCommand } from './estimate-participants.js';
 const subcommands = [
     listCommand,
     getCommand,
+    analyzeCommand,
     searchCommand,
     createCommand,
     cloneCommand,

package/dist/commands/experiments/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/commands/experiments/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AAEzE,MAAM,WAAW,GAAG;IAClB,WAAW;IACX,UAAU;IACV,aAAa;IACb,aAAa;IACb,YAAY;IACZ,aAAa;IACb,YAAY;IACZ,WAAW;IACX,cAAc;IACd,eAAe;IACf,uBAAuB;IACvB,kBAAkB;IAClB,cAAc;IACd,aAAa;IACb,eAAe;IACf,cAAc;IACd,aAAa;IACb,aAAa;IACb,aAAa;IACb,eAAe;IACf,kBAAkB;IAClB,aAAa;IACb,sBAAsB;IACtB,aAAa;IACb,oBAAoB;IACpB,oBAAoB;IACpB,WAAW;IACX,YAAY;IACZ,WAAW;IACX,mBAAmB;IACnB,2BAA2B;CAC5B,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,OAAO,CAAC,aAAa,CAAC;KACzD,KAAK,CAAC,KAAK,CAAC;KACZ,KAAK,CAAC,YAAY,CAAC;KACnB,KAAK,CAAC,UAAU,CAAC;KACjB,KAAK,CAAC,SAAS,CAAC;KAChB,WAAW,CAAC,sCAAsC,CAAC,CAAC;AAEvD,KAAK,MAAM,GAAG,IAAI,WAAW;IAAE,kBAAkB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/commands/experiments/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AAEzE,MAAM,WAAW,GAAG;IAClB,WAAW;IACX,UAAU;IACV,cAAc;IACd,aAAa;IACb,aAAa;IACb,YAAY;IACZ,aAAa;IACb,YAAY;IACZ,WAAW;IACX,cAAc;IACd,eAAe;IACf,uBAAuB;IACvB,kBAAkB;IAClB,cAAc;IACd,aAAa;IACb,eAAe;IACf,cAAc;IACd,aAAa;IACb,aAAa;IACb,aAAa;IACb,eAAe;IACf,kBAAkB;IAClB,aAAa;IACb,sBAAsB;IACtB,aAAa;IACb,oBAAoB;IACpB,oBAAoB;IACpB,WAAW;IACX,YAAY;IACZ,WAAW;IACX,mBAAmB;IACnB,2BAA2B;CAC5B,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,OAAO,CAAC,aAAa,CAAC;KACzD,KAAK,CAAC,KAAK,CAAC;KACZ,KAAK,CAAC,YAAY,CAAC;KACnB,KAAK,CAAC,UAAU,CAAC;KACjB,KAAK,CAAC,SAAS,CAAC;KAChB,WAAW,CAAC,sCAAsC,CAAC,CAAC;AAEvD,KAAK,MAAM,GAAG,IAAI,WAAW;IAAE,kBAAkB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC"}

package/dist/core/experiments/analyze/extract-signals.d.ts

@@ -0,0 +1,42 @@
+import type { LeadingVariant, MetricSignal } from './types.js';
+interface Snapshot {
+    columnNames?: string[];
+    rows?: unknown[][];
+}
+interface VariantRef {
+    variant?: number;
+    name?: string;
+}
+interface MetricRef {
+    metric_id?: number;
+    type?: string;
+    metric?: {
+        id?: number;
+        name?: string;
+        lower_is_better?: boolean;
+    };
+}
+interface Exp {
+    primary_metric_id?: number;
+    primary_metric?: {
+        id?: number;
+        name?: string;
+        lower_is_better?: boolean;
+    };
+    secondary_metrics?: MetricRef[];
+    variants?: VariantRef[];
+    required_alpha?: string | number | null;
+    metrics_snapshot?: Snapshot;
+}
+interface ExtractResult {
+    metricSignals: MetricSignal[];
+    leadingVariant: LeadingVariant | null;
+    participantCount: number | null;
+}
+export declare function extractSignals(experiment: Exp): ExtractResult;
+export declare function leadingPrimaryImpactFromSnapshot(snapshot: {
+    columnNames?: string[];
+    rows?: unknown[][];
+} | null | undefined, primaryMetricId: number): number | null;
+export {};
+//# sourceMappingURL=extract-signals.d.ts.map

package/dist/core/experiments/analyze/extract-signals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-signals.d.ts","sourceRoot":"","sources":["../../../../src/core/experiments/analyze/extract-signals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAc,YAAY,EAAgB,MAAM,YAAY,CAAC;AAEzF,UAAU,QAAQ;IAChB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,UAAU,SAAS;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;CACpE;AAED,UAAU,GAAG;IACX,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IAC3E,iBAAiB,CAAC,EAAE,SAAS,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,UAAU,EAAE,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,gBAAgB,CAAC,EAAE,QAAQ,CAAC;CAC7B;AAED,UAAU,aAAa;IACrB,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;IACtC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AAKD,wBAAgB,cAAc,CAAC,UAAU,EAAE,GAAG,GAAG,aAAa,CAqH7D;AA6BD,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE;IAAE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,CAAA;CAAE,GAAG,IAAI,GAAG,SAAS,EAC3E,eAAe,EAAE,MAAM,GACtB,MAAM,GAAG,IAAI,CAgBf"}

package/dist/core/experiments/analyze/extract-signals.js

@@ -0,0 +1,154 @@
+const FLAT_PCT = 1; // |percent_change| under 1% counts as flat
+const FLAT_PVALUE = 0.5;
+export function extractSignals(experiment) {
+    const snap = experiment.metrics_snapshot;
+    if (!snap || !Array.isArray(snap.rows) || snap.rows.length === 0) {
+        return { metricSignals: [], leadingVariant: null, participantCount: null };
+    }
+    const cols = snap.columnNames ?? [];
+    const idx = (name) => cols.indexOf(name);
+    const cMetric = idx('metric_id');
+    const cVariant = idx('variant');
+    const cPct = idx('percent_change');
+    const cP = idx('p_value');
+    const cLow = idx('confidence_interval_low');
+    const cHigh = idx('confidence_interval_high');
+    const cUnits = idx('cum_unit_count');
+    const alpha = parseAlpha(experiment.required_alpha) ?? 0.1;
+    const variantNames = new Map();
+    for (const v of experiment.variants ?? []) {
+        if (typeof v.variant === 'number')
+            variantNames.set(v.variant, v.name ?? `Variant ${v.variant}`);
+    }
+    const primaryMetricId = experiment.primary_metric_id ?? experiment.primary_metric?.id ?? null;
+    const metricKindById = new Map();
+    const metricNameById = new Map();
+    const lowerIsBetter = new Map();
+    if (primaryMetricId !== null) {
+        metricKindById.set(primaryMetricId, 'primary');
+        metricNameById.set(primaryMetricId, experiment.primary_metric?.name ?? `metric_${primaryMetricId}`);
+        if (experiment.primary_metric?.lower_is_better)
+            lowerIsBetter.set(primaryMetricId, true);
+    }
+    for (const sm of experiment.secondary_metrics ?? []) {
+        const mid = sm.metric_id ?? sm.metric?.id;
+        if (typeof mid !== 'number')
+            continue;
+        const kind = sm.type === 'guardrail'
+            ? 'guardrail'
+            : sm.type === 'exploratory'
+                ? 'exploratory'
+                : 'secondary';
+        if (!metricKindById.has(mid))
+            metricKindById.set(mid, kind);
+        if (!metricNameById.has(mid))
+            metricNameById.set(mid, sm.metric?.name ?? `metric_${mid}`);
+        if (sm.metric?.lower_is_better)
+            lowerIsBetter.set(mid, true);
+    }
+    const signals = [];
+    let bestPrimary = null;
+    let participantCount = null;
+    for (const row of snap.rows) {
+        if (!Array.isArray(row))
+            continue;
+        const metricId = numAt(row, cMetric);
+        const variantId = numAt(row, cVariant);
+        if (metricId === null || variantId === null)
+            continue;
+        if (variantId === 0) {
+            // baseline / control row — skip signal emission, but still use for participant count
+            const u = numAt(row, cUnits);
+            if (primaryMetricId !== null && metricId === primaryMetricId && u !== null) {
+                participantCount = participantCount === null ? u : Math.max(participantCount, u);
+            }
+            continue;
+        }
+        const percent = numAt(row, cPct);
+        const pValue = numAt(row, cP);
+        const ciLow = numAt(row, cLow);
+        const ciHigh = numAt(row, cHigh);
+        const units = numAt(row, cUnits);
+        if (primaryMetricId !== null && metricId === primaryMetricId && units !== null) {
+            participantCount = participantCount === null ? units : Math.max(participantCount, units);
+        }
+        const kind = metricKindById.get(metricId) ?? 'secondary';
+        const status = classifyStatus(percent, pValue, alpha, lowerIsBetter.get(metricId) ?? false);
+        signals.push({
+            metric_id: metricId,
+            metric_name: metricNameById.get(metricId) ?? `metric_${metricId}`,
+            metric_type: kind,
+            variant_id: variantId,
+            variant_name: variantNames.get(variantId) ?? `Variant ${variantId}`,
+            percent_change: percent,
+            p_value: pValue,
+            ci_low: ciLow,
+            ci_high: ciHigh,
+            status,
+        });
+        if (primaryMetricId !== null &&
+            metricId === primaryMetricId &&
+            typeof percent === 'number' &&
+            Number.isFinite(percent)) {
+            if (bestPrimary === null || percent > bestPrimary.impact) {
+                bestPrimary = { variantId, impact: percent, pValue };
+            }
+        }
+    }
+    const leadingVariant = bestPrimary
+        ? {
+            variant_id: bestPrimary.variantId,
+            variant_name: variantNames.get(bestPrimary.variantId) ?? `Variant ${bestPrimary.variantId}`,
+            impact_percent: bestPrimary.impact,
+            p_value: bestPrimary.pValue,
+        }
+        : null;
+    return { metricSignals: signals, leadingVariant, participantCount };
+}
+function numAt(row, i) {
+    if (i < 0 || i >= row.length)
+        return null;
+    const v = row[i];
+    return typeof v === 'number' && Number.isFinite(v) ? v : null;
+}
+function parseAlpha(value) {
+    if (value === null || value === undefined)
+        return null;
+    const n = typeof value === 'string' ? parseFloat(value) : value;
+    return Number.isFinite(n) ? n : null;
+}
+function classifyStatus(percent, pValue, alpha, lowerIsBetter) {
+    if (percent === null || pValue === null)
+        return 'inconclusive';
+    const significant = pValue < alpha;
+    const directionGood = lowerIsBetter ? percent < 0 : percent > 0;
+    if (significant && directionGood)
+        return 'improves';
+    if (significant && !directionGood)
+        return 'contradicts';
+    if (Math.abs(percent) < FLAT_PCT && pValue > FLAT_PVALUE)
+        return 'flat';
+    return 'inconclusive';
+}
+export function leadingPrimaryImpactFromSnapshot(snapshot, primaryMetricId) {
+    if (!snapshot || !Array.isArray(snapshot.rows))
+        return null;
+    const cols = snapshot.columnNames ?? [];
+    const cMetric = cols.indexOf('metric_id');
+    const cVariant = cols.indexOf('variant');
+    const cPct = cols.indexOf('percent_change');
+    let best = null;
+    for (const row of snapshot.rows) {
+        if (!Array.isArray(row))
+            continue;
+        const m = numAt(row, cMetric);
+        const v = numAt(row, cVariant);
+        const p = numAt(row, cPct);
+        if (m !== primaryMetricId || v === null || v === 0 || p === null)
+            continue;
+        if (best === null || p > best)
+            best = p;
+    }
+    return best;
+}
+//# sourceMappingURL=extract-signals.js.map

package/dist/core/experiments/analyze/extract-signals.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-signals.js","sourceRoot":"","sources":["../../../../src/core/experiments/analyze/extract-signals.ts"],"names":[],"mappings":"AAiCA,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,2CAA2C;AAC/D,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,UAAU,cAAc,CAAC,UAAe;IAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,gBAAgB,CAAC;IACzC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjE,OAAO,EAAE,aAAa,EAAE,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;IAC7E,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjD,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACnC,MAAM,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;IAC1B,MAAM,IAAI,GAAG,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAErC,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,GAAG,CAAC;IAE3D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ;YAC/B,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,IAAI,WAAW,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,eAAe,GAAG,UAAU,CAAC,iBAAiB,IAAI,UAAU,CAAC,cAAc,EAAE,EAAE,IAAI,IAAI,CAAC;IAC9F,MAAM,cAAc,GAAG,IAAI,GAAG,EAAsB,CAAC;IACrD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEjD,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;QAC7B,cAAc,CAAC,GAAG,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QAC/C,cAAc,CAAC,GAAG,CAChB,eAAe,EACf,UAAU,CAAC,cAAc,EAAE,IAAI,IAAI,UAAU,eAAe,EAAE,CAC/D,CAAC;QACF,IAAI,UAAU,CAAC,cAAc,EAAE,eAAe;YAAE,aAAa,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAC3F,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,iBAAiB,IAAI,EAAE,EAAE,CAAC;QACpD,MAAM,GAAG,GAAG,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,SAAS;QACtC,MAAM,IAAI,GACR,EAAE,CAAC,IAAI,KAAK,WAAW;YACrB,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,EAAE,CAAC,IAAI,KAAK,aAAa;gBACzB,CAAC,CAAC,aAAa;gBACf,CAAC,CAAC,WAAW,CAAC;QACpB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,EAAE,IAAI,IAAI,UAAU,GAAG,EAAE,CAAC,CAAC;QAC1F,IAAI,EAAE,CAAC,MAAM,EAAE,eAAe;YAAE,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,IAAI,WAAW,GAAwE,IAAI,CAAC;IAC5F,IAAI,gBAAgB,GAAkB,IAAI,CAAC;IAE3C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;YAAE,SAAS;QAClC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACvC,IAAI,QAAQ,KAAK,IAAI,IAAI,SAAS,KAAK,IAAI;YAAE,SAAS;QACtD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,qFAAqF;YACrF,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC7B,IAAI,eAAe,KAAK,IAAI,IAAI,QAAQ,KAAK,eAAe,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC3E,gBAAgB,GAAG,gBAAgB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;YACnF,CAAC;YACD,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAEjC,IAAI,eAAe,KAAK,IAAI,IAAI,QAAQ,KAAK,eAAe,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAC/E,gBAAgB,GAAG,gBAAgB,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;QAC3F,CAAC;QAED,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,WAAW,CAAC;QACzD,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;QAE5F,OAAO,CAAC,IAAI,CAAC;YACX,SAAS,EAAE,QAAQ;YACnB,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,UAAU,QAAQ,EAAE;YACjE,WAAW,EAAE,IAAI;YACjB,UAAU,EAAE,SAAS;YACrB,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,WAAW,SAAS,EAAE;YACnE,cAAc,EAAE,OAAO;YACvB,OAAO,EAAE,MAAM;YACf,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,MAAM;YACf,MAAM;SACP,CAAC,CAAC;QAEH,IACE,eAAe,KAAK,IAAI;YACxB,QAAQ,KAAK,eAAe;YAC5B,OAAO,OAAO,KAAK,QAAQ;YAC3B,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EACxB,CAAC;YACD,IAAI,WAAW,KAAK,IAAI,IAAI,OAAO,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;gBACzD,WAAW,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;YACvD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAA0B,WAAW;QACvD,CAAC,CAAC;YACE,UAAU,EAAE,WAAW,CAAC,SAAS;YACjC,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,WAAW,WAAW,CAAC,SAAS,EAAE;YAC3F,cAAc,EAAE,WAAW,CAAC,MAAM;YAClC,OAAO,EAAE,WAAW,CAAC,MAAM;SAC5B;QACH,CAAC,CAAC,IAAI,CAAC;IAET,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC;AACtE,CAAC;AAED,SAAS,KAAK,CAAC,GAAc,EAAE,CAAS;IACtC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC1C,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACjB,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAChE,CAAC;AAED,SAAS,UAAU,CAAC,KAAyC;IAC3D,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACvD,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAChE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,cAAc,CACrB,OAAsB,EACtB,MAAqB,EACrB,KAAa,EACb,aAAsB;IAEtB,IAAI,OAAO,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,cAAc,CAAC;IAC/D,MAAM,WAAW,GAAG,MAAM,GAAG,KAAK,CAAC;IACnC,MAAM,aAAa,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;IAChE,IAAI,WAAW,IAAI,aAAa;QAAE,OAAO,UAAU,CAAC;IACpD,IAAI,WAAW,IAAI,CAAC,aAAa;QAAE,OAAO,aAAa,CAAC;IACxD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,QAAQ,IAAI,MAAM,GAAG,WAAW;QAAE,OAAO,MAAM,CAAC;IACxE,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,gCAAgC,CAC9C,QAA2E,EAC3E,eAAuB;IAEvB,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5D,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC5C,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;YAAE,SAAS;QAClC,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC9B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC/B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,KAAK,eAAe,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI;YAAE,SAAS;QAC3E,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI;YAAE,IAAI,GAAG,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/core/experiments/analyze/heuristics.d.ts

@@ -0,0 +1,43 @@
+import type { AnalysisConfidence, DesignReadout, HeuristicEntry, LeadingVariant, MetricSignal, Recommendation } from './types.js';
+interface AlertLite {
+    id: number;
+    type: string;
+    dismissed: boolean;
+}
+interface ExpLite {
+    state?: string;
+    hypothesis?: string | null;
+    primary_metric_id?: number | null;
+    required_alpha?: string | number | null;
+    required_power?: string | number | null;
+    analysis_type?: string | null;
+    minimum_detectable_effect?: number | null;
+    baseline_primary_metric_mean?: number | null;
+    baseline_participants_per_day?: number | null;
+    percentage_of_traffic?: number | null;
+    secondary_metrics?: Array<{
+        type?: string;
+    }>;
+    recommended_action?: unknown;
+    started_at?: string | null;
+    stopped_at?: string | null;
+}
+export interface HeuristicsInput {
+    experiment: ExpLite;
+    alerts: AlertLite[];
+    metricSignals: MetricSignal[];
+    leadingVariant: LeadingVariant | null;
+    participantCount: number | null;
+    benchmark: {
+        observed_impacts: number[];
+        median_abs_impact: number | null;
+    } | null;
+}
+export declare function runHeuristics(input: HeuristicsInput): {
+    heuristicOutput: HeuristicEntry[];
+    recommendation: Recommendation | null;
+};
+export declare function computeAnalysisConfidence(input: HeuristicsInput): AnalysisConfidence;
+export declare function computeDesignReadout(input: HeuristicsInput): DesignReadout;
+export {};
+//# sourceMappingURL=heuristics.d.ts.map

package/dist/core/experiments/analyze/heuristics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"heuristics.d.ts","sourceRoot":"","sources":["../../../../src/core/experiments/analyze/heuristics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAClB,aAAa,EACb,cAAc,EACd,cAAc,EACd,YAAY,EACZ,cAAc,EAEf,MAAM,YAAY,CAAC;AAYpB,UAAU,SAAS;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,UAAU,OAAO;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,cAAc,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,cAAc,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,yBAAyB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,4BAA4B,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,6BAA6B,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9C,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,iBAAiB,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC7C,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;IACtC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE;QAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;CACpF;AAsID,wBAAgB,aAAa,CAAC,KAAK,EAAE,eAAe,GAAG;IACrD,eAAe,EAAE,cAAc,EAAE,CAAC;IAClC,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;CACvC,CAOA;AAUD,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,eAAe,GAAG,kBAAkB,CAoCpF;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,eAAe,GAAG,aAAa,CA+C1E"}