@apex-inc/mcp-server 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/resources.d.ts +13 -0
- package/dist/resources.d.ts.map +1 -1
- package/dist/resources.js +21 -0
- package/dist/resources.js.map +1 -1
- package/dist/tools.d.ts +203 -4
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +364 -38
- package/dist/tools.js.map +1 -1
- package/package.json +1 -1
- package/skills/apex-adoption/SKILL.md +47 -0
- package/skills/apex-experimentation/SKILL.md +1 -1
package/dist/tools.js
CHANGED
|
@@ -455,15 +455,23 @@ export const toolDefinitions = {
|
|
|
455
455
|
guardrailType: z.enum(["rate", "revenue", "duration"]).optional().describe("How the guardrail is measured. 'rate' (default) = how often the harm event fires. 'revenue' = sum a value (e.g. refund $)."),
|
|
456
456
|
guardrailThreshold: z.number().optional().describe("Harm margin for the guardrail, RELATIVE (0.25 = flag a >25% rise). Defaults to 0.25."),
|
|
457
457
|
predictionMagnitude: z.string().optional().describe("Your committed prediction of the effect, e.g. '10% lift'. Recorded with 'agent' provenance and used for calibration. Required before the experiment can be ACTIVATED."),
|
|
458
|
-
mode: z.enum(["sdk", "snippet"]).optional().describe("Experiment mode: 'sdk' for code-level (default via MCP), 'snippet' for runtime DOM"),
|
|
459
458
|
surface: z.enum(["web", "mobile"]).optional().describe("The PROPERTY the experiment runs in (not the SDK used): 'web' for a website, 'mobile' for a native / Capacitor / React-Native app. OPTIONAL — when omitted it is auto-detected from repo signals (capacitor.config.* / @apex-inc/capacitor-plugin → mobile) and the workspace's registered data sources; pass it explicitly only to override or when both web and mobile sources exist. Sets the dashboard data-source label (Website vs Mobile app)."),
|
|
460
459
|
dataSourceId: z.string().optional().describe("Optional explicit Data Source (property) id this experiment runs in, e.g. 'ds_website' or 'ds_ios'. Usually inferred from surface; pass it to bind to a specific registered source. Tenant-validated server-side."),
|
|
461
460
|
randomizationUnit: z.enum(["device", "visitor", "person", "account", "org"]).optional().describe("What to bucket on. Default 'visitor' (per-device). Use 'person' to give one stitched user the same variant across their devices, or 'account'/'org' for B2B account-level tests where everyone in an account should see the same variant."),
|
|
461
|
+
minSamplePerArm: z.number().int().min(30).optional().describe("Minimum exposed subjects PER ARM before a winner can be claimed (the sample gate). Defaults to 50. Raise it to demand more evidence (detect a smaller lift); lower it (floor 30) for low-traffic properties that would otherwise stall. The real winner-gate is still the 95% probability-to-be-best — this only sets the traffic floor. Pre-registration: frozen once the experiment is running."),
|
|
462
|
+
runtimeDays: z.number().int().min(1).max(365).optional().describe("How long to run the experiment, in days (the decision window). Apex force-concludes at this point — decisive winner if reached, else inconclusive. Defaults to 14. Set e.g. 28 to run for four weeks. This is the max runtime, not a minimum: a decisive winner can conclude sooner. Pre-registration: frozen once the experiment is running (fork to change it)."),
|
|
462
463
|
preview: z.boolean().optional().describe("If true, returns a preview without creating. Default: true."),
|
|
463
464
|
}),
|
|
464
465
|
handler: async (args) => {
|
|
465
466
|
const split = args.trafficSplit ?? 50;
|
|
466
|
-
|
|
467
|
+
// MCP experiments are ALWAYS code-native (SDK). Client-side snippet /
|
|
468
|
+
// runtime-DOM experiments are intentionally NOT offered here (founder
|
|
469
|
+
// decision 2026-07-05): the MCP audience is developers in an IDE with the
|
|
470
|
+
// code open, where `useApexVariant` is more reliable (no flicker, no
|
|
471
|
+
// fragile selectors, deterministic exposure) and plugs into the
|
|
472
|
+
// graduation pipeline. Snippet experiments live in the Apex web app's
|
|
473
|
+
// visual editor for the no-code / non-repo case.
|
|
474
|
+
const mode = "sdk";
|
|
467
475
|
// Surface auto-detect (QA 2026-06-18): when the agent doesn't pass a
|
|
468
476
|
// surface, infer it from repo signals + registered data sources instead of
|
|
469
477
|
// silently defaulting to web (which mislabels mobile apps). Ambiguous →
|
|
@@ -482,16 +490,23 @@ export const toolDefinitions = {
|
|
|
482
490
|
}
|
|
483
491
|
const experimentSurface = surfaceResolution.surface;
|
|
484
492
|
const isPreview = args.preview !== false;
|
|
485
|
-
// p1-metric — resolve the primary metric
|
|
486
|
-
//
|
|
487
|
-
//
|
|
493
|
+
// p1-metric — resolve the primary metric (the conversion goal).
|
|
494
|
+
//
|
|
495
|
+
// Founder decision 2026-07-05: DO NOT silently default the conversion
|
|
496
|
+
// event to `form_submit`. A guessed goal ("Signups" that secretly counts
|
|
497
|
+
// form submits) is worse than none. When the caller doesn't name a
|
|
498
|
+
// conversion event, the draft is created UNWIRED (no `source`): it's a
|
|
499
|
+
// valid draft, but the activation gate blocks starting it until a real
|
|
500
|
+
// outcome event is set — and we nudge loudly to set one. An explicit
|
|
501
|
+
// primaryMetricEvent is validated against the workspace event spec.
|
|
502
|
+
const conversionEventSet = Boolean(args.primaryMetricEvent);
|
|
488
503
|
let primaryMetric = {
|
|
489
|
-
key: "
|
|
490
|
-
label: "Conversion
|
|
504
|
+
key: "conversion_goal",
|
|
505
|
+
label: "Conversion goal (not set)",
|
|
491
506
|
type: "rate",
|
|
492
507
|
unit: "%",
|
|
493
508
|
direction: "increase",
|
|
494
|
-
source
|
|
509
|
+
// No `source` — UNWIRED on purpose. Can't start until set.
|
|
495
510
|
};
|
|
496
511
|
if (args.primaryMetricEvent) {
|
|
497
512
|
const event = args.primaryMetricEvent.trim();
|
|
@@ -505,7 +520,7 @@ export const toolDefinitions = {
|
|
|
505
520
|
return {
|
|
506
521
|
content: [{
|
|
507
522
|
type: "text",
|
|
508
|
-
text: `Could not load the event spec to validate primaryMetricEvent "${event}" (${message}). Fix the connection or omit primaryMetricEvent to
|
|
523
|
+
text: `Could not load the event spec to validate primaryMetricEvent "${event}" (${message}). Fix the connection, or omit primaryMetricEvent to create the draft without a conversion goal (you'll set it before starting).`,
|
|
509
524
|
}],
|
|
510
525
|
isError: true,
|
|
511
526
|
};
|
|
@@ -545,7 +560,7 @@ export const toolDefinitions = {
|
|
|
545
560
|
target_url: args.targetUrl,
|
|
546
561
|
target_anchor: args.targetAnchor,
|
|
547
562
|
data_source_id: args.dataSourceId,
|
|
548
|
-
primary_metric_event: primaryMetric.source
|
|
563
|
+
primary_metric_event: primaryMetric.source?.eventType,
|
|
549
564
|
});
|
|
550
565
|
if (cc.data.sameMetric.length || cc.data.differentMetric.length) {
|
|
551
566
|
conflictAdvisory = renderConflictAdvisory(cc.data);
|
|
@@ -564,14 +579,23 @@ export const toolDefinitions = {
|
|
|
564
579
|
targetComponent: args.targetComponent || null,
|
|
565
580
|
mode,
|
|
566
581
|
trafficSplit: { control: split, variant: 100 - split },
|
|
582
|
+
decisionWindowDays: args.runtimeDays ?? 14,
|
|
583
|
+
minSamplePerArm: args.minSamplePerArm ?? 50,
|
|
567
584
|
control: args.controlContent,
|
|
568
585
|
variant: args.variantContent,
|
|
569
|
-
|
|
586
|
+
conversionGoal: conversionEventSet
|
|
587
|
+
? { event: primaryMetric.source?.eventType, type: primaryMetric.type }
|
|
588
|
+
: "NOT SET — required before the experiment can start",
|
|
570
589
|
beliefId: args.beliefId || null,
|
|
571
590
|
beliefStatement: args.beliefStatement || null,
|
|
572
591
|
predictionId: args.predictionId || null,
|
|
573
592
|
...(conflictAdvisory ? { _conflict_advisory: conflictAdvisory } : {}),
|
|
574
|
-
|
|
593
|
+
...(conversionEventSet
|
|
594
|
+
? {}
|
|
595
|
+
: {
|
|
596
|
+
_conversion_goal_warning: "No conversion event set. The draft will be created, but it CANNOT be started until you set the outcome you're optimizing. Call list_events to see what's firing (prefer a real outcome like signup/purchase over form_submit), then pass primaryMetricEvent. Confirm the goal with the user before creating.",
|
|
597
|
+
}),
|
|
598
|
+
_instructions: `Before proposing variant copy, ground every factual claim in .apex/brand-truth.md (read it; cite the line each claim traces to; if a claim isn't backed there, ask the user instead of inventing it).${conflictAdvisory ? " IMPORTANT: _conflict_advisory is set — show it to the user and resolve the overlap (mutex / variant / sequence) before creating." : ""}${conversionEventSet ? "" : " IMPORTANT: _conversion_goal_warning is set — surface it and set a conversion event (list_events → primaryMetricEvent) before creating, unless the user explicitly wants a draft to fill in later."} Then present this as a summary and ask the user to confirm (1), adjust (2), or cancel (3). Do NOT create the experiment until confirmed.`,
|
|
575
599
|
};
|
|
576
600
|
return { content: [{ type: "text", text: JSON.stringify(recipe, null, 2) }] };
|
|
577
601
|
}
|
|
@@ -669,6 +693,16 @@ export const toolDefinitions = {
|
|
|
669
693
|
status: "draft",
|
|
670
694
|
...(args.dataSourceId ? { dataSourceId: args.dataSourceId } : {}),
|
|
671
695
|
...(args.randomizationUnit ? { randomizationUnit: args.randomizationUnit } : {}),
|
|
696
|
+
// Sample-size floor (minPerVariant) and/or run duration (durationDays).
|
|
697
|
+
// The API merges evaluationWindow, so sending one never clobbers the other.
|
|
698
|
+
...(() => {
|
|
699
|
+
const ew = {};
|
|
700
|
+
if (args.minSamplePerArm)
|
|
701
|
+
ew.minPerVariant = args.minSamplePerArm;
|
|
702
|
+
if (args.runtimeDays)
|
|
703
|
+
ew.durationDays = args.runtimeDays;
|
|
704
|
+
return Object.keys(ew).length > 0 ? { evaluationWindow: ew } : {};
|
|
705
|
+
})(),
|
|
672
706
|
hypothesis: resolvedHypothesis,
|
|
673
707
|
// Agent-authored: tagged "agent" provenance so calibration attributes it
|
|
674
708
|
// correctly. When the agent supplies predictionMagnitude it satisfies the
|
|
@@ -710,7 +744,8 @@ export const toolDefinitions = {
|
|
|
710
744
|
],
|
|
711
745
|
}
|
|
712
746
|
: {}),
|
|
713
|
-
|
|
747
|
+
// VJATTR D1: 7-day exposure→conversion attribution window (was 24h).
|
|
748
|
+
attributionWindow: { unit: "days", value: 7, startFrom: "first_interaction" },
|
|
714
749
|
variants,
|
|
715
750
|
});
|
|
716
751
|
let previewUrl = "";
|
|
@@ -760,7 +795,9 @@ export const toolDefinitions = {
|
|
|
760
795
|
`7. Show the user the diff and ask them to preview at: ${previewUrl}`,
|
|
761
796
|
`8. After preview approval, commit and push`,
|
|
762
797
|
`9. Call track_deployment with the experiment ID and commit SHA`,
|
|
763
|
-
`10. Call
|
|
798
|
+
`10. Call activate_experiment (with the user's go-ahead). If the code isn't live yet, the`,
|
|
799
|
+
` experiment holds in "Waiting on deploy" and starts automatically at the first exposure —`,
|
|
800
|
+
` verify with verify_experiment_wiring after the deploy lands.`,
|
|
764
801
|
];
|
|
765
802
|
const mobileSteps = [
|
|
766
803
|
"IMPLEMENT THE EXPERIMENT IN CODE (mobile / Capacitor):",
|
|
@@ -790,6 +827,18 @@ export const toolDefinitions = {
|
|
|
790
827
|
"- Before calling activate_experiment, tell the user the expected metered volume and get their explicit go-ahead.",
|
|
791
828
|
"- Activation also requires a measurable primary goal, a guardrail, and a committed prediction (the server enforces this). If you didn't pass predictionMagnitude, add one first.",
|
|
792
829
|
];
|
|
830
|
+
const conversionGoalNudge = conversionEventSet
|
|
831
|
+
? []
|
|
832
|
+
: [
|
|
833
|
+
"",
|
|
834
|
+
"⚠️ CONVERSION GOAL NOT SET — this draft cannot be started yet:",
|
|
835
|
+
"- No outcome event is wired, so there's nothing to measure a winner on.",
|
|
836
|
+
"- Pick the real outcome: call list_events to see what's firing in this",
|
|
837
|
+
" workspace, then update_experiment({ experiment_id, primary_metric_event }).",
|
|
838
|
+
"- Prefer a true outcome (signup / purchase / checkout_completed) over a",
|
|
839
|
+
" proxy like form_submit. Confirm the goal with the user.",
|
|
840
|
+
"- activate_experiment will refuse until a measurable goal is set.",
|
|
841
|
+
];
|
|
793
842
|
const recipe = {
|
|
794
843
|
_apex: true,
|
|
795
844
|
_type: "experiment_created",
|
|
@@ -798,7 +847,11 @@ export const toolDefinitions = {
|
|
|
798
847
|
surface: experimentSurface,
|
|
799
848
|
status: exp.status,
|
|
800
849
|
mode,
|
|
801
|
-
|
|
850
|
+
conversionGoal: conversionEventSet
|
|
851
|
+
? { event: primaryMetric.source?.eventType, type: primaryMetric.type }
|
|
852
|
+
: "NOT SET — required before start (see _instructions)",
|
|
853
|
+
decisionWindowDays: args.runtimeDays ?? 14,
|
|
854
|
+
minSamplePerArm: args.minSamplePerArm ?? 50,
|
|
802
855
|
beliefId: beliefId || null,
|
|
803
856
|
hypothesisId: hypothesisId || null,
|
|
804
857
|
predictionId: args.predictionId || null,
|
|
@@ -822,20 +875,13 @@ export const toolDefinitions = {
|
|
|
822
875
|
} : null,
|
|
823
876
|
previewUrl,
|
|
824
877
|
trafficSplit: { control: split, variant: 100 - split },
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
.concat([
|
|
833
|
-
"SNIPPET MODE — no code changes needed.",
|
|
834
|
-
"The experiment will be applied via the Apex snippet at runtime.",
|
|
835
|
-
`Preview: ${previewUrl}`,
|
|
836
|
-
])
|
|
837
|
-
.concat(activationPolicy)
|
|
838
|
-
.join("\n"),
|
|
878
|
+
// Always code-native (SDK) — see the `mode` note above.
|
|
879
|
+
_instructions: groundingGuidance
|
|
880
|
+
.concat(isMobile ? mobileSteps : webSteps)
|
|
881
|
+
.concat(styleGuidance)
|
|
882
|
+
.concat(activationPolicy)
|
|
883
|
+
.concat(conversionGoalNudge)
|
|
884
|
+
.join("\n"),
|
|
839
885
|
};
|
|
840
886
|
return { content: [{ type: "text", text: JSON.stringify(recipe, null, 2) }] };
|
|
841
887
|
},
|
|
@@ -889,6 +935,38 @@ export const toolDefinitions = {
|
|
|
889
935
|
};
|
|
890
936
|
},
|
|
891
937
|
},
|
|
938
|
+
recapture_experiment_screenshots: {
|
|
939
|
+
description: `${APEX} — Re-run the hosted screenshot capture for a WEB experiment's arms (renders the target URL for control + variant and stores the images shown on the dashboard). Use when screenshots are missing or stale — e.g. the page changed, or an earlier capture failed. Web experiments only (mobile arms capture on-device via the SDK).`,
|
|
940
|
+
schema: z.object({
|
|
941
|
+
experimentId: z.string().describe("The experiment ID to re-capture screenshots for."),
|
|
942
|
+
}),
|
|
943
|
+
handler: async ({ experimentId }) => {
|
|
944
|
+
try {
|
|
945
|
+
const res = await apiPost(`/api/experiments/${encodeURIComponent(experimentId)}/recapture`, {});
|
|
946
|
+
const { captured, errors } = res.data;
|
|
947
|
+
const lines = [
|
|
948
|
+
`${APEX} Re-captured ${captured} screenshot${captured === 1 ? "" : "s"} for ${experimentId}.`,
|
|
949
|
+
];
|
|
950
|
+
if (errors.length)
|
|
951
|
+
lines.push(` Arm(s) that failed: ${errors.join("; ")}`);
|
|
952
|
+
if (captured === 0) {
|
|
953
|
+
lines.push(` No images were produced — confirm the target URL is public and reachable.`);
|
|
954
|
+
}
|
|
955
|
+
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
956
|
+
}
|
|
957
|
+
catch (err) {
|
|
958
|
+
return {
|
|
959
|
+
content: [
|
|
960
|
+
{
|
|
961
|
+
type: "text",
|
|
962
|
+
text: `${APEX} ${err instanceof Error ? err.message : String(err)}`,
|
|
963
|
+
},
|
|
964
|
+
],
|
|
965
|
+
isError: true,
|
|
966
|
+
};
|
|
967
|
+
}
|
|
968
|
+
},
|
|
969
|
+
},
|
|
892
970
|
verify_experiment_wiring: {
|
|
893
971
|
description: `${APEX} — Verify an experiment is actually wired up before launch: check whether \`experiment_exposure\` events are arriving for BOTH arms. Use this before activate_experiment to avoid launching an experiment that silently collects no data. Returns a per-arm exposure breakdown and an actionable next step.`,
|
|
894
972
|
schema: z.object({
|
|
@@ -922,8 +1000,14 @@ export const toolDefinitions = {
|
|
|
922
1000
|
case "no_exposures":
|
|
923
1001
|
default:
|
|
924
1002
|
verdictLine = "NO EXPOSURES — no exposures recorded for any arm";
|
|
925
|
-
nextStep =
|
|
926
|
-
"No `experiment_exposure` events
|
|
1003
|
+
nextStep = wiring.sdkMode
|
|
1004
|
+
? "No `experiment_exposure` events have reached Apex yet. This experiment is " +
|
|
1005
|
+
"code-wired (SDK hook), so you can activate it NOW: activate_experiment will " +
|
|
1006
|
+
"hold it in `pending_deployment` (\"Waiting on deploy\") and it starts " +
|
|
1007
|
+
"automatically the moment the deployed `useApexVariant(\"" +
|
|
1008
|
+
wiring.experimentId +
|
|
1009
|
+
"\")` code records its first exposure. Ship the code, and the test starts itself."
|
|
1010
|
+
: "No `experiment_exposure` events are arriving. The variant-serving code " +
|
|
927
1011
|
"(useApexVariant / Apex.getVariant) probably isn't deployed or running yet, " +
|
|
928
1012
|
"so nothing is being assigned. Deploy the experiment code and confirm it runs " +
|
|
929
1013
|
"on the target surface, then re-run verify_experiment_wiring.";
|
|
@@ -958,7 +1042,7 @@ export const toolDefinitions = {
|
|
|
958
1042
|
},
|
|
959
1043
|
},
|
|
960
1044
|
activate_experiment: {
|
|
961
|
-
description: `${APEX} — Launch an experiment. Changes its status from draft to running. Traffic will be split immediately. Before flipping to running, this verifies BOTH arms are producing \`experiment_exposure\` events (wiring check) and
|
|
1045
|
+
description: `${APEX} — Launch an experiment. Changes its status from draft to running. Traffic will be split immediately. Before flipping to running, this verifies BOTH arms are producing \`experiment_exposure\` events (wiring check). Code-wired (SDK-hook) experiments may be activated before their code ships: the server holds them in \`pending_deployment\` ("Waiting on deploy") and starts on the first exposure. Snippet experiments are applied by the Apex snippet at runtime, so they also don't need pre-activation exposures — they launch and wiring is verified afterward. Only wired experiments that genuinely can't self-start are refused unless force is true. Ask the user to confirm before calling.`,
|
|
962
1046
|
schema: z.object({
|
|
963
1047
|
experimentId: z.string().describe("The experiment ID to activate"),
|
|
964
1048
|
force: z
|
|
@@ -977,7 +1061,17 @@ export const toolDefinitions = {
|
|
|
977
1061
|
catch (err) {
|
|
978
1062
|
wiringError = err instanceof Error ? err.message : String(err);
|
|
979
1063
|
}
|
|
980
|
-
|
|
1064
|
+
// Code-wired (SDK-hook) AND snippet-applied experiments skip the hard
|
|
1065
|
+
// refusal. SDK: the server holds them in `pending_deployment` and
|
|
1066
|
+
// auto-starts on first exposure. Snippet: the Apex snippet only applies
|
|
1067
|
+
// the experiment ONCE IT'S RUNNING, so it can't have pre-activation
|
|
1068
|
+
// exposures — blocking on them is a chicken-and-egg. Both are safe to
|
|
1069
|
+
// launch "early"; wiring is verified AFTER via verify_experiment_wiring.
|
|
1070
|
+
if (wiring &&
|
|
1071
|
+
!wiring.bothArmsLive &&
|
|
1072
|
+
force !== true &&
|
|
1073
|
+
!wiring.sdkMode &&
|
|
1074
|
+
!wiring.snippetMode) {
|
|
981
1075
|
const armLines = wiring.perArm.map(formatArmLine);
|
|
982
1076
|
const zeroArms = wiring.perArm
|
|
983
1077
|
.filter((a) => a.exposures === 0)
|
|
@@ -1023,7 +1117,56 @@ export const toolDefinitions = {
|
|
|
1023
1117
|
const exp = await apiPatch("/api/experiments", {
|
|
1024
1118
|
id: experimentId,
|
|
1025
1119
|
status: "running",
|
|
1120
|
+
// Deploy-wait override: the server holds an unwired code-wired
|
|
1121
|
+
// experiment in `pending_deployment` unless the caller forces.
|
|
1122
|
+
...(force === true ? { force: true } : {}),
|
|
1026
1123
|
});
|
|
1124
|
+
// Deploy-wait hold — the server accepted the start but is waiting for
|
|
1125
|
+
// the experiment code to go live. Report the held state precisely
|
|
1126
|
+
// (claiming "LIVE" here would repeat the 2026-07-04 confusion).
|
|
1127
|
+
if (exp.status === "pending_deployment") {
|
|
1128
|
+
const held = [
|
|
1129
|
+
`${APEX} Experiment Waiting on Deploy`,
|
|
1130
|
+
`${"═".repeat(40)}`,
|
|
1131
|
+
``,
|
|
1132
|
+
` "${exp.name}" is armed but NOT collecting yet.`,
|
|
1133
|
+
` ID: ${exp.id}`,
|
|
1134
|
+
` Status: pending_deployment ("Waiting on deploy")`,
|
|
1135
|
+
``,
|
|
1136
|
+
` Apex hasn't received any \`experiment_exposure\` events from this`,
|
|
1137
|
+
` experiment's code, so the start is on hold. Nothing is wrong —`,
|
|
1138
|
+
` the experiment starts AUTOMATICALLY the moment the deployed code`,
|
|
1139
|
+
` records its first exposure.`,
|
|
1140
|
+
``,
|
|
1141
|
+
` To go live:`,
|
|
1142
|
+
` 1. Render the variants through the SDK hook:`,
|
|
1143
|
+
` const variant = useApexVariant("${exp.id}");`,
|
|
1144
|
+
` 2. Deploy that code to the target page.`,
|
|
1145
|
+
` 3. Done — the first visitor flips it to running. Verify with`,
|
|
1146
|
+
` verify_experiment_wiring or the experiment page.`,
|
|
1147
|
+
``,
|
|
1148
|
+
` Certain the code is already live? Re-run activate_experiment`,
|
|
1149
|
+
` with force: true to start immediately.`,
|
|
1150
|
+
``,
|
|
1151
|
+
`${"═".repeat(40)}`,
|
|
1152
|
+
].join("\n");
|
|
1153
|
+
return {
|
|
1154
|
+
content: [
|
|
1155
|
+
{ type: "text", text: held },
|
|
1156
|
+
{
|
|
1157
|
+
type: "text",
|
|
1158
|
+
text: JSON.stringify({
|
|
1159
|
+
_apex: true,
|
|
1160
|
+
_type: "activation_held",
|
|
1161
|
+
reason: "code_not_live",
|
|
1162
|
+
experimentId: exp.id,
|
|
1163
|
+
status: exp.status,
|
|
1164
|
+
autoStartsOnFirstExposure: true,
|
|
1165
|
+
}, null, 2),
|
|
1166
|
+
},
|
|
1167
|
+
],
|
|
1168
|
+
};
|
|
1169
|
+
}
|
|
1027
1170
|
// PATCH returns a minimal/legacy body — re-read the unified record to
|
|
1028
1171
|
// derive the live traffic split from the allocation weights.
|
|
1029
1172
|
let controlPct = 50;
|
|
@@ -1041,6 +1184,9 @@ export const toolDefinitions = {
|
|
|
1041
1184
|
else if (wiring && !wiring.bothArmsLive && force === true) {
|
|
1042
1185
|
notes.push(` ⚠ Launched with force: true despite an incomplete wiring check`, ` (verdict: ${wiring.verdict}, ${wiring.armsLive}/${wiring.totalArms} arm(s) live).`, ` Some arm(s) may not be collecting data yet.`, ``);
|
|
1043
1186
|
}
|
|
1187
|
+
else if (wiring && !wiring.bothArmsLive && wiring.snippetMode) {
|
|
1188
|
+
notes.push(` ℹ Snippet experiment — the Apex snippet applies it to visitors now`, ` that it's running, so exposures start flowing on the next visits.`, ` Confirm both arms record exposures shortly with`, ` verify_experiment_wiring (a mismatched selector would silently`, ` collect nothing).`, ``);
|
|
1189
|
+
}
|
|
1044
1190
|
// Author-time conflict guard (advisory): if this now-running experiment
|
|
1045
1191
|
// overlaps another live one on the same surface, surface the fix.
|
|
1046
1192
|
try {
|
|
@@ -1175,7 +1321,7 @@ export const toolDefinitions = {
|
|
|
1175
1321
|
},
|
|
1176
1322
|
},
|
|
1177
1323
|
archive_experiment: {
|
|
1178
|
-
description: `${APEX} — Archive an experiment. Removes it from active views but preserves all data, learnings, and belief graph connections. Archived experiments still contribute to intelligence scoring and recommendations. Ask the user to confirm before calling.`,
|
|
1324
|
+
description: `${APEX} — Archive an experiment. Removes it from active views but preserves all data, learnings, and belief graph connections. Archived experiments still contribute to intelligence scoring and recommendations. A RUNNING experiment cannot be archived — end it first (conclude_experiment / promote_winner) so its result + belief are captured; the server returns 409 archive_requires_conclusion otherwise. Ask the user to confirm before calling.`,
|
|
1179
1325
|
schema: z.object({
|
|
1180
1326
|
experimentId: z.string().describe("The experiment ID to archive"),
|
|
1181
1327
|
}),
|
|
@@ -1359,6 +1505,64 @@ export const toolDefinitions = {
|
|
|
1359
1505
|
};
|
|
1360
1506
|
},
|
|
1361
1507
|
},
|
|
1508
|
+
list_events: {
|
|
1509
|
+
description: `${APEX} — List the workspace's ACTIVE events: the canonical event names actually firing, with total volume and the platforms they arrive on, from the ingest catalog. Use this to discover which events exist / are wired before choosing a primary metric or guardrail in create_experiment, or to confirm an event is live. Distinct from list_metrics (which lists typed rate/revenue/duration GOALS) — this is the raw observed-event catalog.`,
|
|
1510
|
+
schema: z.object({
|
|
1511
|
+
limit: z
|
|
1512
|
+
.number()
|
|
1513
|
+
.int()
|
|
1514
|
+
.min(1)
|
|
1515
|
+
.max(200)
|
|
1516
|
+
.optional()
|
|
1517
|
+
.describe("Max events to return, by volume descending (default 50)."),
|
|
1518
|
+
}),
|
|
1519
|
+
handler: async (args) => {
|
|
1520
|
+
const catalog = await apiGet("/api/events/catalog");
|
|
1521
|
+
const rows = catalog?.events ?? [];
|
|
1522
|
+
// Aggregate per event name across platforms (the catalog is keyed by
|
|
1523
|
+
// (eventName, platform); the agent wants one row per event).
|
|
1524
|
+
const byName = new Map();
|
|
1525
|
+
for (const r of rows) {
|
|
1526
|
+
const e = byName.get(r.eventName) ?? { count: 0, platforms: new Set() };
|
|
1527
|
+
e.count += r.count ?? 0;
|
|
1528
|
+
if (r.platform)
|
|
1529
|
+
e.platforms.add(r.platform);
|
|
1530
|
+
byName.set(r.eventName, e);
|
|
1531
|
+
}
|
|
1532
|
+
const events = [...byName.entries()]
|
|
1533
|
+
.map(([name, v]) => ({
|
|
1534
|
+
event: name,
|
|
1535
|
+
count: v.count,
|
|
1536
|
+
platforms: [...v.platforms].sort(),
|
|
1537
|
+
}))
|
|
1538
|
+
.sort((a, b) => b.count - a.count)
|
|
1539
|
+
.slice(0, args.limit ?? 50);
|
|
1540
|
+
if (events.length === 0) {
|
|
1541
|
+
return {
|
|
1542
|
+
content: [
|
|
1543
|
+
{
|
|
1544
|
+
type: "text",
|
|
1545
|
+
text: `${APEX} No active events yet — nothing has been tracked in this workspace. Install the snippet / SDK and fire events, then re-check.`,
|
|
1546
|
+
},
|
|
1547
|
+
],
|
|
1548
|
+
};
|
|
1549
|
+
}
|
|
1550
|
+
const lines = events.map((e) => ` • ${e.event} — ${e.count.toLocaleString()} events (${e.platforms.join(", ") || "web"})`);
|
|
1551
|
+
return {
|
|
1552
|
+
content: [
|
|
1553
|
+
{
|
|
1554
|
+
type: "text",
|
|
1555
|
+
text: [
|
|
1556
|
+
`${APEX} Active events (${events.length}, by volume):`,
|
|
1557
|
+
...lines,
|
|
1558
|
+
``,
|
|
1559
|
+
`Pick a measurable OUTCOME event (e.g. purchase/signup) as primaryMetricEvent in create_experiment. Use list_metrics to see which are typed goals.`,
|
|
1560
|
+
].join("\n"),
|
|
1561
|
+
},
|
|
1562
|
+
],
|
|
1563
|
+
};
|
|
1564
|
+
},
|
|
1565
|
+
},
|
|
1362
1566
|
list_metrics: {
|
|
1363
1567
|
description: `${APEX} — List the canonical metric/goal catalog for this workspace: typed metrics (rate/revenue/duration) with whether each is DETECTED (firing, with volume) and its outcome tier. Use this to pick a measurable OUTCOME goal (purchase/signup) over a proxy (email_open) before create_experiment, or role:'guardrail' to see protective harm signals.`,
|
|
1364
1568
|
schema: z.object({
|
|
@@ -1421,6 +1625,8 @@ export const toolDefinitions = {
|
|
|
1421
1625
|
primaryMetricEvent: z.string().optional().describe("Change the primary metric's canonical event — draft only."),
|
|
1422
1626
|
primaryMetricType: z.enum(["rate", "count", "revenue", "duration"]).optional(),
|
|
1423
1627
|
guardrailEvent: z.string().optional().describe("Replace the guardrail with this canonical protective event — draft only."),
|
|
1628
|
+
minSamplePerArm: z.number().int().min(30).optional().describe("Set the minimum exposed subjects PER ARM before a winner can be claimed (the sample gate; floor 30). Draft only — it's part of the pre-registration and freezes at launch (a running experiment returns experiment_locked; fork to change it)."),
|
|
1629
|
+
runtimeDays: z.number().int().min(1).max(365).optional().describe("Set how long to run the experiment, in days (the decision window Apex force-concludes at). Draft only — part of the pre-registration; freezes at launch (fork to change it)."),
|
|
1424
1630
|
}),
|
|
1425
1631
|
handler: async (args) => {
|
|
1426
1632
|
const body = { id: args.experimentId };
|
|
@@ -1430,6 +1636,16 @@ export const toolDefinitions = {
|
|
|
1430
1636
|
body.surface = args.surface;
|
|
1431
1637
|
if (args.hypothesis !== undefined)
|
|
1432
1638
|
body.hypothesis = args.hypothesis;
|
|
1639
|
+
// evaluationWindow patch — the API merges it, so setting one of
|
|
1640
|
+
// minPerVariant / durationDays never clobbers the other.
|
|
1641
|
+
if (args.minSamplePerArm !== undefined || args.runtimeDays !== undefined) {
|
|
1642
|
+
const ew = {};
|
|
1643
|
+
if (args.minSamplePerArm !== undefined)
|
|
1644
|
+
ew.minPerVariant = args.minSamplePerArm;
|
|
1645
|
+
if (args.runtimeDays !== undefined)
|
|
1646
|
+
ew.durationDays = args.runtimeDays;
|
|
1647
|
+
body.evaluationWindow = ew;
|
|
1648
|
+
}
|
|
1433
1649
|
if (args.primaryMetricEvent) {
|
|
1434
1650
|
const t = args.primaryMetricType ?? "rate";
|
|
1435
1651
|
const isValue = t === "revenue" || t === "duration";
|
|
@@ -3507,20 +3723,31 @@ Events fired through this tool carry a stable synthetic visitorId (mcp-agent-*,
|
|
|
3507
3723
|
}),
|
|
3508
3724
|
handler: async ({ journeyId, confirmLive }) => {
|
|
3509
3725
|
const path = `/api/journeys/${encodeURIComponent(journeyId)}/publish`;
|
|
3726
|
+
// Non-blocking coverage warnings ("we couldn't verify these until the
|
|
3727
|
+
// journey fires") — surfaced so an agent can preview before publishing.
|
|
3728
|
+
const warnText = (warnings) => {
|
|
3729
|
+
const list = Array.isArray(warnings) ? warnings.filter((w) => w?.reason) : [];
|
|
3730
|
+
if (list.length === 0)
|
|
3731
|
+
return "";
|
|
3732
|
+
return (`\n\n${list.length} thing${list.length === 1 ? "" : "s"} we couldn't verify until it fires (preview with example data to check):\n` +
|
|
3733
|
+
list.map((w) => ` • ${w.reason}`).join("\n"));
|
|
3734
|
+
};
|
|
3510
3735
|
if (!confirmLive) {
|
|
3511
3736
|
try {
|
|
3512
|
-
await apiPost(`${path}?dryRun=true`, {});
|
|
3513
|
-
return { content: [{ type: "text", text: `${APEX} Dry-run passed — journey ${journeyId} is valid and ready
|
|
3737
|
+
const dr = await apiPost(`${path}?dryRun=true`, {});
|
|
3738
|
+
return { content: [{ type: "text", text: `${APEX} Dry-run passed — journey ${journeyId} is valid and ready.${warnText(dr?.warnings)}\nRe-call with confirmLive:true to publish it to live customers.` }] };
|
|
3514
3739
|
}
|
|
3515
3740
|
catch (err) {
|
|
3516
|
-
|
|
3741
|
+
// A variable-coverage block arrives here with a plain message like
|
|
3742
|
+
// "This send uses {{first_name}}, but the trigger doesn't provide it."
|
|
3743
|
+
return { content: [{ type: "text", text: `${APEX} Dry-run found a problem before publishing: ${errMsg(err)}\nIf a variable won't resolve, map it to a field the trigger provides or remove it, then preview with example data.` }], isError: true };
|
|
3517
3744
|
}
|
|
3518
3745
|
}
|
|
3519
3746
|
try {
|
|
3520
3747
|
const res = await apiPost(path, {});
|
|
3521
3748
|
if (!tenantOk(res.workspaceKey))
|
|
3522
3749
|
return tenantMismatch();
|
|
3523
|
-
return { content: [{ type: "text", text: `${APEX} Published journey ${journeyId} — it is now live on trigger events
|
|
3750
|
+
return { content: [{ type: "text", text: `${APEX} Published journey ${journeyId} — it is now live on trigger events.${warnText(res?.warnings)} View it: ${journeyLink(journeyId)}` }] };
|
|
3524
3751
|
}
|
|
3525
3752
|
catch (err) {
|
|
3526
3753
|
return { content: [{ type: "text", text: `${APEX} ${errMsg(err)}` }], isError: true };
|
|
@@ -4406,5 +4633,104 @@ _Suggest the next step the user should tackle based on what's incomplete in the
|
|
|
4406
4633
|
};
|
|
4407
4634
|
},
|
|
4408
4635
|
},
|
|
4636
|
+
// ─── Adoption Engine ──────────────────────────────────────────────────────
|
|
4637
|
+
list_adoption_milestones: {
|
|
4638
|
+
description: `${APEX} — List the workspace's adoption milestones (per-feature "has this user adopted X?" definitions). A milestone is NOT a goal.`,
|
|
4639
|
+
schema: z.object({}),
|
|
4640
|
+
handler: async () => {
|
|
4641
|
+
const data = await apiGet("/api/adoption/milestones");
|
|
4642
|
+
return {
|
|
4643
|
+
content: [
|
|
4644
|
+
{ type: "text", text: JSON.stringify(data?.data?.milestones ?? [], null, 2) },
|
|
4645
|
+
],
|
|
4646
|
+
};
|
|
4647
|
+
},
|
|
4648
|
+
},
|
|
4649
|
+
get_adoption_milestone: {
|
|
4650
|
+
description: `${APEX} — Fetch one adoption milestone by id.`,
|
|
4651
|
+
schema: z.object({ milestoneId: z.string().describe("The milestone id") }),
|
|
4652
|
+
handler: async ({ milestoneId }) => {
|
|
4653
|
+
const data = await apiGet(`/api/adoption/milestones/${encodeURIComponent(milestoneId)}`);
|
|
4654
|
+
return {
|
|
4655
|
+
content: [{ type: "text", text: JSON.stringify(data?.data?.milestone ?? null, null, 2) }],
|
|
4656
|
+
};
|
|
4657
|
+
},
|
|
4658
|
+
},
|
|
4659
|
+
create_adoption_milestone: {
|
|
4660
|
+
description: `${APEX} — Create an adoption milestone. "Adopted" is defined by one of the workspace's own events (adoptedWhenKind="event", eventName) or a trait (adoptedWhenKind="trait", traitPath + traitValue). Starts OFF unless active=true.`,
|
|
4661
|
+
schema: z.object({
|
|
4662
|
+
name: z.string().describe("Human label, e.g. 'Ran first report'"),
|
|
4663
|
+
featureKey: z.string().describe("Stable feature key, e.g. 'reporting'"),
|
|
4664
|
+
adoptedWhenKind: z.enum(["event", "trait"]).describe("How adoption is detected"),
|
|
4665
|
+
eventName: z.string().optional().describe("event kind: the event whose firing means adopted"),
|
|
4666
|
+
traitPath: z.string().optional().describe("trait kind: dotted trait path, e.g. 'plan'"),
|
|
4667
|
+
traitValue: z.string().optional().describe("trait kind: value to match (equals)"),
|
|
4668
|
+
priority: z.number().optional().describe("Lower is nudged first. Default 100."),
|
|
4669
|
+
gapDays: z.number().optional().describe("Nudge this many days after signup if not adopted. Default 3."),
|
|
4670
|
+
active: z.boolean().optional().describe("Turn the milestone on. Default false."),
|
|
4671
|
+
}),
|
|
4672
|
+
handler: async (args) => {
|
|
4673
|
+
const body = {
|
|
4674
|
+
name: args.name,
|
|
4675
|
+
featureKey: args.featureKey,
|
|
4676
|
+
priority: args.priority ?? 100,
|
|
4677
|
+
active: args.active ?? false,
|
|
4678
|
+
gapPolicy: { kind: "days_since_signup", days: args.gapDays ?? 3 },
|
|
4679
|
+
adoptedWhen: args.adoptedWhenKind === "event"
|
|
4680
|
+
? { kind: "event", eventName: args.eventName }
|
|
4681
|
+
: { kind: "trait", traitPath: args.traitPath, traitOp: "equals", traitValue: args.traitValue },
|
|
4682
|
+
};
|
|
4683
|
+
const data = await apiPost("/api/adoption/milestones", body);
|
|
4684
|
+
return {
|
|
4685
|
+
content: [{ type: "text", text: JSON.stringify(data?.data?.milestone ?? data, null, 2) }],
|
|
4686
|
+
};
|
|
4687
|
+
},
|
|
4688
|
+
},
|
|
4689
|
+
update_adoption_milestone: {
|
|
4690
|
+
description: `${APEX} — Update an adoption milestone (name, priority, active, gapDays, nudgeJourneyId).`,
|
|
4691
|
+
schema: z.object({
|
|
4692
|
+
milestoneId: z.string(),
|
|
4693
|
+
name: z.string().optional(),
|
|
4694
|
+
priority: z.number().optional(),
|
|
4695
|
+
active: z.boolean().optional(),
|
|
4696
|
+
gapDays: z.number().optional(),
|
|
4697
|
+
nudgeJourneyId: z.string().optional(),
|
|
4698
|
+
}),
|
|
4699
|
+
handler: async (args) => {
|
|
4700
|
+
const updates = {};
|
|
4701
|
+
if (args.name !== undefined)
|
|
4702
|
+
updates.name = args.name;
|
|
4703
|
+
if (args.priority !== undefined)
|
|
4704
|
+
updates.priority = args.priority;
|
|
4705
|
+
if (args.active !== undefined)
|
|
4706
|
+
updates.active = args.active;
|
|
4707
|
+
if (args.nudgeJourneyId !== undefined)
|
|
4708
|
+
updates.nudgeJourneyId = args.nudgeJourneyId;
|
|
4709
|
+
if (args.gapDays !== undefined)
|
|
4710
|
+
updates.gapPolicy = { days: args.gapDays };
|
|
4711
|
+
const data = await apiPatch(`/api/adoption/milestones/${encodeURIComponent(args.milestoneId)}`, updates);
|
|
4712
|
+
return {
|
|
4713
|
+
content: [{ type: "text", text: JSON.stringify(data?.data?.milestone ?? data, null, 2) }],
|
|
4714
|
+
};
|
|
4715
|
+
},
|
|
4716
|
+
},
|
|
4717
|
+
delete_adoption_milestone: {
|
|
4718
|
+
description: `${APEX} — Delete an adoption milestone.`,
|
|
4719
|
+
schema: z.object({ milestoneId: z.string() }),
|
|
4720
|
+
handler: async ({ milestoneId }) => {
|
|
4721
|
+
await apiDelete(`/api/adoption/milestones/${encodeURIComponent(milestoneId)}`);
|
|
4722
|
+
return { content: [{ type: "text", text: `Deleted milestone ${milestoneId}.` }] };
|
|
4723
|
+
},
|
|
4724
|
+
},
|
|
4725
|
+
get_adoption_metrics: {
|
|
4726
|
+
description: `${APEX} — Aggregated adoption metrics: per-milestone funnel + lift-vs-holdout, plus workspace health. Counts only (no user PII).`,
|
|
4727
|
+
schema: z.object({}),
|
|
4728
|
+
handler: async () => {
|
|
4729
|
+
const data = await apiGet("/api/adoption/metrics");
|
|
4730
|
+
return {
|
|
4731
|
+
content: [{ type: "text", text: JSON.stringify(data?.data ?? {}, null, 2) }],
|
|
4732
|
+
};
|
|
4733
|
+
},
|
|
4734
|
+
},
|
|
4409
4735
|
};
|
|
4410
4736
|
//# sourceMappingURL=tools.js.map
|