@delmaredigital/payload-experiments 0.2.0-rc.5 → 0.2.0-rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin/copy/arms.d.ts +1 -7
- package/dist/admin/copy/arms.js +1 -7
- package/dist/admin/copy/testPage.d.ts +2 -0
- package/dist/admin/copy/testPage.js +2 -0
- package/dist/admin/labels.d.ts +2 -7
- package/dist/admin/labels.js +2 -7
- package/dist/admin/results/results.model.d.ts +1 -1
- package/dist/admin/results/results.model.js +2 -3
- package/dist/admin/surface/ResultsRoute.js +6 -2
- package/dist/admin/surface/contracts.d.ts +1 -0
- package/dist/admin/surface/readiness/Banner.js +7 -5
- package/dist/admin/surface/readiness/SaveStatus.js +1 -1
- package/dist/admin/surface/results/ResultsBody.d.ts +5 -2
- package/dist/admin/surface/results/ResultsBody.js +8 -5
- package/dist/admin/surface/test/TestHeader.d.ts +1 -1
- package/dist/admin/surface/test/TestHeader.js +3 -2
- package/dist/admin/surface/test/TestPage.js +11 -2
- package/dist/admin/surface/useTestDocument.d.ts +1 -0
- package/dist/admin/surface/useTestDocument.js +32 -7
- package/dist/collections/experiments/config.js +19 -1
- package/dist/collections/experiments/lockHook.js +13 -1
- package/dist/constants.d.ts +2 -0
- package/dist/constants.js +2 -0
- package/dist/db/staleEdit.d.ts +6 -0
- package/dist/db/staleEdit.js +10 -0
- package/dist/endpoints/events.js +3 -3
- package/dist/endpoints/recompute.js +7 -1
- package/dist/exports/adminClient.d.ts +1 -0
- package/dist/exports/adminClient.js +1 -0
- package/dist/hooks/hostHook.js +3 -2
- package/dist/labels/arms.d.ts +7 -0
- package/dist/labels/arms.js +7 -0
- package/dist/log.d.ts +7 -0
- package/dist/log.js +35 -0
- package/dist/puck/copy.d.ts +6 -0
- package/dist/puck/copy.js +7 -0
- package/dist/puck/editor/ExperimentSlotEditor.js +2 -0
- package/dist/puck/slotConfig.js +14 -9
- package/dist/results/analyse.d.ts +1 -1
- package/dist/results/analyse.js +7 -3
- package/dist/results/verdict.d.ts +2 -0
- package/dist/results/verdict.js +18 -1
- package/dist/server/linkIdentity.js +3 -3
- package/dist/server/recordContentChange.js +2 -1
- package/dist/server/recordExposures.js +3 -3
- package/dist/server/runResults.d.ts +1 -0
- package/dist/server/runResults.js +8 -3
- package/dist/server/trackConversion.js +5 -6
- package/dist/transitions/deleteExperiment.js +2 -1
- package/dist/types/endpoints.d.ts +1 -1
- package/package.json +1 -1
|
@@ -1,12 +1,6 @@
|
|
|
1
1
|
import type { ArmIssueCode } from '../../types/endpoints.js';
|
|
2
|
+
export { ARM_LABELS as GROUP_LABELS } from '../../labels/arms.js';
|
|
2
3
|
export declare const ISSUE_COPY: Record<ArmIssueCode, string>;
|
|
3
|
-
export declare const GROUP_LABELS: {
|
|
4
|
-
readonly control: "Original";
|
|
5
|
-
readonly variantA: "Variant A";
|
|
6
|
-
readonly variantB: "Variant B";
|
|
7
|
-
readonly variantC: "Variant C";
|
|
8
|
-
readonly holdout: "Holdout";
|
|
9
|
-
};
|
|
10
4
|
export declare const ARMS_COPY: {
|
|
11
5
|
readonly holdout: "The holdout sees the Original and is measured but not compared during the test. After you decide, it keeps seeing the Original so the winner can be compared with it.";
|
|
12
6
|
readonly pageRequired: "Enter a page path for this group.";
|
package/dist/admin/copy/arms.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { ARM_LABELS as GROUP_LABELS } from '../../labels/arms.js';
|
|
1
2
|
export const ISSUE_COPY = {
|
|
2
3
|
'weights-total': 'Weights must add up to 100%. They add up to {total}% now.',
|
|
3
4
|
'control-count': 'Include exactly one Original.',
|
|
@@ -10,13 +11,6 @@ export const ISSUE_COPY = {
|
|
|
10
11
|
'slot-required': 'Choose the matching content slot for this group.',
|
|
11
12
|
'invalid-key': 'Use the standard group names: Original, Variant A–C, Holdout.'
|
|
12
13
|
};
|
|
13
|
-
export const GROUP_LABELS = {
|
|
14
|
-
control: 'Original',
|
|
15
|
-
variantA: 'Variant A',
|
|
16
|
-
variantB: 'Variant B',
|
|
17
|
-
variantC: 'Variant C',
|
|
18
|
-
holdout: 'Holdout'
|
|
19
|
-
};
|
|
20
14
|
export const ARMS_COPY = {
|
|
21
15
|
holdout: 'The holdout sees the Original and is measured but not compared during the test. After you decide, it keeps seeing the Original so the winner can be compared with it.',
|
|
22
16
|
pageRequired: 'Enter a page path for this group.',
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export declare const TEST_PAGE: {
|
|
2
|
+
readonly staleEdit: "Someone else changed this test. Reload to see their changes.";
|
|
3
|
+
readonly reload: "Reload";
|
|
2
4
|
readonly loading: "Loading test…";
|
|
3
5
|
readonly forbidden: "You don't have access to this test.";
|
|
4
6
|
readonly missing: "This test no longer exists.";
|
package/dist/admin/labels.d.ts
CHANGED
|
@@ -1,12 +1,7 @@
|
|
|
1
1
|
import type { ExperimentKind, Status } from '../types/contracts.js';
|
|
2
2
|
import type { VerdictCode } from '../types/snapshot.js';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
variantA: string;
|
|
6
|
-
variantB: string;
|
|
7
|
-
variantC: string;
|
|
8
|
-
holdout: string;
|
|
9
|
-
};
|
|
3
|
+
import { ARM_LABELS } from '../labels/arms.js';
|
|
4
|
+
export { ARM_LABELS };
|
|
10
5
|
export declare const STATUS_LABELS: {
|
|
11
6
|
draft: string;
|
|
12
7
|
running: string;
|
package/dist/admin/labels.js
CHANGED
|
@@ -1,10 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
variantA: 'Variant A',
|
|
4
|
-
variantB: 'Variant B',
|
|
5
|
-
variantC: 'Variant C',
|
|
6
|
-
holdout: 'Holdout'
|
|
7
|
-
};
|
|
1
|
+
import { ARM_LABELS } from '../labels/arms.js';
|
|
2
|
+
export { ARM_LABELS };
|
|
8
3
|
export const STATUS_LABELS = {
|
|
9
4
|
draft: 'Draft',
|
|
10
5
|
running: 'Running',
|
|
@@ -92,7 +92,7 @@ export interface ResultsContext {
|
|
|
92
92
|
allocation?: number;
|
|
93
93
|
}
|
|
94
94
|
/** Only selects stored counts; never estimates sample sizes or power in the UI. */
|
|
95
|
-
export declare function collectingLine({ snapshot, phaseKind
|
|
95
|
+
export declare function collectingLine({ snapshot, phaseKind }: RoundLineInput, ctx?: ResultsContext): string | null;
|
|
96
96
|
export interface DataQualityItem {
|
|
97
97
|
id: string;
|
|
98
98
|
message: string;
|
|
@@ -117,9 +117,8 @@ function smallest(values) {
|
|
|
117
117
|
if (!values.length || values.some((value)=>value === undefined || !Number.isFinite(value))) return undefined;
|
|
118
118
|
return values.reduce((min, value)=>min === undefined || value !== undefined && value < min ? value : min, undefined);
|
|
119
119
|
}
|
|
120
|
-
/** Only selects stored counts; never estimates sample sizes or power in the UI. */ export function collectingLine({ snapshot, phaseKind
|
|
121
|
-
if (phaseKind
|
|
122
|
-
if (snapshot.verdict.code !== (phaseKind === 'rollout' ? 'rollout-collecting' : 'collecting')) return null;
|
|
120
|
+
/** Only selects stored counts; never estimates sample sizes or power in the UI. */ export function collectingLine({ snapshot, phaseKind }, ctx = {}) {
|
|
121
|
+
if (phaseKind !== 'rollout' || snapshot.verdict.code !== 'rollout-collecting') return null;
|
|
123
122
|
const keys = comparedKeys(snapshot, ctx.phase);
|
|
124
123
|
const primary = snapshot.metrics.find((metric)=>metric.role === 'primary');
|
|
125
124
|
return collectingCopy(formatCount(smallest(keys.map((key)=>snapshot.exposures.perArm[key]))), formatCount(snapshot.minExposuresPerArm ?? ctx.minExposuresPerArm), formatCount(smallest(keys.map((key)=>primary?.perArm.find((arm)=>arm.arm === key)?.converters))), formatCount(primary ? snapshot.params.minSampleSizes[primary.metricId] : undefined));
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
'use client';
|
|
2
2
|
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
3
|
+
import { useState } from 'react';
|
|
3
4
|
import { useTestDocument } from './useTestDocument.js';
|
|
4
5
|
import { TestHeader } from './test/TestHeader.js';
|
|
5
6
|
import { ResultsBody } from './results/ResultsBody.js';
|
|
@@ -17,6 +18,7 @@ function ManagedResults({ id, surface }) {
|
|
|
17
18
|
const test = useTestDocument(id, {
|
|
18
19
|
apiRoute: surface.apiRoute
|
|
19
20
|
});
|
|
21
|
+
const [resultsVersion, setResultsVersion] = useState(0);
|
|
20
22
|
// Add-goal changes metrics while status, phase and allocation stay the same.
|
|
21
23
|
const bodyKey = test.doc ? JSON.stringify([
|
|
22
24
|
test.doc.status,
|
|
@@ -31,7 +33,8 @@ function ManagedResults({ id, surface }) {
|
|
|
31
33
|
test.loadState === 'ready' && test.doc ? /*#__PURE__*/ _jsx(TestHeader, {
|
|
32
34
|
test: test,
|
|
33
35
|
surface: surface,
|
|
34
|
-
view: "results"
|
|
36
|
+
view: "results",
|
|
37
|
+
resultsVersion: resultsVersion
|
|
35
38
|
}) : /*#__PURE__*/ _jsx("p", {
|
|
36
39
|
role: "status",
|
|
37
40
|
children: test.loadState === 'forbidden' ? C.forbidden : test.loadState === 'missing' ? C.missing : test.loadState === 'error' ? C.error : C.loading
|
|
@@ -51,7 +54,8 @@ function ManagedResults({ id, surface }) {
|
|
|
51
54
|
}) : null,
|
|
52
55
|
test.loadState !== 'loading' && test.loadState !== 'missing' && test.loadState !== 'forbidden' ? /*#__PURE__*/ _jsx(ResultsBody, {
|
|
53
56
|
id: id,
|
|
54
|
-
surface: surface
|
|
57
|
+
surface: surface,
|
|
58
|
+
onRecomputed: ()=>setResultsVersion((value)=>value + 1)
|
|
55
59
|
}, bodyKey) : null
|
|
56
60
|
]
|
|
57
61
|
});
|
|
@@ -25,6 +25,7 @@ export interface SurfaceProps {
|
|
|
25
25
|
export interface SectionProps {
|
|
26
26
|
test: TestDocApi;
|
|
27
27
|
surface: SurfaceProps;
|
|
28
|
+
resultsVersion?: number;
|
|
28
29
|
}
|
|
29
30
|
/** Kind-specific pieces the generic test page composes (Setting kind: ADM-22). */
|
|
30
31
|
export interface KindParts {
|
|
@@ -19,21 +19,22 @@ export function Banner(props) {
|
|
|
19
19
|
...props
|
|
20
20
|
}, `${props.test.doc.id}/${props.test.doc.status}`) : null;
|
|
21
21
|
}
|
|
22
|
-
function BannerContent({ test, surface }) {
|
|
22
|
+
function BannerContent({ test, surface, resultsVersion }) {
|
|
23
23
|
const [selected, setSelected] = useState(null);
|
|
24
24
|
const [people, setPeople] = useState(null);
|
|
25
25
|
const transition = useTransition(test);
|
|
26
26
|
const doc = test.doc;
|
|
27
27
|
useEffect(()=>{
|
|
28
28
|
let active = true;
|
|
29
|
-
|
|
29
|
+
// The id/status key resets the count; refreshes retain it while loading.
|
|
30
30
|
if (surface.canReadResults && doc.status === 'running') {
|
|
31
31
|
void createApi(surface.apiRoute).summary([
|
|
32
32
|
doc.id
|
|
33
33
|
]).then((result)=>{
|
|
34
|
-
|
|
34
|
+
if (!result.ok) return;
|
|
35
|
+
const count = result.data?.[doc.id]?.exposures;
|
|
35
36
|
if (active) setPeople(typeof count === 'number' && Number.isFinite(count) ? count : null);
|
|
36
|
-
});
|
|
37
|
+
}).catch(()=>{});
|
|
37
38
|
}
|
|
38
39
|
return ()=>{
|
|
39
40
|
active = false;
|
|
@@ -42,7 +43,8 @@ function BannerContent({ test, surface }) {
|
|
|
42
43
|
doc.id,
|
|
43
44
|
doc.status,
|
|
44
45
|
surface.apiRoute,
|
|
45
|
-
surface.canReadResults
|
|
46
|
+
surface.canReadResults,
|
|
47
|
+
resultsVersion
|
|
46
48
|
]);
|
|
47
49
|
const decision = doc.decision;
|
|
48
50
|
const winner = actionGroupLabel(decision?.winnerArm ?? 'control', doc.arms.find((item)=>item.key === decision?.winnerArm)?.label);
|
|
@@ -17,7 +17,7 @@ export function SaveStatus({ test }) {
|
|
|
17
17
|
setRetrying(false);
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
|
-
const message = test.saving === 'saving' ? C.saving : test.saving === 'saved' ? C.saved : test.saving === 'error' ? C.saveError : null;
|
|
20
|
+
const message = test.saving === 'saving' ? C.saving : test.saving === 'saved' ? C.saved : test.saving === 'error' && !test.stale ? C.saveError : null;
|
|
21
21
|
return /*#__PURE__*/ _jsxs("div", {
|
|
22
22
|
className: "xp-save-status",
|
|
23
23
|
"aria-live": "polite",
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import '../../styles.js';
|
|
2
2
|
import type { DocId } from '../../../types/contracts.js';
|
|
3
3
|
import type { SurfaceProps } from '../contracts.js';
|
|
4
|
-
|
|
4
|
+
type ResultsBodyProps = {
|
|
5
5
|
id: DocId;
|
|
6
6
|
surface: SurfaceProps;
|
|
7
|
-
|
|
7
|
+
onRecomputed?: () => void;
|
|
8
|
+
};
|
|
9
|
+
export declare function ResultsBody({ id, surface, onRecomputed }: ResultsBodyProps): import("react").JSX.Element;
|
|
10
|
+
export {};
|
|
@@ -18,14 +18,15 @@ import { armRows, collectingLine, dataQuality, defaultRound, guardrailChips, pow
|
|
|
18
18
|
import { recommendation } from './recommendation.model.js';
|
|
19
19
|
import { useResults } from './useResults.js';
|
|
20
20
|
import { DecideDialog } from '../../actions/DecideDialog.js';
|
|
21
|
-
export function ResultsBody({ id, surface }) {
|
|
21
|
+
export function ResultsBody({ id, surface, onRecomputed }) {
|
|
22
22
|
// A different test owns an independent selection and action lifecycle.
|
|
23
23
|
return /*#__PURE__*/ _jsx(ResultsContent, {
|
|
24
24
|
id: id,
|
|
25
|
-
surface: surface
|
|
25
|
+
surface: surface,
|
|
26
|
+
onRecomputed: onRecomputed
|
|
26
27
|
}, `${surface.apiRoute}/${id}`);
|
|
27
28
|
}
|
|
28
|
-
function ResultsContent({ id, surface }) {
|
|
29
|
+
function ResultsContent({ id, surface, onRecomputed }) {
|
|
29
30
|
const results = useResults(id, surface);
|
|
30
31
|
const { api, doc, round, phases, state, data } = results;
|
|
31
32
|
const manage = surface.canManage && surface.canReadResults;
|
|
@@ -69,8 +70,10 @@ function ResultsContent({ id, surface }) {
|
|
|
69
70
|
try {
|
|
70
71
|
const result = await api.recompute(id);
|
|
71
72
|
if (!mounted.current) return;
|
|
72
|
-
if (result.ok)
|
|
73
|
-
|
|
73
|
+
if (result.ok) {
|
|
74
|
+
results.reload();
|
|
75
|
+
onRecomputed?.();
|
|
76
|
+
} else setActionError(result.message);
|
|
74
77
|
} finally{
|
|
75
78
|
inFlight.current = false;
|
|
76
79
|
if (mounted.current) setBusy(false);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import type { SectionProps } from '../contracts.js';
|
|
2
|
-
export declare function TestHeader({ test, surface, view }: SectionProps & {
|
|
2
|
+
export declare function TestHeader({ test, surface, resultsVersion, view }: SectionProps & {
|
|
3
3
|
view?: 'setup' | 'results';
|
|
4
4
|
}): import("react").JSX.Element | null;
|
|
@@ -6,7 +6,7 @@ import { readinessParts } from '../readiness/parts.js';
|
|
|
6
6
|
import { statusPill } from '../../status.model.js';
|
|
7
7
|
import { TEST_PAGE as C } from '../../copy/testPage.js';
|
|
8
8
|
import { FieldErrors, reasonFor, save, TextEdit } from './fields.js';
|
|
9
|
-
export function TestHeader({ test, surface, view = 'setup' }) {
|
|
9
|
+
export function TestHeader({ test, surface, resultsVersion, view = 'setup' }) {
|
|
10
10
|
const doc = test.doc;
|
|
11
11
|
if (!doc) return null;
|
|
12
12
|
const pill = statusPill({
|
|
@@ -62,7 +62,8 @@ export function TestHeader({ test, surface, view = 'setup' }) {
|
|
|
62
62
|
}),
|
|
63
63
|
/*#__PURE__*/ _jsx(readinessParts.Banner, {
|
|
64
64
|
test: test,
|
|
65
|
-
surface: surface
|
|
65
|
+
surface: surface,
|
|
66
|
+
resultsVersion: resultsVersion
|
|
66
67
|
})
|
|
67
68
|
]
|
|
68
69
|
});
|
|
@@ -43,9 +43,18 @@ export function TestPage({ id, surface }) {
|
|
|
43
43
|
/*#__PURE__*/ _jsx(TestHeader, {
|
|
44
44
|
...props
|
|
45
45
|
}),
|
|
46
|
-
test.errors._section ? /*#__PURE__*/
|
|
46
|
+
test.errors._section ? /*#__PURE__*/ _jsxs("div", {
|
|
47
47
|
role: "alert",
|
|
48
|
-
children:
|
|
48
|
+
children: [
|
|
49
|
+
/*#__PURE__*/ _jsx("p", {
|
|
50
|
+
children: test.errors._section
|
|
51
|
+
}),
|
|
52
|
+
test.stale ? /*#__PURE__*/ _jsx("button", {
|
|
53
|
+
type: "button",
|
|
54
|
+
onClick: ()=>void test.reload(),
|
|
55
|
+
children: C.reload
|
|
56
|
+
}) : null
|
|
57
|
+
]
|
|
49
58
|
}) : null,
|
|
50
59
|
test.preflightError ? /*#__PURE__*/ _jsx("p", {
|
|
51
60
|
role: "alert",
|
|
@@ -17,6 +17,7 @@ export declare function useTestDocument(id: DocId, { apiRoute }: {
|
|
|
17
17
|
hasUnsaved: boolean;
|
|
18
18
|
loadState: "loading" | "ready" | "forbidden" | "missing" | "error";
|
|
19
19
|
saving: "idle" | "saving" | "saved" | "error";
|
|
20
|
+
stale: boolean;
|
|
20
21
|
/** Path-specific validation errors; _section is the fallback for a section-wide refusal. */
|
|
21
22
|
errors: Record<string, string>;
|
|
22
23
|
};
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
'use client';
|
|
2
2
|
import { useCallback, useEffect, useRef, useState } from 'react';
|
|
3
3
|
import { API_MESSAGES, refusalMessage } from '../copy/common.js';
|
|
4
|
+
import { XP_IF_UPDATED_AT_HEADER } from '../../constants.js';
|
|
5
|
+
import { TEST_PAGE } from '../copy/testPage.js';
|
|
4
6
|
import { changedPatch, editablePatch, payloadFieldErrors } from './testDocument.model.js';
|
|
5
7
|
// Saves outlive their mounted reader. Reopening the same document must wait for its queue.
|
|
6
8
|
const drainingSaves = new Map();
|
|
@@ -12,6 +14,7 @@ const initialView = ()=>({
|
|
|
12
14
|
hasUnsaved: false,
|
|
13
15
|
loadState: 'loading',
|
|
14
16
|
saving: 'idle',
|
|
17
|
+
stale: false,
|
|
15
18
|
errors: {}
|
|
16
19
|
});
|
|
17
20
|
async function waitForDrain(draining, signal) {
|
|
@@ -49,7 +52,7 @@ function clearErrors(errors, patch) {
|
|
|
49
52
|
return Object.fromEntries(Object.entries(errors).filter(([path])=>!Object.hasOwn(patch, path.split('.')[0])));
|
|
50
53
|
}
|
|
51
54
|
function stage(session, partial) {
|
|
52
|
-
if (!session.saved || session.reloading || session.view.loadState !== 'ready' || session.controller.signal.aborted) return;
|
|
55
|
+
if (!session.saved || session.view.stale || session.reloading || session.view.loadState !== 'ready' || session.controller.signal.aborted) return;
|
|
53
56
|
const patch = editablePatch(session.saved.status, partial);
|
|
54
57
|
if (!Object.keys(patch).length) return;
|
|
55
58
|
session.pending = {
|
|
@@ -165,13 +168,14 @@ function stage(session, partial) {
|
|
|
165
168
|
}
|
|
166
169
|
// Capture each commit now, then diff against the latest server document when its turn arrives.
|
|
167
170
|
function commit() {
|
|
168
|
-
if (!session.saved || session.reloading || controller.signal.aborted || !Object.keys(session.pending).length) return session.tail;
|
|
171
|
+
if (!session.saved || session.view.stale || session.reloading || controller.signal.aborted || !Object.keys(session.pending).length) return session.tail;
|
|
169
172
|
const queued = session.pending;
|
|
170
173
|
session.pending = {};
|
|
171
174
|
session.queue.push(queued);
|
|
172
175
|
session.view.saving = 'saving';
|
|
173
176
|
session.publish();
|
|
174
177
|
const step = async ()=>{
|
|
178
|
+
if (session.view.stale) return;
|
|
175
179
|
let saved = false;
|
|
176
180
|
let changed = false;
|
|
177
181
|
let failed = false;
|
|
@@ -180,17 +184,35 @@ function stage(session, partial) {
|
|
|
180
184
|
const patch = changedPatch(session.saved, queued);
|
|
181
185
|
changed = Object.keys(patch).length > 0;
|
|
182
186
|
if (changed) {
|
|
187
|
+
const headers = {
|
|
188
|
+
'Content-Type': 'application/json'
|
|
189
|
+
};
|
|
190
|
+
if (typeof session.saved.updatedAt === 'string') headers[XP_IF_UPDATED_AT_HEADER] = session.saved.updatedAt;
|
|
183
191
|
const response = await fetch(url, {
|
|
184
192
|
// Submitted saves belong to the queue, not the mounted reader's lifetime.
|
|
185
193
|
method: 'PATCH',
|
|
186
194
|
credentials: 'include',
|
|
187
|
-
headers
|
|
188
|
-
'Content-Type': 'application/json'
|
|
189
|
-
},
|
|
195
|
+
headers,
|
|
190
196
|
body: JSON.stringify(patch)
|
|
191
197
|
});
|
|
192
198
|
const body = await response.json().catch(()=>undefined);
|
|
193
|
-
|
|
199
|
+
const errors = body?.errors;
|
|
200
|
+
const staleError = response.status === 409 && Array.isArray(errors) ? errors.find((error)=>error?.code === 'stale-edit') : undefined;
|
|
201
|
+
if (staleError) {
|
|
202
|
+
session.view.stale = true;
|
|
203
|
+
// Keep every typed value visible, but discard all queued sends until Reload.
|
|
204
|
+
session.pending = [
|
|
205
|
+
...session.queue,
|
|
206
|
+
session.pending
|
|
207
|
+
].reduce((draft, edit)=>({
|
|
208
|
+
...draft,
|
|
209
|
+
...edit
|
|
210
|
+
}), {});
|
|
211
|
+
session.queue = [];
|
|
212
|
+
session.view.errors = {
|
|
213
|
+
_section: staleError.message || TEST_PAGE.staleEdit
|
|
214
|
+
};
|
|
215
|
+
} else if (!response.ok) session.view.errors = {
|
|
194
216
|
...session.view.errors,
|
|
195
217
|
...payloadFieldErrors(body)
|
|
196
218
|
};
|
|
@@ -199,9 +221,12 @@ function stage(session, partial) {
|
|
|
199
221
|
const doc = body?.doc;
|
|
200
222
|
const readable = doc && doc.id != null && doc.status;
|
|
201
223
|
// A 2xx has applied even if its response body was lost. Preserve it while re-reading.
|
|
224
|
+
// The old timestamp is no longer a valid baseline. Omit the header until a
|
|
225
|
+
// readable PATCH or GET restores it, including when this read is aborted.
|
|
202
226
|
session.saved = readable ? doc : {
|
|
203
227
|
...session.saved,
|
|
204
|
-
...patch
|
|
228
|
+
...patch,
|
|
229
|
+
updatedAt: undefined
|
|
205
230
|
};
|
|
206
231
|
saved = true;
|
|
207
232
|
// Keep readiness pending even while recovering an unreadable successful PATCH body.
|
|
@@ -7,7 +7,7 @@ import { flagControlParams } from './flagControlParams.js';
|
|
|
7
7
|
import { lockHook } from './lockHook.js';
|
|
8
8
|
import { deleteGuard, deleteGuardAfter } from './deleteGuard.js';
|
|
9
9
|
import { checkArms } from './armInvariants.js';
|
|
10
|
-
import { XpNotImplemented } from '../../errors.js';
|
|
10
|
+
import { XpNotImplemented, XpRefusal } from '../../errors.js';
|
|
11
11
|
const positiveInteger = (value)=>value != null && Number.isInteger(value) && value > 0 || 'Enter a positive whole number.';
|
|
12
12
|
export function experimentsCollection(options) {
|
|
13
13
|
const flagPoints = serializeFlagPoints(options);
|
|
@@ -91,6 +91,24 @@ export function experimentsCollection(options) {
|
|
|
91
91
|
],
|
|
92
92
|
afterDelete: [
|
|
93
93
|
deleteGuardAfter
|
|
94
|
+
],
|
|
95
|
+
afterError: [
|
|
96
|
+
({ error })=>{
|
|
97
|
+
// Payload preserves public Error status/message, but omits custom codes.
|
|
98
|
+
if (error instanceof XpRefusal && error.code === 'stale-edit') {
|
|
99
|
+
return {
|
|
100
|
+
status: error.status,
|
|
101
|
+
response: {
|
|
102
|
+
errors: [
|
|
103
|
+
{
|
|
104
|
+
code: error.code,
|
|
105
|
+
message: error.message
|
|
106
|
+
}
|
|
107
|
+
]
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
}
|
|
94
112
|
]
|
|
95
113
|
},
|
|
96
114
|
fields: [
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import { XpLockedFieldError } from '../../errors.js';
|
|
1
|
+
import { XpLockedFieldError, XpRefusal } from '../../errors.js';
|
|
2
|
+
import { XP_IF_UPDATED_AT_HEADER } from '../../constants.js';
|
|
3
|
+
import { txExecutor } from '../../db/executor.js';
|
|
4
|
+
import { lockExperimentUpdatedAt } from '../../db/staleEdit.js';
|
|
2
5
|
import { xpOptionsFromReq } from '../../options/fromReq.js';
|
|
3
6
|
import { EDITABLE_BY_STATUS } from './editability.js';
|
|
4
7
|
import { defaultArms } from './armInvariants.js';
|
|
@@ -60,6 +63,15 @@ const dates = new Set([
|
|
|
60
63
|
const same = (a, b, path)=>JSON.stringify(normalize(a, path)) === JSON.stringify(normalize(b, path));
|
|
61
64
|
export const lockHook = async ({ data, originalDoc, operation, req })=>{
|
|
62
65
|
if (req.context.xpTransition === true) return data;
|
|
66
|
+
const baseline = req.headers.get(XP_IF_UPDATED_AT_HEADER);
|
|
67
|
+
if (operation === 'update' && baseline !== null) {
|
|
68
|
+
// originalDoc predates beforeChange; read again under a transaction row lock
|
|
69
|
+
// so a competing save observes the preceding save's committed timestamp.
|
|
70
|
+
const stored = await lockExperimentUpdatedAt(txExecutor(req), originalDoc.id);
|
|
71
|
+
if (stored === null || new Date(baseline).getTime() !== new Date(stored).getTime()) {
|
|
72
|
+
throw new XpRefusal('stale-edit', 'Someone else changed this test. Reload to see their changes.', undefined, 409);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
63
75
|
const admin = async ()=>{
|
|
64
76
|
const options = xpOptionsFromReq(req);
|
|
65
77
|
return await (options.access.admin ?? options.access.manage)({
|
package/dist/constants.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export declare const DEFAULT_SEQUENTIAL_TUNING_PARAMETER = 5000;
|
|
1
2
|
export declare const MAX_EVENTS_PER_BODY = 20;
|
|
2
3
|
export declare const DEFAULT_MAX_BODY_BYTES = 8192;
|
|
3
4
|
export declare const XP_COOKIE = "xp_id";
|
|
@@ -5,3 +6,4 @@ export declare const XP_HEADER = "x-xp-assignments";
|
|
|
5
6
|
export declare const XP_PREVIEW_PARAM = "xp_preview";
|
|
6
7
|
export declare const XP_MANIFEST_KEY_HEADER = "x-xp-manifest-key";
|
|
7
8
|
export declare const XP_REQUEST_HEADER = "x-xp-request";
|
|
9
|
+
export declare const XP_IF_UPDATED_AT_HEADER = "x-xp-if-updated-at";
|
package/dist/constants.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export const DEFAULT_SEQUENTIAL_TUNING_PARAMETER = 5000;
|
|
1
2
|
export const MAX_EVENTS_PER_BODY = 20;
|
|
2
3
|
export const DEFAULT_MAX_BODY_BYTES = 8192;
|
|
3
4
|
export const XP_COOKIE = 'xp_id';
|
|
@@ -5,3 +6,4 @@ export const XP_HEADER = 'x-xp-assignments';
|
|
|
5
6
|
export const XP_PREVIEW_PARAM = 'xp_preview';
|
|
6
7
|
export const XP_MANIFEST_KEY_HEADER = 'x-xp-manifest-key';
|
|
7
8
|
export const XP_REQUEST_HEADER = 'x-xp-request';
|
|
9
|
+
export const XP_IF_UPDATED_AT_HEADER = 'x-xp-if-updated-at';
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Executor } from './executor.js';
|
|
2
|
+
import type { DocId } from '../types/contracts.js';
|
|
3
|
+
/** Request transaction only: hold the row lock through the experiment write. */
|
|
4
|
+
export declare function lockExperimentUpdatedAt(exec: Executor & {
|
|
5
|
+
schemaName?: string;
|
|
6
|
+
}, id: DocId): Promise<Date | string | null>;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { COLUMNS, TABLES } from './names.js';
|
|
2
|
+
const quote = (name)=>`"${name.replaceAll('"', '""')}"`;
|
|
3
|
+
/** Request transaction only: hold the row lock through the experiment write. */ export async function lockExperimentUpdatedAt(exec, id) {
|
|
4
|
+
const table = `${exec.schemaName ? `${quote(exec.schemaName)}.` : ''}${quote(TABLES.experiments)}`;
|
|
5
|
+
const columns = COLUMNS.experiments;
|
|
6
|
+
const result = await exec.query(`SELECT ${quote(columns.updatedAt)} AS "updatedAt" FROM ${table} WHERE ${quote(columns.id)} = $1 FOR NO KEY UPDATE`, [
|
|
7
|
+
id
|
|
8
|
+
]);
|
|
9
|
+
return result.rows[0]?.updatedAt ?? null;
|
|
10
|
+
}
|
package/dist/endpoints/events.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { safeError } from '../log.js';
|
|
1
2
|
import { NextRequest } from 'next/server.js';
|
|
2
3
|
import { DEFAULT_MAX_BODY_BYTES } from '../constants.js';
|
|
3
4
|
import { pooledExecutor } from '../db/executor.js';
|
|
@@ -89,9 +90,8 @@ export async function handleEvents(req, { options }) {
|
|
|
89
90
|
} catch (error) {
|
|
90
91
|
try {
|
|
91
92
|
req.payload.logger.error({
|
|
92
|
-
err: error
|
|
93
|
-
|
|
94
|
-
});
|
|
93
|
+
err: safeError(error)
|
|
94
|
+
}, 'Experiments: events endpoint failed');
|
|
95
95
|
} catch {}
|
|
96
96
|
}
|
|
97
97
|
return accepted();
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { safeError } from '../log.js';
|
|
1
2
|
import { runResults } from '../server/runResults.js';
|
|
2
3
|
import { readSnapshots, resultsError } from './results.js';
|
|
3
4
|
import { routeId } from './routeId.js';
|
|
@@ -22,6 +23,7 @@ export async function handleRecompute(req, _ctx) {
|
|
|
22
23
|
experimentId: docs[0].id,
|
|
23
24
|
sweep: false
|
|
24
25
|
});
|
|
26
|
+
if (result.failed) return resultsError(500, 'analysis-failed', 'Results could not be computed; this has been logged.');
|
|
25
27
|
if (result.skipped || result.budgetExhausted) return resultsError(503, 'recompute-failed', 'Results could not be recomputed. Try again.');
|
|
26
28
|
const latest = (await readSnapshots(req, [
|
|
27
29
|
docs[0].id
|
|
@@ -36,7 +38,11 @@ export async function handleRecompute(req, _ctx) {
|
|
|
36
38
|
'Cache-Control': 'no-store'
|
|
37
39
|
}
|
|
38
40
|
});
|
|
39
|
-
} catch
|
|
41
|
+
} catch (error) {
|
|
42
|
+
req.payload.logger.error({
|
|
43
|
+
err: safeError(error),
|
|
44
|
+
experimentId: docs[0].id
|
|
45
|
+
}, 'Experiment recompute failed');
|
|
40
46
|
return resultsError(503, 'recompute-failed', 'Results could not be recomputed. Try again.');
|
|
41
47
|
}
|
|
42
48
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
'use client';
|
|
2
|
+
export { XP_IF_UPDATED_AT_HEADER } from '../constants.js';
|
|
2
3
|
export { LayerUsage } from '../admin/LayerUsage.js';
|
|
3
4
|
export { MetricSourceSelect } from '../admin/MetricSourceSelect.js';
|
|
4
5
|
export { ArmSelectField } from '../admin/armSelect/ArmSelectField.js';
|
package/dist/hooks/hostHook.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import { safeError } from '../log.js';
|
|
1
2
|
/** Host saves must survive synchronous and asynchronous plugin hook failures. */ export function neverThrow(name, hook) {
|
|
2
3
|
return async (args)=>{
|
|
3
4
|
try {
|
|
4
5
|
return await hook(args);
|
|
5
6
|
} catch (error) {
|
|
6
|
-
console.error(`[payload-experiments] ${name} failed; the document was saved without it`, error);
|
|
7
|
+
console.error(`[payload-experiments] ${name} failed; the document was saved without it`, safeError(error));
|
|
7
8
|
return args.doc;
|
|
8
9
|
}
|
|
9
10
|
};
|
|
@@ -13,7 +14,7 @@
|
|
|
13
14
|
try {
|
|
14
15
|
return await hook(args);
|
|
15
16
|
} catch (error) {
|
|
16
|
-
console.error(`[payload-experiments] ${name} failed; the document was saved without it`, error);
|
|
17
|
+
console.error(`[payload-experiments] ${name} failed; the document was saved without it`, safeError(error));
|
|
17
18
|
return args.data;
|
|
18
19
|
}
|
|
19
20
|
};
|
package/dist/log.d.ts
ADDED
package/dist/log.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { XpNoTransactionError, XpNotImplemented, XpRefusal } from './errors.js';
|
|
2
|
+
import { ResultsBudgetError } from './db/pgExecutor.js';
|
|
3
|
+
/** Deliberately construct a plain object: host serializers must never see the error. */ export function safeError(err) {
|
|
4
|
+
try {
|
|
5
|
+
if (!(err instanceof Error)) return {
|
|
6
|
+
name: typeof err
|
|
7
|
+
};
|
|
8
|
+
const name = err.name;
|
|
9
|
+
const result = {
|
|
10
|
+
name: typeof name === 'string' ? name : 'Error'
|
|
11
|
+
};
|
|
12
|
+
const code = 'code' in err ? err.code : undefined;
|
|
13
|
+
if (typeof code === 'string' && (/^[0-9A-Z]{5}$/.test(code) || err instanceof XpRefusal)) result.code = code;
|
|
14
|
+
const message = err.message;
|
|
15
|
+
const stack = err.stack;
|
|
16
|
+
// Remove the entire V8 header first: message lines can impersonate frames.
|
|
17
|
+
// Unknown stack formats fail closed. Never traverse cause or database fields.
|
|
18
|
+
if (typeof name === 'string' && typeof message === 'string' && typeof stack === 'string') {
|
|
19
|
+
const header = message ? `${name}: ${message}` : name;
|
|
20
|
+
const frames = stack.startsWith(`${header}\n`) ? stack.slice(header.length + 1).split('\n').filter((line)=>/^\s+at /.test(line)) : [];
|
|
21
|
+
if (frames.length) result.stack = frames.join('\n');
|
|
22
|
+
}
|
|
23
|
+
// XpRefusal includes stored slugs; XpLockedFieldError includes request keys
|
|
24
|
+
// and stored status. Neither class's message is safe to send to host logs.
|
|
25
|
+
if (err instanceof XpNoTransactionError || err instanceof XpNotImplemented || err instanceof ResultsBudgetError) {
|
|
26
|
+
if (typeof message === 'string') result.message = message;
|
|
27
|
+
}
|
|
28
|
+
return result;
|
|
29
|
+
} catch {
|
|
30
|
+
// Even instanceof, property existence checks and getters can throw on hosts' errors.
|
|
31
|
+
return {
|
|
32
|
+
name: 'Error'
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
}
|
package/dist/puck/copy.d.ts
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
export declare const slotLabels: {
|
|
2
|
+
readonly control: "Original";
|
|
3
|
+
readonly variantA: "Variant A";
|
|
4
|
+
readonly variantB: "Variant B";
|
|
5
|
+
readonly variantC: "Variant C";
|
|
6
|
+
};
|
|
1
7
|
export declare const pickerCopy: {
|
|
2
8
|
readonly label: "Experiment";
|
|
3
9
|
readonly none: "No experiment";
|
package/dist/puck/copy.js
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
import { ARM_LABELS } from '../labels/arms.js';
|
|
2
|
+
export const slotLabels = {
|
|
3
|
+
control: ARM_LABELS.control,
|
|
4
|
+
variantA: ARM_LABELS.variantA,
|
|
5
|
+
variantB: ARM_LABELS.variantB,
|
|
6
|
+
variantC: ARM_LABELS.variantC
|
|
7
|
+
};
|
|
1
8
|
export const pickerCopy = {
|
|
2
9
|
label: 'Experiment',
|
|
3
10
|
none: 'No experiment',
|
|
@@ -156,6 +156,8 @@ function EditorRender(props) {
|
|
|
156
156
|
experiment ? /*#__PURE__*/ _jsx("a", {
|
|
157
157
|
className: "xp-slot-button xp-slot-button-primary",
|
|
158
158
|
href: `/admin/experiments/${encodeURIComponent(String(experimentId))}${experiment.status === 'draft' ? '' : '/results'}`,
|
|
159
|
+
target: "_blank",
|
|
160
|
+
rel: "noopener noreferrer",
|
|
159
161
|
children: experiment.status === 'draft' ? editorCopy.setup : editorCopy.viewResults
|
|
160
162
|
}) : null
|
|
161
163
|
]
|
package/dist/puck/slotConfig.js
CHANGED
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
import { createExperimentSlotField } from './experimentSlotField.js';
|
|
2
|
-
import { pickerCopy } from './copy.js';
|
|
2
|
+
import { pickerCopy, slotLabels } from './copy.js';
|
|
3
3
|
export const experimentSlotLabel = pickerCopy.label;
|
|
4
4
|
export const experimentSlotFields = {
|
|
5
5
|
experiment: createExperimentSlotField(),
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
type: 'slot'
|
|
6
|
+
// Puck 0.24's defaultSlots prepends each slot, reversing declaration order in the outline.
|
|
7
|
+
variantC: {
|
|
8
|
+
type: 'slot',
|
|
9
|
+
label: slotLabels.variantC
|
|
11
10
|
},
|
|
12
11
|
variantB: {
|
|
13
|
-
type: 'slot'
|
|
12
|
+
type: 'slot',
|
|
13
|
+
label: slotLabels.variantB
|
|
14
14
|
},
|
|
15
|
-
|
|
16
|
-
type: 'slot'
|
|
15
|
+
variantA: {
|
|
16
|
+
type: 'slot',
|
|
17
|
+
label: slotLabels.variantA
|
|
18
|
+
},
|
|
19
|
+
control: {
|
|
20
|
+
type: 'slot',
|
|
21
|
+
label: slotLabels.control
|
|
17
22
|
}
|
|
18
23
|
};
|
|
19
24
|
export const experimentSlotDefaultProps = {
|
|
@@ -33,7 +33,7 @@ export interface AnalysisPhase {
|
|
|
33
33
|
analysis: {
|
|
34
34
|
method: 'sequential' | 'bayesian';
|
|
35
35
|
alpha?: number;
|
|
36
|
-
sequentialTuningParameter
|
|
36
|
+
sequentialTuningParameter?: number | null;
|
|
37
37
|
cuped: 'auto' | 'off';
|
|
38
38
|
cupedLookbackDays?: number;
|
|
39
39
|
minExposuresPerArm: number;
|
package/dist/results/analyse.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DEFAULT_GAUSSIAN_PRIOR, bonferroniAlpha, capValues, createThetaAdjustedStatistics, effectBayesianABTest, sequentialOneSidedTreatmentGreaterTTest, sequentialOneSidedTreatmentLesserTTest, sequentialTwoSidedTTest, statisticMean, statisticVariance } from '@delmaredigital/experiments-stats';
|
|
2
|
+
import { DEFAULT_SEQUENTIAL_TUNING_PARAMETER } from '../constants.js';
|
|
2
3
|
import { timestampMs } from './aggregate.js';
|
|
3
4
|
import { outcomeStatistic } from './cuped.js';
|
|
4
5
|
import { guardrailStatus } from './guardrails.js';
|
|
@@ -17,6 +18,9 @@ const severity = {
|
|
|
17
18
|
*/ export function analysePhase(input) {
|
|
18
19
|
const { phase, units } = input;
|
|
19
20
|
const { analysis } = phase;
|
|
21
|
+
// Start freezes this optional collection field as-is. Match the stats engine's
|
|
22
|
+
// default and record the effective value in every snapshot, including n = 0.
|
|
23
|
+
const sequentialTuningParameter = analysis.sequentialTuningParameter ?? DEFAULT_SEQUENTIAL_TUNING_PARAMETER;
|
|
20
24
|
const alpha = analysis.alpha ?? 0.05;
|
|
21
25
|
const prior = analysis.prior ?? DEFAULT_GAUSSIAN_PRIOR;
|
|
22
26
|
const bayesian = analysis.bayesian ?? {
|
|
@@ -73,7 +77,7 @@ const severity = {
|
|
|
73
77
|
capped: {}
|
|
74
78
|
};
|
|
75
79
|
const params = {
|
|
76
|
-
sequentialTuningParameter
|
|
80
|
+
sequentialTuningParameter,
|
|
77
81
|
prior: {
|
|
78
82
|
...prior
|
|
79
83
|
},
|
|
@@ -158,7 +162,7 @@ const severity = {
|
|
|
158
162
|
metric.cupedApplied ||= a.type === 'regressionAdjusted' && b.type === 'regressionAdjusted';
|
|
159
163
|
const config = {
|
|
160
164
|
alpha,
|
|
161
|
-
sequentialTuningParameter
|
|
165
|
+
sequentialTuningParameter
|
|
162
166
|
};
|
|
163
167
|
const inverse = definition.direction === 'decrease';
|
|
164
168
|
let comparison;
|
|
@@ -342,7 +346,7 @@ const severity = {
|
|
|
342
346
|
alpha: alphaAdjusted,
|
|
343
347
|
method: analysis.method,
|
|
344
348
|
prior,
|
|
345
|
-
sequentialTuningParameter
|
|
349
|
+
sequentialTuningParameter
|
|
346
350
|
});
|
|
347
351
|
verdict = computeVerdict({
|
|
348
352
|
...verdictInput,
|
|
@@ -78,6 +78,8 @@ interface VerdictInputFields {
|
|
|
78
78
|
originalLabel: string;
|
|
79
79
|
controlArm: string;
|
|
80
80
|
}
|
|
81
|
+
/** Resolve a stored group label without exposing an internal key in verdict copy. */
|
|
82
|
+
export declare function verdictGroupLabel(key: string, label?: string): string;
|
|
81
83
|
/** Apply spec §8.5 in order to already-computed results; no inference is performed here. */
|
|
82
84
|
export declare function computeVerdict(input: VerdictInput): Snapshot['verdict'];
|
|
83
85
|
export {};
|
package/dist/results/verdict.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ARM_LABELS } from '../labels/arms.js';
|
|
1
2
|
function percent(value) {
|
|
2
3
|
if (value === null) return '…';
|
|
3
4
|
const rounded = Math.round(value * 100);
|
|
@@ -19,6 +20,9 @@ function chance(value) {
|
|
|
19
20
|
if (value === undefined) return '';
|
|
20
21
|
return value >= 0.995 ? ' (more than 99%)' : ` (${Math.floor(value * 100)}%)`;
|
|
21
22
|
}
|
|
23
|
+
/** Resolve a stored group label without exposing an internal key in verdict copy. */ export function verdictGroupLabel(key, label = '') {
|
|
24
|
+
return label.trim() || (Object.hasOwn(ARM_LABELS, key) ? ARM_LABELS[key] : 'This group');
|
|
25
|
+
}
|
|
22
26
|
/** Apply spec §8.5 in order to already-computed results; no inference is performed here. */ export function computeVerdict(input) {
|
|
23
27
|
const rollout = input.phaseKind === 'rollout';
|
|
24
28
|
const make = (row, code, finding, action, detail, recommendedArm, line, worstArm)=>({
|
|
@@ -60,7 +64,20 @@ function chance(value) {
|
|
|
60
64
|
}
|
|
61
65
|
const failing = input.guardrails.find((item)=>item.status === 'failing');
|
|
62
66
|
if (failing) return make(3, 'guardrail-failing', `Stop: ${failing.worstArmLabel} harms ${failing.name}.`, null, 'An outcome that must stay safe has become worse.', input.controlArm, undefined, failing.worstArm);
|
|
63
|
-
if (collecting)
|
|
67
|
+
if (collecting) {
|
|
68
|
+
const smallest = compared.find((item)=>item.n === minimum);
|
|
69
|
+
const fewest = converters.find((item)=>item.converters === minimumConverters);
|
|
70
|
+
const groupLabel = (group)=>{
|
|
71
|
+
if (!group || group.kind === 'control') return 'Original';
|
|
72
|
+
const label = input.exposures.perArm.find((item)=>item.arm === group.arm)?.label ?? input.primary.comparisons.find((item)=>item.arm === group.arm)?.label;
|
|
73
|
+
return verdictGroupLabel(group.arm, label);
|
|
74
|
+
};
|
|
75
|
+
const detail = [
|
|
76
|
+
minimum < input.exposures.minExposuresPerArm ? `Smallest group so far: ${groupLabel(smallest)}, ${minimum} of the ${input.exposures.minExposuresPerArm} people it needs.` : '',
|
|
77
|
+
minimumConverters < input.primary.minSampleSize ? `Fewest conversions so far: ${groupLabel(fewest)}, ${minimumConverters} of the ${input.primary.minSampleSize} needed.` : ''
|
|
78
|
+
].filter(Boolean).join(' ') || 'More people or conversions are needed before drawing a conclusion.';
|
|
79
|
+
return make(4, 'collecting', input.status === 'stopped' ? 'Stopped before enough data was collected.' : 'Collecting data.', null, input.status === 'stopped' ? 'More people or conversions are needed before drawing a conclusion.' : detail, null);
|
|
80
|
+
}
|
|
64
81
|
const variantKeys = new Set(compared.filter((item)=>item.kind === 'variant').map((item)=>item.arm));
|
|
65
82
|
const variants = input.primary.comparisons.filter((item)=>variantKeys.has(item.arm));
|
|
66
83
|
const better = variants.filter((item)=>item.significant === 'better');
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { safeError } from '../log.js';
|
|
1
2
|
import { NextRequest } from 'next/server.js';
|
|
2
3
|
import { insertIdentityOnOwnConnection } from '../db/identities.js';
|
|
3
4
|
import { readUnitCookie } from '../runtime/cookie.js';
|
|
@@ -26,9 +27,8 @@ export async function linkIdentity(serverRuntime, opts) {
|
|
|
26
27
|
} catch (error) {
|
|
27
28
|
try {
|
|
28
29
|
payload?.logger.error({
|
|
29
|
-
err: error
|
|
30
|
-
|
|
31
|
-
});
|
|
30
|
+
err: safeError(error)
|
|
31
|
+
}, 'Experiments: linkIdentity failed');
|
|
32
32
|
} catch {}
|
|
33
33
|
}
|
|
34
34
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { safeError } from '../log.js';
|
|
1
2
|
import { isDeepStrictEqual } from 'node:util';
|
|
2
3
|
import { appendContentChange, contentChangeSavepoint, previousPublishedContent, runningContentExperiment } from '../db/appends.js';
|
|
3
4
|
import { txExecutor } from '../db/executor.js';
|
|
@@ -75,7 +76,7 @@ const baselineKey = (collection, id)=>`${collection}:${id}`;
|
|
|
75
76
|
try {
|
|
76
77
|
await auditContentChange(args);
|
|
77
78
|
} catch (error) {
|
|
78
|
-
console.error('[payload-experiments] contentChanges failed; the document was saved without it', error);
|
|
79
|
+
console.error('[payload-experiments] contentChanges failed; the document was saved without it', safeError(error));
|
|
79
80
|
}
|
|
80
81
|
}
|
|
81
82
|
async function auditContentChange({ doc, previousDoc, req, collection }) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { safeError } from '../log.js';
|
|
1
2
|
import { pooledExecutor } from '../db/executor.js';
|
|
2
3
|
import { insertEvents } from '../db/events.js';
|
|
3
4
|
import { exposureDimensions } from '../events/dimensions.js';
|
|
@@ -48,9 +49,8 @@ export async function recordExposures(serverRuntime, rendered, ctx) {
|
|
|
48
49
|
} catch (error) {
|
|
49
50
|
try {
|
|
50
51
|
payload?.logger.error({
|
|
51
|
-
err: error
|
|
52
|
-
|
|
53
|
-
});
|
|
52
|
+
err: safeError(error)
|
|
53
|
+
}, 'Experiments: recordExposures failed');
|
|
54
54
|
} catch {}
|
|
55
55
|
}
|
|
56
56
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { safeError } from '../log.js';
|
|
1
2
|
import { resultsCandidates, resultsExecutor, writeSnapshots } from '../db/snapshots.js';
|
|
2
3
|
import { sweepEvents } from '../db/sweep.js';
|
|
3
4
|
import { armMismatchCount, identityRows, metricEventRows, phaseExposureRows, rolloutExposureRows } from '../db/results.js';
|
|
@@ -219,6 +220,7 @@ export async function runResults(payload, { budgetMs = 20_000, experimentId, swe
|
|
|
219
220
|
const summary = {
|
|
220
221
|
ran: 0,
|
|
221
222
|
skipped: 0,
|
|
223
|
+
failed: 0,
|
|
222
224
|
swept: 0,
|
|
223
225
|
processedIds: [],
|
|
224
226
|
budgetExhausted: false,
|
|
@@ -243,9 +245,10 @@ export async function runResults(payload, { budgetMs = 20_000, experimentId, swe
|
|
|
243
245
|
});
|
|
244
246
|
} catch (error) {
|
|
245
247
|
summary.skipped++;
|
|
248
|
+
if (!(error instanceof ResultsBudgetError)) summary.failed++;
|
|
246
249
|
summary.budgetExhausted = error instanceof ResultsBudgetError;
|
|
247
250
|
payload.logger.error({
|
|
248
|
-
err: error,
|
|
251
|
+
err: safeError(error),
|
|
249
252
|
experimentId
|
|
250
253
|
}, 'Experiment scheduling failed');
|
|
251
254
|
}
|
|
@@ -277,9 +280,10 @@ export async function runResults(payload, { budgetMs = 20_000, experimentId, swe
|
|
|
277
280
|
if (summary.processedIds.length < 100) summary.processedIds.push(id);
|
|
278
281
|
} catch (error) {
|
|
279
282
|
summary.skipped++;
|
|
283
|
+
if (!(error instanceof ResultsBudgetError)) summary.failed++;
|
|
280
284
|
summary.budgetExhausted ||= error instanceof ResultsBudgetError;
|
|
281
285
|
payload.logger.error({
|
|
282
|
-
err: error,
|
|
286
|
+
err: safeError(error),
|
|
283
287
|
experimentId: id
|
|
284
288
|
}, 'Experiment results failed');
|
|
285
289
|
}
|
|
@@ -296,9 +300,10 @@ export async function runResults(payload, { budgetMs = 20_000, experimentId, swe
|
|
|
296
300
|
logger: payload.logger
|
|
297
301
|
});
|
|
298
302
|
} catch (error) {
|
|
303
|
+
if (!(error instanceof ResultsBudgetError)) summary.failed++;
|
|
299
304
|
summary.budgetExhausted ||= error instanceof ResultsBudgetError;
|
|
300
305
|
payload.logger.error({
|
|
301
|
-
err: error
|
|
306
|
+
err: safeError(error)
|
|
302
307
|
}, 'Experiment retention sweep failed');
|
|
303
308
|
}
|
|
304
309
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { safeError } from '../log.js';
|
|
1
2
|
import { NextRequest } from 'next/server.js';
|
|
2
3
|
import { pooledExecutor } from '../db/executor.js';
|
|
3
4
|
import { insertEvents } from '../db/events.js';
|
|
@@ -39,9 +40,8 @@ export async function trackConversion(serverRuntime, event) {
|
|
|
39
40
|
} catch (error) {
|
|
40
41
|
try {
|
|
41
42
|
payload.logger.error({
|
|
42
|
-
err: error
|
|
43
|
-
|
|
44
|
-
});
|
|
43
|
+
err: safeError(error)
|
|
44
|
+
}, 'Experiments: trackConversion getUser failed');
|
|
45
45
|
} catch {}
|
|
46
46
|
}
|
|
47
47
|
}
|
|
@@ -61,9 +61,8 @@ export async function trackConversion(serverRuntime, event) {
|
|
|
61
61
|
} catch (error) {
|
|
62
62
|
try {
|
|
63
63
|
payload?.logger.error({
|
|
64
|
-
err: error
|
|
65
|
-
|
|
66
|
-
});
|
|
64
|
+
err: safeError(error)
|
|
65
|
+
}, 'Experiments: trackConversion failed');
|
|
67
66
|
} catch {}
|
|
68
67
|
}
|
|
69
68
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { safeError } from '../log.js';
|
|
1
2
|
import { txExecutor } from '../db/executor.js';
|
|
2
3
|
import { deleteExperimentSnapshots, serializeDelete } from '../db/delete.js';
|
|
3
4
|
import { deleteExperimentEvents } from '../db/events.js';
|
|
@@ -45,7 +46,7 @@ import { readExperiment, refuse } from './start.js';
|
|
|
45
46
|
} catch (error) {
|
|
46
47
|
req.payload.logger.error({
|
|
47
48
|
experimentId,
|
|
48
|
-
err: error
|
|
49
|
+
err: safeError(error)
|
|
49
50
|
}, 'Experiment cascade failed; aborting delete');
|
|
50
51
|
throw error;
|
|
51
52
|
}
|
|
@@ -4,7 +4,7 @@ import type { ReadinessCode } from '../transitions/checkStart.js';
|
|
|
4
4
|
export type { ReadinessCode } from '../transitions/checkStart.js';
|
|
5
5
|
import type { Snapshot, VerdictCode } from './snapshot.js';
|
|
6
6
|
export type ArmIssueCode = 'weights-total' | 'control-count' | 'control-first' | 'holdout-last' | 'holdout-count' | 'variant-count' | 'duplicate-key' | 'duplicate-slot' | 'slot-required' | 'invalid-key';
|
|
7
|
-
export type RefusalCode = ArmIssueCode | 'not-found' | 'no-planned-duration' | 'no-primary-metric' | 'no-placements' | 'empty-slot' | 'page-no-targets' | 'page-user-unit' | 'flag-no-point' | 'flag-params-invalid' | 'flag-point-busy' | 'flag-placement-missing' | 'layer-full' | 'layer-unit-mismatch' | 'unknown-metric-source' | 'unit-source-mismatch' | 'no-reach-per-day' | 'invalid-status' | 'stale-draft' | 'commit-unverified' | 'allocation-lower' | 'confirmation-required' | 'invalid-winner' | 'slices-taken' | 'locked-placement' | 'not-implemented' | 'forbidden' | 'unauthenticated' | 'body-too-large' | 'invalid-body' | 'too-many-ids' | 'invalid-phase' | 'invalid-ids' | 'recompute-failed' | 'undeclared-targeting-key' | 'invalid-targeting-rule' | 'duplicate-metric';
|
|
7
|
+
export type RefusalCode = ArmIssueCode | 'not-found' | 'no-planned-duration' | 'no-primary-metric' | 'no-placements' | 'empty-slot' | 'page-no-targets' | 'page-user-unit' | 'flag-no-point' | 'flag-params-invalid' | 'flag-point-busy' | 'flag-placement-missing' | 'layer-full' | 'layer-unit-mismatch' | 'unknown-metric-source' | 'unit-source-mismatch' | 'no-reach-per-day' | 'invalid-status' | 'stale-draft' | 'stale-edit' | 'commit-unverified' | 'allocation-lower' | 'confirmation-required' | 'invalid-winner' | 'slices-taken' | 'locked-placement' | 'not-implemented' | 'forbidden' | 'unauthenticated' | 'body-too-large' | 'invalid-body' | 'too-many-ids' | 'invalid-phase' | 'invalid-ids' | 'recompute-failed' | 'analysis-failed' | 'undeclared-targeting-key' | 'invalid-targeting-rule' | 'duplicate-metric';
|
|
8
8
|
export type ErrorEnvelope = {
|
|
9
9
|
errors: Array<{
|
|
10
10
|
code: RefusalCode;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@delmaredigital/payload-experiments",
|
|
3
|
-
"version": "0.2.0-rc.
|
|
3
|
+
"version": "0.2.0-rc.7",
|
|
4
4
|
"description": "No-code A/B testing for Payload CMS and Next.js: block, page and flag experiments with holdouts, layers, exposure logging and honest statistics inside the admin — no vendor",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|