@delmaredigital/payload-experiments 0.0.0-stage → 0.1.0-rc.3

package/dist/transitions/phaseRecord.js

@@ -0,0 +1,81 @@
+import { XpRefusal } from '../errors.js';
+export async function readPhaseRecord(req, experimentId, phase) {
+    // §7.8: frozen configuration lookup serves an authorised transition.
+    const { docs } = await req.payload.find({
+        collection: 'xp-phases',
+        req,
+        overrideAccess: true,
+        depth: 0,
+        limit: 1,
+        where: {
+            and: [
+                {
+                    experiment: {
+                        equals: experimentId
+                    }
+                },
+                {
+                    phase: {
+                        equals: phase
+                    }
+                }
+            ]
+        }
+    });
+    if (!docs[0]) throw new XpRefusal('not-found', 'The phase record was not found.', {
+        experimentId,
+        phase
+    }, 404);
+    return docs[0];
+}
+/** Caller holds the experiment lock and passes the kernel's transaction req. */ export async function writePhaseRecord(req, record) {
+    // §7.8: transition endpoints' writes to plugin collections.
+    return req.payload.create({
+        collection: 'xp-phases',
+        data: record,
+        req,
+        overrideAccess: true
+    });
+}
+/** End only an open phase, preserving the original stop time on a later decide.
+ * Caller holds the experiment lock and passes the kernel's transaction req.
+ */ export async function setPhaseEndedAt(req, experimentId, phase, at) {
+    // §7.8: phase lookup for transition endpoints' writes to plugin collections.
+    const { docs } = await req.payload.find({
+        collection: 'xp-phases',
+        req,
+        limit: 1,
+        depth: 0,
+        where: {
+            and: [
+                {
+                    experiment: {
+                        equals: experimentId
+                    }
+                },
+                {
+                    phase: {
+                        equals: phase
+                    }
+                }
+            ]
+        },
+        overrideAccess: true
+    });
+    const record = docs[0];
+    if (!record) throw new XpRefusal('not-found', 'The phase record was not found.', {
+        experimentId,
+        phase
+    }, 404);
+    if (record.endedAt) return record;
+    // §7.8: transition endpoints' writes to plugin collections.
+    return req.payload.update({
+        collection: 'xp-phases',
+        req,
+        id: record.id,
+        data: {
+            endedAt: at
+        },
+        overrideAccess: true
+    });
+}

package/dist/transitions/placements.d.ts

@@ -0,0 +1,26 @@
+import type { PayloadRequest } from 'payload';
+import type { Arm, DocId, FlagPoint, ResolvedOptions } from '../types/contracts.js';
+import type { ArmSlot } from '../types/puck.js';
+export interface BlockPlacement {
+    collection: string;
+    id: DocId;
+    title: string;
+    published: boolean;
+    slots: Record<ArmSlot, number>;
+    locked?: boolean;
+}
+export type FlagPlacement = Awaited<ReturnType<NonNullable<FlagPoint['placements']>>>[number];
+/** Each matching ExperimentSlot is checked separately, including multiple slots
+ * in one document: content elsewhere in the document cannot hide an empty slot.
+ */
+export declare function scanBlockPlacements(req: PayloadRequest, { slug, puckCollections }: {
+    slug: string;
+    puckCollections: NonNullable<ResolvedOptions['puck']>['collections'];
+}): Promise<BlockPlacement[]>;
+export declare function flagPlacements<TUser extends {
+    id: DocId;
+}>(options: ResolvedOptions<TUser>, experiment: {
+    id: DocId;
+    flagPoint?: string | null;
+    arms: Arm[];
+}, req: PayloadRequest): Promise<FlagPlacement[]>;

package/dist/transitions/placements.js

@@ -0,0 +1,89 @@
+import { puckCollections as normalisePuckCollections } from '../options.js';
+const slots = [
+    'control',
+    'variantA',
+    'variantB',
+    'variantC'
+];
+/** Each matching ExperimentSlot is checked separately, including multiple slots
+ * in one document: content elsewhere in the document cannot hide an empty slot.
+ */ export async function scanBlockPlacements(req, { slug, puckCollections }) {
+    const placements = [];
+    for (const { slug: collection, field } of normalisePuckCollections({
+        puck: {
+            enabled: true,
+            collections: puckCollections
+        }
+    })){
+        const publishedIds = new Set();
+        const draftsEnabled = req.payload.config.collections?.some((config)=>config.slug === collection && config.versions && config.versions.drafts);
+        for (const draft of [
+            false,
+            true
+        ]){
+            let page = 1;
+            while(true){
+                const result = await req.payload.find({
+                    collection,
+                    req,
+                    overrideAccess: false,
+                    limit: 100,
+                    page,
+                    depth: 0,
+                    draft,
+                    includeLockStatus: true,
+                    ...!draft && draftsEnabled ? {
+                        where: {
+                            _status: {
+                                equals: 'published'
+                            }
+                        }
+                    } : {}
+                });
+                for (const doc of result.docs){
+                    if (!draft && doc._status === 'draft') continue;
+                    if (draft && doc._status !== 'draft' && publishedIds.has(String(doc.id))) continue;
+                    if (!draft) publishedIds.add(String(doc.id));
+                    function walk(value) {
+                        if (!value || typeof value !== 'object') return;
+                        if (Array.isArray(value)) {
+                            value.forEach(walk);
+                            return;
+                        }
+                        const node = value;
+                        const props = node.props;
+                        const ref = props?.experiment;
+                        const experiment = typeof ref === 'string' ? ref : ref && typeof ref === 'object' && 'slug' in ref ? ref.slug : null;
+                        if (node.type === 'ExperimentSlot' && props && experiment === slug) {
+                            placements.push({
+                                collection,
+                                id: doc.id,
+                                title: String(doc.title ?? doc.name ?? doc.id),
+                                published: !draft || doc._status === 'published',
+                                slots: Object.fromEntries(slots.map((slot)=>[
+                                        slot,
+                                        Array.isArray(props[slot]) ? props[slot].length : 0
+                                    ])),
+                                locked: doc._isLocked === true || doc.locked === true
+                            });
+                        }
+                        Object.values(node).forEach(walk);
+                    }
+                    walk(doc[field]);
+                }
+                if (!result.hasNextPage) break;
+                page++;
+            }
+        }
+    }
+    return placements;
+}
+export async function flagPlacements(options, experiment, req) {
+    const point = options.flagPoints?.find((point)=>point.id === experiment.flagPoint);
+    if (!point?.placements) return [];
+    return point.placements({
+        experimentId: experiment.id,
+        arms: experiment.arms.filter((arm)=>arm.kind !== 'holdout').map((arm)=>arm.key),
+        req
+    });
+}

package/dist/transitions/preflightData.d.ts

@@ -0,0 +1,21 @@
+import type { PayloadRequest } from 'payload';
+import type { PowerWeeksInput, PowerWeeksResult } from '@delmaredigital/experiments-stats';
+import type { DocId, ResolvedOptions } from '../types/contracts.js';
+import type { PreflightResult } from '../types/endpoints.js';
+import type { CheckStartInput } from './checkStart.js';
+export interface PreflightDependencies {
+    now?: Date;
+    estimate?: (input: PowerWeeksInput) => PowerWeeksResult | Promise<PowerWeeksResult>;
+}
+/** Read-only inputs shared with Start's pre-transaction checks. Start must re-read
+ * the document and lock-sensitive facts inside its transition (CORE-10b).
+ */
+export declare function loadStartInputs<TUser extends {
+    id: DocId;
+}>(req: PayloadRequest, options: ResolvedOptions<TUser>, experimentId: DocId): Promise<CheckStartInput | null>;
+export declare function preflightData<TUser extends {
+    id: DocId;
+}>(req: PayloadRequest, { experimentId, options }: {
+    experimentId: DocId;
+    options: ResolvedOptions<TUser>;
+}, { now, estimate }?: PreflightDependencies): Promise<PreflightResult | null>;

package/dist/transitions/preflightData.js

@@ -0,0 +1,317 @@
+import { powerMetricWeeks } from '@delmaredigital/experiments-stats';
+import { checkStart, relationshipId } from './checkStart.js';
+import { flagPlacements, scanBlockPlacements } from './placements.js';
+const failureDetail = (cause)=>cause instanceof Error ? cause.message : 'Placement lookup failed.';
+/** Read-only inputs shared with Start's pre-transaction checks. Start must re-read
+ * the document and lock-sensitive facts inside its transition (CORE-10b).
+ */ export async function loadStartInputs(req, options, experimentId) {
+    const found = await req.payload.find({
+        collection: 'xp-experiments',
+        req,
+        overrideAccess: false,
+        depth: 0,
+        limit: 1,
+        where: {
+            id: {
+                equals: experimentId
+            }
+        }
+    });
+    if (!found.docs[0]) return null;
+    const experiment = found.docs[0];
+    const ids = [
+        experiment.metrics?.primary,
+        ...experiment.metrics?.secondary ?? [],
+        ...experiment.metrics?.guardrails ?? []
+    ].map(relationshipId).filter((id)=>id !== undefined);
+    const metrics = [];
+    if (ids.length) {
+        let page = 1;
+        while(true){
+            const result = await req.payload.find({
+                collection: 'xp-metrics',
+                req,
+                overrideAccess: false,
+                depth: 0,
+                limit: 100,
+                page,
+                where: {
+                    id: {
+                        in: ids
+                    }
+                }
+            });
+            metrics.push(...result.docs);
+            if (!result.hasNextPage) break;
+            page++;
+        }
+    }
+    const input = {
+        experiment,
+        metrics,
+        metricSources: options.metricSources,
+        flagPoints: options.flagPoints,
+        attributeKeys: options.attributeKeys
+    };
+    if (experiment.kind === 'block') {
+        try {
+            input.placements = await scanBlockPlacements(req, {
+                slug: experiment.slug,
+                puckCollections: options.puck?.collections ?? []
+            });
+        } catch (cause) {
+            input.placementError = failureDetail(cause);
+        }
+    }
+    if (experiment.kind === 'flag') {
+        const point = options.flagPoints?.find((point)=>point.id === experiment.flagPoint);
+        if (point) {
+            // Match the save hook: the control always uses effective point defaults.
+            let params = point.defaultParams;
+            try {
+                const state = await req.payload.findGlobal({
+                    slug: 'xp-state',
+                    req,
+                    overrideAccess: false,
+                    depth: 0
+                });
+                const defaults = state.flagDefaults;
+                params = defaults?.[point.id]?.params ?? params;
+            } catch (cause) {
+                const error = cause;
+                if (error?.status !== 403 && error?.name !== 'Forbidden') throw cause;
+            }
+            experiment.arms = experiment.arms.map((arm)=>arm.kind === 'control' ? {
+                    ...arm,
+                    params
+                } : arm);
+        }
+        if (point?.placements) {
+            try {
+                const placements = await flagPlacements(options, experiment, req);
+                if (Array.isArray(placements) && placements.every((p)=>p && typeof p === 'object' && typeof p.arm === 'string')) input.flagPlacements = placements;
+                else input.flagPlacementError = 'Placement lookup returned an invalid result.';
+            } catch (cause) {
+                input.flagPlacementError = failureDetail(cause);
+            }
+        }
+        if (experiment.flagPoint) {
+            // At most one row can be this experiment; two rows suffice to find a peer.
+            const busy = await req.payload.find({
+                collection: 'xp-experiments',
+                req,
+                overrideAccess: false,
+                depth: 0,
+                limit: 2,
+                where: {
+                    and: [
+                        {
+                            flagPoint: {
+                                equals: experiment.flagPoint
+                            }
+                        },
+                        {
+                            status: {
+                                in: [
+                                    'running',
+                                    'decided'
+                                ]
+                            }
+                        }
+                    ]
+                }
+            });
+            input.flagPointBusy = busy.docs.some((doc)=>String(doc.id) !== String(experiment.id));
+        }
+    }
+    const layerId = relationshipId(experiment.layer);
+    if (layerId !== undefined) {
+        const layers = await req.payload.find({
+            collection: 'xp-layers',
+            req,
+            overrideAccess: false,
+            depth: 0,
+            limit: 1,
+            where: {
+                id: {
+                    equals: layerId
+                }
+            }
+        });
+        const layer = layers.docs[0];
+        const usedSlices = [];
+        let page = 1;
+        while(true){
+            const result = await req.payload.find({
+                collection: 'xp-experiments',
+                req,
+                overrideAccess: false,
+                depth: 0,
+                limit: 100,
+                page,
+                where: {
+                    and: [
+                        {
+                            layer: {
+                                equals: layerId
+                            }
+                        },
+                        {
+                            status: {
+                                in: [
+                                    'running',
+                                    'decided'
+                                ]
+                            }
+                        }
+                    ]
+                }
+            });
+            for (const doc of result.docs)if (Array.isArray(doc.layerSlices)) usedSlices.push(...doc.layerSlices);
+            if (!result.hasNextPage) break;
+            page++;
+        }
+        input.layer = layer ? {
+            unitType: layer.unitType,
+            usedSlices
+        } : null;
+    }
+    return input;
+}
+async function primaryHistory(source, now) {
+    if (!source) return null;
+    const from = new Date(now.getTime() - 28 * 86400000);
+    try {
+        const rows = await source.query({
+            from,
+            to: now
+        });
+        const units = new Map();
+        const dailyUnits = new Set();
+        for (const row of rows){
+            if (!Number.isFinite(row.value) || !Number.isFinite(new Date(row.ts).getTime())) return null;
+            if (new Date(row.ts) < from || new Date(row.ts) >= now) continue;
+            units.set(row.unitId, (units.get(row.unitId) ?? 0) + row.value);
+            dailyUnits.add(JSON.stringify([
+                new Date(row.ts).toISOString().slice(0, 10),
+                row.unitId
+            ]));
+        }
+        return {
+            values: [
+                ...units.values()
+            ],
+            dailyConverters: dailyUnits.size
+        };
+    } catch  {
+        return null;
+    }
+}
+export async function preflightData(req, { experimentId, options }, { now = new Date(), estimate = powerMetricWeeks } = {}) {
+    const input = await loadStartInputs(req, options, experimentId);
+    if (!input) return null;
+    const { experiment: e } = input;
+    const primaryId = relationshipId(e.metrics?.primary);
+    const metric = input.metrics.find((metric)=>String(metric.id) === String(primaryId));
+    const source = options.metricSources?.find((source)=>source.id === metric?.source);
+    const historical = await primaryHistory(source, now);
+    const history = historical?.values ?? null;
+    const reach = e.analysis?.reachPerDay;
+    const trafficPerDay = typeof reach === 'number' && Number.isFinite(reach) && reach > 0 ? reach : null;
+    const value = history === null ? null : source?.kind === 'binomial' ? trafficPerDay === null ? null : historical.dailyConverters / 28 / trafficPerDay : history.length ? history.reduce((sum, value)=>sum + value, 0) / history.length : 0;
+    // A rate of 1 or more means reachPerDay is below the converters; no honest baseline then.
+    const baseline = primaryId === undefined || value === null || source?.kind === 'binomial' && value >= 1 ? null : {
+        metricId: primaryId,
+        value,
+        days: 28
+    };
+    let power = null;
+    if (trafficPerDay !== null && baseline && source && history) {
+        const compared = e.arms.filter((arm)=>arm.kind !== 'holdout');
+        const control = compared.find((arm)=>arm.kind === 'control');
+        const variant = compared.find((arm)=>arm.kind === 'variant');
+        const savedDuration = e.analysis?.plannedDurationDays && e.analysis.plannedDurationDays > 0 ? e.analysis.plannedDurationDays : undefined;
+        const planned = savedDuration ?? 14;
+        const enrolled = trafficPerDay * e.allocation / 100;
+        const pairShare = ((control?.weight ?? 0) + (variant?.weight ?? 0)) / 100;
+        const tuning = enrolled * planned * pairShare;
+        // Power's equal-arm model uses the smallest compared arm, so unequal weights
+        // and holdout traffic never overstate the information in a comparison.
+        const usersPerWeek = enrolled * 7 * Math.min(...compared.map((arm)=>arm.weight / 100)) * compared.length;
+        const n = Math.max(trafficPerDay * 28, history.length);
+        const mean = history.reduce((sum, value)=>sum + value, 0) / n;
+        const variance = history.reduce((sum, value)=>sum + value * value, 0) / n - mean * mean;
+        const conversionRate = baseline.value;
+        // v1 planning assumption: 10% relative lift at 80% power, alpha fixed at 5%.
+        const params = source.kind === 'binomial' ? {
+            type: 'binomial',
+            conversionRate,
+            effectSize: 0.1
+        } : {
+            type: 'mean',
+            mean,
+            standardDeviation: Math.sqrt(Math.max(0, variance)),
+            effectSize: 0.1
+        };
+        if (control && variant && usersPerWeek > 0 && tuning > 0 && (source.kind !== 'binomial' || conversionRate > 0 && conversionRate < 1) && (source.kind === 'binomial' || mean > 0)) {
+            try {
+                const alpha = e.analysis?.method === 'bayesian' ? 0.05 : 0.05 / Math.max(1, compared.filter((arm)=>arm.kind === 'variant').length);
+                const estimateAt = (sequentialTuningParameter)=>estimate({
+                        metrics: {
+                            primary: params
+                        },
+                        nVariations: compared.length,
+                        nWeeks: 13,
+                        alpha,
+                        usersPerWeek,
+                        targetPower: 0.8,
+                        method: e.analysis?.method === 'bayesian' ? {
+                            type: 'bayesian'
+                        } : {
+                            type: 'frequentist',
+                            sequentialTesting: sequentialTuningParameter
+                        }
+                    });
+                let result = await estimateAt(tuning);
+                let weeks = result.sampleSizeAndRuntime.primary?.weeks ?? result.weekThreshold ?? 13;
+                const plannedDurationDays = savedDuration ?? Math.ceil(weeks * 7);
+                const sequentialTuningParameter = enrolled * plannedDurationDays * pairShare;
+                if (savedDuration === undefined) {
+                    // Re-evaluate using exactly the duration and tuning Review will prefill.
+                    result = await estimateAt(sequentialTuningParameter);
+                    weeks = result.sampleSizeAndRuntime.primary?.weeks ?? result.weekThreshold ?? 13;
+                }
+                power = {
+                    weeks,
+                    plannedDurationDays,
+                    sequentialTuningParameter,
+                    feasible: Number.isFinite(weeks) && weeks * 7 <= 90 && result.sampleSizeAndRuntime.primary !== undefined
+                };
+            } catch  {}
+        }
+    }
+    const requiredSlots = e.arms.filter((arm)=>arm.kind !== 'holdout').map((arm)=>arm.slot);
+    const placements = (input.placements ?? []).map((placement)=>({
+            collection: placement.collection,
+            id: placement.id,
+            title: placement.title,
+            ok: requiredSlots.every((slot)=>slot && placement.slots[slot] > 0),
+            detail: placement.published ? 'Published' : 'Draft'
+        }));
+    for (const placement of input.flagPlacements ?? [])placements.push({
+        collection: e.flagPoint ?? '',
+        id: placement.arm,
+        title: placement.arm,
+        ok: placement.ok,
+        ...placement.detail ? {
+            detail: placement.detail
+        } : {}
+    });
+    return {
+        checks: checkStart(input),
+        placements,
+        baseline,
+        trafficPerDay,
+        estimate: power
+    };
+}

package/dist/transitions/releaseHoldout.d.ts

@@ -0,0 +1,8 @@
+import type { PayloadRequest } from 'payload';
+import type { DocId } from '../types/contracts.js';
+export declare function releaseHoldout(req: PayloadRequest, { experimentId }: {
+    experimentId: DocId;
+}): Promise<{
+    doc: import("payload").JsonObject & import("payload").TypeWithID;
+    revision: number;
+}>;

package/dist/transitions/releaseHoldout.js

@@ -0,0 +1,30 @@
+import { relationshipId } from './checkStart.js';
+import { withTransition } from './kernel.js';
+import { readPhaseRecord, setPhaseEndedAt } from './phaseRecord.js';
+import { readExperiment, refuse, updateExperiment } from './start.js';
+export async function releaseHoldout(req, { experimentId }) {
+    const before = await readExperiment(req, experimentId);
+    const { result: doc, revision } = await withTransition(req, {
+        experimentId,
+        layerId: relationshipId(before.layer)
+    }, async ({ req })=>{
+        const current = await readExperiment(req, experimentId);
+        const decision = current.decision;
+        if (current.status !== 'decided' || !decision?.winnerArm || decision.holdoutReleasedAt) refuse('invalid-status');
+        const phase = await readPhaseRecord(req, experimentId, current.phase);
+        if (phase.phaseKind !== 'rollout' || phase.endedAt) refuse('invalid-status');
+        const at = new Date().toISOString();
+        await setPhaseEndedAt(req, experimentId, phase.phase, at);
+        return updateExperiment(req, experimentId, {
+            layerSlices: [],
+            decision: {
+                ...decision,
+                holdoutReleasedAt: at
+            }
+        });
+    });
+    return {
+        doc,
+        revision: revision
+    };
+}

package/dist/transitions/resolveCaps.d.ts

@@ -0,0 +1,28 @@
+import type { DocId, MetricSource } from '../types/contracts.js';
+export interface MetricDefinition extends Record<string, unknown> {
+    id: DocId;
+    source: string;
+    attributionWindowHours?: number | null;
+    minSampleSize?: number | null;
+    guardrailThreshold?: number | null;
+    capping?: {
+        type?: 'percentile' | 'absolute' | null;
+        value?: number | null;
+    } | null;
+}
+/** Resolve revenue caps before taking transition locks. Host history failures
+ * disable capping for that metric and travel back as an explanatory detail.
+ */
+export declare function resolveCaps(metrics: MetricDefinition[], sources: MetricSource[], start: Date): Promise<{
+    definitions: (MetricDefinition & {
+        kind?: MetricSource["kind"];
+        keyedBy?: MetricSource["keyedBy"];
+        resolvedCap: number | null;
+        historyUnits?: number;
+        capNotice?: "no-history";
+    })[];
+    detail: {
+        metricId: DocId;
+        message: string;
+    }[];
+}>;

package/dist/transitions/resolveCaps.js

@@ -0,0 +1,68 @@
+import { resolvePercentileCap } from '@delmaredigital/experiments-stats';
+import { perUnitHistory } from '../results/history.js';
+/** Resolve revenue caps before taking transition locks. Host history failures
+ * disable capping for that metric and travel back as an explanatory detail.
+ */ export async function resolveCaps(metrics, sources, start) {
+    const detail = [];
+    const definitions = [];
+    const from = new Date(start.getTime() - 90 * 86400000);
+    for (const metric of metrics){
+        const source = sources.find((source)=>source.id === metric.source);
+        const kind = metric.source.startsWith('events:') ? 'binomial' : source?.kind;
+        const definition = {
+            ...structuredClone(metric),
+            ...source ? {
+                keyedBy: source.keyedBy
+            } : {},
+            kind,
+            resolvedCap: null,
+            historyUnits: undefined,
+            capNotice: undefined
+        };
+        const cap = metric.capping?.type ? metric.capping : kind === 'revenue' ? {
+            type: 'percentile',
+            value: 0.99
+        } : null;
+        if (kind === 'revenue' || !source && cap && !metric.source.startsWith('events:')) {
+            definition.capping = cap;
+            if (cap?.type === 'absolute' && typeof cap.value === 'number' && Number.isFinite(cap.value) && cap.value >= 0) {
+                definition.resolvedCap = cap.value;
+            } else if (cap?.type === 'percentile') {
+                definition.historyUnits = 0;
+                try {
+                    if (!source) throw new Error('Metric source unavailable');
+                    const rows = (await source.query({
+                        from,
+                        to: start
+                    })).filter((row)=>new Date(row.ts) >= from && new Date(row.ts) < start);
+                    if (rows.some((row)=>!Number.isFinite(row.value))) throw new Error('Invalid metric history');
+                    const history = perUnitHistory(rows, {
+                        windowHours: metric.attributionWindowHours ?? 168
+                    });
+                    definition.historyUnits = history.units;
+                    if (history.units >= 20) {
+                        // The collection permits 1 (the maximum); the stats percentile API requires p < 1.
+                        definition.resolvedCap = cap.value === 1 ? Math.max(...history.values) : resolvePercentileCap(history.values, cap.value ?? 0.99);
+                    } else {
+                        definition.capNotice = 'no-history';
+                        detail.push({
+                            metricId: metric.id,
+                            message: 'Fewer than 20 history units; no cap was applied.'
+                        });
+                    }
+                } catch  {
+                    definition.capNotice = 'no-history';
+                    detail.push({
+                        metricId: metric.id,
+                        message: 'Metric history is unavailable; no cap was applied.'
+                    });
+                }
+            }
+        }
+        definitions.push(definition);
+    }
+    return {
+        definitions,
+        detail
+    };
+}