@delmaredigital/payload-experiments 0.0.0-stage → 0.1.0-rc.6

package/dist/server/runResults.js

@@ -0,0 +1,307 @@
+import { resultsCandidates, resultsExecutor, writeSnapshots } from '../db/snapshots.js';
+import { sweepEvents } from '../db/sweep.js';
+import { armMismatchCount, identityRows, metricEventRows, phaseExposureRows, rolloutExposureRows } from '../db/results.js';
+import { raceDeadline, ResultsBudgetError, RESULTS_TIMEOUT_MARGIN_MS } from '../db/executor.js';
+import { analysePhase } from '../results/analyse.js';
+import { firstExposures, perUnitValues, timestampMs } from '../results/aggregate.js';
+import { rolloutAnalysisUnits } from '../results/rolloutUnits.js';
+import { creditUserEvents } from '../results/identityCredit.js';
+async function analyseCandidate(exec, candidate, options, now, deadline, unavailableMetrics) {
+    const { phase } = candidate;
+    // Rollout assignment preserves the original test ranges, but its SRM compares
+    // only the shipped group against holdout with weights {100-h, h}.
+    const analysisPhase = phase.phaseKind === 'rollout' ? {
+        ...phase,
+        arms: phase.arms.map((arm)=>({
+                ...arm,
+                weight: arm.kind === 'holdout' ? candidate.testArms.at(-1).weight : 100 - candidate.testArms.at(-1).weight
+            }))
+    } : phase;
+    const to = new Date(Math.min(+now, phase.endedAt == null ? Infinity : timestampMs(phase.endedAt)));
+    const query = {
+        experimentId: phase.experiment,
+        phase: phase.phase,
+        endedAt: to
+    };
+    const exposed = firstExposures(await phaseExposureRows(exec, query), {
+        endedAt: to
+    });
+    let units = exposed.units;
+    if (phase.phaseKind === 'rollout') {
+        const eligible = rolloutAnalysisUnits({
+            exposuresAllPhases: await rolloutExposureRows(exec, {
+                ...query,
+                respectPhaseEnds: true
+            }),
+            rolloutPhase: phase.phase,
+            seed: phase.seed,
+            testWeights: candidate.testArms.map((arm)=>arm.weight),
+            coverage: candidate.layerSlug ? 1 : phase.allocation / 100,
+            ...candidate.layerSlug && candidate.layerSlices?.length ? {
+                layer: {
+                    slug: candidate.layerSlug,
+                    slices: candidate.layerSlices
+                }
+            } : {}
+        });
+        units = new Map([
+            ...units
+        ].filter(([id])=>eligible.has(id)));
+    }
+    const ids = [
+        ...units.keys()
+    ];
+    const identities = candidate.unitType === 'visitor' ? await identityRows(exec, {
+        anonymousIds: ids
+    }) : [];
+    const userIds = [
+        ...new Set(identities.map((identity)=>identity.userId))
+    ];
+    const linked = new Set(userIds);
+    const credit = (events, userKeyed)=>{
+        if (candidate.unitType === 'user') return events.map((event)=>({
+                ...event,
+                unitId: event.userId ?? event.unitId
+            }));
+        const userEvents = events.filter((event)=>userKeyed || event.userId != null && linked.has(event.userId)).map((event)=>({
+                userId: String(event.userId ?? event.unitId),
+                value: event.value,
+                ts: event.ts
+            }));
+        const credited = creditUserEvents(units, identities, userEvents).events;
+        return userKeyed ? credited : [
+            ...events.filter((event)=>event.userId == null || !linked.has(event.userId)),
+            ...credited
+        ];
+    };
+    const diagnostics = creditUserEvents(units, identities, []);
+    const metricData = [];
+    for (const definition of phase.metricDefinitions){
+        try {
+            if (Date.now() >= deadline) throw new ResultsBudgetError();
+            const kind = definition.kind;
+            const windowHours = phase.analysis.attributionWindowHours ?? definition.attributionWindowHours ?? 168;
+            const from = new Date([
+                ...units.values()
+            ].reduce((earliest, unit)=>Math.min(earliest, +unit.firstTs), timestampMs(phase.startedAt)));
+            const preFrom = new Date(+from - 14 * 86_400_000);
+            let events, covariates;
+            if (definition.source.startsWith('events:')) {
+                const eventName = definition.source.slice(7);
+                if (!options.eventSources?.some((source)=>source.name === eventName)) throw new Error('Metric source unavailable');
+                const read = async (start)=>{
+                    if (candidate.unitType === 'user') return metricEventRows(exec, {
+                        eventName,
+                        userIds: ids,
+                        from: start,
+                        to
+                    });
+                    const direct = await metricEventRows(exec, {
+                        eventName,
+                        unitIds: ids,
+                        from: start,
+                        to
+                    });
+                    const signedIn = await metricEventRows(exec, {
+                        eventName,
+                        userIds,
+                        from: start,
+                        to
+                    });
+                    return [
+                        ...direct,
+                        ...signedIn
+                    ];
+                };
+                events = credit(await read(phase.analysis.cuped === 'auto' ? preFrom : from), false);
+                if (phase.analysis.cuped === 'auto') {
+                    const preUnits = new Map([
+                        ...units
+                    ].map(([id, unit])=>[
+                            id,
+                            {
+                                arm: unit.arm,
+                                firstTs: new Date(+unit.firstTs - 14 * 86_400_000)
+                            }
+                        ]));
+                    covariates = perUnitValues(preUnits, events, {
+                        kind,
+                        windowHours: 14 * 24
+                    });
+                }
+            } else {
+                const source = options.metricSources?.find((source)=>source.id === definition.source);
+                if (!source || source.kind !== kind || definition.keyedBy && source.keyedBy !== definition.keyedBy) throw new Error('Metric source unavailable');
+                const userKeyed = source.keyedBy === 'user' && candidate.unitType === 'visitor';
+                if (candidate.unitType === 'user' && source.keyedBy !== 'user') throw new Error('Metric unit type unavailable');
+                const sourceIds = userKeyed ? userIds : ids;
+                events = credit(await raceDeadline(deadline, ()=>source.query({
+                        from,
+                        to,
+                        unitIds: sourceIds
+                    })), userKeyed);
+                if (phase.analysis.cuped === 'auto' && source.covariate) {
+                    const sourceUnits = userKeyed ? creditUserEvents(units, identities, userIds.map((userId)=>({
+                            userId,
+                            ts: now,
+                            value: 0
+                        }))).events.map((event)=>({
+                            unitId: identities.find((identity)=>identity.anonymousId === event.unitId).userId,
+                            firstExposureTs: units.get(event.unitId).firstTs
+                        })) : [
+                        ...units
+                    ].map(([unitId, unit])=>({
+                            unitId,
+                            firstExposureTs: unit.firstTs
+                        }));
+                    const pre = await raceDeadline(deadline, ()=>source.covariate({
+                            units: sourceUnits,
+                            lookbackDays: 14
+                        }));
+                    if (pre.some((row)=>!Number.isFinite(row.value))) throw new Error('Covariates must be finite');
+                    const credited = credit(pre.map((row)=>({
+                            ...row,
+                            ts: now
+                        })), userKeyed);
+                    covariates = new Map(credited.map((row)=>[
+                            row.unitId,
+                            row.value
+                        ]));
+                }
+            }
+            if (events.some((event)=>!Number.isFinite(event.value))) throw new Error('Metric values must be finite');
+            metricData.push({
+                metricId: definition.metricId,
+                values: perUnitValues(units, events, {
+                    kind,
+                    windowHours,
+                    endedAt: to
+                }),
+                ...covariates ? {
+                    covariates
+                } : {}
+            });
+        } catch (error) {
+            const reason = error instanceof Error ? error.message : 'Metric source unavailable';
+            metricData.push({
+                metricId: definition.metricId,
+                values: new Map(),
+                unavailable: reason
+            });
+            if (unavailableMetrics.length < 100) unavailableMetrics.push({
+                experimentId: phase.experiment,
+                phase: phase.phase,
+                metricId: definition.metricId,
+                reason
+            });
+            if (error instanceof ResultsBudgetError || Date.now() >= deadline) throw new ResultsBudgetError();
+        }
+    }
+    const status = phase.endedAt != null || candidate.status === 'archived' ? 'stopped' : candidate.status;
+    const snapshot = analysePhase({
+        phase: analysisPhase,
+        now,
+        units,
+        metricData,
+        status,
+        excludedMultiArm: exposed.excludedMultiArm,
+        armMismatch: await armMismatchCount(exec, query),
+        identityConflictShare: userIds.length ? diagnostics.conflicts / userIds.length : 0,
+        ...phase.phaseKind === 'rollout' ? {
+            winnerArm: phase.arms.find((arm)=>arm.kind !== 'holdout').key
+        } : {}
+    });
+    return snapshot;
+}
+export async function runResults(payload, { budgetMs = 20_000, experimentId, sweep = true } = {}) {
+    if (!Number.isFinite(budgetMs) || budgetMs < 0) throw new RangeError('budgetMs must be finite and nonnegative');
+    const started = Date.now(), deadline = started + budgetMs, now = new Date(started);
+    const summary = {
+        ran: 0,
+        skipped: 0,
+        swept: 0,
+        processedIds: [],
+        budgetExhausted: false,
+        unavailableMetrics: [],
+        sweepAttempted: false
+    };
+    if (budgetMs === 0) return {
+        ...summary,
+        budgetExhausted: true
+    };
+    const options = payload.config.custom?.xp;
+    const retentionDays = options.events?.retentionDays ?? 180;
+    // Reserve up to one second (25% of short runs) for retention, even when host
+    // code never settles. The safety margin also leaves time to return the summary.
+    const analysisDeadline = deadline - (sweep ? Math.min(1000, budgetMs / 4) : 0) - RESULTS_TIMEOUT_MARGIN_MS;
+    const exec = resultsExecutor(payload, ()=>analysisDeadline - Date.now());
+    let candidates = [];
+    try {
+        candidates = await resultsCandidates(exec, {
+            cutoff: new Date(started - retentionDays * 86_400_000).toISOString(),
+            experimentId
+        });
+    } catch (error) {
+        summary.skipped++;
+        summary.budgetExhausted = error instanceof ResultsBudgetError;
+        payload.logger.error({
+            err: error,
+            experimentId
+        }, 'Experiment scheduling failed');
+    }
+    // Query order is oldest phase snapshot first; grouping keeps each experiment's
+    // phases together while preserving that scheduling priority.
+    const experiments = new Map();
+    for (const candidate of candidates){
+        const id = candidate.phase.experiment;
+        experiments.set(id, [
+            ...experiments.get(id) ?? [],
+            candidate
+        ]);
+    }
+    for (const [id, phases] of experiments){
+        if (Date.now() >= analysisDeadline) {
+            summary.budgetExhausted = true;
+            summary.skipped += experiments.size - summary.ran - summary.skipped;
+            break;
+        }
+        try {
+            const snapshots = [];
+            for (const phase of phases){
+                if (Date.now() >= analysisDeadline) throw new ResultsBudgetError();
+                snapshots.push(await analyseCandidate(exec, phase, options, now, analysisDeadline, summary.unavailableMetrics));
+            }
+            if (Date.now() >= analysisDeadline) throw new ResultsBudgetError();
+            await writeSnapshots(exec, snapshots);
+            summary.ran++;
+            if (summary.processedIds.length < 100) summary.processedIds.push(id);
+        } catch (error) {
+            summary.skipped++;
+            summary.budgetExhausted ||= error instanceof ResultsBudgetError;
+            payload.logger.error({
+                err: error,
+                experimentId: id
+            }, 'Experiment results failed');
+        }
+    }
+    if (sweep) {
+        summary.sweepAttempted = true;
+        const sweepDeadline = deadline - RESULTS_TIMEOUT_MARGIN_MS;
+        try {
+            const sweepExec = resultsExecutor(payload, ()=>sweepDeadline - Date.now());
+            summary.swept = await sweepEvents(sweepExec, {
+                now,
+                retentionDays,
+                shouldContinue: ()=>Date.now() < sweepDeadline,
+                logger: payload.logger
+            });
+        } catch (error) {
+            summary.budgetExhausted ||= error instanceof ResultsBudgetError;
+            payload.logger.error({
+                err: error
+            }, 'Experiment retention sweep failed');
+        }
+    }
+    summary.budgetExhausted ||= Date.now() >= deadline;
+    return summary;
+}

package/dist/server/serverRuntime.d.ts

@@ -0,0 +1,7 @@
+import type { ExperimentsProxyRuntime, ExperimentsServerRuntime, ExperimentsServerRuntimeOptions } from '../types/runtime.js';
+export declare function hasHostAttributesOf(runtime: object): boolean;
+export declare function defineExperimentsServerRuntime<TUser extends {
+    id: string | number;
+} = {
+    id: string | number;
+}>(proxyRuntime: ExperimentsProxyRuntime, opts: ExperimentsServerRuntimeOptions<TUser>): ExperimentsServerRuntime<TUser>;

package/dist/server/serverRuntime.js

@@ -0,0 +1,20 @@
+const hostAttributesOf = Symbol('hostAttributesOf');
+export function hasHostAttributesOf(runtime) {
+    return Object.getOwnPropertyDescriptor(runtime, hostAttributesOf)?.value === true;
+}
+export function defineExperimentsServerRuntime(proxyRuntime, opts) {
+    const runtime = {
+        proxyRuntime,
+        getPayload: opts.getPayload,
+        getUser: opts.getUser,
+        userIdOf: opts.userIdOf ?? ((u)=>String(u.id)),
+        attributesOf: opts.attributesOf ?? ((u)=>({
+                loggedIn: Boolean(u)
+            }))
+    };
+    if (typeof opts.attributesOf === 'function') Object.defineProperty(runtime, hostAttributesOf, {
+        value: true,
+        enumerable: false
+    });
+    return runtime;
+}

package/dist/server/sitemap.d.ts

@@ -0,0 +1,3 @@
+import type { Manifest } from '../types/manifest.js';
+/** Build-time filter: pass a server-loaded manifest when generating the sitemap. */
+export declare function experimentSitemapFilter(path: string, manifest: Manifest): boolean;

package/dist/server/sitemap.js

@@ -0,0 +1,6 @@
+import { findNormalisedPathKey, normalisePath } from '../runtime/normalisePath.js';
+/** Build-time filter: pass a server-loaded manifest when generating the sitemap. */ export function experimentSitemapFilter(path, manifest) {
+    const live = manifest.experiments.find((e)=>e.kind === 'page' && e.arms.some((a)=>a.pageTarget !== undefined && normalisePath(a.pageTarget) === normalisePath(path)));
+    if (live) return live.arms.some((a)=>a.kind === 'control' && a.pageTarget !== undefined && normalisePath(a.pageTarget) === normalisePath(path));
+    return findNormalisedPathKey(manifest.archivedPages, path) === undefined;
+}

package/dist/server/toClientAssignments.d.ts

@@ -0,0 +1,7 @@
+import type { Assignments, ClientAssignments } from '../types/assignments.js';
+export declare function toClientAssignments(assignments: Assignments, opts: {
+    pathname: string;
+    apiRoute?: string;
+    flagPoints?: string[];
+    experiments?: string[];
+}): ClientAssignments;

package/dist/server/toClientAssignments.js

@@ -0,0 +1,56 @@
+import { normalisePath } from '../runtime/normalisePath.js';
+import { manifestOf, suppressExposure } from './context.js';
+export function toClientAssignments(assignments, opts) {
+    const blocked = suppressExposure(assignments);
+    const manifest = manifestOf(assignments);
+    const experiments = Object.fromEntries((opts.experiments ?? []).filter((slug)=>Object.hasOwn(assignments.bySlug, slug)).map((slug)=>{
+        const a = assignments.bySlug[slug];
+        const experiment = manifest?.experiments.find((e)=>e.slug === slug);
+        const arm = experiment?.arms.find((arm)=>arm.key === a.arm && arm.kind !== 'holdout') ?? experiment?.arms.find((arm)=>arm.kind === 'control');
+        return [
+            slug,
+            {
+                arm: a.arm,
+                experimentId: a.experimentId,
+                phase: a.phase,
+                params: {
+                    ...arm?.params
+                },
+                policy: a.policy,
+                exposable: a.exposable && !blocked
+            }
+        ];
+    }));
+    const flagPoints = Object.fromEntries((opts.flagPoints ?? []).filter((id)=>Object.hasOwn(assignments.flagPoints, id)).map((id)=>{
+        const point = assignments.flagPoints[id];
+        return [
+            id,
+            {
+                ...point,
+                params: {
+                    ...point.params
+                },
+                exposable: point.exposable && !blocked
+            }
+        ];
+    }));
+    const result = {
+        v: 1,
+        revision: assignments.revision,
+        apiRoute: opts.apiRoute ?? '/api',
+        experiments,
+        flagPoints,
+        pageviews: []
+    };
+    if (!blocked) for (const e of manifest?.experiments ?? []){
+        const a = assignments.bySlug[e.slug];
+        if (e.kind !== 'page' || !a?.exposable) continue;
+        const served = e.arms.find((arm)=>arm.key === a.arm && arm.kind !== 'holdout') ?? e.arms.find((arm)=>arm.kind === 'control');
+        if (served?.pageTarget !== undefined && normalisePath(served.pageTarget) === normalisePath(opts.pathname)) result.pageviews.push({
+            slug: e.slug,
+            arm: a.arm,
+            phase: a.phase
+        });
+    }
+    return result;
+}

package/dist/server/trackConversion.d.ts

@@ -0,0 +1,12 @@
+import type { ExperimentsServerRuntime } from '../types/runtime.js';
+export declare function trackConversion<TUser extends {
+    id: string | number;
+} = {
+    id: string | number;
+}>(serverRuntime: ExperimentsServerRuntime<TUser>, event: {
+    eventName: string;
+    value?: number;
+    idempotencyKey: string;
+    userId?: string;
+    headers?: Headers;
+}): Promise<void>;

package/dist/server/trackConversion.js

@@ -0,0 +1,69 @@
+import { NextRequest } from 'next/server.js';
+import { pooledExecutor } from '../db/executor.js';
+import { insertEvents } from '../db/events.js';
+import { readUnitCookie } from '../runtime/cookie.js';
+export async function trackConversion(serverRuntime, event) {
+    let payload;
+    try {
+        let headers = event.headers;
+        if (!headers) {
+            try {
+                headers = await (await import('next/headers.js')).headers();
+            } catch  {}
+        }
+        const runtime = serverRuntime.proxyRuntime;
+        // Consent sees cookies and headers only; the URL is a placeholder.
+        let consent = true;
+        try {
+            consent = runtime.cookie.consent?.(new NextRequest('http://consent.invalid/', {
+                headers
+            })) !== false;
+        } catch  {
+            console.error('[payload-experiments] consent failed; treating consent as false');
+            consent = false;
+        }
+        if (!consent) return;
+        payload = await serverRuntime.getPayload();
+        if (typeof event.idempotencyKey !== 'string' || !event.idempotencyKey.trim()) {
+            try {
+                payload.logger.warn('Experiments: trackConversion requires a non-empty idempotencyKey');
+            } catch  {}
+            return;
+        }
+        const visitorId = headers ? readUnitCookie(headers, runtime) : null;
+        let userId = event.userId;
+        if (!userId && serverRuntime.getUser) {
+            try {
+                const user = await serverRuntime.getUser();
+                if (user) userId = serverRuntime.userIdOf(user);
+            } catch (error) {
+                try {
+                    payload.logger.error({
+                        err: error,
+                        msg: 'Experiments: trackConversion getUser failed'
+                    });
+                } catch  {}
+            }
+        }
+        if (!visitorId && !userId) return;
+        await insertEvents(pooledExecutor(payload), [
+            {
+                type: 'conversion',
+                eventName: event.eventName,
+                value: event.value ?? 1,
+                unitId: visitorId ?? userId,
+                unitType: visitorId ? 'visitor' : 'user',
+                userId,
+                dedupeKey: `conversion:s:${event.idempotencyKey}`,
+                ts: new Date().toISOString()
+            }
+        ], (message)=>payload.logger.error(message));
+    } catch (error) {
+        try {
+            payload?.logger.error({
+                err: error,
+                msg: 'Experiments: trackConversion failed'
+            });
+        } catch  {}
+    }
+}

package/dist/targeting/ops.d.ts

@@ -0,0 +1,8 @@
+export declare const TARGETING_OPS: {
+    readonly string: readonly ["eq", "neq", "in", "nin", "contains"];
+    readonly number: readonly ["eq", "neq", "gt", "gte", "lt", "lte"];
+    readonly boolean: readonly ["eq"];
+};
+export type TargetingOp = (typeof TARGETING_OPS)[keyof typeof TARGETING_OPS][number];
+export declare function isOpValid(type: string, op: string): op is TargetingOp;
+export declare function evaluateOp(type: string, op: string, actual: unknown, expected: unknown): boolean;

package/dist/targeting/ops.js

@@ -0,0 +1,58 @@
+export const TARGETING_OPS = {
+    string: [
+        'eq',
+        'neq',
+        'in',
+        'nin',
+        'contains'
+    ],
+    number: [
+        'eq',
+        'neq',
+        'gt',
+        'gte',
+        'lt',
+        'lte'
+    ],
+    boolean: [
+        'eq'
+    ]
+};
+export function isOpValid(type, op) {
+    return Object.hasOwn(TARGETING_OPS, type) && TARGETING_OPS[type].includes(op);
+}
+export function evaluateOp(type, op, actual, expected) {
+    if (!isOpValid(type, op) || typeof actual !== type) return false;
+    if (type === 'string') {
+        const value = actual.toLowerCase();
+        if (op === 'in' || op === 'nin') {
+            if (!Array.isArray(expected) || !expected.every((v)=>typeof v === 'string')) return false;
+            const found = expected.some((v)=>v.toLowerCase() === value);
+            return op === 'in' ? found : !found;
+        }
+        if (typeof expected !== 'string') return false;
+        const target = expected.toLowerCase();
+        if (op === 'contains') return value.includes(target);
+        return op === 'eq' ? value === target : value !== target;
+    }
+    if (typeof expected !== type) return false;
+    if (type === 'boolean') return actual === expected;
+    if (type !== 'number' || !Number.isFinite(actual) || !Number.isFinite(expected)) return false;
+    const a = actual, b = expected;
+    switch(op){
+        case 'eq':
+            return a === b;
+        case 'neq':
+            return a !== b;
+        case 'gt':
+            return a > b;
+        case 'gte':
+            return a >= b;
+        case 'lt':
+            return a < b;
+        case 'lte':
+            return a <= b;
+        default:
+            return false;
+    }
+}

package/dist/transitions/addMetric.d.ts

@@ -0,0 +1,16 @@
+import type { PayloadRequest } from 'payload';
+import type { DocId, ResolvedOptions } from '../types/contracts.js';
+import type { AddMetricBody } from '../types/endpoints.js';
+export declare function addMetric<TUser extends {
+    id: DocId;
+}>(req: PayloadRequest, { experimentId, metric, role, options }: AddMetricBody & {
+    experimentId: DocId;
+    options: ResolvedOptions<TUser>;
+}): Promise<{
+    detail?: {
+        metricId: DocId;
+        message: string;
+    }[] | undefined;
+    doc: import("payload").JsonObject & import("payload").TypeWithID;
+    revision: number;
+}>;