@delmaredigital/payload-experiments 0.0.0-stage → 0.1.0-rc.3

package/dist/transitions/start.d.ts

@@ -0,0 +1,26 @@
+import type { PayloadRequest } from 'payload';
+import type { DocId, ResolvedOptions } from '../types/contracts.js';
+import type { RefusalCode } from '../types/endpoints.js';
+import { type StartExperiment } from './checkStart.js';
+export type TransitionExperiment = StartExperiment & Record<string, unknown> & {
+    phase?: number;
+    seed?: string;
+    updatedAt?: string;
+    layerSlices?: number[] | null;
+};
+export declare function readExperiment(req: PayloadRequest, id: DocId): Promise<TransitionExperiment>;
+export declare function refuse(code: RefusalCode, detail?: unknown): never;
+export declare function updateExperiment(req: PayloadRequest, id: DocId, data: Record<string, unknown>): Promise<import("payload").JsonObject & import("payload").TypeWithID>;
+export declare function start<TUser extends {
+    id: DocId;
+}>(req: PayloadRequest, { experimentId, options }: {
+    experimentId: DocId;
+    options: ResolvedOptions<TUser>;
+}): Promise<{
+    detail?: {
+        metricId: DocId;
+        message: string;
+    }[] | undefined;
+    doc: import("payload").JsonObject & import("payload").TypeWithID;
+    revision: number;
+}>;

package/dist/transitions/start.js

@@ -0,0 +1,220 @@
+import { randomBytes } from 'node:crypto';
+import { claimSlices } from '../db/slices.js';
+import { XpRefusal } from '../errors.js';
+import { isOpValid } from '../targeting/ops.js';
+import { checkStart, relationshipId } from './checkStart.js';
+import { withTransition } from './kernel.js';
+import { writePhaseRecord } from './phaseRecord.js';
+import { loadStartInputs } from './preflightData.js';
+import { resolveCaps } from './resolveCaps.js';
+export async function readExperiment(req, id) {
+    // §7.8: internal lookup serves the authorised transition write. Preserve
+    // admin-only analysis.method/kind for freezing; return access-filtered docs.
+    const { docs } = await req.payload.find({
+        collection: 'xp-experiments',
+        req,
+        overrideAccess: true,
+        depth: 0,
+        limit: 1,
+        where: {
+            id: {
+                equals: id
+            }
+        }
+    });
+    if (!docs[0]) throw new XpRefusal('not-found', 'Experiment not found', undefined, 404);
+    return docs[0];
+}
+export function refuse(code, detail) {
+    const messages = {
+        'invalid-status': 'This action is not available in the current status.',
+        'stale-draft': 'The experiment changed. Review it again before starting.',
+        'flag-point-busy': 'Another experiment is running or decided on this flag point.',
+        'layer-full': 'There is not enough room in this layer.',
+        'allocation-lower': 'Allocation can only increase while running. Stop and start a new round to lower it.',
+        'invalid-body': 'Check the experiment settings and try again.'
+    };
+    const conflict = [
+        'invalid-status',
+        'stale-draft',
+        'flag-point-busy',
+        'layer-full',
+        'allocation-lower'
+    ].includes(code);
+    throw new XpRefusal(code, messages[code] ?? 'The experiment is not ready to start.', detail, conflict ? 409 : 422);
+}
+export async function updateExperiment(req, id, data) {
+    // §7.8: access-authorised transition writes locked plugin fields.
+    await req.payload.update({
+        collection: 'xp-experiments',
+        req,
+        id,
+        data,
+        overrideAccess: true
+    });
+    const { docs } = await req.payload.find({
+        collection: 'xp-experiments',
+        req,
+        overrideAccess: false,
+        depth: 0,
+        limit: 1,
+        where: {
+            id: {
+                equals: id
+            }
+        }
+    });
+    if (!docs[0]) throw new XpRefusal('not-found', 'Experiment not found', undefined, 404);
+    return docs[0];
+}
+export async function start(req, { experimentId, options }) {
+    const input = await loadStartInputs(req, options, experimentId);
+    if (!input) throw new XpRefusal('not-found', 'Experiment not found', undefined, 404);
+    const full = await readExperiment(req, experimentId);
+    if (full.updatedAt !== input.experiment.updatedAt) refuse('stale-draft');
+    // Review reads respect field access; internal writes must preserve hidden config.
+    input.experiment = {
+        ...input.experiment,
+        kind: full.kind,
+        analysis: full.analysis
+    };
+    const failed = checkStart(input).find((check)=>!check.ok);
+    if (failed) refuse(failed.code, failed.detail);
+    const before = input.experiment;
+    const updatedAt = before.updatedAt;
+    const at = new Date();
+    const caps = await resolveCaps(input.metrics, options.metricSources ?? [], at);
+    const { result: doc, revision } = await withTransition(req, {
+        experimentId,
+        layerId: relationshipId(before.layer),
+        flagPointId: before.kind === 'flag' ? before.flagPoint : undefined
+    }, async ({ req, exec })=>{
+        const current = await readExperiment(req, experimentId);
+        if (current.updatedAt !== updatedAt) refuse('stale-draft');
+        if (current.status !== 'draft' && current.status !== 'stopped') refuse('invalid-status');
+        if (current.kind === 'flag' && current.flagPoint) {
+            const busy = await req.payload.find({
+                collection: 'xp-experiments',
+                req,
+                overrideAccess: false,
+                depth: 0,
+                limit: 2,
+                where: {
+                    and: [
+                        {
+                            flagPoint: {
+                                equals: current.flagPoint
+                            }
+                        },
+                        {
+                            status: {
+                                in: [
+                                    'running',
+                                    'decided'
+                                ]
+                            }
+                        }
+                    ]
+                }
+            });
+            if (busy.docs.some((doc)=>String(doc.id) !== String(experimentId))) refuse('flag-point-busy');
+            const point = options.flagPoints.find((point)=>point.id === current.flagPoint);
+            const state = await req.payload.findGlobal({
+                slug: 'xp-state',
+                req,
+                overrideAccess: false,
+                depth: 0
+            });
+            const defaults = state.flagDefaults;
+            current.arms = current.arms.map((arm)=>arm.kind === 'control' ? {
+                    ...arm,
+                    params: defaults?.[point.id]?.params ?? point.defaultParams
+                } : arm);
+            const recheck = checkStart({
+                ...input,
+                experiment: current
+            }).find((check)=>!check.ok);
+            if (recheck) refuse(recheck.code, recheck.detail);
+        }
+        const targeting = (current.targeting ?? []).map((rule)=>{
+            const type = Object.hasOwn(options.attributeKeys ?? {}, rule.attribute) ? options.attributeKeys[rule.attribute].type : undefined;
+            const validValue = type === 'string' && [
+                'in',
+                'nin'
+            ].includes(rule.op) ? Array.isArray(rule.value) && rule.value.length > 0 && rule.value.every((value)=>typeof value === 'string') : typeof rule.value === type && (type !== 'number' || Number.isFinite(rule.value));
+            if (!type) refuse('undeclared-targeting-key', {
+                attribute: rule.attribute
+            });
+            if (!isOpValid(type, rule.op) || !validValue) refuse('invalid-targeting-rule', {
+                attribute: rule.attribute
+            });
+            return {
+                ...rule,
+                type
+            };
+        });
+        const layerId = relationshipId(current.layer);
+        const layerSlices = layerId === undefined ? [] : await claimSlices(exec, {
+            layerId,
+            experimentId,
+            count: current.allocation
+        });
+        const phase = (current.phase ?? 0) + 1;
+        const seed = `${randomBytes(16).toString('hex')}|`;
+        const startedAt = at.toISOString();
+        const definitions = caps.definitions.map((metric)=>({
+                ...metric,
+                metricId: metric.id,
+                addedAt: startedAt,
+                role: String(relationshipId(current.metrics?.primary)) === String(metric.id) ? 'primary' : current.metrics?.guardrails?.some((id)=>String(relationshipId(id)) === String(metric.id)) ? 'guardrail' : 'secondary'
+            }));
+        await writePhaseRecord(req, {
+            experiment: experimentId,
+            phase,
+            phaseKind: 'test',
+            seed,
+            startedAt,
+            arms: current.arms,
+            allocation: current.allocation,
+            allocationChanges: [],
+            targeting,
+            metricDefinitions: definitions,
+            analysis: {
+                ...current.analysis,
+                method: current.analysis?.method ?? 'sequential',
+                alpha: 0.05,
+                cupedLookbackDays: 14,
+                layerSlices,
+                minSampleSizes: Object.fromEntries(definitions.map((metric)=>[
+                        metric.id,
+                        metric.minSampleSize ?? 150
+                    ])),
+                guardrailThresholds: Object.fromEntries(definitions.filter((metric)=>metric.role === 'guardrail').map((metric)=>[
+                        metric.id,
+                        metric.guardrailThreshold ?? 0.05
+                    ])),
+                bayesian: {
+                    ctw: 0.95,
+                    risk: 0.0125
+                }
+            },
+            contentChanges: []
+        });
+        return updateExperiment(req, experimentId, {
+            status: 'running',
+            phase,
+            seed,
+            startedAt,
+            stoppedAt: null,
+            layerSlices,
+            arms: current.arms
+        });
+    });
+    return {
+        doc,
+        revision: revision,
+        ...caps.detail.length ? {
+            detail: caps.detail
+        } : {}
+    };
+}

package/dist/transitions/stop.d.ts

@@ -0,0 +1,8 @@
+import type { PayloadRequest } from 'payload';
+import type { DocId } from '../types/contracts.js';
+export declare function stop(req: PayloadRequest, { experimentId }: {
+    experimentId: DocId;
+}): Promise<{
+    doc: import("payload").JsonObject & import("payload").TypeWithID;
+    revision: number;
+}>;

package/dist/transitions/stop.js

@@ -0,0 +1,25 @@
+import { relationshipId } from './checkStart.js';
+import { withTransition } from './kernel.js';
+import { setPhaseEndedAt } from './phaseRecord.js';
+import { readExperiment, refuse, updateExperiment } from './start.js';
+export async function stop(req, { experimentId }) {
+    const before = await readExperiment(req, experimentId);
+    const { result: doc, revision } = await withTransition(req, {
+        experimentId,
+        layerId: relationshipId(before.layer)
+    }, async ({ req })=>{
+        const current = await readExperiment(req, experimentId);
+        if (current.status !== 'running') refuse('invalid-status');
+        const at = new Date().toISOString();
+        await setPhaseEndedAt(req, experimentId, current.phase, at);
+        return updateExperiment(req, experimentId, {
+            status: 'stopped',
+            layerSlices: [],
+            stoppedAt: at
+        });
+    });
+    return {
+        doc,
+        revision: revision
+    };
+}

package/dist/types/admin.d.ts

@@ -0,0 +1,20 @@
+import type { MetricSource, ParamField } from './contracts.js';
+export type SerializedFlagPoint = {
+    id: string;
+    label: string;
+    params: Record<string, ParamField>;
+    defaultParams: Record<string, unknown>;
+    effectiveDefaults: Record<string, unknown>;
+};
+export type SerializedMetricSource = {
+    id: string;
+    label: string;
+    kind: MetricSource['kind'];
+    keyedBy: MetricSource['keyedBy'];
+    clientWritable: boolean;
+};
+export type SerializedAttributeKeys = Record<string, {
+    type: 'string' | 'number' | 'boolean';
+    values?: string[];
+}>;
+export type EditableField = 'name' | 'hypothesis' | 'arms' | 'targeting' | 'allocation' | 'flagPoint' | 'metrics' | 'analysis' | 'exposurePolicy';

package/dist/types/admin.js

@@ -0,0 +1 @@
+export { };

package/dist/types/assignments.d.ts

@@ -0,0 +1,57 @@
+import type { DocId, ExposurePolicy } from './contracts.js';
+export interface Assignment {
+    slug: string;
+    experimentId: DocId;
+    arm: string;
+    phase: number;
+    exposable: boolean;
+    policy: ExposurePolicy;
+    source: 'header' | 'recomputed' | 'preview' | 'fallback';
+}
+export interface Assignments {
+    archivedWinners: Record<string, string>;
+    revision: number;
+    visitorId?: string;
+    userId?: string;
+    bySlug: Record<string, Assignment>;
+    flagPoints: Record<string, FlagPointResult>;
+    staff: boolean;
+    bot: boolean;
+    preview: boolean;
+    sessionUnknown: boolean;
+    method: string;
+}
+export interface FlagPointResult {
+    arm: string;
+    params: Record<string, unknown>;
+    source: 'experiment' | 'default';
+    experiment?: string;
+    experimentId?: DocId;
+    phase?: number;
+    policy?: ExposurePolicy;
+    exposable: boolean;
+}
+export interface ClientAssignments {
+    v: 1;
+    revision: number;
+    apiRoute: string;
+    experiments: Record<string, {
+        arm: string;
+        experimentId: DocId;
+        phase: number;
+        params: Record<string, unknown>;
+        policy: ExposurePolicy;
+        exposable: boolean;
+    }>;
+    flagPoints: Record<string, FlagPointResult>;
+    pageviews: Array<{
+        slug: string;
+        phase: number;
+        arm: string;
+    }>;
+}
+export interface RenderedExposure {
+    slug: string;
+    arm: string;
+    phase: number;
+}

package/dist/types/assignments.js

@@ -0,0 +1 @@
+export { };

package/dist/types/contracts.d.ts

@@ -0,0 +1,133 @@
+import type { PayloadRequest } from 'payload';
+import type { ExperimentsServerRuntime } from './runtime.js';
+import type { ArmSlot } from './puck.js';
+import type { TargetingOp } from '../targeting/ops.js';
+export type DocId = string | number;
+export type Attributes = Record<string, string | number | boolean>;
+export type ExposurePolicy = 'render' | 'viewport' | 'pageview' | 'manual';
+export type ExperimentKind = 'block' | 'page' | 'flag' | 'code';
+export type Status = 'draft' | 'running' | 'stopped' | 'decided' | 'archived';
+export type UnitType = 'visitor' | 'user';
+export type ArmKey = 'control' | 'variantA' | 'variantB' | 'variantC' | 'holdout';
+export type ArmKind = 'control' | 'variant' | 'holdout';
+export interface TargetingRule {
+    attribute: string;
+    op: TargetingOp;
+    value: string | number | boolean | string[];
+}
+export interface Arm {
+    key: ArmKey;
+    label: string;
+    weight: number;
+    kind: ArmKind;
+    slot?: ArmSlot;
+    params?: Record<string, unknown>;
+    pageTarget?: string;
+}
+export interface ParamField {
+    type: 'text' | 'textarea' | 'number' | 'checkbox' | 'select';
+    label: string;
+    required?: boolean;
+    options?: Array<{
+        label: string;
+        value: string;
+    }>;
+    min?: number;
+    max?: number;
+}
+export interface MetricSource {
+    id: string;
+    label: string;
+    kind: 'binomial' | 'count' | 'revenue';
+    keyedBy: UnitType;
+    clientWritable?: boolean;
+    query(args: {
+        from: Date;
+        to: Date;
+        unitIds?: string[];
+    }): Promise<Array<{
+        unitId: string;
+        value: number;
+        ts: Date;
+    }>>;
+    covariate?(args: {
+        units: Array<{
+            unitId: string;
+            firstExposureTs: Date;
+        }>;
+        lookbackDays: number;
+    }): Promise<Array<{
+        unitId: string;
+        value: number;
+    }>>;
+}
+export interface FlagPoint {
+    id: string;
+    label: string;
+    params: Record<string, ParamField>;
+    defaultParams: Record<string, unknown>;
+    placements?(args: {
+        experimentId: DocId;
+        arms: string[];
+        req: PayloadRequest;
+    }): Promise<Array<{
+        arm: string;
+        ok: boolean;
+        detail?: string;
+    }>>;
+    makeContent?(args: {
+        experimentId: DocId;
+        winner: {
+            arm: string;
+            params: Record<string, unknown>;
+        };
+        req: PayloadRequest;
+    }): Promise<void>;
+}
+/** Host adapter: true permits the request; false drops it. */
+export type RateLimiter = (args: {
+    req: PayloadRequest;
+    unitId: string;
+}) => boolean | Promise<boolean>;
+export type ExperimentsAccess = (args: {
+    req: PayloadRequest;
+}) => boolean | Promise<boolean>;
+export interface ExperimentsPluginOptions<TUser extends {
+    id: string | number;
+} = {
+    id: string | number;
+}> {
+    runtime: ExperimentsServerRuntime<TUser>;
+    access: {
+        manage: ExperimentsAccess;
+        admin?: ExperimentsAccess;
+        readResults?: ExperimentsAccess;
+        readIdentities?: ExperimentsAccess;
+    };
+    metricSources?: MetricSource[];
+    eventSources?: string[];
+    flagPoints?: FlagPoint[];
+    attributeKeys?: Record<string, {
+        type: 'string' | 'number' | 'boolean';
+        values?: string[];
+    }>;
+    puck?: {
+        enabled: boolean;
+        collections: Array<string | {
+            slug: string;
+            field?: string;
+        }>;
+    };
+    events?: {
+        retentionDays?: number;
+        maxBodyBytes?: number;
+        rateLimit?: RateLimiter;
+        dimensions?: string[];
+    };
+}
+/** Options required by the route table; CORE-02c resolves optional defaults. */
+export type ResolvedOptions<TUser extends {
+    id: string | number;
+} = {
+    id: string | number;
+}> = ExperimentsPluginOptions<TUser>;

package/dist/types/contracts.js

@@ -0,0 +1,2 @@
+/** Options required by the route table; CORE-02c resolves optional defaults. */ // CORE-02c may narrow this alias (pre-authorised exception to the src/types freeze)
+export { };

package/dist/types/endpoints.d.ts

@@ -0,0 +1,135 @@
+import type { ArmKind, DocId, ExposurePolicy, MetricSource, ResolvedOptions } from './contracts.js';
+import type { Snapshot, VerdictCode } from './snapshot.js';
+export type ArmIssueCode = 'weights-total' | 'control-count' | 'control-first' | 'holdout-last' | 'holdout-count' | 'variant-count' | 'duplicate-key' | 'duplicate-slot' | 'slot-required' | 'invalid-key';
+export type RefusalCode = ArmIssueCode | 'not-found' | 'no-planned-duration' | 'no-primary-metric' | 'no-placements' | 'empty-slot' | 'page-no-targets' | 'page-user-unit' | 'flag-no-point' | 'flag-params-invalid' | 'flag-point-busy' | 'flag-placement-missing' | 'layer-full' | 'layer-unit-mismatch' | 'unknown-metric-source' | 'unit-source-mismatch' | 'no-reach-per-day' | 'invalid-status' | 'stale-draft' | 'commit-unverified' | 'allocation-lower' | 'confirmation-required' | 'invalid-winner' | 'slices-taken' | 'locked-placement' | 'not-implemented' | 'forbidden' | 'unauthenticated' | 'body-too-large' | 'invalid-body' | 'too-many-ids' | 'invalid-phase' | 'invalid-ids' | 'recompute-failed' | 'undeclared-targeting-key' | 'invalid-targeting-rule' | 'duplicate-metric';
+export type ErrorEnvelope = {
+    errors: Array<{
+        code: RefusalCode;
+        message: string;
+        detail?: unknown;
+    }>;
+};
+export type TransitionSuccess = {
+    doc: Record<string, unknown>;
+    revision: number;
+};
+export type AllocateBody = {
+    allocation: number;
+};
+export type AddMetricBody = {
+    metric: DocId;
+    role: 'secondary' | 'guardrail';
+};
+export type DecideBody = {
+    winnerArm: string;
+    note?: string;
+    confirmation?: string;
+};
+export type ArchiveBody = {
+    mode: 'stopped' | 'after-content' | 'discard' | 'hard-coded';
+    confirmation?: string;
+};
+export type MakeContentResult = TransitionSuccess & {
+    docs: Array<{
+        collection: string;
+        id: DocId;
+        ok: boolean;
+        error?: string;
+    }>;
+};
+export type PreflightResult = {
+    checks: Array<{
+        code: RefusalCode;
+        ok: boolean;
+        detail?: unknown;
+    }>;
+    placements: Array<{
+        collection: string;
+        id: DocId;
+        title: string;
+        url?: string;
+        ok: boolean;
+        detail?: string;
+    }>;
+    baseline: {
+        metricId: DocId;
+        value: number;
+        days: 28;
+    } | null;
+    trafficPerDay: number | null;
+    estimate: {
+        weeks: number;
+        plannedDurationDays: number;
+        sequentialTuningParameter: number;
+        feasible: boolean;
+    } | null;
+};
+export type PreviewTokenResult = {
+    token: string;
+};
+export interface PhaseSummary {
+    phase: number;
+    phaseKind: 'test' | 'rollout';
+    startedAt: string;
+    endedAt: string | null;
+    arms: Array<{
+        key: string;
+        label: string;
+        kind: ArmKind;
+    }>;
+    metrics: Array<{
+        metricId: DocId;
+        name: string;
+        kind: MetricSource['kind'];
+        role: 'primary' | 'secondary' | 'guardrail';
+        resolvedCap?: number | null;
+        historyUnits?: number;
+        capNotice?: 'no-history';
+    }>;
+}
+export type ResultsResponse = {
+    phases: PhaseSummary[];
+    snapshot: Snapshot | null;
+    decisionSnapshot?: Snapshot;
+    series?: Array<{
+        computedAt: string;
+        metricId: DocId;
+        arm: string;
+        lift: number;
+        ci: [number | null, number | null];
+    }>;
+};
+export type SummaryResponse = Record<DocId, {
+    exposures: number;
+    days: number;
+    verdict: {
+        code: VerdictCode;
+        headline: string;
+    } | null;
+}>;
+export type EventBody = {
+    events: Array<{
+        type: 'exposure';
+        experiment: string;
+        phase: number;
+        arm: string;
+        policy: ExposurePolicy;
+    } | {
+        type: 'conversion';
+        eventName: string;
+        value?: number;
+        idempotencyKey: string;
+        attributes?: Record<string, string | number | boolean>;
+    }>;
+};
+export type AccessKind = 'manage' | 'readResults' | 'manifestKey' | 'signedCookie';
+export type EndpointContext<TUser extends {
+    id: string | number;
+} = {
+    id: string | number;
+}> = {
+    options: ResolvedOptions<TUser>;
+    unit?: {
+        visitorId: string;
+    };
+};

package/dist/types/endpoints.js

@@ -0,0 +1 @@
+export { };

package/dist/types/index.d.ts

@@ -0,0 +1,8 @@
+export * from './contracts.js';
+export * from './runtime.js';
+export * from './manifest.js';
+export * from './assignments.js';
+export * from './puck.js';
+export * from './admin.js';
+export * from './endpoints.js';
+export * from './snapshot.js';

package/dist/types/index.js

@@ -0,0 +1,8 @@
+export * from './contracts.js';
+export * from './runtime.js';
+export * from './manifest.js';
+export * from './assignments.js';
+export * from './puck.js';
+export * from './admin.js';
+export * from './endpoints.js';
+export * from './snapshot.js';