@cursor/july 0.1.1

package/dist/ab.d.ts

@@ -0,0 +1,290 @@
+/**
+ * Live A/B metrics plug-in for agent-serve.
+ *
+ * Author `agent/ab.ts` (or `agent/ab/<name>.ts`) with {@link defineAB}. At
+ * session creation the engine runs {@link ABConfig.split} and appends durable
+ * `ab.assigned` events; arms are exposed on {@link SessionInfo.abs} for the
+ * agent loop (tools / optional per-variant `instructions`). {@link ABCollector}
+ * folds the event stream into metrics and calls {@link ABConfig.onSample}.
+ * There is no `agent-serve ab` test runner.
+ *
+ * Assignments and foldable metrics live in each session's `events.ndjson`
+ * (survive serve restart). Optional project defaults in `agent/ab.config.ts`
+ * ({@link defineABConfig}) let authors retain the playground fold window and
+ * persist samples / aggregate snapshots wherever they like — same idea as
+ * eval `persistRuns`.
+ *
+ * ```ts
+ * import { defineAB, splitBySessionHash } from "@anysphere/agent-serve/ab";
+ *
+ * export default defineAB({
+ *   name: "concise-instructions",
+ *   variants: {
+ *     control: { label: "Baseline" },
+ *     treatment: { label: "Shorter" },
+ *   },
+ *   split: splitBySessionHash({ holdout: 0.1 }),
+ *   onSample(sample) {
+ *     console.log(sample.variant, sample.metrics);
+ *   },
+ * });
+ * ```
+ */
+import type { ABSnapshot } from "./internal/ab-snapshot.js";
+import type { SessionEvent, SessionInfo } from "./types.js";
+export type { ABSnapshot } from "./internal/ab-snapshot.js";
+export interface ABVariantMeta {
+    label?: string;
+    description?: string;
+    /**
+     * Optional prompt overlay for this arm. Appended to the agent's
+     * instructions for the session (local `AGENTS.md` + cloud first-turn
+     * preamble) after sticky enrollment.
+     */
+    instructions?: string;
+}
+export interface ABSplitContext {
+    agent: {
+        name: string;
+    };
+    channel: {
+        id: string;
+    };
+    /**
+     * Live sessions only — the engine never calls `split` for
+     * `purpose: "eval"` harness sessions.
+     */
+    session: SessionInfo;
+    experiment: string;
+    /** Variant keys from {@link ABConfig.variants}, in declaration order. */
+    variants: string[];
+}
+export interface ABCollectContext {
+    agent: {
+        name: string;
+    };
+    channel: {
+        id: string;
+    };
+    session: SessionInfo;
+    experiment: string;
+    variant: string;
+}
+/** Cumulative per-session metrics for one experiment arm. */
+export interface ABMetrics {
+    turns: number;
+    turnFailures: number;
+    toolCalls: number;
+    toolErrors: number;
+    inputTokens: number;
+    outputTokens: number;
+    cacheReadTokens: number;
+    cacheWriteTokens: number;
+    /** Sum of turn wall times (turn.started → turn.completed/failed). */
+    wallTimeMs: number;
+    /** Custom counters from {@link ABConfig.derive}. */
+    custom: Record<string, number>;
+}
+export type ABSampleReason = "turn.completed" | "turn.failed";
+export interface ABMetricSample {
+    experiment: string;
+    variant: string;
+    variantLabel?: string;
+    sessionId: string;
+    channelId: string;
+    metrics: ABMetrics;
+    reason: ABSampleReason;
+    at: string;
+}
+/** Variant key to enroll, or `null` to leave the session out (sticky). */
+export type ABSplitResult = string | null;
+/**
+ * Sticky enrollment decision. Return a variant key, or `null` to skip.
+ * Prefer helpers ({@link splitBySessionHash}, {@link splitIf}, …).
+ */
+export type ABSplit = (ctx: ABSplitContext) => ABSplitResult | Promise<ABSplitResult>;
+export interface ABSplitOptions {
+    /** Relative weights per variant key (default: equal). */
+    weights?: Record<string, number>;
+    /** Fraction of sessions left out (`0`–`1`, sticky). */
+    holdout?: number;
+    /** Salt so a new rollout reshuffles without renaming the experiment. */
+    salt?: string;
+}
+/** Sticky hash of session id → arm (recommended default). */
+export declare function splitBySessionHash(options?: ABSplitOptions): ABSplit;
+/** Random draw once per new session (then sticky via persistence). */
+export declare function splitByRandom(options?: Omit<ABSplitOptions, "salt">): ABSplit;
+/** Pin every new session to one arm. */
+export declare function splitAlways(variant: string): ABSplit;
+/** Skip every session (pause without deleting the experiment). */
+export declare function splitNone(): ABSplit;
+/** Run `inner` only when `predicate` is true; otherwise skip. */
+export declare function splitIf(predicate: (ctx: ABSplitContext) => boolean | Promise<boolean>, inner: ABSplit): ABSplit;
+export interface ABConfig {
+    /** Experiment id. Defaults to the file stem (`ab/concise.ts` → `concise`). */
+    name?: string;
+    /**
+     * Named arms (at least two). Prefer `control` / `treatment` over `a` / `b`.
+     */
+    variants: Record<string, ABVariantMeta>;
+    /**
+     * Choose an arm at session creation, or return `null` to skip. The engine
+     * appends a durable `ab.assigned` event; the collector folds that stream.
+     */
+    split: ABSplit;
+    /**
+     * Optional custom metric extractors. Return a numeric delta, or
+     * `null`/`undefined` to skip. Called on **every** session event (including
+     * high-frequency `message.appended` deltas) — keep these synchronous and
+     * cheap; do not await I/O.
+     */
+    derive?: Record<string, (event: SessionEvent, ctx: ABCollectContext) => number | null | undefined | Promise<number | null | undefined>>;
+    /**
+     * Called on turn complete/fail for enrolled sessions only. Observe-only;
+     * errors are logged and never fatal.
+     */
+    onSample: (sample: ABMetricSample) => void | Promise<void>;
+}
+export type ABDefinition = ABConfig & {
+    readonly __agentServe: "ab";
+};
+/** Author a live A/B metrics experiment under `agent/ab`. */
+export declare function defineAB(config: ABConfig): ABDefinition;
+/** Resolve {@link ABConfig.split} to a variant key or `null`. */
+export declare function resolveABSplit(split: ABSplit, ctx: ABSplitContext): Promise<ABSplitResult>;
+/** FNV-1a → [0, 1). */
+export declare function hashToUnitInterval(input: string): number;
+export declare function pickWeightedVariant(variantKeys: readonly string[], weights: Record<string, number> | undefined, unitInterval: number): string;
+export declare function emptyABMetrics(): ABMetrics;
+export declare function cloneABMetrics(metrics: ABMetrics): ABMetrics;
+/**
+ * Append enrolled per-variant `instructions` overlays onto the base agent
+ * prompt. Used by the session engine before materializing the harness.
+ */
+export declare function mergeABInstructions(base: string | undefined, abs: Readonly<Record<string, string | null>> | undefined, experiments: ReadonlyArray<{
+    name: string;
+    definition: ABDefinition;
+}>): string | undefined;
+/** Default sessions folded into `GET /v1/abs` / the A/Bs tab. */
+export declare const AB_DEFAULT_MAX_PLAYGROUND_SESSIONS = 200;
+/** Hard ceiling for {@link ABConfigFile.maxPlaygroundSessions}. */
+export declare const AB_MAX_PLAYGROUND_SESSIONS = 10000;
+/** Default snapshot files kept when {@link ABConfigFile.persistSnapshots} is set. */
+export declare const AB_DEFAULT_RETAIN_SNAPSHOTS = 20;
+/** Hard ceiling for {@link ABConfigFile.retainSnapshots}. */
+export declare const AB_MAX_RETAIN_SNAPSHOTS = 500;
+/** Context passed to A/B persistence hooks. */
+export interface ABPersistContext {
+    /** Absolute agent project root (directory that contains `agent/`). */
+    projectRoot: string;
+}
+/**
+ * Durable sink for {@link ABMetricSample} rows (same payload as
+ * {@link ABConfig.onSample}). Session `events.ndjson` remains the source of
+ * truth for assignment + fold; this is for analytics export.
+ */
+export interface ABSamplePersistence {
+    save(sample: ABMetricSample, ctx: ABPersistContext): void | Promise<void>;
+}
+/**
+ * Durable store for playground-style aggregate snapshots (`GET /v1/abs`).
+ * Without this, aggregates are always recomputed from session logs (nothing
+ * is kept after sessions are pruned).
+ *
+ * **Runtime note:** `serve` currently only calls {@link save} (after each
+ * `GET /v1/abs`). {@link load} / {@link delete} are for author tooling or a
+ * custom store; they are not wired into the playground hydrate path.
+ */
+export interface ABSnapshotPersistence {
+    load(ctx: ABPersistContext): ABSnapshot[] | Promise<ABSnapshot[]>;
+    save(snapshot: ABSnapshot, ctx: ABPersistContext): void | Promise<void>;
+    delete?(snapshotId: string, ctx: ABPersistContext): void | Promise<void>;
+}
+/**
+ * Project-wide A/B defaults (`agent/ab.config.ts`).
+ *
+ * **Assignments and foldable metrics** already persist in each session's
+ * `events.ndjson` under `--state-root`. Use this config to:
+ * - cap how many sessions the playground / `GET /v1/abs` folds
+ * - export samples and/or aggregate snapshots to author-chosen storage
+ */
+export interface ABConfigFile {
+    /**
+     * Optional. Max newest sessions folded into **`GET /v1/abs` / the A/Bs
+     * tab** (and into {@link persistSnapshots} payloads). Does not prune
+     * session logs, change assignment, or limit {@link persistSamples}.
+     *
+     * Default {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}; hard-capped at
+     * {@link AB_MAX_PLAYGROUND_SESSIONS}.
+     */
+    maxPlaygroundSessions?: number;
+    /**
+     * Snapshot-file budget when {@link persistSnapshots} is set.
+     * Default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}; hard-capped at
+     * {@link AB_MAX_RETAIN_SNAPSHOTS}.
+     *
+     * When using {@link persistABSnapshotsToDir}, this value controls on-disk
+     * pruning (overriding the helper's `{ retain }` default). Custom stores
+     * should apply the same budget themselves.
+     */
+    retainSnapshots?: number;
+    /**
+     * Optional sink for cumulative metric samples (in addition to each
+     * experiment's {@link ABConfig.onSample}).
+     *
+     * ```ts
+     * persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
+     * ```
+     */
+    persistSamples?: ABSamplePersistence;
+    /**
+     * Optional store for aggregate snapshots written when `GET /v1/abs` runs.
+     *
+     * ```ts
+     * persistSnapshots: persistABSnapshotsToDir(".agent-serve/ab-snapshots"),
+     * ```
+     */
+    persistSnapshots?: ABSnapshotPersistence;
+}
+/**
+ * Defaults for every experiment under `agent/ab*` (`agent/ab.config.ts`).
+ *
+ * Session event logs remain the assignment/metrics source of truth; this
+ * file controls the playground fold window and optional durable exports.
+ */
+export declare function defineABConfig(config: ABConfigFile): ABConfigFile;
+/** Validate {@link ABConfigFile.maxPlaygroundSessions}. */
+export declare function normalizeABMaxPlaygroundSessions(value: unknown): number;
+/**
+ * Effective `GET /v1/abs` fold window. Invalid values fall back to
+ * {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}.
+ */
+export declare function resolveABMaxPlaygroundSessions(value: unknown): number;
+/** Validate {@link ABConfigFile.retainSnapshots}. */
+export declare function normalizeABRetainSnapshots(value: unknown): number;
+/**
+ * Append-only NDJSON {@link ABSamplePersistence} under `dir` (relative to
+ * the agent project root).
+ *
+ * ```ts
+ * // agent/ab.config.ts
+ * export default defineABConfig({
+ *   persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
+ * });
+ * ```
+ */
+export declare function persistABSamplesToDir(dir: string): ABSamplePersistence;
+/**
+ * Filesystem {@link ABSnapshotPersistence}: one JSON file per snapshot under
+ * `dir`, pruned to `retain` files (default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}).
+ *
+ * Prefer setting {@link ABConfigFile.retainSnapshots} in `defineABConfig` —
+ * that value is applied to this helper's prune budget. Pass `{ retain }` here
+ * only when constructing the helper outside `defineABConfig`, or as a default
+ * that `retainSnapshots` may override.
+ */
+export declare function persistABSnapshotsToDir(dir: string, options?: {
+    retain?: number;
+}): ABSnapshotPersistence;
+//# sourceMappingURL=ab.d.ts.map

package/dist/ab.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ab.d.ts","sourceRoot":"","sources":["../src/ab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAKH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAW5D,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE5D,YAAY,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAE5D,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB;;;OAGG;IACH,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,6DAA6D;AAC7D,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,cAAc,GAAG,gBAAgB,GAAG,aAAa,CAAC;AAE9D,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,SAAS,CAAC;IACnB,MAAM,EAAE,cAAc,CAAC;IACvB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,0EAA0E;AAC1E,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,IAAI,CAAC;AAE1C;;;GAGG;AACH,MAAM,MAAM,OAAO,GAAG,CACpB,GAAG,EAAE,cAAc,KAChB,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AAE5C,MAAM,WAAW,cAAc;IAC7B,yDAAyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,6DAA6D;AAC7D,wBAAgB,kBAAkB,CAAC,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CASpE;AAED,sEAAsE;AACtE,wBAAgB,aAAa,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,GAAG,OAAO,CAI7E;AAED,wCAAwC;AACxC,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAEpD;AAED,kEAAkE;AAClE,wBAAgB,SAAS,IAAI,OAAO,CAEnC;AAED,iEAAiE;AACjE,wBAAgB,OAAO,CACrB,SAAS,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,EAC9D,KAAK,EAAE,OAAO,GACb,OAAO,CAET;AAED,MAAM,WAAW,QAAQ;IACvB,8EAA8E;IAC9E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACxC;;;OAGG;IACH,KAAK,EAAE,OAAO,CAAC;IACf;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CACb,MAAM,EACN,CACE,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,gBAAgB,KAClB,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,CACpE,CAAC;IACF;;;OAGG;IACH,QAAQ,EAAE,CAAC,MAAM,EAAE,cAAc,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5D;AAED,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG;IAAE,QAAQ,CAAC,YAAY,EAAE,IAAI,CAAA;CAAE,CAAC;AAEtE,6DAA6D;AAC7D,wBAAgB,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,YAAY,CA8BvD;AAED,iEAAiE;AACjE,wBAAsB,cAAc,CAClC,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,cAAc,GAClB,OAAO,CAAC,aAAa,CAAC,CAWxB;AA2BD,uBAAuB;AACvB,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAOxD;AAED,wBAAgB,mBAAmB,CACjC,WAAW,EAAE,SAAS,MAAM,EAAE,EAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,EAC3C,YAAY,EAAE,MAAM,GACnB,MAAM,CAsBR;AAMD,wBAAgB,cAAc,IAAI,SAAS,CAa1C;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,GAAG,SAAS,CAE5D;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,GAAG,SAAS,EACxD,WAAW,EAAE,aAAa,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,YAAY,CAAA;CAAE,CAAC,GACrE,MAAM,GAAG,SAAS,CAsBpB;AAMD,iEAAiE;AACjE,eAAO,MAAM,kCAAkC,MAAM,CAAC;AAEtD,mEAAmE;AACnE,eAAO,MAAM,0BAA0B,QAAS,CAAC;AAEjD,qFAAqF;AACrF,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAE9C,6DAA6D;AAC7D,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,+CAA+C;AAC/C,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IAClC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,EAAE,gBAAgB,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3E;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,qBAAqB;IACpC,IAAI,CAAC,GAAG,EAAE,gBAAgB,GAAG,UAAU,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAClE,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,gBAAgB,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,MAAM,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,gBAAgB,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAQD;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;;;;OAOG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,qBAAqB,CAAC;CAC1C;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,YAAY,GAAG,YAAY,CA4BjE;AAED,2DAA2D;AAC3D,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAKvE;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASrE;AAED,qDAAqD;AACrD,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAKjE;AA0CD;;;;;;;;;;GAUG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAgBtE;AAED;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CACrC,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5B,qBAAqB,CAwDvB"}

package/dist/ab.js

@@ -0,0 +1,386 @@
+/**
+ * Live A/B metrics plug-in for agent-serve.
+ *
+ * Author `agent/ab.ts` (or `agent/ab/<name>.ts`) with {@link defineAB}. At
+ * session creation the engine runs {@link ABConfig.split} and appends durable
+ * `ab.assigned` events; arms are exposed on {@link SessionInfo.abs} for the
+ * agent loop (tools / optional per-variant `instructions`). {@link ABCollector}
+ * folds the event stream into metrics and calls {@link ABConfig.onSample}.
+ * There is no `agent-serve ab` test runner.
+ *
+ * Assignments and foldable metrics live in each session's `events.ndjson`
+ * (survive serve restart). Optional project defaults in `agent/ab.config.ts`
+ * ({@link defineABConfig}) let authors retain the playground fold window and
+ * persist samples / aggregate snapshots wherever they like — same idea as
+ * eval `persistRuns`.
+ *
+ * ```ts
+ * import { defineAB, splitBySessionHash } from "@anysphere/agent-serve/ab";
+ *
+ * export default defineAB({
+ *   name: "concise-instructions",
+ *   variants: {
+ *     control: { label: "Baseline" },
+ *     treatment: { label: "Shorter" },
+ *   },
+ *   split: splitBySessionHash({ holdout: 0.1 }),
+ *   onSample(sample) {
+ *     console.log(sample.variant, sample.metrics);
+ *   },
+ * });
+ * ```
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { randomUUID } from "node:crypto";
+import { appendFile, mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { normalizePositiveInt } from "./internal/bounded-int.js";
+import { brandDefinition } from "./internal/brand.js";
+import { assertNonEmptyDir, deleteFileInDir, listJsonBasenames, readJsonDir, resolveProjectRelativeDir, writeJsonFile, } from "./internal/json-dir-store.js";
+/** Sticky hash of session id → arm (recommended default). */
+export function splitBySessionHash(options) {
+    assertHoldout(options === null || options === void 0 ? void 0 : options.holdout);
+    return (ctx) => { var _a; return pickArm(ctx.variants, options === null || options === void 0 ? void 0 : options.weights, hashToUnitInterval(`${(_a = options === null || options === void 0 ? void 0 : options.salt) !== null && _a !== void 0 ? _a : ""}\0${ctx.session.id}`), options === null || options === void 0 ? void 0 : options.holdout); };
+}
+/** Random draw once per new session (then sticky via persistence). */
+export function splitByRandom(options) {
+    assertHoldout(options === null || options === void 0 ? void 0 : options.holdout);
+    return (ctx) => pickArm(ctx.variants, options === null || options === void 0 ? void 0 : options.weights, Math.random(), options === null || options === void 0 ? void 0 : options.holdout);
+}
+/** Pin every new session to one arm. */
+export function splitAlways(variant) {
+    return () => variant;
+}
+/** Skip every session (pause without deleting the experiment). */
+export function splitNone() {
+    return () => null;
+}
+/** Run `inner` only when `predicate` is true; otherwise skip. */
+export function splitIf(predicate, inner) {
+    return (ctx) => __awaiter(this, void 0, void 0, function* () { return ((yield predicate(ctx)) ? inner(ctx) : null); });
+}
+/** Author a live A/B metrics experiment under `agent/ab`. */
+export function defineAB(config) {
+    var _a;
+    const keys = Object.keys((_a = config.variants) !== null && _a !== void 0 ? _a : {});
+    if (keys.length < 2) {
+        throw new Error(`defineAB: variants needs at least two arms (got ${keys.length})`);
+    }
+    for (const key of keys) {
+        if (key.length === 0 || key.includes("/") || key.includes("\\")) {
+            throw new Error(`defineAB: invalid variant key ${JSON.stringify(key)} (single segment)`);
+        }
+    }
+    if (typeof config.split !== "function") {
+        throw new Error("defineAB: split must be a function (splitBySessionHash(), splitIf(...), or custom)");
+    }
+    if (typeof config.onSample !== "function") {
+        throw new Error("defineAB: onSample(sample) is required");
+    }
+    if (config.derive !== undefined) {
+        for (const [key, fn] of Object.entries(config.derive)) {
+            if (typeof fn !== "function") {
+                throw new Error(`defineAB: derive.${key} must be a function`);
+            }
+        }
+    }
+    return brandDefinition("ab", config);
+}
+/** Resolve {@link ABConfig.split} to a variant key or `null`. */
+export function resolveABSplit(split, ctx) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const result = yield split(ctx);
+        if (result === null || result === undefined) {
+            return null;
+        }
+        if (typeof result === "string") {
+            return result;
+        }
+        throw new Error(`defineAB: split must return a variant key or null (got ${typeof result})`);
+    });
+}
+function assertHoldout(holdout) {
+    if (holdout === undefined) {
+        return;
+    }
+    if (!Number.isFinite(holdout) || holdout < 0 || holdout > 1) {
+        throw new Error("defineAB: holdout must be a finite number in [0, 1]");
+    }
+}
+/** Pick a variant (or `null` via holdout) from a unit interval in `[0, 1)`. */
+function pickArm(variantKeys, weights, unitInterval, holdout) {
+    const u = clampUnit(unitInterval);
+    const out = holdout !== null && holdout !== void 0 ? holdout : 0;
+    if (out >= 1 || (out > 0 && u < out)) {
+        return null;
+    }
+    const remapped = out > 0 ? (u - out) / (1 - out) : u;
+    return pickWeightedVariant(variantKeys, weights, remapped);
+}
+/** FNV-1a → [0, 1). */
+export function hashToUnitInterval(input) {
+    let h = 2166136261;
+    for (let i = 0; i < input.length; i++) {
+        h ^= input.charCodeAt(i);
+        h = Math.imul(h, 16777619);
+    }
+    return (h >>> 0) / 4294967296;
+}
+export function pickWeightedVariant(variantKeys, weights, unitInterval) {
+    var _a;
+    const u = clampUnit(unitInterval);
+    const rows = [];
+    let total = 0;
+    for (const key of variantKeys) {
+        const weight = (_a = weights === null || weights === void 0 ? void 0 : weights[key]) !== null && _a !== void 0 ? _a : 1;
+        if (weight > 0 && Number.isFinite(weight)) {
+            rows.push({ key, weight });
+            total += weight;
+        }
+    }
+    if (rows.length === 0 || total <= 0) {
+        throw new Error("defineAB: no positive-weight variants to split across");
+    }
+    let cursor = u * total;
+    for (const row of rows) {
+        cursor -= row.weight;
+        if (cursor < 0) {
+            return row.key;
+        }
+    }
+    return rows[rows.length - 1].key;
+}
+function clampUnit(unitInterval) {
+    return Math.min(Math.max(unitInterval, 0), 0.999999999);
+}
+export function emptyABMetrics() {
+    return {
+        turns: 0,
+        turnFailures: 0,
+        toolCalls: 0,
+        toolErrors: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+        wallTimeMs: 0,
+        custom: {},
+    };
+}
+export function cloneABMetrics(metrics) {
+    return Object.assign(Object.assign({}, metrics), { custom: Object.assign({}, metrics.custom) });
+}
+/**
+ * Append enrolled per-variant `instructions` overlays onto the base agent
+ * prompt. Used by the session engine before materializing the harness.
+ */
+export function mergeABInstructions(base, abs, experiments) {
+    var _a;
+    var _b;
+    if (abs === undefined) {
+        return base;
+    }
+    const extras = [];
+    for (const experiment of experiments) {
+        const variant = abs[experiment.name];
+        if (variant === undefined || variant === null) {
+            continue;
+        }
+        const extra = (_a = experiment.definition.variants[variant]) === null || _a === void 0 ? void 0 : _a.instructions;
+        if (typeof extra === "string" && extra.trim() !== "") {
+            extras.push(extra.trim());
+        }
+    }
+    if (extras.length === 0) {
+        return base;
+    }
+    const baseText = (_b = base === null || base === void 0 ? void 0 : base.trim()) !== null && _b !== void 0 ? _b : "";
+    return baseText === ""
+        ? extras.join("\n\n")
+        : `${baseText}\n\n${extras.join("\n\n")}`;
+}
+// ---------------------------------------------------------------------------
+// Project config (`agent/ab.config.ts`) — fold window + optional persistence
+// ---------------------------------------------------------------------------
+/** Default sessions folded into `GET /v1/abs` / the A/Bs tab. */
+export const AB_DEFAULT_MAX_PLAYGROUND_SESSIONS = 200;
+/** Hard ceiling for {@link ABConfigFile.maxPlaygroundSessions}. */
+export const AB_MAX_PLAYGROUND_SESSIONS = 10000;
+/** Default snapshot files kept when {@link ABConfigFile.persistSnapshots} is set. */
+export const AB_DEFAULT_RETAIN_SNAPSHOTS = 20;
+/** Hard ceiling for {@link ABConfigFile.retainSnapshots}. */
+export const AB_MAX_RETAIN_SNAPSHOTS = 500;
+/** Lets {@link defineABConfig} push `retainSnapshots` into {@link persistABSnapshotsToDir}. */
+const abSnapshotRetainState = new WeakMap();
+/**
+ * Defaults for every experiment under `agent/ab*` (`agent/ab.config.ts`).
+ *
+ * Session event logs remain the assignment/metrics source of truth; this
+ * file controls the playground fold window and optional durable exports.
+ */
+export function defineABConfig(config) {
+    var _a, _b;
+    const persistSamples = normalizeSamplePersistence(config.persistSamples);
+    const persistSnapshots = normalizeSnapshotPersistence(config.persistSnapshots);
+    const helperRetain = persistSnapshots !== undefined
+        ? abSnapshotRetainState.get(persistSnapshots)
+        : undefined;
+    let retainSnapshots;
+    if (config.retainSnapshots !== undefined) {
+        retainSnapshots = normalizeABRetainSnapshots(config.retainSnapshots);
+        if (helperRetain !== undefined) {
+            helperRetain.retain = retainSnapshots;
+        }
+    }
+    else if (persistSnapshots !== undefined) {
+        retainSnapshots = (_a = helperRetain === null || helperRetain === void 0 ? void 0 : helperRetain.retain) !== null && _a !== void 0 ? _a : normalizeABRetainSnapshots(AB_DEFAULT_RETAIN_SNAPSHOTS);
+    }
+    return Object.assign(Object.assign(Object.assign({ maxPlaygroundSessions: normalizeABMaxPlaygroundSessions((_b = config.maxPlaygroundSessions) !== null && _b !== void 0 ? _b : AB_DEFAULT_MAX_PLAYGROUND_SESSIONS) }, (retainSnapshots === undefined ? {} : { retainSnapshots })), (persistSamples === undefined ? {} : { persistSamples })), (persistSnapshots === undefined ? {} : { persistSnapshots }));
+}
+/** Validate {@link ABConfigFile.maxPlaygroundSessions}. */
+export function normalizeABMaxPlaygroundSessions(value) {
+    return normalizePositiveInt(value, {
+        label: "ab.config.ts: maxPlaygroundSessions",
+        max: AB_MAX_PLAYGROUND_SESSIONS,
+    });
+}
+/**
+ * Effective `GET /v1/abs` fold window. Invalid values fall back to
+ * {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}.
+ */
+export function resolveABMaxPlaygroundSessions(value) {
+    if (value === undefined) {
+        return AB_DEFAULT_MAX_PLAYGROUND_SESSIONS;
+    }
+    try {
+        return normalizeABMaxPlaygroundSessions(value);
+    }
+    catch (_a) {
+        return AB_DEFAULT_MAX_PLAYGROUND_SESSIONS;
+    }
+}
+/** Validate {@link ABConfigFile.retainSnapshots}. */
+export function normalizeABRetainSnapshots(value) {
+    return normalizePositiveInt(value, {
+        label: "ab.config.ts: retainSnapshots",
+        max: AB_MAX_RETAIN_SNAPSHOTS,
+    });
+}
+function normalizeSamplePersistence(value) {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (typeof value !== "object" ||
+        value === null ||
+        typeof value.save !== "function") {
+        throw new Error("ab.config.ts: persistSamples must define save(sample, ctx)");
+    }
+    return value;
+}
+function normalizeSnapshotPersistence(value) {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (typeof value !== "object" ||
+        value === null ||
+        typeof value.load !== "function" ||
+        typeof value.save !== "function") {
+        throw new Error("ab.config.ts: persistSnapshots must define load(ctx) and save(snapshot, ctx)");
+    }
+    if (value.delete !== undefined && typeof value.delete !== "function") {
+        throw new Error("ab.config.ts: persistSnapshots.delete must be a function");
+    }
+    return value;
+}
+/**
+ * Append-only NDJSON {@link ABSamplePersistence} under `dir` (relative to
+ * the agent project root).
+ *
+ * ```ts
+ * // agent/ab.config.ts
+ * export default defineABConfig({
+ *   persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
+ * });
+ * ```
+ */
+export function persistABSamplesToDir(dir) {
+    const relativeDir = assertNonEmptyDir(dir, "persistABSamplesToDir");
+    const resolveDir = (projectRoot) => resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
+    return {
+        save(sample, ctx) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const root = resolveDir(ctx.projectRoot);
+                yield mkdir(root, { recursive: true });
+                yield appendFile(join(root, "samples.ndjson"), `${JSON.stringify(sample)}\n`, "utf8");
+            });
+        },
+    };
+}
+/**
+ * Filesystem {@link ABSnapshotPersistence}: one JSON file per snapshot under
+ * `dir`, pruned to `retain` files (default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}).
+ *
+ * Prefer setting {@link ABConfigFile.retainSnapshots} in `defineABConfig` —
+ * that value is applied to this helper's prune budget. Pass `{ retain }` here
+ * only when constructing the helper outside `defineABConfig`, or as a default
+ * that `retainSnapshots` may override.
+ */
+export function persistABSnapshotsToDir(dir, options) {
+    var _a;
+    const relativeDir = assertNonEmptyDir(dir, "persistABSnapshotsToDir");
+    const retainState = {
+        retain: normalizeABRetainSnapshots((_a = options === null || options === void 0 ? void 0 : options.retain) !== null && _a !== void 0 ? _a : AB_DEFAULT_RETAIN_SNAPSHOTS),
+    };
+    const resolveDir = (projectRoot) => resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
+    let saveChain = Promise.resolve();
+    const persist = {
+        load(ctx) {
+            return __awaiter(this, void 0, void 0, function* () {
+                return readJsonDir(resolveDir(ctx.projectRoot), {
+                    sortBasenames: "desc",
+                });
+            });
+        },
+        save(snapshot, ctx) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const run = () => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    var _b;
+                    const root = resolveDir(ctx.projectRoot);
+                    const snapshotId = (_b = (_a = snapshot.config) === null || _a === void 0 ? void 0 : _a.snapshotId) !== null && _b !== void 0 ? _b : `absnap_${randomUUID().replace(/-/g, "")}`;
+                    const toWrite = Object.assign(Object.assign({}, snapshot), { config: Object.assign(Object.assign({}, snapshot.config), { snapshotId }) });
+                    yield writeJsonFile({
+                        root,
+                        fileName: `${snapshotId}.json`,
+                        value: toWrite,
+                    });
+                    const names = (yield listJsonBasenames(root)).sort().reverse();
+                    for (const stale of names.slice(retainState.retain)) {
+                        yield deleteFileInDir({ root, fileName: stale });
+                    }
+                });
+                const next = saveChain.then(run, run);
+                saveChain = next.then(() => undefined, () => undefined);
+                yield next;
+            });
+        },
+        delete(snapshotId, ctx) {
+            return __awaiter(this, void 0, void 0, function* () {
+                yield deleteFileInDir({
+                    root: resolveDir(ctx.projectRoot),
+                    fileName: `${snapshotId}.json`,
+                });
+            });
+        },
+    };
+    abSnapshotRetainState.set(persist, retainState);
+    return persist;
+}