@cursor/july 0.1.108 → 0.1.109

package/src/ab.ts

@@ -1,430 +0,0 @@
-/**
- * Live A/B metrics plug-in for agent-serve.
- *
- * Author `agent/ab.ts` (or `agent/ab/<name>.ts`) with {@link defineAB}. At
- * session creation the engine runs {@link ABConfig.split} and appends durable
- * `ab.assigned` events; arms are exposed on {@link SessionInfo.abs} for the
- * agent loop (tools / optional per-variant `instructions`). {@link ABCollector}
- * folds the event stream into metrics and calls {@link ABConfig.onSample}.
- * There is no `agent-sdk ab` test runner.
- *
- * Assignments and foldable metrics live in each session's `events.ndjson`
- * (survive serve restart). Optional project defaults in `agent/ab.config.ts`
- * ({@link defineABConfig}) cap the playground fold window; durable
- * sample/snapshot export lives on the `defineStorage` `abs` table.
- *
- * ```ts
- * import { defineAB, splitBySessionHash } from "@cursor/july/ab";
- *
- * export default defineAB({
- *   name: "concise-instructions",
- *   variants: {
- *     control: { label: "Baseline" },
- *     treatment: { label: "Shorter" },
- *   },
- *   split: splitBySessionHash({ holdout: 0.1 }),
- *   onSample(sample) {
- *     console.log(sample.variant, sample.metrics);
- *   },
- * });
- * ```
- */
-
-import { normalizePositiveInt } from "./internal/bounded-int.js";
-import { brandDefinition } from "./internal/brand.js";
-import type { SessionEvent, SessionInfo } from "./types.js";
-
-export type { ABSnapshot } from "./internal/ab-snapshot.js";
-
-export interface ABVariantMeta {
-  label?: string;
-  description?: string;
-  /**
-   * Optional prompt overlay for this arm. Appended to the agent's
-   * instructions for the session (local `AGENTS.md` + cloud first-turn
-   * preamble) after sticky enrollment.
-   */
-  instructions?: string;
-}
-
-export interface ABSplitContext {
-  agent: { name: string };
-  channel: { id: string };
-  /**
-   * Live sessions only — the engine never calls `split` for
-   * `purpose: "eval"` harness sessions.
-   */
-  session: SessionInfo;
-  experiment: string;
-  /** Variant keys from {@link ABConfig.variants}, in declaration order. */
-  variants: string[];
-}
-
-export interface ABCollectContext {
-  agent: { name: string };
-  channel: { id: string };
-  session: SessionInfo;
-  experiment: string;
-  variant: string;
-}
-
-/** Cumulative per-session metrics for one experiment arm. */
-export interface ABMetrics {
-  turns: number;
-  turnFailures: number;
-  toolCalls: number;
-  toolErrors: number;
-  inputTokens: number;
-  outputTokens: number;
-  cacheReadTokens: number;
-  cacheWriteTokens: number;
-  /**
-   * Sum of estimated turn costs in USD (the `cost.totalUsd` recorded on
-   * `turn.completed` events). Turns whose model has no known rates
-   * contribute 0.
-   */
-  costUsd: number;
-  /** Sum of turn wall times (turn.started → turn.completed/failed). */
-  wallTimeMs: number;
-  /** Custom counters from {@link ABConfig.derive}. */
-  custom: Record<string, number>;
-}
-
-export type ABSampleReason = "turn.completed" | "turn.failed";
-
-export interface ABMetricSample {
-  experiment: string;
-  variant: string;
-  variantLabel?: string;
-  sessionId: string;
-  channelId: string;
-  metrics: ABMetrics;
-  reason: ABSampleReason;
-  at: string;
-}
-
-/** Variant key to enroll, or `null` to leave the session out (sticky). */
-export type ABSplitResult = string | null;
-
-/**
- * Sticky enrollment decision. Return a variant key, or `null` to skip.
- * Prefer helpers ({@link splitBySessionHash}, {@link splitIf}, …).
- */
-export type ABSplit = (
-  ctx: ABSplitContext
-) => ABSplitResult | Promise<ABSplitResult>;
-
-export interface ABSplitOptions {
-  /** Relative weights per variant key (default: equal). */
-  weights?: Record<string, number>;
-  /** Fraction of sessions left out (`0`–`1`, sticky). */
-  holdout?: number;
-  /** Salt so a new rollout reshuffles without renaming the experiment. */
-  salt?: string;
-}
-
-/** Sticky hash of session id → arm (recommended default). */
-export function splitBySessionHash(options?: ABSplitOptions): ABSplit {
-  assertHoldout(options?.holdout);
-  return (ctx) =>
-    pickArm(
-      ctx.variants,
-      options?.weights,
-      hashToUnitInterval(`${options?.salt ?? ""}\0${ctx.session.id}`),
-      options?.holdout
-    );
-}
-
-/** Random draw once per new session (then sticky via persistence). */
-export function splitByRandom(options?: Omit<ABSplitOptions, "salt">): ABSplit {
-  assertHoldout(options?.holdout);
-  return (ctx) =>
-    pickArm(ctx.variants, options?.weights, Math.random(), options?.holdout);
-}
-
-/** Pin every new session to one arm. */
-export function splitAlways(variant: string): ABSplit {
-  return () => variant;
-}
-
-/** Skip every session (pause without deleting the experiment). */
-export function splitNone(): ABSplit {
-  return () => null;
-}
-
-/** Run `inner` only when `predicate` is true; otherwise skip. */
-export function splitIf(
-  predicate: (ctx: ABSplitContext) => boolean | Promise<boolean>,
-  inner: ABSplit
-): ABSplit {
-  return async (ctx) => ((await predicate(ctx)) ? inner(ctx) : null);
-}
-
-export interface ABConfig {
-  /** Experiment id. Defaults to the file stem (`ab/concise.ts` → `concise`). */
-  name?: string;
-  /**
-   * Named arms (at least two). Prefer `control` / `treatment` over `a` / `b`.
-   */
-  variants: Record<string, ABVariantMeta>;
-  /**
-   * Choose an arm at session creation, or return `null` to skip. The engine
-   * appends a durable `ab.assigned` event; the collector folds that stream.
-   */
-  split: ABSplit;
-  /**
-   * Optional custom metric extractors. Return a numeric delta, or
-   * `null`/`undefined` to skip. Called on **every** session event (including
-   * high-frequency `message.appended` deltas) — keep these synchronous and
-   * cheap; do not await I/O.
-   */
-  derive?: Record<
-    string,
-    (
-      event: SessionEvent,
-      ctx: ABCollectContext
-    ) => number | null | undefined | Promise<number | null | undefined>
-  >;
-  /**
-   * Called on turn complete/fail for enrolled sessions only. Observe-only;
-   * errors are logged and never fatal.
-   */
-  onSample: (sample: ABMetricSample) => void | Promise<void>;
-}
-
-export type ABDefinition = ABConfig & { readonly __agentServe: "ab" };
-
-/** Author a live A/B metrics experiment under `agent/ab`. */
-export function defineAB(config: ABConfig): ABDefinition {
-  const keys = Object.keys(config.variants ?? {});
-  if (keys.length < 2) {
-    throw new Error(
-      `defineAB: variants needs at least two arms (got ${keys.length})`
-    );
-  }
-  for (const key of keys) {
-    if (key.length === 0 || key.includes("/") || key.includes("\\")) {
-      throw new Error(
-        `defineAB: invalid variant key ${JSON.stringify(key)} (single segment)`
-      );
-    }
-  }
-  if (typeof config.split !== "function") {
-    throw new Error(
-      "defineAB: split must be a function (splitBySessionHash(), splitIf(...), or custom)"
-    );
-  }
-  if (typeof config.onSample !== "function") {
-    throw new Error("defineAB: onSample(sample) is required");
-  }
-  if (config.derive !== undefined) {
-    for (const [key, fn] of Object.entries(config.derive)) {
-      if (typeof fn !== "function") {
-        throw new Error(`defineAB: derive.${key} must be a function`);
-      }
-    }
-  }
-  return brandDefinition("ab", config);
-}
-
-/** Resolve {@link ABConfig.split} to a variant key or `null`. */
-export async function resolveABSplit(
-  split: ABSplit,
-  ctx: ABSplitContext
-): Promise<ABSplitResult> {
-  const result = await split(ctx);
-  if (result === null || result === undefined) {
-    return null;
-  }
-  if (typeof result === "string") {
-    return result;
-  }
-  throw new Error(
-    `defineAB: split must return a variant key or null (got ${typeof result})`
-  );
-}
-
-function assertHoldout(holdout: number | undefined): void {
-  if (holdout === undefined) {
-    return;
-  }
-  if (!Number.isFinite(holdout) || holdout < 0 || holdout > 1) {
-    throw new Error("defineAB: holdout must be a finite number in [0, 1]");
-  }
-}
-
-/** Pick a variant (or `null` via holdout) from a unit interval in `[0, 1)`. */
-function pickArm(
-  variantKeys: readonly string[],
-  weights: Record<string, number> | undefined,
-  unitInterval: number,
-  holdout: number | undefined
-): ABSplitResult {
-  const u = clampUnit(unitInterval);
-  const out = holdout ?? 0;
-  if (out >= 1 || (out > 0 && u < out)) {
-    return null;
-  }
-  const remapped = out > 0 ? (u - out) / (1 - out) : u;
-  return pickWeightedVariant(variantKeys, weights, remapped);
-}
-
-/** FNV-1a → [0, 1). */
-export function hashToUnitInterval(input: string): number {
-  let h = 2166136261;
-  for (let i = 0; i < input.length; i++) {
-    h ^= input.charCodeAt(i);
-    h = Math.imul(h, 16777619);
-  }
-  return (h >>> 0) / 0x1_0000_0000;
-}
-
-export function pickWeightedVariant(
-  variantKeys: readonly string[],
-  weights: Record<string, number> | undefined,
-  unitInterval: number
-): string {
-  const u = clampUnit(unitInterval);
-  const rows: Array<{ key: string; weight: number }> = [];
-  let total = 0;
-  for (const key of variantKeys) {
-    const weight = weights?.[key] ?? 1;
-    if (weight > 0 && Number.isFinite(weight)) {
-      rows.push({ key, weight });
-      total += weight;
-    }
-  }
-  if (rows.length === 0 || total <= 0) {
-    throw new Error("defineAB: no positive-weight variants to split across");
-  }
-  let cursor = u * total;
-  for (const row of rows) {
-    cursor -= row.weight;
-    if (cursor < 0) {
-      return row.key;
-    }
-  }
-  return rows[rows.length - 1]!.key;
-}
-
-function clampUnit(unitInterval: number): number {
-  return Math.min(Math.max(unitInterval, 0), 0.999999999);
-}
-
-export function emptyABMetrics(): ABMetrics {
-  return {
-    turns: 0,
-    turnFailures: 0,
-    toolCalls: 0,
-    toolErrors: 0,
-    inputTokens: 0,
-    outputTokens: 0,
-    cacheReadTokens: 0,
-    cacheWriteTokens: 0,
-    costUsd: 0,
-    wallTimeMs: 0,
-    custom: {},
-  };
-}
-
-export function cloneABMetrics(metrics: ABMetrics): ABMetrics {
-  return { ...metrics, custom: { ...metrics.custom } };
-}
-
-/**
- * Append enrolled per-variant `instructions` overlays onto the base agent
- * prompt. Used by the session engine before materializing the harness.
- */
-export function mergeABInstructions(
-  base: string | undefined,
-  abs: Readonly<Record<string, string | null>> | undefined,
-  experiments: ReadonlyArray<{ name: string; definition: ABDefinition }>
-): string | undefined {
-  if (abs === undefined) {
-    return base;
-  }
-  const extras: string[] = [];
-  for (const experiment of experiments) {
-    const variant = abs[experiment.name];
-    if (variant === undefined || variant === null) {
-      continue;
-    }
-    const extra = experiment.definition.variants[variant]?.instructions;
-    if (typeof extra === "string" && extra.trim() !== "") {
-      extras.push(extra.trim());
-    }
-  }
-  if (extras.length === 0) {
-    return base;
-  }
-  const baseText = base?.trim() ?? "";
-  return baseText === ""
-    ? extras.join("\n\n")
-    : `${baseText}\n\n${extras.join("\n\n")}`;
-}
-
-// ---------------------------------------------------------------------------
-// Project config (`agent/ab.config.ts`) — playground fold window
-// ---------------------------------------------------------------------------
-
-/** Default sessions folded into `GET /v1/abs` / the A/Bs tab. */
-export const AB_DEFAULT_MAX_PLAYGROUND_SESSIONS = 200;
-
-/** Hard ceiling for {@link ABConfigFile.maxPlaygroundSessions}. */
-export const AB_MAX_PLAYGROUND_SESSIONS = 10_000;
-
-/**
- * Project-wide A/B defaults (`agent/ab.config.ts`).
- *
- * **Assignments and foldable metrics** already persist in each session's
- * `events.ndjson` under `--state-root`; durable sample/snapshot export
- * lives on the `defineStorage` `abs` table (`agent/storage.ts`). This
- * config only caps how many sessions the playground / `GET /v1/abs` folds.
- */
-export interface ABConfigFile {
-  /**
-   * Optional. Max newest sessions folded into **`GET /v1/abs` / the A/Bs
-   * tab**. Does not prune session logs or change assignment.
-   *
-   * Default {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}; hard-capped at
-   * {@link AB_MAX_PLAYGROUND_SESSIONS}.
-   */
-  maxPlaygroundSessions?: number;
-}
-
-/**
- * Defaults for every experiment under `agent/ab*` (`agent/ab.config.ts`).
- *
- * Session event logs remain the assignment/metrics source of truth;
- * durable exports go through the `defineStorage` `abs` table.
- */
-export function defineABConfig(config: ABConfigFile): ABConfigFile {
-  return {
-    maxPlaygroundSessions: normalizeABMaxPlaygroundSessions(
-      config.maxPlaygroundSessions ?? AB_DEFAULT_MAX_PLAYGROUND_SESSIONS
-    ),
-  };
-}
-
-/** Validate {@link ABConfigFile.maxPlaygroundSessions}. */
-export function normalizeABMaxPlaygroundSessions(value: unknown): number {
-  return normalizePositiveInt(value, {
-    label: "ab.config.ts: maxPlaygroundSessions",
-    max: AB_MAX_PLAYGROUND_SESSIONS,
-  });
-}
-
-/**
- * Effective `GET /v1/abs` fold window. Invalid values fall back to
- * {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}.
- */
-export function resolveABMaxPlaygroundSessions(value: unknown): number {
-  if (value === undefined) {
-    return AB_DEFAULT_MAX_PLAYGROUND_SESSIONS;
-  }
-  try {
-    return normalizeABMaxPlaygroundSessions(value);
-  } catch {
-    return AB_DEFAULT_MAX_PLAYGROUND_SESSIONS;
-  }
-}

package/src/internal/ab-collector.ts

@@ -1,200 +0,0 @@
-/**
- * Stream-native collector for {@link defineAB} experiments.
- *
- * Pure fold over the session event log:
- * - sticky arm = last `ab.assigned` for that experiment
- * - metrics accumulate from the stream; `onSample` on turn complete/fail
- *
- * Enrollment is written by the engine at session creation (not here).
- * In-memory state is a cache: on cold start (or after park eviction) the
- * collector replays `events.ndjson` so counters survive restart.
- *
- * Hydration note: the engine appends before dispatch, so the first-touch
- * replay already includes the current event — after rebuild we return
- * without folding it again (and without re-firing `onSample`).
- */
-
-import {
-  type ABDefinition,
-  type ABMetricSample,
-  type ABSampleReason,
-  cloneABMetrics,
-} from "../ab.js";
-import type { DiscoveredAB, SessionEvent, SessionInfo } from "../types.js";
-import {
-  applyDeriveForEvent,
-  emptyFoldedABSession,
-  ensureArmAccum,
-  type FoldedABSession,
-  foldABEvent,
-} from "./ab-fold.js";
-import { describeError } from "./describe-error.js";
-
-export type { DiscoveredAB };
-
-export interface ABCollectorOptions {
-  /**
-   * Durable sample export (the `defineStorage` `abs` table, via the
-   * coordinator). Fire-and-forget; must never throw.
-   */
-  persistSample?: (sample: ABMetricSample) => void;
-}
-
-/** Drop cache after these — next event rehydrates from the durable log. */
-const EVICT_ON: ReadonlySet<SessionEvent["type"]> = new Set([
-  "session.waiting",
-  "session.completed",
-  "session.failed",
-]);
-
-export class ABCollector {
-  private readonly state = new Map<string, FoldedABSession>();
-  private readonly hydrated = new Set<string>();
-  private readonly persistSample:
-    | ((sample: ABMetricSample) => void)
-    | undefined;
-
-  constructor(
-    private readonly experiments: readonly DiscoveredAB[],
-    private readonly logger: (line: string) => void,
-    private readonly loadEvents: (sessionId: string) => Promise<SessionEvent[]>,
-    options?: ABCollectorOptions
-  ) {
-    this.persistSample = options?.persistSample;
-  }
-
-  get enabled(): boolean {
-    return this.experiments.length > 0;
-  }
-
-  /**
-   * Fold one live event. On first touch (or after eviction), replays the
-   * durable log first so enrollment + metrics match disk.
-   */
-  async onEvent(
-    event: SessionEvent,
-    sessionInfo: SessionInfo,
-    agentName: string
-  ): Promise<void> {
-    if (!this.enabled) {
-      return;
-    }
-
-    if (!this.hydrated.has(event.sessionId)) {
-      const history = await this.loadEvents(event.sessionId);
-      // Rebuild arms + metrics only — do not re-fire onSample for past turns.
-      // History already includes `event` (append-before-dispatch).
-      for (const past of history) {
-        await this.fold(past, sessionInfo, agentName, { emitSamples: false });
-      }
-      this.hydrated.add(event.sessionId);
-      if (EVICT_ON.has(event.type)) {
-        this.drop(event.sessionId);
-      }
-      return;
-    }
-
-    await this.fold(event, sessionInfo, agentName, { emitSamples: true });
-    if (EVICT_ON.has(event.type)) {
-      this.drop(event.sessionId);
-    }
-  }
-
-  private drop(sessionId: string): void {
-    this.state.delete(sessionId);
-    this.hydrated.delete(sessionId);
-  }
-
-  private async fold(
-    event: SessionEvent,
-    sessionInfo: SessionInfo,
-    agentName: string,
-    options: { emitSamples: boolean }
-  ): Promise<void> {
-    const session = this.sessionAccum(event.sessionId);
-
-    if (event.type === "ab.assigned") {
-      foldABEvent(session, event);
-      return;
-    }
-
-    if (session.arms.size === 0) {
-      return;
-    }
-
-    // Built-ins for every enrolled arm (shared fold).
-    foldABEvent(session, event);
-
-    for (const experiment of this.experiments) {
-      const variant = session.arms.get(experiment.name);
-      if (variant === undefined || variant === null) {
-        continue;
-      }
-      const def = experiment.definition as ABDefinition;
-      const accum = ensureArmAccum(session, experiment.name, variant);
-
-      if (def.derive !== undefined) {
-        await applyDeriveForEvent(
-          accum.metrics.custom,
-          def.derive,
-          event,
-          {
-            agent: { name: agentName },
-            channel: { id: sessionInfo.channelId },
-            session: sessionInfo,
-            experiment: experiment.name,
-            variant,
-          },
-          (key, error) => {
-            this.logger(
-              `[agent-sdk] ab "${experiment.name}" derive.${key} threw: ${describeError(error)}`
-            );
-          }
-        );
-      }
-
-      if (!options.emitSamples) {
-        continue;
-      }
-      const reason = sampleReason(event);
-      if (reason !== undefined) {
-        const sample: ABMetricSample = {
-          experiment: experiment.name,
-          variant,
-          ...(def.variants[variant]?.label !== undefined
-            ? { variantLabel: def.variants[variant].label }
-            : {}),
-          sessionId: event.sessionId,
-          channelId: sessionInfo.channelId,
-          metrics: cloneABMetrics(accum.metrics),
-          reason,
-          at: event.at,
-        };
-        try {
-          await def.onSample(sample);
-        } catch (error) {
-          this.logger(
-            `[agent-sdk] ab "${experiment.name}" onSample threw: ${describeError(error)}`
-          );
-        }
-        this.persistSample?.(sample);
-      }
-    }
-  }
-
-  private sessionAccum(sessionId: string): FoldedABSession {
-    let session = this.state.get(sessionId);
-    if (session === undefined) {
-      session = emptyFoldedABSession();
-      this.state.set(sessionId, session);
-    }
-    return session;
-  }
-}
-
-function sampleReason(event: SessionEvent): ABSampleReason | undefined {
-  if (event.type === "turn.completed" || event.type === "turn.failed") {
-    return event.type;
-  }
-  return undefined;
-}