@cursor/july 0.1.107 → 0.1.109

package/src/internal/ab-fold.ts

@@ -1,232 +0,0 @@
-/**
- * Pure fold of a session event stream into A/B arms + cumulative metrics.
- * Shared by {@link ABCollector} (live) and {@link buildABSnapshot} (read API).
- */
-
-import {
-  type ABCollectContext,
-  type ABDefinition,
-  type ABMetrics,
-  cloneABMetrics,
-  emptyABMetrics,
-} from "../ab.js";
-import type { DiscoveredAB, SessionEvent, SessionInfo } from "../types.js";
-import { describeError } from "./describe-error.js";
-
-export interface ABArmAccum {
-  variant: string;
-  metrics: ABMetrics;
-  turnStartedAt: string | null;
-}
-
-export interface FoldedABSession {
-  /** experiment → variant key, or `null` for sticky skip */
-  arms: Map<string, string | null>;
-  /** experiment → metrics (only for enrolled arms) */
-  metrics: Map<string, ABArmAccum>;
-}
-
-export function emptyFoldedABSession(): FoldedABSession {
-  return { arms: new Map(), metrics: new Map() };
-}
-
-/** Ensure a metrics bucket exists for an enrolled arm. */
-export function ensureArmAccum(
-  state: FoldedABSession,
-  experiment: string,
-  variant: string
-): ABArmAccum {
-  let accum = state.metrics.get(experiment);
-  if (accum === undefined) {
-    accum = {
-      variant,
-      metrics: emptyABMetrics(),
-      turnStartedAt: null,
-    };
-    state.metrics.set(experiment, accum);
-  }
-  return accum;
-}
-
-/** Fold one event's built-in metrics + assignment. Does not run `derive`. */
-export function foldABEvent(state: FoldedABSession, event: SessionEvent): void {
-  if (event.type === "ab.assigned") {
-    const { experiment, variant } = event.data;
-    state.arms.set(experiment, variant);
-    if (variant === null) {
-      state.metrics.delete(experiment);
-    } else {
-      state.metrics.set(experiment, {
-        variant,
-        metrics: emptyABMetrics(),
-        turnStartedAt: null,
-      });
-    }
-    return;
-  }
-
-  if (state.arms.size === 0) {
-    return;
-  }
-
-  for (const [experiment, variant] of state.arms) {
-    if (variant === null) {
-      continue;
-    }
-    applyBuiltinMetrics(ensureArmAccum(state, experiment, variant), event);
-  }
-}
-
-export function foldABStream(events: readonly SessionEvent[]): FoldedABSession {
-  const state = emptyFoldedABSession();
-  for (const event of events) {
-    foldABEvent(state, event);
-  }
-  return state;
-}
-
-/**
- * Apply `derive` extractors for one event into `custom` counters.
- * Errors are reported via `onError` and skipped.
- */
-export async function applyDeriveForEvent(
-  custom: Record<string, number>,
-  derive: NonNullable<ABDefinition["derive"]>,
-  event: SessionEvent,
-  collectCtx: ABCollectContext,
-  onError: (key: string, error: unknown) => void
-): Promise<void> {
-  for (const [key, extract] of Object.entries(derive)) {
-    try {
-      const delta = await extract(event, collectCtx);
-      if (typeof delta === "number" && Number.isFinite(delta)) {
-        custom[key] = (custom[key] ?? 0) + delta;
-      }
-    } catch (error) {
-      onError(key, error);
-    }
-  }
-}
-
-function collectContext(
-  experiment: string,
-  variant: string,
-  sessionInfo: SessionInfo,
-  agentName: string
-): ABCollectContext {
-  return {
-    agent: { name: agentName },
-    channel: { id: sessionInfo.channelId },
-    session: sessionInfo,
-    experiment,
-    variant,
-  };
-}
-
-/**
- * Re-run `derive` extractors over the stream for enrolled arms.
- * Mutates `state.metrics` in place. Errors are logged and skipped.
- */
-export async function applyABDerive(
-  state: FoldedABSession,
-  events: readonly SessionEvent[],
-  experiments: readonly DiscoveredAB[],
-  sessionInfo: SessionInfo,
-  agentName: string,
-  logger: (line: string) => void
-): Promise<void> {
-  for (const experiment of experiments) {
-    const variant = state.arms.get(experiment.name);
-    if (variant === undefined || variant === null) {
-      continue;
-    }
-    const def = experiment.definition as ABDefinition;
-    if (def.derive === undefined) {
-      continue;
-    }
-    const accum = ensureArmAccum(state, experiment.name, variant);
-    // Reset custom so a re-fold is idempotent.
-    accum.metrics.custom = {};
-    const ctx = collectContext(
-      experiment.name,
-      variant,
-      sessionInfo,
-      agentName
-    );
-    for (const event of events) {
-      await applyDeriveForEvent(
-        accum.metrics.custom,
-        def.derive,
-        event,
-        ctx,
-        (key, error) => {
-          logger(
-            `[agent-sdk] ab "${experiment.name}" derive.${key} threw: ${describeError(error)}`
-          );
-        }
-      );
-    }
-  }
-}
-
-export function applyBuiltinMetrics(
-  accum: ABArmAccum,
-  event: SessionEvent
-): void {
-  const m = accum.metrics;
-  switch (event.type) {
-    case "turn.started":
-      accum.turnStartedAt = event.at;
-      break;
-    case "turn.completed": {
-      m.turns += 1;
-      const usage = event.data.usage;
-      if (usage !== undefined) {
-        m.inputTokens += usage.inputTokens;
-        m.outputTokens += usage.outputTokens;
-        m.cacheReadTokens += usage.cacheReadTokens;
-        m.cacheWriteTokens += usage.cacheWriteTokens;
-      }
-      const cost = event.data.cost;
-      if (cost !== undefined) {
-        m.costUsd += cost.totalUsd;
-      }
-      addWallTime(accum, event.at);
-      break;
-    }
-    case "turn.failed":
-      m.turns += 1;
-      m.turnFailures += 1;
-      addWallTime(accum, event.at);
-      break;
-    case "action.result":
-      m.toolCalls += 1;
-      if (event.data.isError) {
-        m.toolErrors += 1;
-      }
-      break;
-    default:
-      break;
-  }
-}
-
-function addWallTime(accum: ABArmAccum, endedAt: string): void {
-  if (accum.turnStartedAt === null) {
-    return;
-  }
-  const start = Date.parse(accum.turnStartedAt);
-  const end = Date.parse(endedAt);
-  accum.turnStartedAt = null;
-  if (Number.isFinite(start) && Number.isFinite(end) && end >= start) {
-    accum.metrics.wallTimeMs += end - start;
-  }
-}
-
-export function cloneFoldedMetrics(
-  accum: ABArmAccum | undefined
-): ABMetrics | undefined {
-  if (accum === undefined) {
-    return undefined;
-  }
-  return cloneABMetrics(accum.metrics);
-}

package/src/internal/ab-snapshot.ts

@@ -1,331 +0,0 @@
-/**
- * Read-only A/B metrics snapshot: fold durable session event streams into
- * per-session assignments and aggregate arm totals. Used by `GET /v1/abs`.
- */
-
-import {
-  type ABDefinition,
-  type ABMetrics,
-  type ABVariantMeta,
-  cloneABMetrics,
-  emptyABMetrics,
-} from "../ab.js";
-import type {
-  DiscoveredAB,
-  SessionEvent,
-  SessionInfo,
-  SessionRecord,
-  SessionSummary,
-} from "../types.js";
-import { applyABDerive, cloneFoldedMetrics, foldABStream } from "./ab-fold.js";
-import { mapPool } from "./map-pool.js";
-
-export interface ABSnapshotAssignment {
-  experiment: string;
-  variant: string | null;
-  variantLabel?: string;
-  /** Present when enrolled (`variant` non-null). */
-  metrics?: ABMetrics;
-}
-
-export interface ABSnapshotSession {
-  sessionId: string;
-  channelId: string;
-  mode: string;
-  title?: string;
-  turnCount: number;
-  updatedAt?: string;
-  assignments: ABSnapshotAssignment[];
-}
-
-export interface ABSnapshotArm {
-  variant: string;
-  variantLabel?: string;
-  description?: string;
-  /** Sessions enrolled in this arm. */
-  sessions: number;
-  metrics: ABMetrics;
-}
-
-export interface ABSnapshotExperiment {
-  name: string;
-  variants: Record<string, ABVariantMeta>;
-  /** Sessions with sticky skip (`variant: null`). */
-  skipped: number;
-  arms: ABSnapshotArm[];
-}
-
-export interface ABSnapshotConfig {
-  /** Newest-first session fold window for this snapshot (`GET /v1/abs`). */
-  maxPlaygroundSessions?: number;
-  /** True when the `defineStorage` `abs` table is configured. */
-  durableSamples?: boolean;
-  /** True when the `abs` table also stores the aggregate snapshot. */
-  durableSnapshots?: boolean;
-}
-
-export interface ABSnapshot {
-  experiments: ABSnapshotExperiment[];
-  sessions: ABSnapshotSession[];
-  config?: ABSnapshotConfig;
-}
-
-export interface BuildABSnapshotInput {
-  experiments: readonly DiscoveredAB[];
-  agentName: string;
-  /** Newest-first session list (same order as `GET /v1/sessions`). */
-  sessions: readonly SessionSummary[];
-  loadRecord: (sessionId: string) => Promise<SessionRecord | undefined>;
-  loadEvents: (sessionId: string) => Promise<SessionEvent[]>;
-  sessionInfo: (record: SessionRecord) => SessionInfo;
-  logger: (line: string) => void;
-  /** Max concurrent event-log reads (default 8). */
-  concurrency?: number;
-  /** Attached to the returned snapshot (playground / persistence meta). */
-  config?: ABSnapshotConfig;
-}
-
-export async function buildABSnapshot(
-  input: BuildABSnapshotInput
-): Promise<ABSnapshot> {
-  const experiments = input.experiments;
-
-  const armTotals = new Map<
-    string,
-    Map<string, { sessions: number; metrics: ABMetrics }>
-  >();
-  const skipped = new Map<string, number>();
-  for (const experiment of experiments) {
-    armTotals.set(experiment.name, new Map());
-    skipped.set(experiment.name, 0);
-    for (const variant of Object.keys(
-      (experiment.definition as ABDefinition).variants
-    )) {
-      armTotals.get(experiment.name)!.set(variant, {
-        sessions: 0,
-        metrics: emptyABMetrics(),
-      });
-    }
-  }
-
-  const concurrency = Math.max(1, input.concurrency ?? 8);
-  const sessionRows: ABSnapshotSession[] = [];
-
-  async function processOne(summary: SessionSummary): Promise<void> {
-    const record = await input.loadRecord(summary.sessionId);
-    if (record === undefined) {
-      return;
-    }
-    // Eval harness sessions never enroll; keep them out of aggregates too.
-    if (record.purpose === "eval") {
-      return;
-    }
-    const events = await input.loadEvents(summary.sessionId);
-    const folded = foldABStream(events);
-    if (experiments.length > 0) {
-      await applyABDerive(
-        folded,
-        events,
-        experiments,
-        input.sessionInfo(record),
-        input.agentName,
-        input.logger
-      );
-    }
-
-    const assignments: ABSnapshotAssignment[] = [];
-    // Prefer declared experiment order; include unknown arms from the log after.
-    const seen = new Set<string>();
-    for (const experiment of experiments) {
-      seen.add(experiment.name);
-      const variant = folded.arms.get(experiment.name);
-      if (variant === undefined) {
-        continue;
-      }
-      const def = experiment.definition as ABDefinition;
-      const label = variant === null ? undefined : def.variants[variant]?.label;
-      const metrics =
-        variant === null
-          ? undefined
-          : cloneFoldedMetrics(folded.metrics.get(experiment.name));
-      assignments.push({
-        experiment: experiment.name,
-        variant,
-        ...(label !== undefined ? { variantLabel: label } : {}),
-        ...(metrics !== undefined ? { metrics } : {}),
-      });
-      if (variant === null) {
-        skipped.set(experiment.name, (skipped.get(experiment.name) ?? 0) + 1);
-      } else {
-        const bucket = armTotals.get(experiment.name)?.get(variant);
-        if (bucket !== undefined && metrics !== undefined) {
-          bucket.sessions += 1;
-          addMetrics(bucket.metrics, metrics);
-        } else if (bucket === undefined) {
-          // Assigned to an arm no longer in the definition — still surface it.
-          const arms = armTotals.get(experiment.name) ?? new Map();
-          const created = {
-            sessions: 1,
-            metrics:
-              metrics !== undefined
-                ? cloneABMetrics(metrics)
-                : emptyABMetrics(),
-          };
-          arms.set(variant, created);
-          armTotals.set(experiment.name, arms);
-        }
-      }
-    }
-    for (const [experiment, variant] of folded.arms) {
-      if (seen.has(experiment)) {
-        continue;
-      }
-      const metrics =
-        variant === null
-          ? undefined
-          : cloneFoldedMetrics(folded.metrics.get(experiment));
-      assignments.push({
-        experiment,
-        variant,
-        ...(metrics !== undefined ? { metrics } : {}),
-      });
-    }
-
-    if (assignments.length === 0) {
-      return;
-    }
-
-    sessionRows.push({
-      sessionId: summary.sessionId,
-      channelId: summary.channelId,
-      mode: summary.mode,
-      ...(summary.title === undefined ? {} : { title: summary.title }),
-      turnCount: summary.turnCount,
-      ...(summary.updatedAt === undefined
-        ? {}
-        : { updatedAt: summary.updatedAt }),
-      assignments,
-    });
-  }
-
-  await mapPool(input.sessions, concurrency, async (summary) => {
-    await processOne(summary);
-  });
-
-  // Preserve newest-first order from the sessions list.
-  const order = new Map(
-    input.sessions.map((summary, index) => [summary.sessionId, index])
-  );
-  sessionRows.sort(
-    (a, b) => (order.get(a.sessionId) ?? 0) - (order.get(b.sessionId) ?? 0)
-  );
-
-  const experimentRows: ABSnapshotExperiment[] = experiments.map(
-    (experiment) => {
-      const def = experiment.definition as ABDefinition;
-      const arms: ABSnapshotArm[] = [];
-      for (const [variant, meta] of Object.entries(def.variants)) {
-        const bucket = armTotals.get(experiment.name)?.get(variant) ?? {
-          sessions: 0,
-          metrics: emptyABMetrics(),
-        };
-        arms.push({
-          variant,
-          ...(meta.label !== undefined ? { variantLabel: meta.label } : {}),
-          ...(meta.description !== undefined
-            ? { description: meta.description }
-            : {}),
-          sessions: bucket.sessions,
-          metrics: cloneABMetrics(bucket.metrics),
-        });
-      }
-      // Orphan arms from the log (no longer in definition).
-      for (const [variant, bucket] of armTotals.get(experiment.name) ?? []) {
-        if (variant in def.variants) {
-          continue;
-        }
-        arms.push({
-          variant,
-          sessions: bucket.sessions,
-          metrics: cloneABMetrics(bucket.metrics),
-        });
-      }
-      return {
-        name: experiment.name,
-        variants: def.variants,
-        skipped: skipped.get(experiment.name) ?? 0,
-        arms,
-      };
-    }
-  );
-
-  // Experiments that appear only in logs (definition removed).
-  const known = new Set(experiments.map((experiment) => experiment.name));
-  const orphanNames = new Set<string>();
-  for (const row of sessionRows) {
-    for (const assignment of row.assignments) {
-      if (!known.has(assignment.experiment)) {
-        orphanNames.add(assignment.experiment);
-      }
-    }
-  }
-  for (const name of orphanNames) {
-    const armsMap = new Map<string, { sessions: number; metrics: ABMetrics }>();
-    let skipCount = 0;
-    for (const row of sessionRows) {
-      for (const assignment of row.assignments) {
-        if (assignment.experiment !== name) {
-          continue;
-        }
-        if (assignment.variant === null) {
-          skipCount += 1;
-          continue;
-        }
-        const bucket = armsMap.get(assignment.variant) ?? {
-          sessions: 0,
-          metrics: emptyABMetrics(),
-        };
-        bucket.sessions += 1;
-        if (assignment.metrics !== undefined) {
-          addMetrics(bucket.metrics, assignment.metrics);
-        }
-        armsMap.set(assignment.variant, bucket);
-      }
-    }
-    experimentRows.push({
-      name,
-      variants: {},
-      skipped: skipCount,
-      arms: [...armsMap.entries()].map(([variant, bucket]) => ({
-        variant,
-        sessions: bucket.sessions,
-        metrics: cloneABMetrics(bucket.metrics),
-      })),
-    });
-  }
-
-  return {
-    experiments: experimentRows,
-    sessions: sessionRows,
-    ...(input.config === undefined ? {} : { config: input.config }),
-  };
-}
-
-function addMetrics(target: ABMetrics, delta: ABMetrics): void {
-  target.turns += delta.turns;
-  target.turnFailures += delta.turnFailures;
-  target.toolCalls += delta.toolCalls;
-  target.toolErrors += delta.toolErrors;
-  target.inputTokens += delta.inputTokens;
-  target.outputTokens += delta.outputTokens;
-  target.cacheReadTokens += delta.cacheReadTokens;
-  target.cacheWriteTokens += delta.cacheWriteTokens;
-  target.costUsd += delta.costUsd;
-  target.wallTimeMs += delta.wallTimeMs;
-  for (const [key, value] of Object.entries(delta.custom)) {
-    target.custom[key] = (target.custom[key] ?? 0) + value;
-  }
-}
-
-/** Narrow helper so callers don't need the unused DiscoveredAB import above. */
-export type { DiscoveredAB };

package/src/internal/discovery/ab.ts

@@ -1,131 +0,0 @@
-/**
- * `agent/ab.ts` or `agent/ab/*.ts` plus `agent/ab.config.ts`: live A/B experiments.
- */
-
-import { basename, extname, join, relative } from "node:path";
-import type { ABConfigFile } from "../../ab.js";
-import type { DiscoveredAB } from "../../types.js";
-import { getDefinitionKind } from "../brand.js";
-import { type DiscoveryContext, displayPath } from "./context.js";
-import {
-  findModuleFile,
-  isDirectory,
-  isModuleFile,
-  toPosix,
-  walkFiles,
-} from "./fs.js";
-import { importDefault } from "./modules.js";
-
-export async function loadABs(
-  ctx: DiscoveryContext,
-  agentDir: string,
-  onDisable?: (source: string, path: string) => void
-): Promise<DiscoveredAB[]> {
-  const abs: DiscoveredAB[] = [];
-  const seen = new Set<string>();
-
-  const single = await findModuleFile(agentDir, "ab");
-  if (single !== undefined) {
-    const definition = await importDefault(
-      ctx,
-      single,
-      onDisable === undefined ? "ab" : ["ab", "disabled"]
-    );
-    if (definition !== undefined) {
-      if (getDefinitionKind(definition) === "disabled") {
-        onDisable?.("ab", single);
-      } else {
-        const name = resolveABName(definition, "ab");
-        seen.add(name);
-        abs.push({
-          name,
-          source: "ab",
-          definition: definition as DiscoveredAB["definition"],
-        });
-      }
-    }
-  }
-
-  const absDir = join(agentDir, "ab");
-  if (await isDirectory(absDir)) {
-    for (const filePath of await walkFiles(absDir)) {
-      if (!isModuleFile(basename(filePath))) {
-        continue;
-      }
-      const rel = toPosix(relative(absDir, filePath));
-      const stem = rel.slice(0, -extname(rel).length);
-      const definition = await importDefault(
-        ctx,
-        filePath,
-        onDisable === undefined ? "ab" : ["ab", "disabled"]
-      );
-      if (definition === undefined) {
-        continue;
-      }
-      if (getDefinitionKind(definition) === "disabled") {
-        onDisable?.(stem, filePath);
-        continue;
-      }
-      const name = resolveABName(definition, stem);
-      if (seen.has(name)) {
-        ctx.diagnostics.push({
-          severity: "error",
-          path: displayPath(ctx, filePath),
-          message: `Duplicate A/B experiment name ${JSON.stringify(name)}.`,
-        });
-        continue;
-      }
-      seen.add(name);
-      abs.push({
-        name,
-        source: stem,
-        definition: definition as DiscoveredAB["definition"],
-      });
-    }
-  }
-
-  return abs;
-}
-
-export async function loadABConfig(
-  ctx: DiscoveryContext,
-  agentDir: string
-): Promise<ABConfigFile | undefined> {
-  const path = await findModuleFile(agentDir, "ab.config");
-  if (path === undefined) {
-    return undefined;
-  }
-  try {
-    const mod = await ctx.importModule(path);
-    if (mod.default === null || typeof mod.default !== "object") {
-      ctx.diagnostics.push({
-        severity: "error",
-        path: displayPath(ctx, path),
-        message:
-          "ab.config.ts must default-export defineABConfig({ ... }) from @cursor/july/ab.",
-      });
-      return undefined;
-    }
-    return mod.default as ABConfigFile;
-  } catch (error) {
-    ctx.diagnostics.push({
-      severity: "error",
-      path: displayPath(ctx, path),
-      message: `Failed to load ab.config: ${error instanceof Error ? error.message : String(error)}`,
-    });
-    return undefined;
-  }
-}
-
-function resolveABName(definition: unknown, fallback: string): string {
-  if (
-    typeof definition === "object" &&
-    definition !== null &&
-    "name" in definition &&
-    typeof (definition as { name: unknown }).name === "string" &&
-    (definition as { name: string }).name.length > 0
-  ) {
-    return (definition as { name: string }).name;
-  }
-  return fallback;
-}