@vauban-org/agent-sdk 1.0.0 → 1.3.0

package/src/retry/index.ts

@@ -0,0 +1,227 @@
+/**
+ * Retry primitives with exponential backoff and optional jitter.
+ *
+ * Provides three usage modes:
+ *   - `retry(fn, opts)` — wrap an async call.
+ *   - `RetryContext` — manual control flow for granular retry logic.
+ *   - {@link ./presets.js} — 4 named configurations for common scenarios.
+ *
+ * Sleep is injectable via `RetryOptions.sleepFn` for deterministic tests.
+ * BaseError.retryable flag is honored by default presets (see ./presets.ts).
+ *
+ * @public @since 1.2.0
+ */
+
+import { RETRY_TRANSIENT } from "./presets.js";
+
+export type SleepFn = (ms: number) => Promise<void>;
+
+/**
+ * Retry policy. `jitter: true` adds ±25% randomization to each delay
+ * (anti-thundering-herd when multiple callers retry simultaneously).
+ *
+ * Selection precedence on each error:
+ *   1. `retryIf` predicate if provided
+ *   2. `retryOn` class membership if provided
+ *   3. Default: retry on any error
+ *
+ * @public
+ */
+export interface RetryConfig {
+  /** Total attempts including the first try. Must be >= 1. */
+  readonly maxAttempts: number;
+  /** Initial delay before retry 1, in ms. */
+  readonly baseDelayMs: number;
+  /** Upper bound on any computed delay, in ms. */
+  readonly maxDelayMs: number;
+  /** Exponent for `baseDelayMs * base^attempt`. Typically 2.0. */
+  readonly exponentialBase: number;
+  /** Whether to add ±25% random jitter to each delay. */
+  readonly jitter: boolean;
+  /** If set, only retry when the error is an instance of one of these classes. */
+  readonly retryOn?: ReadonlyArray<new (...args: never[]) => Error>;
+  /** If set, called per error; return true to retry. Overrides retryOn. */
+  readonly retryIf?: (err: unknown) => boolean;
+}
+
+/**
+ * Options to {@link retry}.
+ * @public
+ */
+export interface RetryOptions {
+  /** Retry policy. Defaults to RETRY_TRANSIENT. */
+  config?: RetryConfig;
+  /** Called before each retry sleep. Receives the error, the 0-indexed attempt that failed, and the upcoming delay. */
+  onRetry?: (err: unknown, attempt: number, delayMs: number) => void;
+  /** Sleep injection — defaults to setTimeout. Tests pass a zero-delay or mock. */
+  sleepFn?: SleepFn;
+}
+
+/**
+ * Thrown after all retry attempts have been exhausted.
+ * @public
+ */
+export class RetryExhaustedError extends Error {
+  readonly attempts: number;
+  readonly lastError: unknown;
+  constructor(attempts: number, lastError: unknown) {
+    const reason =
+      lastError instanceof Error ? lastError.message : String(lastError);
+    super(`All ${attempts} retry attempts exhausted: ${reason}`);
+    this.name = "RetryExhaustedError";
+    this.attempts = attempts;
+    this.lastError = lastError;
+  }
+}
+
+const defaultSleep: SleepFn = (ms) =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Compute the delay before retry `attempt` (0-indexed). Caps at `maxDelayMs`.
+ * Jitter, when enabled, adds a uniform-random ±25% offset.
+ *
+ * Visible for tests and for callers that need to inspect timing.
+ * @public
+ */
+export function calculateDelay(config: RetryConfig, attempt: number): number {
+  const raw = config.baseDelayMs * Math.pow(config.exponentialBase, attempt);
+  const capped = Math.min(raw, config.maxDelayMs);
+  if (!config.jitter) return Math.max(0, capped);
+  const jitterRange = capped * 0.25;
+  const offset = (Math.random() * 2 - 1) * jitterRange;
+  return Math.max(0, capped + offset);
+}
+
+/**
+ * Decide whether to retry after `err` at 0-indexed `attempt`.
+ *
+ * Returns false on the final attempt (caller should throw exhaustion).
+ *
+ * @public
+ */
+export function shouldRetry(
+  config: RetryConfig,
+  err: unknown,
+  attempt: number
+): boolean {
+  if (attempt >= config.maxAttempts - 1) return false;
+  if (config.retryIf) return config.retryIf(err);
+  if (config.retryOn && config.retryOn.length > 0) {
+    return config.retryOn.some((Ctor) => err instanceof Ctor);
+  }
+  return true;
+}
+
+/**
+ * Execute `fn` with retry and exponential backoff.
+ *
+ * @throws The non-retryable error as-is if `shouldRetry` returns false mid-loop.
+ * @throws {@link RetryExhaustedError} when all attempts have failed.
+ *
+ * @public
+ */
+export async function retry<T>(
+  fn: () => Promise<T>,
+  opts: RetryOptions = {}
+): Promise<T> {
+  const config = opts.config ?? RETRY_TRANSIENT;
+  const sleep = opts.sleepFn ?? defaultSleep;
+
+  let lastError: unknown;
+
+  for (let attempt = 0; attempt < config.maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastError = err;
+      if (!shouldRetry(config, err, attempt)) {
+        if (attempt >= config.maxAttempts - 1) {
+          throw new RetryExhaustedError(config.maxAttempts, err);
+        }
+        throw err;
+      }
+      const delay = calculateDelay(config, attempt);
+      opts.onRetry?.(err, attempt, delay);
+      await sleep(delay);
+    }
+  }
+
+  throw new RetryExhaustedError(config.maxAttempts, lastError);
+}
+
+/**
+ * Manual retry control. Use when the operation doesn't fit a single `fn()` —
+ * e.g. when multiple endpoints are tried per attempt, or when partial state
+ * must be reset between tries.
+ *
+ * @example
+ * ```ts
+ * const ctx = new RetryContext({ config: RETRY_AGGRESSIVE });
+ * while (ctx.shouldContinue) {
+ *   try {
+ *     return await op();
+ *   } catch (err) {
+ *     await ctx.handleError(err);
+ *   }
+ * }
+ * ```
+ *
+ * @public
+ */
+export class RetryContext {
+  private readonly config: RetryConfig;
+  private readonly sleep: SleepFn;
+  private readonly onRetry?: (
+    err: unknown,
+    attempt: number,
+    delayMs: number
+  ) => void;
+  private attemptCount = 0;
+  private exhausted = false;
+
+  constructor(opts: RetryOptions & { config: RetryConfig }) {
+    this.config = opts.config;
+    this.sleep = opts.sleepFn ?? defaultSleep;
+    this.onRetry = opts.onRetry;
+  }
+
+  /** True while attempts remain and no non-retryable error has been thrown. */
+  get shouldContinue(): boolean {
+    return !this.exhausted && this.attemptCount < this.config.maxAttempts;
+  }
+
+  /** 0-indexed count of completed (failed) attempts. */
+  get attempt(): number {
+    return this.attemptCount;
+  }
+
+  /**
+   * Record an error and sleep before the next try.
+   *
+   * @throws The non-retryable error as-is.
+   * @throws {@link RetryExhaustedError} if attempts are exhausted.
+   */
+  async handleError(err: unknown): Promise<void> {
+    const failedAttempt = this.attemptCount;
+    this.attemptCount += 1;
+    if (this.attemptCount >= this.config.maxAttempts) {
+      this.exhausted = true;
+      throw new RetryExhaustedError(this.config.maxAttempts, err);
+    }
+    if (!shouldRetry(this.config, err, failedAttempt)) {
+      this.exhausted = true;
+      throw err;
+    }
+    const delay = calculateDelay(this.config, failedAttempt);
+    this.onRetry?.(err, failedAttempt, delay);
+    await this.sleep(delay);
+  }
+}
+
+export {
+  RETRY_TRANSIENT,
+  RETRY_AGGRESSIVE,
+  RETRY_PATIENT,
+  NO_RETRY,
+} from "./presets.js";

package/src/retry/presets.ts

@@ -0,0 +1,75 @@
+/**
+ * Named retry presets covering common scenarios.
+ *
+ * All presets honor a `retryable: boolean` flag on the thrown error when
+ * present (compatible with {@link ../errors.js#BaseError}). When the flag is
+ * absent, defaults fall back to: transient = retry, no_retry = no retry.
+ *
+ * @public @since 1.2.0
+ */
+
+import type { RetryConfig } from "./index.js";
+
+function honorsRetryableFlag(err: unknown, defaultIfMissing: boolean): boolean {
+  if (typeof err === "object" && err !== null && "retryable" in err) {
+    return (err as { retryable: unknown }).retryable === true;
+  }
+  return defaultIfMissing;
+}
+
+/**
+ * 3 attempts, 1s base, 10s cap, jitter on. Honors `retryable` flag (defaults
+ * to retry when absent). Use for short-lived network calls.
+ *
+ * @public
+ */
+export const RETRY_TRANSIENT: RetryConfig = {
+  maxAttempts: 3,
+  baseDelayMs: 1000,
+  maxDelayMs: 10_000,
+  exponentialBase: 2.0,
+  jitter: true,
+  retryIf: (err) => honorsRetryableFlag(err, true),
+};
+
+/**
+ * 5 attempts, 500ms base, 30s cap, jitter on. Use for flaky external APIs
+ * where rapid recovery is plausible.
+ *
+ * @public
+ */
+export const RETRY_AGGRESSIVE: RetryConfig = {
+  maxAttempts: 5,
+  baseDelayMs: 500,
+  maxDelayMs: 30_000,
+  exponentialBase: 2.0,
+  jitter: true,
+};
+
+/**
+ * 10 attempts, 2s base, 120s cap, jitter on. Use for long-recovery
+ * dependencies (database failover, restart-storm tolerant calls).
+ *
+ * @public
+ */
+export const RETRY_PATIENT: RetryConfig = {
+  maxAttempts: 10,
+  baseDelayMs: 2000,
+  maxDelayMs: 120_000,
+  exponentialBase: 2.0,
+  jitter: true,
+};
+
+/**
+ * Single attempt, no retry. Use as an explicit policy to disable retry at
+ * call sites without scattering booleans.
+ *
+ * @public
+ */
+export const NO_RETRY: RetryConfig = {
+  maxAttempts: 1,
+  baseDelayMs: 0,
+  maxDelayMs: 0,
+  exponentialBase: 2.0,
+  jitter: false,
+};

package/src/skill-loop/ab-runner.ts

@@ -0,0 +1,196 @@
+/**
+ * src/skill-loop/ab-runner.ts
+ *
+ * A/B test runner with failure budgets for SkillCandidates.
+ *
+ * Invariants:
+ *   - Max 3 candidates simultaneously (4th is rejected).
+ *   - Traffic fraction cap: 10% per candidate.
+ *   - Rollback triggered when a candidate's mean score drops >5% vs incumbent.
+ *   - Winner declared when: p < 0.05 AND delta >= 10% AND no rollback triggered.
+ *
+ * Statistical significance uses the same Welch approach as evaluator.ts.
+ *
+ * @module skill-loop/ab-runner
+ */
+
+import type { SkillCandidate } from "./candidate.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ABConfig {
+  /** Maximum concurrent candidates. Hard cap: 3. */
+  maxCandidates: number;
+  /** Traffic fraction routed to each candidate. Hard cap: 0.10. */
+  trafficPct: number;
+  /** Rollback threshold: if candidate mean drops by this fraction vs incumbent. */
+  rollbackThresholdPct: number;
+}
+
+export interface ABSlot {
+  candidateId: string;
+  trafficFraction: number;
+  outcomes: number[];
+  rollbackTriggered: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Statistical helpers (inline — avoid circular dep with evaluator)
+// ---------------------------------------------------------------------------
+
+function mean(xs: number[]): number {
+  if (xs.length === 0) return 0;
+  return xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+
+function variance(xs: number[], m: number): number {
+  if (xs.length < 2) return 0;
+  return xs.reduce((acc, x) => acc + (x - m) ** 2, 0) / (xs.length - 1);
+}
+
+function normalCdf(z: number): number {
+  const t = 1 / (1 + 0.2316419 * z);
+  const d = 0.3989423 * Math.exp((-z * z) / 2);
+  const p =
+    d *
+    t *
+    (0.3193815 +
+      t * (-0.3565638 + t * (1.7814779 + t * (-1.8212559 + t * 1.3302744))));
+  return 1 - p;
+}
+
+function welchPValue(a: number[], b: number[]): number {
+  const n1 = a.length;
+  const n2 = b.length;
+  if (n1 < 2 || n2 < 2) return 1.0;
+
+  const m1 = mean(a);
+  const m2 = mean(b);
+  const v1 = variance(a, m1);
+  const v2 = variance(b, m2);
+  const se = Math.sqrt(v1 / n1 + v2 / n2);
+  if (se === 0) return m1 === m2 ? 1.0 : 0.0;
+
+  const tStat = Math.abs((m1 - m2) / se);
+  const df =
+    (v1 / n1 + v2 / n2) ** 2 /
+    ((v1 / n1) ** 2 / (n1 - 1) + (v2 / n2) ** 2 / (n2 - 1));
+
+  if (df >= 30) {
+    return Math.max(0, Math.min(1, 2 * (1 - normalCdf(tStat))));
+  }
+  // Conservative: approximate p via critical value at df ≈ 30 (t-crit = 2.042)
+  return tStat > 2.042 ? 0.04 : 0.1;
+}
+
+// ---------------------------------------------------------------------------
+// ABRunner
+// ---------------------------------------------------------------------------
+
+export class ABRunner {
+  private readonly config: Readonly<ABConfig>;
+  private readonly slots = new Map<string, ABSlot>();
+  private readonly candidates = new Map<string, SkillCandidate>();
+  private readonly incumbentOutcomes: number[] = [];
+
+  constructor(config: ABConfig) {
+    // Enforce hard caps
+    this.config = {
+      maxCandidates: Math.min(config.maxCandidates, 3),
+      trafficPct: Math.min(config.trafficPct, 0.1),
+      rollbackThresholdPct: config.rollbackThresholdPct,
+    };
+  }
+
+  /**
+   * Register a new candidate for A/B testing.
+   *
+   * @returns true if added, false if maxCandidates already reached.
+   */
+  addCandidate(candidate: SkillCandidate): boolean {
+    if (this.slots.size >= this.config.maxCandidates) {
+      return false;
+    }
+    this.candidates.set(candidate.id, candidate);
+    this.slots.set(candidate.id, {
+      candidateId: candidate.id,
+      trafficFraction: this.config.trafficPct,
+      outcomes: [],
+      rollbackTriggered: false,
+    });
+    return true;
+  }
+
+  /**
+   * Record an outcome score for a candidate or the incumbent.
+   *
+   * @param candidateId - Candidate ID or "incumbent".
+   * @param score       - Outcome score [0, 1].
+   */
+  recordOutcome(candidateId: string | "incumbent", score: number): void {
+    if (candidateId === "incumbent") {
+      this.incumbentOutcomes.push(score);
+      return;
+    }
+    const slot = this.slots.get(candidateId);
+    if (!slot) return;
+
+    slot.outcomes.push(score);
+
+    // Check rollback condition after each new data point
+    if (!slot.rollbackTriggered && this.shouldRollback(candidateId)) {
+      slot.rollbackTriggered = true;
+    }
+  }
+
+  /**
+   * Returns true if the candidate's mean score has dropped >rollbackThresholdPct
+   * vs the incumbent mean.
+   */
+  shouldRollback(candidateId: string): boolean {
+    const slot = this.slots.get(candidateId);
+    if (!slot || slot.outcomes.length === 0) return false;
+    if (this.incumbentOutcomes.length === 0) return false;
+
+    const candMean = mean(slot.outcomes);
+    const incMean = mean(this.incumbentOutcomes);
+
+    // Degradation: candidate is worse than incumbent by more than threshold
+    const degradation = incMean - candMean;
+    return degradation > this.config.rollbackThresholdPct;
+  }
+
+  /**
+   * Return the winning candidate if statistical significance is established.
+   *
+   * Winner criteria:
+   *   - p < 0.05 (Welch's t-test vs incumbent)
+   *   - delta >= 10% vs incumbent mean
+   *   - rollback NOT triggered
+   *
+   * Returns null if no candidate meets all criteria.
+   */
+  getWinner(): SkillCandidate | null {
+    if (this.incumbentOutcomes.length < 2) return null;
+
+    const incMean = mean(this.incumbentOutcomes);
+
+    for (const [id, slot] of this.slots) {
+      if (slot.rollbackTriggered) continue;
+      if (slot.outcomes.length < 2) continue;
+
+      const candMean = mean(slot.outcomes);
+      const delta = candMean - incMean;
+      if (delta < 0.1) continue;
+
+      const p = welchPValue(slot.outcomes, this.incumbentOutcomes);
+      if (p < 0.05) {
+        return this.candidates.get(id) ?? null;
+      }
+    }
+
+    return null;
+  }
+}

package/src/skill-loop/adoption.ts

@@ -0,0 +1,188 @@
+/**
+ * src/skill-loop/adoption.ts
+ *
+ * Skill adoption — auto-promote and deprecate logic gated by sign-off.
+ *
+ * Promotion conditions (ALL must hold):
+ *   1. EvalResult.pValue < 0.05  (statistical significance)
+ *   2. EvalResult.deltaVsIncumbent >= 0.10  (>=10% improvement)
+ *   3. A valid approved SignOffRecord exists for the candidateId
+ *
+ * Violation of condition 3 → throws PromotionWithoutSignOffError.
+ *
+ * Underperformers (deltaVsIncumbent < -0.05 OR pValue > 0.5) are
+ * automatically deprecated via deprecateUnderperformers().
+ *
+ * @module skill-loop/adoption
+ */
+
+import type { SkillCandidate } from "./candidate.js";
+import type { EvalResult } from "./evaluator.js";
+import type { SignOffRecord } from "./sign-off.js";
+
+// ---------------------------------------------------------------------------
+// Errors
+// ---------------------------------------------------------------------------
+
+export class PromotionWithoutSignOffError extends Error {
+  readonly candidateId: string;
+
+  constructor(candidateId: string) {
+    super(
+      `adoption: cannot promote candidate "${candidateId}" without a valid approved sign-off. ` +
+        "Request human approval via SignOffManager.requestSignOff() first."
+    );
+    this.name = "PromotionWithoutSignOffError";
+    this.candidateId = candidateId;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type AdoptionStatus =
+  | "promoted"
+  | "rejected"
+  | "deprecated"
+  | "pending_signoff"
+  | "insufficient_signal";
+
+export interface AdoptionRecord {
+  candidateId: string;
+  status: AdoptionStatus;
+  evalResult: EvalResult;
+  signOff: SignOffRecord | null;
+  promotedAt: Date | null;
+  deprecatedAt: Date | null;
+  reason: string;
+}
+
+// ---------------------------------------------------------------------------
+// Adoption helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Attempt to promote a candidate to production.
+ *
+ * @param candidate  - The skill candidate.
+ * @param evalResult - Evaluation result against the verifier set.
+ * @param signOff    - Approved sign-off record (must be decision === 'approved').
+ * @param nowDate    - Optional date override for deterministic testing.
+ *
+ * @throws PromotionWithoutSignOffError if signOff is null or not approved.
+ */
+export function promoteCandidate(
+  candidate: SkillCandidate,
+  evalResult: EvalResult,
+  signOff: SignOffRecord | null,
+  nowDate?: Date
+): AdoptionRecord {
+  // Gate 1: sign-off required — never auto-promote without human approval
+  if (!signOff || signOff.decision !== "approved") {
+    throw new PromotionWithoutSignOffError(candidate.id);
+  }
+
+  // Gate 2: statistical significance
+  if (!evalResult.significant) {
+    return {
+      candidateId: candidate.id,
+      status: "insufficient_signal",
+      evalResult,
+      signOff,
+      promotedAt: null,
+      deprecatedAt: null,
+      reason: `insufficient signal: p=${evalResult.pValue.toFixed(
+        4
+      )} delta=${evalResult.deltaVsIncumbent.toFixed(
+        4
+      )} (need p<0.05 AND delta>=0.10)`,
+    };
+  }
+
+  // All gates passed → promote
+  return {
+    candidateId: candidate.id,
+    status: "promoted",
+    evalResult,
+    signOff,
+    promotedAt: nowDate ?? new Date(),
+    deprecatedAt: null,
+    reason: `promoted: p=${evalResult.pValue.toFixed(4)} delta=${(
+      evalResult.deltaVsIncumbent * 100
+    ).toFixed(1)}% sign-off by ${signOff.approverId}`,
+  };
+}
+
+/**
+ * Reject a candidate explicitly (e.g. sign-off decision = rejected).
+ */
+export function rejectCandidate(
+  candidate: SkillCandidate,
+  evalResult: EvalResult,
+  signOff: SignOffRecord | null,
+  reason: string
+): AdoptionRecord {
+  return {
+    candidateId: candidate.id,
+    status: "rejected",
+    evalResult,
+    signOff,
+    promotedAt: null,
+    deprecatedAt: null,
+    reason,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Underperformer deprecation
+// ---------------------------------------------------------------------------
+
+/**
+ * Deprecation threshold: candidate's delta < -5% OR p-value > 0.5 with negative delta.
+ */
+const UNDERPERFORMER_DELTA_THRESHOLD = -0.05;
+const UNDERPERFORMER_PVALUE_THRESHOLD = 0.5;
+
+/**
+ * Automatically deprecate underperforming candidates from a pool.
+ *
+ * A candidate is an underperformer if:
+ *   - deltaVsIncumbent < -0.05 (degrades by more than 5%)
+ *   - OR (deltaVsIncumbent < 0 AND pValue > 0.5) — statistically noisy & negative
+ *
+ * Deprecation is automatic — no sign-off required (sign-off is only for promotion).
+ *
+ * @param candidates  - Pool of (candidate, evalResult) pairs to assess.
+ * @param nowDate     - Optional date override.
+ * @returns AdoptionRecords for every deprecated candidate.
+ */
+export function deprecateUnderperformers(
+  candidates: Array<{ candidate: SkillCandidate; evalResult: EvalResult }>,
+  nowDate?: Date
+): AdoptionRecord[] {
+  const deprecated: AdoptionRecord[] = [];
+
+  for (const { candidate, evalResult } of candidates) {
+    const isUnderperformer =
+      evalResult.deltaVsIncumbent < UNDERPERFORMER_DELTA_THRESHOLD ||
+      (evalResult.deltaVsIncumbent < 0 &&
+        evalResult.pValue > UNDERPERFORMER_PVALUE_THRESHOLD);
+
+    if (isUnderperformer) {
+      deprecated.push({
+        candidateId: candidate.id,
+        status: "deprecated",
+        evalResult,
+        signOff: null,
+        promotedAt: null,
+        deprecatedAt: nowDate ?? new Date(),
+        reason: `auto-deprecated: delta=${(
+          evalResult.deltaVsIncumbent * 100
+        ).toFixed(1)}% p=${evalResult.pValue.toFixed(4)}`,
+      });
+    }
+  }
+
+  return deprecated;
+}