@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/define-agent-eval-sH24zBfM.d.ts

@@ -0,0 +1,388 @@
+import { c as CostLedgerHandle, f as CostLedgerSummary, m as CostReceipt } from "./cost-ledger-DbQdN3nO.js";
+import { _ as ProposalFinding } from "./types-DLQx4mKU.js";
+import { H as SurfaceProposer, R as Scenario, S as JudgeConfig, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, p as Gate } from "./types-BZ59Ahr8.js";
+import { o as InsightReport } from "./insight-report-CRi-Ufrj.js";
+import { C as RunOptimizationOptions, E as RunEvalOptions, J as OptimizationMethodProvenance, S as PremeasuredOptimizationBaseline, W as OptimizationMethod, Y as OptimizationMethodResult, b as RunImprovementLoopResult, l as LoopProvenanceRecord, xt as CampaignStorage } from "./provenance-CCdxgLDT.js";
+import { o as PowerPreflight } from "./pareto-BqNW3LJR.js";
+import { n as HostedTenant } from "./client-KUbGulm_.js";
+//#region src/contract/self-improve.d.ts
+interface SelfImproveBudget {
+  /** Hard spend cap across the full run. Each paid call reserves its enforced
+   *  maximum before dispatch, so completed spend cannot cross this amount. */
+  dollars?: number;
+  /** Proposer generations. Default: 3. External methods own their rounds and
+   *  require this value to be omitted or set to 1. Set 0 only for a
+   *  proposer-free baseline run. */
+  generations?: number;
+  /** Candidates the proposer emits per generation. Default 2. */
+  populationSize?: number;
+  /** Max concurrent cells across the loop. Default 2. */
+  maxConcurrency?: number;
+  /** Candidate campaigns scored in parallel. Default 1. Total concurrent
+   *  cells are bounded by `candidateConcurrency * maxConcurrency`. */
+  candidateConcurrency?: number;
+  /** Fraction of `scenarios` held out from training, used for the gate.
+   *  Default 0.25. Ignored when `holdoutScenarios` is set explicitly. */
+  holdoutFraction?: number;
+  /** Fraction of the non-final cases reserved for method selection.
+   *  Default 0.25. Used only with `method` and ignored when
+   *  `selectionScenarios` is supplied explicitly. */
+  selectionFraction?: number;
+  /** Explicit held-out scenarios; overrides `holdoutFraction`. */
+  holdoutScenarios?: Scenario[];
+  /** Holdout policy. Default `'measured'`: split, re-score baseline vs winner
+   *  on the held-out set, gate on that comparison. `'deferred'`: run the
+   *  improvement-set campaigns + search promotion, dispatch ZERO holdout cells,
+   *  force the gate to `'hold'`, return `lift: undefined`, and record
+   *  `holdout: 'deferred'` in the provenance record — for callers that measure
+   *  the held-out comparison in a separate later run instead of faking a
+   *  static holdout scenario and recording a meaningless lift. Unless
+   *  `holdoutScenarios` reserves an explicit set, ALL scenarios train. */
+  holdout?: 'measured' | 'deferred';
+  /** Per-scenario replicates per cell — raises bootstrap-CI tightness. Default 1. */
+  reps?: number;
+  /** DEPTH dial forwarded to the proposer's `propose()` as
+   *  `ctx.maxImprovementShots` — max iterations an agentic candidate generator
+   *  may take per candidate (verify-in-session retries). Unset ⇒ the
+   *  proposer's own default. */
+  maxImprovementShots?: number;
+}
+type SelfImproveProgressEvent = {
+  kind: 'baseline.started';
+  scenarios: number;
+} | {
+  kind: 'baseline.completed';
+  compositeMean: number;
+  durationMs: number;
+} | {
+  kind: 'generation.started';
+  index: number;
+  populationSize: number;
+} | {
+  kind: 'generation.completed';
+  index: number;
+  bestComposite: number;
+  durationMs: number;
+} | {
+  kind: 'gate.decided';
+  decision: string;
+  lift?: number;
+} | {
+  kind: 'power.estimated';
+  n: number;
+  sd: number;
+  mde: number;
+  underpowered: boolean;
+};
+interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
+  /**
+   * Your agent — a function that takes the current `MutableSurface`
+   * (typically a system prompt the loop is optimizing) plus the
+   * scenario + cell ctx, and returns the artifact your judge scores.
+   *
+   * Same shape as `RunOptimizationOptions.dispatchWithSurface`. Wrap a
+   * plain `Dispatch` if you don't have a surface seam:
+   *
+   *   agent: (_surface, scenario, ctx) => yourPlainDispatch(scenario, ctx)
+   *
+   * That mode evaluates without mutating any surface — useful as a
+   * baseline-only run (set `budget.generations = 0`).
+   */
+  agent: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
+  /**
+   * Snapshot-bearing model identity for agents that do not report a paid-call
+   * receipt through `ctx.cost.runPaidCall()`.
+   *
+   * Omit this when every cell reports its concrete model in a receipt.
+   */
+  model?: string;
+  /** Scenarios to evaluate against. Train/holdout split is computed from
+   *  these unless `budget.holdoutScenarios` is set explicitly. */
+  scenarios: TScenario[];
+  /** Judge that scores artifacts. Bring your own; use `langchainJudge`
+   *  from `/adapters/langchain` for a Runnable-shaped one. */
+  judge: JudgeConfig<TArtifact, TScenario>;
+  /** Starting surface — system prompt, JSON config, anything `MutableSurface`
+   *  accepts. The proposer mutates this each generation. */
+  baselineSurface: MutableSurface;
+  /** Budget + loop shape. All fields optional. */
+  budget?: SelfImproveBudget;
+  /**
+   * Complete prior measurement of `baselineSurface` over the TRAIN split.
+   * Forwarded to the loop body, which validates its surface hash, scenario
+   * split, seed (42), reps, and coverage, then skips the baseline search
+   * campaign entirely — no baseline dispatch, no resumability lookup. The
+   * train split is `scenarios` minus the holdout split, so premeasure with
+   * exactly that scenario set (explicit `budget.holdoutScenarios`, or
+   * `budget.holdout: 'deferred'` with no reserved set, makes the train split
+   * deterministic). Prior spend stays in the imported campaign aggregates and
+   * is not re-added to this run's cost ledger.
+   */
+  premeasuredBaseline?: PremeasuredOptimizationBaseline<TArtifact, TScenario>;
+  /**
+   * Candidate generator for this local generation loop.
+   * Required when `budget.generations` is greater than zero.
+   */
+  proposer?: SurfaceProposer<ProposalFinding>;
+  /**
+   * Complete optimization method, such as official GEPA or SkillOpt.
+   * The method receives disjoint train and selection cases and never receives
+   * the final comparison cases. Mutually exclusive with `proposer`.
+   */
+  method?: OptimizationMethod<TScenario, TArtifact>;
+  /** Explicit method-selection cases. They must also appear in `scenarios`
+   *  and must not overlap the final comparison cases. */
+  selectionScenarios?: TScenario[];
+  /** Custom gate. Default is `defaultProductionGate` with
+   *  `deltaThreshold: 0.05` on the held-out split. */
+  gate?: Gate<TArtifact, TScenario>;
+  /** Placebo control. When supplied AND the winner differs from baseline, the
+   *  loop scores a THIRD held-out arm: the winner surface with its content
+   *  footprint-matched-blanked by this fn (typically via `neutralizeText`). Its
+   *  scores reach the gate as `ctx.neutralizedJudgeScores`, letting a
+   *  `neutralizationGate` reject a win whose lift survives blanking the content
+   *  (decorative — driven by footprint, not content). Costs one extra held-out
+   *  campaign; omit to skip. Compose `neutralizationGate` into `gate` to act on it. */
+  neutralize?: (winnerSurface: MutableSurface, baselineSurface: MutableSurface) => MutableSurface;
+  /** Storage backend. A filesystem run directory uses `fsCampaignStorage()`;
+   *  a `mem://` directory uses in-memory storage. External methods default to
+   *  a filesystem directory because their official state must survive. */
+  storage?: CampaignStorage;
+  /** Run directory. Proposer mode defaults to
+   *  `mem://selfImprove-<timestamp>`. External method mode defaults to
+   *  `.agent-eval/runs/self-improve-<timestamp>`. */
+  runDir?: string;
+  /** Fires once the durable provenance record + OTel spans are emitted.
+   *  Receives the structured record for inline assertions / custom routing. */
+  onProvenance?: (record: LoopProvenanceRecord) => void;
+  /** Distributed execution seam — same as `RunCampaignOptions.cellPlacement`.
+   *  Returns an opaque placement key the substrate forwards to your agent
+   *  as `ctx.placement`. Combined with `httpDispatch` from
+   *  `/adapters/http`, fans cells across regions. */
+  cellPlacement?: (input: {
+    scenario: TScenario;
+    rep: number;
+    generation?: number;
+  }) => string | undefined;
+  /** Per-cell agent dispatch deadline, applied to baseline, candidate, and
+   *  held-out campaigns. Default 600_000 ms. Set 0 to disable. */
+  dispatchTimeoutMs?: number;
+  /** Streaming hook — fires on baseline + each generation + gate decision.
+   *  Consumer routes events wherever (UI, dashboard, logs). */
+  onProgress?: (event: SelfImproveProgressEvent) => void;
+  /** Auto-promotion behavior on a ship decision. Default `'none'` — we
+   *  return the winner; you ship it however you ship. `'pr'` opens a
+   *  GitHub PR via `openAutoPr`; requires `ghOwner` + `ghRepo`. */
+  autoOnPromote?: 'pr' | 'none';
+  ghOwner?: string;
+  ghRepo?: string;
+  /**
+   * Opt-in: ship eval-run events to a hosted orchestrator (ours, your
+   * self-hosted one, or any compatible implementation of the
+   * `docs/hosted-ingest-spec.md` wire format). When set, the substrate
+   * POSTs the final `EvalRunEvent` to `${endpoint}/v1/ingest/eval-runs`
+   * after the loop completes. Failures are logged but do not fail the
+   * loop — local result is always returned.
+   *
+   * For our orchestrator: `{ endpoint: 'https://orchestrator.tangle.tools/v1', apiKey, tenantId }`.
+   *
+   * For your self-hosted: any URL serving the wire format. See
+   * `examples/hosted-ingest-server/` for the reference receiver.
+   */
+  hostedTenant?: HostedTenant;
+  /** Free-form labels attached to the hosted event (env, branch, model id,
+   *  etc.). Ignored when `hostedTenant` is unset. */
+  hostedLabels?: Record<string, string>;
+  /** Capture every search artifact and judge score to this store.
+   *  The store is output only and is never exposed to candidate generation.
+   *  Pass `'off'` to disable. Default: off. */
+  labeledStore?: LabeledScenarioStore | 'off';
+  /** Capture-source tag for `labeledStore`. Default `'eval-run'`. */
+  captureSource?: 'production-trace' | 'eval-run' | 'manual' | 'red-team' | 'synthetic';
+  /**
+   * Per-cell backend-integrity expectation — the fail-loud guard. A cell that
+   * produced an artifact but reported `costUsd === 0` AND zero tokens is a
+   * stub. Modes: `'assert'` throws on the first such cell, `'warn'` logs it,
+   * `'off'` skips the check (offline/replay). Default `'assert'` — `selfImprove`
+   * is the real-run path, so a stub fails loud rather than scoring a clean 0.
+   */
+  expectUsage?: 'assert' | 'warn' | 'off';
+  /**
+   * Per-generation findings producer. Runs once on the baseline campaign (as
+   * `generation: -1`) before generation 0 proposes — so single-generation runs
+   * propose with trace context — and again after each generation is scored;
+   * whatever it returns REPLACES the proposer's `findings` for the next
+   * `propose()`. Plug a trace-analyst registry / HALO here. When absent,
+   * findings stay `opts.findings`.
+   */
+  analyzeGeneration?: RunOptimizationOptions<TScenario, TArtifact>['analyzeGeneration'];
+  /** Static findings forwarded to the proposer's `propose()` as `ctx.findings`
+   *  (a findings-grounded proposer consumes them). Default: none. */
+  findings?: ProposalFinding[];
+  /** Override how the WINNER is selected among coverage-complete candidates.
+   *  Defaults to the scalar mean composite (historical behavior). A binary-with-
+   *  replicates consumer whose ship-gate counts an instance resolved only when
+   *  every replicate resolved passes a fail-closed lexicographic key here so that
+   *  winner-selection and the ship-gate rank on the identical metric and cannot
+   *  invert. See `RunOptimizationOptions.selectionRankKey`. */
+  selectionRankKey?: RunOptimizationOptions<TScenario, TArtifact>['selectionRankKey'];
+}
+interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
+  /** Composite mean across all scenarios, baseline run. When
+   *  `budget.holdout === 'deferred'` this is measured on the improvement
+   *  (search) split — no holdout campaign ran. */
+  baseline: {
+    compositeMean: number;
+    perScenario: Record<string, number>;
+  };
+  /** Composite mean on the held-out set, winner run. When
+   *  `budget.holdout === 'deferred'` this is the winner's improvement-set
+   *  (search) measurement — no holdout campaign ran. */
+  winner: {
+    compositeMean: number;
+    perScenario: Record<string, number>;
+    surface: MutableSurface;
+    /** Proposer label for the promoted change. Absent ⇒ winner == baseline or
+     *  a bare-surface mutator. */
+    label?: string;
+    /** Proposer rationale — the "because Z" that motivated the promoted change.
+     *  Threaded from the proposer's `ProposedCandidate` through the loop.
+     *  Absent ⇒ winner == baseline. */
+    rationale?: string;
+  };
+  /** `winner.compositeMean - baselineOnHoldout.compositeMean`. Positive
+   *  means the gate observed improvement. Absent iff
+   *  `budget.holdout === 'deferred'` — no held-out measurement ran, so there
+   *  is no lift to report (never a fabricated 0). */
+  lift?: number;
+  /** The explicit baseline→winner unified diff. Always present (empty string
+   *  when winner == baseline). */
+  diff: string;
+  /** Durable, queryable provenance record: candidate→cell→gate→promote chain +
+   *  rationale + diff + backend provenance. The artifact the hosted ingest
+   *  path stores; the +lift RECOMPUTES from `record.heldOutLift`. */
+  provenance: LoopProvenanceRecord;
+  /** `defaultProductionGate.decide()` result. */
+  gateDecision: 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
+  /** Number of generations actually explored (may be less than the
+   *  budget if the proposer gave up early). */
+  generationsExplored: number;
+  /** Wall-clock total. */
+  durationMs: number;
+  /** Total newly observed cost across the full run. */
+  totalCostUsd: number;
+  /** Canonical run-wide spend summary. */
+  cost: CostLedgerSummary;
+  /** Run-wide receipts across proposal, search, holdout, judging, analysis,
+   *  and promotion work, with phase and actor attribution. */
+  receipts: CostReceipt[];
+  /** Exact external method and source identity, when `method` was used. */
+  optimization?: {
+    name: string;
+    cost: OptimizationMethodResult['cost'];
+    durationMs?: number;
+    provenance?: OptimizationMethodProvenance;
+  };
+  /**
+   * Rigor packet: distributional summary, paired-bootstrap lift CI,
+   * judge stats, contamination check, recommendations. Wired through
+   * `analyzeRuns()` on the baseline + winner cells of the campaign.
+   * Hosted-tier dashboards render this as the v3-vs-v4 decision view.
+   */
+  insight: InsightReport;
+  /** Minimum-detectable-lift analysis from the baseline holdout cells: could this
+   *  budget have shipped ANY plausible effect? Absent when the baseline produced
+   *  fewer than 3 scored holdout cells. See `powerPreflight` for the standalone
+   *  pre-run version (run `gate: 'none'` first, budget the real search after). */
+  power?: PowerPreflight;
+  /**
+   * Raw substrate result for advanced inspection — full per-generation
+   * candidates, full campaign artifacts, all judge scores. Useful for
+   * debugging or reporting beyond the summary.
+   */
+  raw: RunImprovementLoopResult<TArtifact, TScenario>;
+}
+/** Failed self-improvement run with an immutable receipt snapshot. */
+declare class SelfImproveRunError extends Error {
+  readonly cost: CostLedgerSummary;
+  readonly receipts: CostReceipt[];
+  constructor(cause: unknown, ledger: CostLedgerHandle);
+}
+/**
+ * One-shot self-improvement loop. See module docstring for defaults +
+ * extension points.
+ *
+ * @example Minimum:
+ *
+ *   const result = await selfImprove({
+ *     agent: (surface, scenario, ctx) => myAgent(surface, scenario, ctx.signal),
+ *     scenarios,
+ *     judge,
+ *     baselineSurface: DEFAULT_PROMPT,
+ *     proposer,
+ *   })
+ *   console.log(`lift: ${result.lift.toFixed(3)} (${result.gateDecision})`)
+ *
+ * @example Distributed (workers in three regions):
+ *
+ *   await selfImprove({
+ *     agent: httpDispatch({ resolveUrl: ({ placement }) => REGION_URLS[placement!] }),
+ *     scenarios,
+ *     judge,
+ *     baselineSurface: DEFAULT_PROMPT,
+ *     cellPlacement: ({ scenario }) => scenario.region,
+ *     budget: { maxConcurrency: 12 },
+ *   })
+ */
+declare function selfImprove<TScenario extends Scenario, TArtifact>(opts: SelfImproveOptions<TScenario, TArtifact>): Promise<SelfImproveResult<TScenario, TArtifact>>;
+//#endregion
+//#region src/contract/define-agent-eval.d.ts
+type AgentEvalAgent<TScenario extends Scenario, TArtifact> = (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
+type DefineAgentEvalOptions<TScenario extends Scenario, TArtifact> = SelfImproveOptions<TScenario, TArtifact>;
+interface AgentEvalEvaluateOptions<TScenario extends Scenario, TArtifact> extends Omit<RunEvalOptions<TScenario, TArtifact>, 'dispatch' | 'judges' | 'runDir' | 'scenarios'> {
+  /** Scenario set to evaluate. Defaults to the scenarios passed to `defineAgentEval`. */
+  scenarios?: TScenario[];
+  /** Surface to evaluate. Defaults to the baseline surface passed to `defineAgentEval`. */
+  surface?: MutableSurface;
+  /** Agent to evaluate. Defaults to the agent passed to `defineAgentEval`. */
+  agent?: AgentEvalAgent<TScenario, TArtifact>;
+  /** Single judge override. Ignored when `judges` is set. */
+  judge?: JudgeConfig<TArtifact, TScenario>;
+  /** Full judge list override. Defaults to the single judge passed to `defineAgentEval`. */
+  judges?: JudgeConfig<TArtifact, TScenario>[];
+  /** Logical or filesystem run directory. Defaults to an in-memory run. */
+  runDir?: string;
+}
+type AgentEvalImproveOptions<TScenario extends Scenario, TArtifact> = Omit<Partial<SelfImproveOptions<TScenario, TArtifact>>, 'budget' | 'hostedTenant'> & {
+  budget?: Partial<SelfImproveBudget>;
+  hostedTenant?: Partial<HostedTenant>;
+};
+interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
+  /** The default scenarios used by `evaluate()` and `improve()`. */
+  readonly scenarios: readonly TScenario[];
+  /** The default baseline surface used by `evaluate()` and `improve()`. */
+  readonly baselineSurface: MutableSurface;
+  /**
+   * Run one scored evaluation. Use this for a baseline score or to score one
+   * candidate surface without running an improvement loop.
+   */
+  evaluate(opts?: AgentEvalEvaluateOptions<TScenario, TArtifact>): Promise<CampaignResult<TArtifact, TScenario>>;
+  /**
+   * Run the closed improvement loop. Per-call overrides replace the definition
+   * except for nested config objects (`budget`, `hostedTenant`), which
+   * are merged field-by-field so callers can override one knob without
+   * repeating secrets or budget defaults.
+   */
+  improve(opts?: AgentEvalImproveOptions<TScenario, TArtifact>): Promise<SelfImproveResult<TScenario, TArtifact>>;
+}
+/**
+ * Define an agent eval once, then either score a surface with `evaluate()` or
+ * run the closed loop with `improve()`.
+ *
+ * This is a DX wrapper only: it delegates to `runEval()` and `selfImprove()` and
+ * returns their native result shapes.
+ */
+declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
+//#endregion
+export { DefinedAgentEval as a, SelfImproveOptions as c, SelfImproveRunError as d, selfImprove as f, DefineAgentEvalOptions as i, SelfImproveProgressEvent as l, AgentEvalEvaluateOptions as n, defineAgentEval as o, AgentEvalImproveOptions as r, SelfImproveBudget as s, AgentEvalAgent as t, SelfImproveResult as u };
+//# sourceMappingURL=define-agent-eval-sH24zBfM.d.ts.map

package/dist/define-agent-eval-sH24zBfM.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"define-agent-eval-sH24zBfM.d.ts","names":[],"sources":["../src/contract/self-improve.ts","../src/contract/define-agent-eval.ts"],"mappings":";;;;;;;;UA+DiB;;;EAGf;;;;EAIA;;EAEA;;EAEA;;;EAGA;;;EAGA;;;;EAIA;;EAEA,mBAAmB;;;;;;;;;EASnB;;EAEA;;;;;EAKA;;KAGU;EACN;EAA0B;;EAC1B;EAA4B;EAAuB;;EACnD;EAA4B;EAAe;;EAC3C;EAA8B;EAAe;EAAuB;;EAGpE;EAAsB;EAAkB;;EACxC;EAAyB;EAAW;EAAY;EAAa;;UAElD,mBAAmB,kBAAkB,UAAU;;;;;;;;;;;;;;EAc9D,QAAQ,SAAS,gBAAgB,UAAU,WAAW,KAAK,oBAAoB,QAAQ;;;;;;;EAQvF;;;EAIA,WAAW;;;EAIX,OAAO,YAAY,WAAW;;;EAI9B,iBAAiB;;EAGjB,SAAS;;;;;;;;;;;;EAaT,sBAAsB,gCAAgC,WAAW;;;;;EAMjE,WAAW,gBAAgB;;;;;;EAO3B,SAAS,mBAAmB,WAAW;;;EAIvC,qBAAqB;;;EAIrB,OAAO,KAAK,WAAW;;;;;;;;EASvB,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;;;EAKjF,UAAU;;;;EAKV;;;EAIA,gBAAgB,QAAQ;;;;;EAMxB,iBAAiB;IACf,UAAU;IACV;IACA;;;;EAKF;;;EAIA,cAAc,OAAO;;;;EAKrB;EACA;EACA;;;;;;;;;;;;;;EAeA,eAAe;;;EAIf,eAAe;;;;EAKf,eAAe;;EAGf;;;;;;;;EASA;;;;;;;;;EAUA,oBAAoB,uBAAuB,WAAW;;;EAItD,WAAW;;;;;;;EAQX,mBAAmB,uBAAuB,WAAW;;UAGtC,kBAAkB,kBAAkB,UAAU;;;;EAI7D;IACE;IACA,aAAa;;;;;EAKf;IACE;IACA,aAAa;IACb,SAAS;;;IAGT;;;;IAIA;;;;;;EAMF;;;EAGA;;;;EAIA,YAAY;;EAEZ;;;EAGA;;EAEA;;EAEA;;EAEA,MAAM;;;EAGN,UAAU;;EAEV;IACE;IACA,MAAM;IACN;IACA,aAAa;;;;;;;;EAQf,SAAS;;;;;EAKT,QAAQ;;;;;;EAMR,KAAK,yBAAyB,WAAW;;;cAI9B,4BAA4B;WAC9B,MAAM;WACN,UAAU;EAEnB,YAAY,gBAAgB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA+LhB,YAAY,kBAAkB,UAAU,WAC5D,MAAM,mBAAmB,WAAW,aACnC,QAAQ,kBAAkB,WAAW;;;KCviB5B,eAAe,kBAAkB,UAAU,cACrD,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;KAED,uBAAuB,kBAAkB,UAAU,aAAa,mBAC1E,WACA;UAGe,yBAAyB,kBAAkB,UAAU,mBAC5D,KACN,eAAe,WAAW;;EAI5B,YAAY;;EAEZ,UAAU;;EAEV,QAAQ,eAAe,WAAW;;EAElC,QAAQ,YAAY,WAAW;;EAE/B,SAAS,YAAY,WAAW;;EAEhC;;KAGU,wBAAwB,kBAAkB,UAAU,aAAa,KAC3E,QAAQ,mBAAmB,WAAW;EAGtC,SAAS,QAAQ;EACjB,eAAe,QAAQ;;UAGR,iBAAiB,kBAAkB,UAAU;;WAEnD,oBAAoB;;WAEpB,iBAAiB;;;;;EAK1B,SACE,OAAO,yBAAyB,WAAW,aAC1C,QAAQ,eAAe,WAAW;;;;;;;EAOrC,QACE,OAAO,wBAAwB,WAAW,aACzC,QAAQ,kBAAkB,WAAW;;;;;;;;;iBAU1B,gBAAgB,kBAAkB,UAAU,WAC1D,UAAU,uBAAuB,WAAW,aAC3C,iBAAiB,WAAW"}

package/dist/descriptive-B5MwKfbf.js

@@ -0,0 +1,144 @@
+import { i as makeRng } from "./internal-BDHPCnjk.js";
+//#region src/statistics/descriptive.ts
+/**
+* Descriptive statistics: means, bootstrap spread, correlation, and the
+* weighted judge-dimension composite. Nothing here is a significance test.
+*/
+/** Weighted mean — falls back to uniform weights when omitted */
+function weightedMean(scores) {
+	if (scores.length === 0) return 0;
+	let totalWeight = 0;
+	let weightedSum = 0;
+	for (const { score, weight } of scores) {
+		const w = weight ?? 1;
+		weightedSum += score * w;
+		totalWeight += w;
+	}
+	return totalWeight > 0 ? weightedSum / totalWeight : 0;
+}
+/**
+* Percentile bootstrap confidence interval on the mean of `scores`.
+*
+* Descriptive spread. It is not a significance test, and at small n its bounds
+* are anti-conservative in the same way {@link pairedBootstrap}'s are — see
+* {@link BOOTSTRAP_GATE_MIN_N}. With no `seed` the resampling is seeded from
+* the scores themselves, so the interval is reproducible either way.
+*/
+function confidenceInterval(scores, confidence = .95, opts = {}) {
+	if (scores.length === 0) return {
+		mean: 0,
+		lower: 0,
+		upper: 0
+	};
+	if (scores.length === 1) return {
+		mean: scores[0],
+		lower: scores[0],
+		upper: scores[0]
+	};
+	const n = scores.length;
+	const mean = scores.reduce((a, b) => a + b, 0) / n;
+	const B = opts.resamples ?? 1e3;
+	const rng = makeRng(opts.seed, scores);
+	const bootstrapMeans = [];
+	for (let i = 0; i < B; i++) {
+		let sum = 0;
+		for (let j = 0; j < n; j++) sum += scores[Math.floor(rng() * n)];
+		bootstrapMeans.push(sum / n);
+	}
+	bootstrapMeans.sort((a, b) => a - b);
+	const alpha = 1 - confidence;
+	const lowerIdx = Math.floor(alpha / 2 * B);
+	const upperIdx = Math.floor((1 - alpha / 2) * B) - 1;
+	return {
+		mean,
+		lower: bootstrapMeans[lowerIdx],
+		upper: bootstrapMeans[Math.min(upperIdx, B - 1)]
+	};
+}
+/** Partial credit: returns 0-1 ratio of current toward target */
+function partialCredit(current, target) {
+	if (target <= 0) return 1;
+	return Math.min(1, Math.max(0, current / target));
+}
+/**
+* Average-rank-with-ties transform (1-indexed). Tied values receive the mean
+* of the ranks they span, the standard correction for Spearman's ρ.
+*/
+function ranks(xs) {
+	const indexed = xs.map((v, i) => ({
+		v,
+		i
+	})).sort((a, b) => a.v - b.v);
+	const r = new Array(xs.length);
+	let i = 0;
+	while (i < indexed.length) {
+		let j = i;
+		while (j + 1 < indexed.length && indexed[j + 1].v === indexed[i].v) j++;
+		const avg = (i + j) / 2 + 1;
+		for (let k = i; k <= j; k++) r[indexed[k].i] = avg;
+		i = j + 1;
+	}
+	return r;
+}
+/**
+* Pearson product-moment correlation coefficient r ∈ [-1, 1] between two
+* equal-length series. See the edge-case contract above: NaN for n < 2 or
+* unequal lengths, 1 when both series are constant, 0 when exactly one is.
+*/
+function pearsonR(a, b) {
+	if (a.length !== b.length || a.length < 2) return NaN;
+	const n = a.length;
+	const meanA = a.reduce((s, v) => s + v, 0) / n;
+	const meanB = b.reduce((s, v) => s + v, 0) / n;
+	let num = 0;
+	let varA = 0;
+	let varB = 0;
+	for (let i = 0; i < n; i++) {
+		const da = a[i] - meanA;
+		const db = b[i] - meanB;
+		num += da * db;
+		varA += da * da;
+		varB += db * db;
+	}
+	if (varA === 0 || varB === 0) return varA === 0 && varB === 0 ? 1 : 0;
+	return num / Math.sqrt(varA * varB);
+}
+/**
+* Spearman's rank correlation ρ — Pearson over the average-rank-with-ties
+* transform of each series. Same edge-case contract as {@link pearsonR}.
+*/
+function spearmanR(a, b) {
+	if (a.length !== b.length || a.length < 2) return NaN;
+	return pearsonR(ranks(a), ranks(b));
+}
+/**
+* Weighted composite over judge dimensions: `Σ(score_d · w_d) / Σ(w_d)` across
+* the weighted dimensions. The canonical replacement for the per-consumer
+* hand-rolled composite math (tax/legal/creative/gtm each ship a copy).
+*
+* Fail-loud: throws if a weighted dimension is missing from `dims`, if any
+* weight is negative, or if the weights sum to 0 — none of which can produce
+* a meaningful composite.
+*/
+function weightedComposite(input) {
+	const entries = Object.entries(input.weights);
+	if (entries.length === 0) throw new Error("weightedComposite: `weights` is empty — nothing to combine");
+	let weightedSum = 0;
+	let weightTotal = 0;
+	for (const [dim, weight] of entries) {
+		if (weight < 0) throw new Error(`weightedComposite: weight for '${dim}' is negative (${weight})`);
+		if (!(dim in input.dims)) throw new Error(`weightedComposite: weighted dimension '${dim}' is absent from \`dims\` — refusing to renormalise onto a different denominator`);
+		weightedSum += input.dims[dim] * weight;
+		weightTotal += weight;
+	}
+	if (weightTotal === 0) throw new Error("weightedComposite: weights sum to 0 — composite is undefined");
+	const composite = weightedSum / weightTotal;
+	return input.threshold === void 0 ? { composite } : {
+		composite,
+		pass: composite >= input.threshold
+	};
+}
+//#endregion
+export { spearmanR as a, ranks as i, partialCredit as n, weightedComposite as o, pearsonR as r, weightedMean as s, confidenceInterval as t };
+
+//# sourceMappingURL=descriptive-B5MwKfbf.js.map

package/dist/descriptive-B5MwKfbf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"descriptive-B5MwKfbf.js","names":[],"sources":["../src/statistics/descriptive.ts"],"sourcesContent":["/**\n * Descriptive statistics: means, bootstrap spread, correlation, and the\n * weighted judge-dimension composite. Nothing here is a significance test.\n */\n\nimport { makeRng } from './internal'\n\n/** Weighted mean — falls back to uniform weights when omitted */\nexport function weightedMean(scores: { score: number; weight?: number }[]): number {\n  if (scores.length === 0) return 0\n  let totalWeight = 0\n  let weightedSum = 0\n  for (const { score, weight } of scores) {\n    const w = weight ?? 1\n    weightedSum += score * w\n    totalWeight += w\n  }\n  return totalWeight > 0 ? weightedSum / totalWeight : 0\n}\n\n/**\n * Percentile bootstrap confidence interval on the mean of `scores`.\n *\n * Descriptive spread. It is not a significance test, and at small n its bounds\n * are anti-conservative in the same way {@link pairedBootstrap}'s are — see\n * {@link BOOTSTRAP_GATE_MIN_N}. With no `seed` the resampling is seeded from\n * the scores themselves, so the interval is reproducible either way.\n */\nexport function confidenceInterval(\n  scores: number[],\n  confidence = 0.95,\n  opts: { seed?: number; resamples?: number } = {},\n): { mean: number; lower: number; upper: number } {\n  if (scores.length === 0) return { mean: 0, lower: 0, upper: 0 }\n  if (scores.length === 1) return { mean: scores[0]!, lower: scores[0]!, upper: scores[0]! }\n\n  const n = scores.length\n  const mean = scores.reduce((a, b) => a + b, 0) / n\n\n  const B = opts.resamples ?? 1000\n  const rng = makeRng(opts.seed, scores)\n  const bootstrapMeans: number[] = []\n\n  for (let i = 0; i < B; i++) {\n    let sum = 0\n    for (let j = 0; j < n; j++) {\n      sum += scores[Math.floor(rng() * n)]!\n    }\n    bootstrapMeans.push(sum / n)\n  }\n\n  bootstrapMeans.sort((a, b) => a - b)\n\n  const alpha = 1 - confidence\n  const lowerIdx = Math.floor((alpha / 2) * B)\n  const upperIdx = Math.floor((1 - alpha / 2) * B) - 1\n\n  return {\n    mean,\n    lower: bootstrapMeans[lowerIdx]!,\n    upper: bootstrapMeans[Math.min(upperIdx, B - 1)]!,\n  }\n}\n\n/** Partial credit: returns 0-1 ratio of current toward target */\nexport function partialCredit(current: number, target: number): number {\n  if (target <= 0) return 1\n  return Math.min(1, Math.max(0, current / target))\n}\n\n// ── Correlation (Pearson / Spearman) ─────────────────────────────────\n//\n// The single source for linear (Pearson) and rank (Spearman) correlation.\n// Edge-case contract is explicit so every caller agrees on what a\n// degenerate input means:\n//   - length mismatch or n < 2  → NaN   (correlation is undefined; not 0)\n//   - both series constant      → 1     (degenerate perfect agreement)\n//   - exactly one series constant → 0   (no covariation to detect)\n// Returning NaN for n < 2 (rather than 0) keeps \"not enough data\" distinct\n// from \"measured zero correlation\", which a 0 would silently conflate.\n\n/**\n * Average-rank-with-ties transform (1-indexed). Tied values receive the mean\n * of the ranks they span, the standard correction for Spearman's ρ.\n */\nexport function ranks(xs: number[]): number[] {\n  const indexed = xs.map((v, i) => ({ v, i })).sort((a, b) => a.v - b.v)\n  const r = new Array<number>(xs.length)\n  let i = 0\n  while (i < indexed.length) {\n    let j = i\n    while (j + 1 < indexed.length && indexed[j + 1]!.v === indexed[i]!.v) j++\n    const avg = (i + j) / 2 + 1\n    for (let k = i; k <= j; k++) r[indexed[k]!.i] = avg\n    i = j + 1\n  }\n  return r\n}\n\n/**\n * Pearson product-moment correlation coefficient r ∈ [-1, 1] between two\n * equal-length series. See the edge-case contract above: NaN for n < 2 or\n * unequal lengths, 1 when both series are constant, 0 when exactly one is.\n */\nexport function pearsonR(a: number[], b: number[]): number {\n  if (a.length !== b.length || a.length < 2) return Number.NaN\n  const n = a.length\n  const meanA = a.reduce((s, v) => s + v, 0) / n\n  const meanB = b.reduce((s, v) => s + v, 0) / n\n  let num = 0\n  let varA = 0\n  let varB = 0\n  for (let i = 0; i < n; i++) {\n    const da = a[i]! - meanA\n    const db = b[i]! - meanB\n    num += da * db\n    varA += da * da\n    varB += db * db\n  }\n  if (varA === 0 || varB === 0) return varA === 0 && varB === 0 ? 1 : 0\n  return num / Math.sqrt(varA * varB)\n}\n\n/**\n * Spearman's rank correlation ρ — Pearson over the average-rank-with-ties\n * transform of each series. Same edge-case contract as {@link pearsonR}.\n */\nexport function spearmanR(a: number[], b: number[]): number {\n  if (a.length !== b.length || a.length < 2) return Number.NaN\n  return pearsonR(ranks(a), ranks(b))\n}\n\nexport interface WeightedCompositeInput {\n  /** Per-dimension scores (typically 0..1). */\n  dims: Record<string, number>\n  /** Weight per dimension. Every weighted dimension MUST be present in\n   *  `dims` — a weight for an absent dimension is a config error and throws,\n   *  because silently dropping it would renormalise the composite onto a\n   *  different denominator than intended. */\n  weights: Record<string, number>\n  /** Optional pass threshold; when set, the result reports `pass`. */\n  threshold?: number\n}\n\nexport interface WeightedCompositeResult {\n  composite: number\n  pass?: boolean\n}\n\n/**\n * Weighted composite over judge dimensions: `Σ(score_d · w_d) / Σ(w_d)` across\n * the weighted dimensions. The canonical replacement for the per-consumer\n * hand-rolled composite math (tax/legal/creative/gtm each ship a copy).\n *\n * Fail-loud: throws if a weighted dimension is missing from `dims`, if any\n * weight is negative, or if the weights sum to 0 — none of which can produce\n * a meaningful composite.\n */\nexport function weightedComposite(input: WeightedCompositeInput): WeightedCompositeResult {\n  const entries = Object.entries(input.weights)\n  if (entries.length === 0) {\n    throw new Error('weightedComposite: `weights` is empty — nothing to combine')\n  }\n  let weightedSum = 0\n  let weightTotal = 0\n  for (const [dim, weight] of entries) {\n    if (weight < 0) {\n      throw new Error(`weightedComposite: weight for '${dim}' is negative (${weight})`)\n    }\n    if (!(dim in input.dims)) {\n      throw new Error(\n        `weightedComposite: weighted dimension '${dim}' is absent from \\`dims\\` — ` +\n          'refusing to renormalise onto a different denominator',\n      )\n    }\n    weightedSum += input.dims[dim]! * weight\n    weightTotal += weight\n  }\n  if (weightTotal === 0) {\n    throw new Error('weightedComposite: weights sum to 0 — composite is undefined')\n  }\n  const composite = weightedSum / weightTotal\n  return input.threshold === undefined\n    ? { composite }\n    : { composite, pass: composite >= input.threshold }\n}\n"],"mappings":";;;;;;;AAQA,SAAgB,aAAa,QAAsD;CACjF,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,IAAI,cAAc;CAClB,IAAI,cAAc;CAClB,KAAK,MAAM,EAAE,OAAO,YAAY,QAAQ;EACtC,MAAM,IAAI,UAAU;EACpB,eAAe,QAAQ;EACvB,eAAe;CACjB;CACA,OAAO,cAAc,IAAI,cAAc,cAAc;AACvD;;;;;;;;;AAUA,SAAgB,mBACd,QACA,aAAa,KACb,OAA8C,CAAC,GACC;CAChD,IAAI,OAAO,WAAW,GAAG,OAAO;EAAE,MAAM;EAAG,OAAO;EAAG,OAAO;CAAE;CAC9D,IAAI,OAAO,WAAW,GAAG,OAAO;EAAE,MAAM,OAAO;EAAK,OAAO,OAAO;EAAK,OAAO,OAAO;CAAI;CAEzF,MAAM,IAAI,OAAO;CACjB,MAAM,OAAO,OAAO,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;CAEjD,MAAM,IAAI,KAAK,aAAa;CAC5B,MAAM,MAAM,QAAQ,KAAK,MAAM,MAAM;CACrC,MAAM,iBAA2B,CAAC;CAElC,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI,MAAM;EACV,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,OAAO,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC;EAEpC,eAAe,KAAK,MAAM,CAAC;CAC7B;CAEA,eAAe,MAAM,GAAG,MAAM,IAAI,CAAC;CAEnC,MAAM,QAAQ,IAAI;CAClB,MAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,CAAC;CAC3C,MAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,CAAC,IAAI;CAEnD,OAAO;EACL;EACA,OAAO,eAAe;EACtB,OAAO,eAAe,KAAK,IAAI,UAAU,IAAI,CAAC;CAChD;AACF;;AAGA,SAAgB,cAAc,SAAiB,QAAwB;CACrE,IAAI,UAAU,GAAG,OAAO;CACxB,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,UAAU,MAAM,CAAC;AAClD;;;;;AAiBA,SAAgB,MAAM,IAAwB;CAC5C,MAAM,UAAU,GAAG,KAAK,GAAG,OAAO;EAAE;EAAG;CAAE,EAAE,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC;CACrE,MAAM,IAAI,IAAI,MAAc,GAAG,MAAM;CACrC,IAAI,IAAI;CACR,OAAO,IAAI,QAAQ,QAAQ;EACzB,IAAI,IAAI;EACR,OAAO,IAAI,IAAI,QAAQ,UAAU,QAAQ,IAAI,EAAE,CAAE,MAAM,QAAQ,EAAE,CAAE,GAAG;EACtE,MAAM,OAAO,IAAI,KAAK,IAAI;EAC1B,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG,KAAK,EAAE,QAAQ,EAAE,CAAE,KAAK;EAChD,IAAI,IAAI;CACV;CACA,OAAO;AACT;;;;;;AAOA,SAAgB,SAAS,GAAa,GAAqB;CACzD,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,GAAG,OAAO;CAClD,MAAM,IAAI,EAAE;CACZ,MAAM,QAAQ,EAAE,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;CAC7C,MAAM,QAAQ,EAAE,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;CAC7C,IAAI,MAAM;CACV,IAAI,OAAO;CACX,IAAI,OAAO;CACX,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,MAAM,KAAK,EAAE,KAAM;EACnB,MAAM,KAAK,EAAE,KAAM;EACnB,OAAO,KAAK;EACZ,QAAQ,KAAK;EACb,QAAQ,KAAK;CACf;CACA,IAAI,SAAS,KAAK,SAAS,GAAG,OAAO,SAAS,KAAK,SAAS,IAAI,IAAI;CACpE,OAAO,MAAM,KAAK,KAAK,OAAO,IAAI;AACpC;;;;;AAMA,SAAgB,UAAU,GAAa,GAAqB;CAC1D,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,GAAG,OAAO;CAClD,OAAO,SAAS,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC;AACpC;;;;;;;;;;AA4BA,SAAgB,kBAAkB,OAAwD;CACxF,MAAM,UAAU,OAAO,QAAQ,MAAM,OAAO;CAC5C,IAAI,QAAQ,WAAW,GACrB,MAAM,IAAI,MAAM,4DAA4D;CAE9E,IAAI,cAAc;CAClB,IAAI,cAAc;CAClB,KAAK,MAAM,CAAC,KAAK,WAAW,SAAS;EACnC,IAAI,SAAS,GACX,MAAM,IAAI,MAAM,kCAAkC,IAAI,iBAAiB,OAAO,EAAE;EAElF,IAAI,EAAE,OAAO,MAAM,OACjB,MAAM,IAAI,MACR,0CAA0C,IAAI,iFAEhD;EAEF,eAAe,MAAM,KAAK,OAAQ;EAClC,eAAe;CACjB;CACA,IAAI,gBAAgB,GAClB,MAAM,IAAI,MAAM,8DAA8D;CAEhF,MAAM,YAAY,cAAc;CAChC,OAAO,MAAM,cAAc,KAAA,IACvB,EAAE,UAAU,IACZ;EAAE;EAAW,MAAM,aAAa,MAAM;CAAU;AACtD"}

package/dist/{dspy-rlm-engine-DbTk4JdR.js → dspy-rlm-engine-B_qhSc21.js}

@@ -1,6 +1,6 @@
-import { s as resolveModelPricing } from "./metrics-C9YY1OcL.js";
-import { D as RawAnalystFindingSchema } from "./kind-factory-B8-r8-y8.js";
-import { D as resolveExternalOptimizerProcessLimits, E as resolveExternalOptimizerCallbackLimits, T as removeCredentialEnvironment, f as runWithCleanup, g as sendJson, h as listenLocal, m as closeServer, p as startExternalOptimizerModelProxy, u as runExternalOptimizerProcess, y as assertExternalOptimizerModelBudget } from "./single-run-lock-DFWHEB09.js";
+import { a as resolveModelPricing } from "./metrics-Cl0L1KUy.js";
+import { _ as resolveExternalOptimizerCallbackLimits, a as closeServer, g as removeCredentialEnvironment, i as startExternalOptimizerModelProxy, o as listenLocal, r as runWithCleanup, s as sendJson, t as runExternalOptimizerProcess, u as assertExternalOptimizerModelBudget, v as resolveExternalOptimizerProcessLimits } from "./external-optimizer-subprocess-DrJ9hR8u.js";
+import { D as RawAnalystFindingSchema } from "./kind-factory-CPmSd58s.js";
 import { randomBytes } from "node:crypto";
 import { createServer } from "node:http";
 //#region src/analyst/trace-tool-callback.ts
@@ -390,4 +390,4 @@ function isRecord(value) {
 //#endregion
 export { createDspyRlmTraceEngine as t };
 
-//# sourceMappingURL=dspy-rlm-engine-DbTk4JdR.js.map
+//# sourceMappingURL=dspy-rlm-engine-B_qhSc21.js.map