@tangle-network/agent-eval 0.144.12 → 0.145.0

package/dist/backend-integrity-CsVin_Wb.d.ts

@@ -0,0 +1,280 @@
+import { t as AgentEvalError } from "./errors-DEE6u6ot.js";
+import { a as CheckerIdentity, n as VerdictCertification, t as DefaultVerdict, x as VerificationStrategySource } from "./verdict-E4eRNf7-.js";
+import { c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
+import { a as RunRecord } from "./run-record-CKiihE6f.js";
+import { it as RawProviderSink, p as ChatClient } from "./types-Cx3YUh2r.js";
+//#region src/artifact-validator.d.ts
+/**
+ * Artifact validators.
+ *
+ * Generic "score a produced artifact" primitive. Tax uses it for PDF form
+ * correctness, research for sourced briefs, browser for task assertions, coding
+ * for social posts. One interface, many validators.
+ *
+ * A validator receives an `Artifact` (file on disk, JSON blob, text, binary)
+ * plus a `ValidationContext` (scenario id, the turns that produced it) and
+ * returns a `ValidationResult` with pass/fail + 0..1 score + structured
+ * issues.
+ */
+interface Artifact {
+  /** Logical kind — validators type-guard on this */
+  kind: 'file' | 'json' | 'text' | 'binary' | string;
+  /** Filesystem-style path, optional */
+  path?: string;
+  /** String content for text/json/file kinds */
+  content?: string;
+  /** Binary content (if kind === 'binary') */
+  bytes?: Uint8Array;
+  /** Caller-supplied metadata (mimeType, sha256, size, etc.) */
+  metadata?: Record<string, unknown>;
+}
+//#endregion
+//#region src/completion-verifier.d.ts
+/** What kind of produced state can satisfy a requirement structurally. */
+type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any';
+interface CompletionRequirement {
+  /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */
+  reqId: string;
+  /** Human-readable description of the required deliverable. */
+  title: string;
+  /** Optional kind/category hint, matched against a produced item's kind. */
+  category?: string;
+  /** What produced state satisfies this requirement. Defaults to 'any'. */
+  satisfiedBy?: SatisfiedBy;
+}
+interface TaskGold {
+  taskId: string;
+  requirements: CompletionRequirement[];
+}
+interface ProducedProposal {
+  id: string;
+  title: string;
+  status: 'pending' | 'approved' | 'rejected';
+  /** Optional persisted body — when present, enables a correctness check. */
+  content?: string;
+}
+/** Everything observable about what a run actually produced. */
+interface ProducedState {
+  /** Persisted vault artifacts. Reuses the shared `Artifact` shape. */
+  artifacts: Artifact[];
+  /** Proposals / filings the agent created. */
+  proposals: ProducedProposal[];
+  /** Names of tools the agent invoked. */
+  toolCalls: string[];
+}
+interface RequirementCheck {
+  reqId: string;
+  title: string;
+  /** A produced item of the right kind matched the requirement, non-empty. */
+  structurallyPresent: boolean;
+  /**
+   * Whether the matched item actually fulfils the requirement. `null` when
+   * not structurally present, when the matched item carries no content
+   * to assess, or when the correctness check itself failed (`unmeasured`).
+   */
+  correct: boolean | null;
+  /** structurallyPresent && !unmeasured && correct !== false. */
+  satisfied: boolean;
+  /**
+   * Set when the correctness check itself errored (LLM call failure or an
+   * unparseable response after retry). The requirement's fulfilment is
+   * UNKNOWN — `correct` stays null, `satisfied` is false, and
+   * `completionVerdict` excludes the row from `completionRate`'s
+   * denominator. Never folded into a zero: a synthetic zero is
+   * indistinguishable from a real failure (see `JudgeParseError`).
+   */
+  unmeasured?: true;
+  /** Why the correctness check could not be measured (present iff `unmeasured`). */
+  unmeasuredReason?: string;
+  /** Human-readable evidence for the verdict. */
+  evidence: string[];
+}
+/** Extends the substrate verdict spine: `valid` = `fullyComplete` and
+ *  `score` = `completionRate` — derived in `completionVerdict()`, the one
+ *  place those equalities hold by construction. */
+interface CompletionVerdict extends DefaultVerdict {
+  taskId: string;
+  requirements: RequirementCheck[];
+  /** satisfied / MEASURABLE requirements (unmeasured rows leave the denominator). */
+  completionRate: number;
+  /** Every measurable requirement satisfied (false when anything is unmeasured). */
+  fullyComplete: boolean;
+  /** Requirements whose correctness check errored — reported, never scored as zero. */
+  unmeasuredCount: number;
+}
+/**
+ * Construct a `CompletionVerdict` from the per-requirement checks, deriving
+ * `completionRate` / `fullyComplete` and the spine fields (`valid` =
+ * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero
+ * requirements — a verdict over nothing is a misconfiguration, mirroring
+ * `verifyCompletion`'s gold-spec guard.
+ */
+declare function completionVerdict(input: {
+  taskId: string;
+  requirements: RequirementCheck[];
+  /** What certified the correctness stage, when anything did. Omitted =
+   *  an uncertified verdict — the honest default for a bare checker. */
+  certification?: VerdictCertification;
+}): CompletionVerdict;
+/**
+ * What a correctness checker declares about itself so the completion
+ * verdict can carry a certification: which strategy member it discharges,
+ * its exact identity, and the steps its answers rest on unverified.
+ */
+interface CorrectnessCheckerAttestation {
+  strategy: VerificationStrategySource;
+  checker: CheckerIdentity;
+  assumptions: string[];
+}
+/**
+ * Decides whether a produced item's content actually fulfils a requirement.
+ * Injected so the structural verifier stays pure and unit-testable; the
+ * production implementation is `createLlmCorrectnessChecker`.
+ *
+ * `attestation` is optional metadata on the function value: a checker that
+ * carries one yields certified completion verdicts; a bare function yields
+ * the same verdict uncertified. A plain arrow function remains a valid
+ * checker.
+ */
+interface CorrectnessChecker {
+  (requirement: CompletionRequirement, content: string): Promise<{
+    correct: boolean;
+    reason: string;
+  }>;
+  attestation?: CorrectnessCheckerAttestation;
+}
+/**
+ * Verify whether a run completed the task. `checkCorrectness` is injected —
+ * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.
+ *
+ * Throws on a gold spec with no requirements: an eval task that requires
+ * nothing is a misconfiguration, not a vacuously-complete task.
+ */
+declare function verifyCompletion(gold: TaskGold, state: ProducedState, checkCorrectness: CorrectnessChecker): Promise<CompletionVerdict>;
+interface LlmCorrectnessCheckerOpts {
+  model?: string;
+  /** Optional ledger for direct use. */
+  costLedger?: CostLedgerHandle;
+  costPhase?: string;
+  costTags?: Record<string, string>;
+  signal?: AbortSignal;
+  /** Max chars of artifact content sent to the checker. */
+  maxContentChars?: number;
+  /**
+   * Checker LLM calls per requirement before giving up (parse failures and
+   * call errors both consume attempts). The failure then surfaces as an
+   * `unmeasured` requirement, never a zero.
+   */
+  maxAttempts?: number;
+  /**
+   * Forensic capture of every checker request/response/error — without it a
+   * checker failure is unauditable (the agent-turn raws never contain the
+   * checker's own calls). Same sink contract as `LlmClient`.
+   */
+  rawSink?: RawProviderSink;
+}
+/**
+ * Production `CorrectnessChecker` — one LLM call per matched artifact,
+ * deterministic (temperature 0), structured JSON out. Judges fulfilment
+ * only: a plan, a gesture, or a description of what should be done does not
+ * fulfil a requirement — the artifact must BE the deliverable.
+ */
+declare function createLlmCorrectnessChecker(chat: ChatClient, opts?: LlmCorrectnessCheckerOpts): CorrectnessChecker;
+//#endregion
+//#region src/produced-state.d.ts
+/** A tool the agent invoked. */
+interface ToolCallEventLike {
+  type: 'tool_call';
+  toolName: string;
+}
+/**
+ * An artifact the agent produced. `content` is the enriched field — the
+ * runtime's base `artifact` event carries only metadata; the completion
+ * oracle needs the body to verify the deliverable, so the runtime emits it.
+ */
+interface ArtifactEventLike {
+  type: 'artifact';
+  artifactId: string;
+  name?: string;
+  mimeType?: string;
+  uri?: string;
+  content?: string;
+}
+/** A proposal / filing the agent created. */
+interface ProposalEventLike {
+  type: 'proposal_created';
+  proposalId: string;
+  title: string;
+  status?: 'pending' | 'approved' | 'rejected';
+  content?: string;
+}
+/**
+ * The subset of runtime stream events `extractProducedState` consumes.
+ * agent-runtime's full `RuntimeStreamEvent` union satisfies this structurally;
+ * the `{ type: string }` catch-all keeps the input permissive so callers can
+ * pass the whole unfiltered telemetry stream — unrecognized events are skipped.
+ */
+type RuntimeEventLike = ToolCallEventLike | ArtifactEventLike | ProposalEventLike | {
+  type: string;
+};
+/**
+ * Normalize a run's runtime event stream into `ProducedState`.
+ *
+ * Pure and total — unrecognized event types are skipped. `toolCalls` is
+ * deduplicated by name in first-seen order (completion cares about a tool's
+ * presence, not its call count). An artifact with neither a name nor a uri
+ * still yields an entry keyed by its `artifactId` so it is never silently
+ * dropped; an artifact with no `content` yields empty content, which the
+ * completion oracle's structural check then rejects on its own.
+ */
+declare function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState;
+//#endregion
+//#region src/integrity/backend-integrity.d.ts
+interface BackendIntegrityReport {
+  /** Total records inspected. */
+  totalRecords: number;
+  /** Records with input=0 AND output=0 (a stub fingerprint). */
+  stubRecords: number;
+  /** Records with nonzero token usage (real LLM activity). */
+  realRecords: number;
+  /** Records where output>0 but costUsd=0 (real LLM, broken cost ledger). */
+  uncostedRecords: number;
+  /** Sum of input tokens across all records. */
+  totalInputTokens: number;
+  /** Sum of output tokens across all records. */
+  totalOutputTokens: number;
+  /** Sum of costUsd across all records. */
+  totalCostUsd: number;
+  /** Worst-case integrity verdict. */
+  verdict: 'real' | 'mixed' | 'stub';
+  /** Human-readable diagnosis suitable for terminal output. */
+  diagnosis: string;
+}
+/**
+ * Error thrown when an integrity assertion fails. Caller can pattern-match
+ * by `code === 'AGENT_EVAL_BACKEND_STUB'` to differentiate from other
+ * errors.
+ */
+declare class BackendIntegrityError extends AgentEvalError {
+  readonly report: BackendIntegrityReport;
+  constructor(message: string, report: BackendIntegrityReport);
+}
+/**
+ * Inspect a batch of RunRecords and return an integrity report. Pure
+ * function — no I/O, no logging. The caller decides what to do with the
+ * verdict (print warning, throw, gate CI, etc.).
+ */
+declare function summarizeBackendIntegrity(records: ReadonlyArray<RunRecord>): BackendIntegrityReport;
+/**
+ * Throw BackendIntegrityError if the verdict is 'stub' — i.e. every record
+ * shows zero LLM activity. Non-strict callers can pass `{ allowMixed: false }`
+ * to also reject mixed verdicts (recommended for CI gates).
+ *
+ * Real backends pass through silently.
+ */
+declare function assertRealBackend(records: ReadonlyArray<RunRecord>, opts?: {
+  allowMixed?: boolean;
+}): BackendIntegrityReport;
+//#endregion
+export { SatisfiedBy as _, ArtifactEventLike as a, createLlmCorrectnessChecker as b, ToolCallEventLike as c, CompletionVerdict as d, CorrectnessChecker as f, RequirementCheck as g, ProducedState as h, summarizeBackendIntegrity as i, extractProducedState as l, ProducedProposal as m, BackendIntegrityReport as n, ProposalEventLike as o, LlmCorrectnessCheckerOpts as p, assertRealBackend as r, RuntimeEventLike as s, BackendIntegrityError as t, CompletionRequirement as u, TaskGold as v, verifyCompletion as x, completionVerdict as y };
+//# sourceMappingURL=backend-integrity-CsVin_Wb.d.ts.map

package/dist/backend-integrity-CsVin_Wb.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backend-integrity-CsVin_Wb.d.ts","names":[],"sources":["../src/artifact-validator.ts","../src/completion-verifier.ts","../src/produced-state.ts","../src/integrity/backend-integrity.ts"],"mappings":";;;;;;;;;;;;;;;;;;UAaiB;;EAEf;;EAEA;;EAEA;;EAEA,QAAQ;;EAER,WAAW;;;;;KCsBD;UAEK;;EAEf;;EAEA;;EAEA;;EAEA,cAAc;;UAGC;EACf;EACA,cAAc;;UAGC;EACf;EACA;EACA;;EAEA;;;UAIe;;EAEf,WAAW;;EAEX,WAAW;;EAEX;;UAGe;EACf;EACA;;EAEA;;;;;;EAMA;;EAEA;;;;;;;;;EASA;;EAEA;;EAEA;;;;;UAMe,0BAA0B;EACzC;EACA,cAAc;;EAEd;;EAEA;;EAEA;;;;;;;;;iBAUc,kBAAkB;EAChC;EACA,cAAc;;;EAGd,gBAAgB;IACd;;;;;;UAqCa;EACf,UAAU;EACV,SAAS;EACT;;;;;;;;;;;;UAae;GAEb,aAAa,uBACb,kBACC;IAAU;IAAkB;;EAC/B,cAAc;;;;;;;;;iBAsLM,iBACpB,MAAM,UACN,OAAO,eACP,kBAAkB,qBACjB,QAAQ;UAyGM;EACf;;EAEA,aAAa;EACb;EACA,WAAW;EACX,SAAS;;EAET;;;;;;EAMA;;;;;;EAMA,UAAU;;;;;;;;iBA2CI,4BACd,MAAM,YACN,OAAM,4BACL;;;;UCnhBc;EACf;EACA;;;;;;;UAQe;EACf;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EAIA;;;;;;;;KASU,mBACR,oBACA,oBACA;EACE;;;;;;;;;;;;iBAmBU,qBAAqB,iBAAiB,qBAAqB;;;UCpD1D;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;cAQW,8BAA8B;WAGvB,QAAQ;EAF1B,YACE,iBACgB,QAAQ;;;;;;;iBAWZ,0BACd,SAAS,cAAc,aACtB;;;;;;;;iBAuHa,kBACd,SAAS,cAAc,YACvB;EAAQ;IACP"}

package/dist/{baseline-C-GocmIW.js → baseline-BhPRQBVn.js}

@@ -1,4 +1,5 @@
-import { K as studentTCdf, q as studentTQuantile, u as cohensD } from "./statistics-ByxzSiOM.js";
+import { n as studentTQuantile, t as studentTCdf } from "./student-t-CvBq2mve.js";
+import { n as cohensD } from "./effect-sizes-DiH8MGOH.js";
 //#region src/baseline.ts
 /**
 * Baseline regression detection.
@@ -146,4 +147,4 @@ function assertFiniteSample(name, sample) {
 //#endregion
 export { iqr as n, welchsTTest as r, compareToBaseline as t };
 
-//# sourceMappingURL=baseline-C-GocmIW.js.map
+//# sourceMappingURL=baseline-BhPRQBVn.js.map

package/dist/{baseline-C-GocmIW.js.map → baseline-BhPRQBVn.js.map}

@@ -1 +1 @@
-{"version":3,"file":"baseline-C-GocmIW.js","names":[],"sources":["../src/baseline.ts"],"sourcesContent":["/**\n * Baseline regression detection.\n *\n * Lifted from ADC baseline.ts. Every promotion-blocking signal boils down\n * to: \"is this run measurably worse than baseline?\" — with enough\n * statistical rigor to distinguish noise from drift.\n *\n * Uses:\n *   - Welch's t-test (unequal variance) for per-metric mean comparison\n *   - Cohen's d for effect size magnitude\n *   - IQR for stability flag (unstable samples can't be trusted for comparisons)\n *\n * Returns a structured verdict: improved | regressed | stable | unstable.\n */\n\nimport { studentTCdf, studentTQuantile } from './math/student-t'\nimport { cohensD } from './statistics'\n\nexport interface MetricSamples {\n  /** Stable metric key (e.g. \"overallScore\", \"firstTokenMs\"). */\n  metric: string\n  /** Whether higher values are better. */\n  higherIsBetter: boolean\n  baseline: number[]\n  candidate: number[]\n}\n\nexport interface MetricVerdict {\n  metric: string\n  baselineMean: number\n  candidateMean: number\n  delta: number\n  /** Null when both samples are constant with unequal means — the\n   *  standardized effect is unbounded there, not zero. */\n  cohensD: number | null\n  welchT: number\n  welchDf: number\n  welchP: number\n  stable: boolean\n  /** IQR of the combined samples — used as a rough stability indicator. */\n  iqr: number\n  verdict: 'improved' | 'regressed' | 'stable' | 'unstable'\n}\n\nexport interface BaselineReport {\n  metrics: MetricVerdict[]\n  /** True if any critical metric regressed. */\n  hasRegression: boolean\n  /** True if any metric is unstable (too noisy to judge). */\n  hasUnstable: boolean\n}\n\nexport interface BaselineOptions {\n  /** Effect size threshold for meaningful delta (default 0.5 — medium effect). */\n  effectThreshold?: number\n  /** p-value threshold for statistical significance (default 0.05). */\n  alpha?: number\n  /** IQR/mean ratio above which samples are flagged unstable (default 0.30). */\n  unstableCvThreshold?: number\n}\n\nexport type WelchTestStatus = 'ok' | 'insufficient-sample' | 'zero-variance'\n\n/**\n * Full unequal-variance comparison for two independent samples.\n *\n * `meanB`, `delta`, `t`, and `cohensD` are oriented as `b - a`. Inferential\n * fields are `NaN` and `ci95` is null when the sample count or observed\n * variance cannot define a Student-t result. `status` makes that state\n * explicit so callers cannot mistake it for evidence of no difference.\n */\ninterface WelchTestResultBase {\n  meanA: number\n  meanB: number\n  delta: number\n}\n\nexport type WelchTestResult =\n  | (WelchTestResultBase & {\n      status: 'ok'\n      standardError: number\n      t: number\n      df: number\n      p: number\n      ci95: [number, number]\n      cohensD: number\n    })\n  | (WelchTestResultBase & {\n      status: 'insufficient-sample' | 'zero-variance'\n      standardError: number\n      t: number\n      df: number\n      p: number\n      ci95: null\n      cohensD: null\n    })\n\n/**\n * Compare candidate samples against baseline per metric. Verdict logic:\n *   - unstable: IQR/|mean| > threshold on either set — not enough signal\n *   - improved: meaningful effect in the \"better\" direction AND p < alpha\n *   - regressed: meaningful effect in the \"worse\" direction AND p < alpha\n *   - stable: otherwise (no significant change)\n */\nexport function compareToBaseline(\n  samples: MetricSamples[],\n  options: BaselineOptions = {},\n): BaselineReport {\n  const effectThreshold = options.effectThreshold ?? 0.5\n  const alpha = options.alpha ?? 0.05\n  const cvThreshold = options.unstableCvThreshold ?? 0.3\n\n  const metrics: MetricVerdict[] = samples.map((s) => {\n    const comparison = welchsTTest(s.baseline, s.candidate)\n    if (comparison.status === 'insufficient-sample') {\n      throw new Error(`compareToBaseline: need ≥2 samples per side for \"${s.metric}\"`)\n    }\n    const { meanA: bMean, meanB: cMean, delta, cohensD: d, t, df, p } = comparison\n    // Stability is per-side: a comparison is trustworthy only when BOTH\n    // samples are internally consistent. Combining the sides would flag\n    // large-but-real deltas as \"unstable\" which is exactly what we want\n    // to detect.\n    const baselineIqr = iqr(s.baseline)\n    const candidateIqr = iqr(s.candidate)\n    const baselineStable = baselineIqr / Math.max(Math.abs(bMean), 1e-9) <= cvThreshold\n    const candidateStable = candidateIqr / Math.max(Math.abs(cMean), 1e-9) <= cvThreshold\n    const stable = baselineStable && candidateStable\n    const reportedIqr = Math.max(baselineIqr, candidateIqr)\n\n    // Both sides are guaranteed ≥ 2 samples above, so a null d means zero\n    // pooled spread across a real mean gap: an unbounded standardized effect,\n    // which clears any finite threshold.\n    const effectClears = d === null || Math.abs(d) >= effectThreshold\n\n    let verdict: MetricVerdict['verdict']\n    if (!stable) {\n      verdict = 'unstable'\n    } else if (comparison.status === 'ok' && p < alpha && effectClears) {\n      const candidateIsBetter = s.higherIsBetter ? delta > 0 : delta < 0\n      verdict = candidateIsBetter ? 'improved' : 'regressed'\n    } else {\n      verdict = 'stable'\n    }\n\n    return {\n      metric: s.metric,\n      baselineMean: bMean,\n      candidateMean: cMean,\n      delta,\n      cohensD: d,\n      welchT: t,\n      welchDf: df,\n      welchP: p,\n      stable,\n      iqr: reportedIqr,\n      verdict,\n    }\n  })\n\n  return {\n    metrics,\n    hasRegression: metrics.some((m) => m.verdict === 'regressed'),\n    hasUnstable: metrics.some((m) => m.verdict === 'unstable'),\n  }\n}\n\nfunction mean(xs: number[]): number {\n  return xs.reduce((a, b) => a + b, 0) / xs.length\n}\n\n/** Inter-quartile range; 0 when the sample has no spread. */\nexport function iqr(xs: number[]): number {\n  if (xs.length === 0) return 0\n  const sorted = [...xs].sort((a, b) => a - b)\n  const q = (p: number) => {\n    const idx = p * (sorted.length - 1)\n    const lo = Math.floor(idx)\n    const hi = Math.ceil(idx)\n    return sorted[lo]! + (sorted[hi]! - sorted[lo]!) * (idx - lo)\n  }\n  return q(0.75) - q(0.25)\n}\n\n/**\n * Welch's t-test and 95% interval for two independent samples.\n *\n * Uses the Student-t distribution at every finite degree of freedom. Fewer\n * than two observations per side and zero observed standard error cannot\n * define a t reference distribution; those cases return an explicit status,\n * `NaN` inferential fields, and no interval rather than fabricated certainty.\n */\nexport function welchsTTest(a: number[], b: number[]): WelchTestResult {\n  assertFiniteSample('a', a)\n  assertFiniteSample('b', b)\n  const mA = mean(a)\n  const mB = mean(b)\n  const delta = mB - mA\n  if (a.length < 2 || b.length < 2) {\n    return {\n      status: 'insufficient-sample',\n      meanA: mA,\n      meanB: mB,\n      delta,\n      standardError: Number.NaN,\n      t: Number.NaN,\n      df: Number.NaN,\n      p: Number.NaN,\n      ci95: null,\n      cohensD: null,\n    }\n  }\n\n  const vA = variance(a, mA)\n  const vB = variance(b, mB)\n  const seSquared = vA / a.length + vB / b.length\n  const d = cohensD(a, b)\n  if (seSquared === 0) {\n    return {\n      status: 'zero-variance',\n      meanA: mA,\n      meanB: mB,\n      delta,\n      standardError: 0,\n      t: delta === 0 ? 0 : Math.sign(delta) * Number.POSITIVE_INFINITY,\n      df: Number.NaN,\n      p: Number.NaN,\n      ci95: null,\n      cohensD: null,\n    }\n  }\n\n  const standardError = Math.sqrt(seSquared)\n  const t = delta / standardError\n  const df =\n    (seSquared * seSquared) /\n    ((vA / a.length) ** 2 / (a.length - 1) + (vB / b.length) ** 2 / (b.length - 1))\n  const p = 2 * (1 - studentTCdf(Math.abs(t), df))\n  if (d === null) {\n    throw new Error('welchsTTest: non-zero standard error produced no pooled effect size')\n  }\n  const halfWidth = studentTQuantile(0.975, df) * standardError\n  return {\n    status: 'ok',\n    meanA: mA,\n    meanB: mB,\n    delta,\n    standardError,\n    t,\n    df,\n    p,\n    ci95: [delta - halfWidth, delta + halfWidth],\n    cohensD: d,\n  }\n}\n\nfunction variance(xs: number[], m: number): number {\n  return xs.reduce((acc, x) => acc + (x - m) ** 2, 0) / (xs.length - 1)\n}\n\nfunction assertFiniteSample(name: string, sample: readonly number[]): void {\n  const invalid = sample.find((value) => !Number.isFinite(value))\n  if (invalid !== undefined) {\n    throw new RangeError(\n      `welchsTTest: sample ${name} must contain only finite values, got ${invalid}`,\n    )\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAwGA,SAAgB,kBACd,SACA,UAA2B,CAAC,GACZ;CAChB,MAAM,kBAAkB,QAAQ,mBAAmB;CACnD,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,cAAc,QAAQ,uBAAuB;CAEnD,MAAM,UAA2B,QAAQ,KAAK,MAAM;EAClD,MAAM,aAAa,YAAY,EAAE,UAAU,EAAE,SAAS;EACtD,IAAI,WAAW,WAAW,uBACxB,MAAM,IAAI,MAAM,oDAAoD,EAAE,OAAO,EAAE;EAEjF,MAAM,EAAE,OAAO,OAAO,OAAO,OAAO,OAAO,SAAS,GAAG,GAAG,IAAI,MAAM;EAKpE,MAAM,cAAc,IAAI,EAAE,QAAQ;EAClC,MAAM,eAAe,IAAI,EAAE,SAAS;EACpC,MAAM,iBAAiB,cAAc,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,KAAK;EACxE,MAAM,kBAAkB,eAAe,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,KAAK;EAC1E,MAAM,SAAS,kBAAkB;EACjC,MAAM,cAAc,KAAK,IAAI,aAAa,YAAY;EAKtD,MAAM,eAAe,MAAM,QAAQ,KAAK,IAAI,CAAC,KAAK;EAElD,IAAI;EACJ,IAAI,CAAC,QACH,UAAU;OACL,IAAI,WAAW,WAAW,QAAQ,IAAI,SAAS,cAEpD,WAD0B,EAAE,iBAAiB,QAAQ,IAAI,QAAQ,KACnC,aAAa;OAE3C,UAAU;EAGZ,OAAO;GACL,QAAQ,EAAE;GACV,cAAc;GACd,eAAe;GACf;GACA,SAAS;GACT,QAAQ;GACR,SAAS;GACT,QAAQ;GACR;GACA,KAAK;GACL;EACF;CACF,CAAC;CAED,OAAO;EACL;EACA,eAAe,QAAQ,MAAM,MAAM,EAAE,YAAY,WAAW;EAC5D,aAAa,QAAQ,MAAM,MAAM,EAAE,YAAY,UAAU;CAC3D;AACF;AAEA,SAAS,KAAK,IAAsB;CAClC,OAAO,GAAG,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;;AAGA,SAAgB,IAAI,IAAsB;CACxC,IAAI,GAAG,WAAW,GAAG,OAAO;CAC5B,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;CAC3C,MAAM,KAAK,MAAc;EACvB,MAAM,MAAM,KAAK,OAAO,SAAS;EACjC,MAAM,KAAK,KAAK,MAAM,GAAG;EACzB,MAAM,KAAK,KAAK,KAAK,GAAG;EACxB,OAAO,OAAO,OAAQ,OAAO,MAAO,OAAO,QAAS,MAAM;CAC5D;CACA,OAAO,EAAE,GAAI,IAAI,EAAE,GAAI;AACzB;;;;;;;;;AAUA,SAAgB,YAAY,GAAa,GAA8B;CACrE,mBAAmB,KAAK,CAAC;CACzB,mBAAmB,KAAK,CAAC;CACzB,MAAM,KAAK,KAAK,CAAC;CACjB,MAAM,KAAK,KAAK,CAAC;CACjB,MAAM,QAAQ,KAAK;CACnB,IAAI,EAAE,SAAS,KAAK,EAAE,SAAS,GAC7B,OAAO;EACL,QAAQ;EACR,OAAO;EACP,OAAO;EACP;EACA,eAAe;EACf,GAAG;EACH,IAAI;EACJ,GAAG;EACH,MAAM;EACN,SAAS;CACX;CAGF,MAAM,KAAK,SAAS,GAAG,EAAE;CACzB,MAAM,KAAK,SAAS,GAAG,EAAE;CACzB,MAAM,YAAY,KAAK,EAAE,SAAS,KAAK,EAAE;CACzC,MAAM,IAAI,QAAQ,GAAG,CAAC;CACtB,IAAI,cAAc,GAChB,OAAO;EACL,QAAQ;EACR,OAAO;EACP,OAAO;EACP;EACA,eAAe;EACf,GAAG,UAAU,IAAI,IAAI,KAAK,KAAK,KAAK,IAAI,OAAO;EAC/C,IAAI;EACJ,GAAG;EACH,MAAM;EACN,SAAS;CACX;CAGF,MAAM,gBAAgB,KAAK,KAAK,SAAS;CACzC,MAAM,IAAI,QAAQ;CAClB,MAAM,KACH,YAAY,cACX,KAAK,EAAE,WAAW,KAAK,EAAE,SAAS,MAAM,KAAK,EAAE,WAAW,KAAK,EAAE,SAAS;CAC9E,MAAM,IAAI,KAAK,IAAI,YAAY,KAAK,IAAI,CAAC,GAAG,EAAE;CAC9C,IAAI,MAAM,MACR,MAAM,IAAI,MAAM,qEAAqE;CAEvF,MAAM,YAAY,iBAAiB,MAAO,EAAE,IAAI;CAChD,OAAO;EACL,QAAQ;EACR,OAAO;EACP,OAAO;EACP;EACA;EACA;EACA;EACA;EACA,MAAM,CAAC,QAAQ,WAAW,QAAQ,SAAS;EAC3C,SAAS;CACX;AACF;AAEA,SAAS,SAAS,IAAc,GAAmB;CACjD,OAAO,GAAG,QAAQ,KAAK,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,KAAK,GAAG,SAAS;AACrE;AAEA,SAAS,mBAAmB,MAAc,QAAiC;CACzE,MAAM,UAAU,OAAO,MAAM,UAAU,CAAC,OAAO,SAAS,KAAK,CAAC;CAC9D,IAAI,YAAY,KAAA,GACd,MAAM,IAAI,WACR,uBAAuB,KAAK,wCAAwC,SACtE;AAEJ"}
+{"version":3,"file":"baseline-BhPRQBVn.js","names":[],"sources":["../src/baseline.ts"],"sourcesContent":["/**\n * Baseline regression detection.\n *\n * Lifted from ADC baseline.ts. Every promotion-blocking signal boils down\n * to: \"is this run measurably worse than baseline?\" — with enough\n * statistical rigor to distinguish noise from drift.\n *\n * Uses:\n *   - Welch's t-test (unequal variance) for per-metric mean comparison\n *   - Cohen's d for effect size magnitude\n *   - IQR for stability flag (unstable samples can't be trusted for comparisons)\n *\n * Returns a structured verdict: improved | regressed | stable | unstable.\n */\n\nimport { studentTCdf, studentTQuantile } from './math/student-t'\nimport { cohensD } from './statistics'\n\nexport interface MetricSamples {\n  /** Stable metric key (e.g. \"overallScore\", \"firstTokenMs\"). */\n  metric: string\n  /** Whether higher values are better. */\n  higherIsBetter: boolean\n  baseline: number[]\n  candidate: number[]\n}\n\nexport interface MetricVerdict {\n  metric: string\n  baselineMean: number\n  candidateMean: number\n  delta: number\n  /** Null when both samples are constant with unequal means — the\n   *  standardized effect is unbounded there, not zero. */\n  cohensD: number | null\n  welchT: number\n  welchDf: number\n  welchP: number\n  stable: boolean\n  /** IQR of the combined samples — used as a rough stability indicator. */\n  iqr: number\n  verdict: 'improved' | 'regressed' | 'stable' | 'unstable'\n}\n\nexport interface BaselineReport {\n  metrics: MetricVerdict[]\n  /** True if any critical metric regressed. */\n  hasRegression: boolean\n  /** True if any metric is unstable (too noisy to judge). */\n  hasUnstable: boolean\n}\n\nexport interface BaselineOptions {\n  /** Effect size threshold for meaningful delta (default 0.5 — medium effect). */\n  effectThreshold?: number\n  /** p-value threshold for statistical significance (default 0.05). */\n  alpha?: number\n  /** IQR/mean ratio above which samples are flagged unstable (default 0.30). */\n  unstableCvThreshold?: number\n}\n\nexport type WelchTestStatus = 'ok' | 'insufficient-sample' | 'zero-variance'\n\n/**\n * Full unequal-variance comparison for two independent samples.\n *\n * `meanB`, `delta`, `t`, and `cohensD` are oriented as `b - a`. Inferential\n * fields are `NaN` and `ci95` is null when the sample count or observed\n * variance cannot define a Student-t result. `status` makes that state\n * explicit so callers cannot mistake it for evidence of no difference.\n */\ninterface WelchTestResultBase {\n  meanA: number\n  meanB: number\n  delta: number\n}\n\nexport type WelchTestResult =\n  | (WelchTestResultBase & {\n      status: 'ok'\n      standardError: number\n      t: number\n      df: number\n      p: number\n      ci95: [number, number]\n      cohensD: number\n    })\n  | (WelchTestResultBase & {\n      status: 'insufficient-sample' | 'zero-variance'\n      standardError: number\n      t: number\n      df: number\n      p: number\n      ci95: null\n      cohensD: null\n    })\n\n/**\n * Compare candidate samples against baseline per metric. Verdict logic:\n *   - unstable: IQR/|mean| > threshold on either set — not enough signal\n *   - improved: meaningful effect in the \"better\" direction AND p < alpha\n *   - regressed: meaningful effect in the \"worse\" direction AND p < alpha\n *   - stable: otherwise (no significant change)\n */\nexport function compareToBaseline(\n  samples: MetricSamples[],\n  options: BaselineOptions = {},\n): BaselineReport {\n  const effectThreshold = options.effectThreshold ?? 0.5\n  const alpha = options.alpha ?? 0.05\n  const cvThreshold = options.unstableCvThreshold ?? 0.3\n\n  const metrics: MetricVerdict[] = samples.map((s) => {\n    const comparison = welchsTTest(s.baseline, s.candidate)\n    if (comparison.status === 'insufficient-sample') {\n      throw new Error(`compareToBaseline: need ≥2 samples per side for \"${s.metric}\"`)\n    }\n    const { meanA: bMean, meanB: cMean, delta, cohensD: d, t, df, p } = comparison\n    // Stability is per-side: a comparison is trustworthy only when BOTH\n    // samples are internally consistent. Combining the sides would flag\n    // large-but-real deltas as \"unstable\" which is exactly what we want\n    // to detect.\n    const baselineIqr = iqr(s.baseline)\n    const candidateIqr = iqr(s.candidate)\n    const baselineStable = baselineIqr / Math.max(Math.abs(bMean), 1e-9) <= cvThreshold\n    const candidateStable = candidateIqr / Math.max(Math.abs(cMean), 1e-9) <= cvThreshold\n    const stable = baselineStable && candidateStable\n    const reportedIqr = Math.max(baselineIqr, candidateIqr)\n\n    // Both sides are guaranteed ≥ 2 samples above, so a null d means zero\n    // pooled spread across a real mean gap: an unbounded standardized effect,\n    // which clears any finite threshold.\n    const effectClears = d === null || Math.abs(d) >= effectThreshold\n\n    let verdict: MetricVerdict['verdict']\n    if (!stable) {\n      verdict = 'unstable'\n    } else if (comparison.status === 'ok' && p < alpha && effectClears) {\n      const candidateIsBetter = s.higherIsBetter ? delta > 0 : delta < 0\n      verdict = candidateIsBetter ? 'improved' : 'regressed'\n    } else {\n      verdict = 'stable'\n    }\n\n    return {\n      metric: s.metric,\n      baselineMean: bMean,\n      candidateMean: cMean,\n      delta,\n      cohensD: d,\n      welchT: t,\n      welchDf: df,\n      welchP: p,\n      stable,\n      iqr: reportedIqr,\n      verdict,\n    }\n  })\n\n  return {\n    metrics,\n    hasRegression: metrics.some((m) => m.verdict === 'regressed'),\n    hasUnstable: metrics.some((m) => m.verdict === 'unstable'),\n  }\n}\n\nfunction mean(xs: number[]): number {\n  return xs.reduce((a, b) => a + b, 0) / xs.length\n}\n\n/** Inter-quartile range; 0 when the sample has no spread. */\nexport function iqr(xs: number[]): number {\n  if (xs.length === 0) return 0\n  const sorted = [...xs].sort((a, b) => a - b)\n  const q = (p: number) => {\n    const idx = p * (sorted.length - 1)\n    const lo = Math.floor(idx)\n    const hi = Math.ceil(idx)\n    return sorted[lo]! + (sorted[hi]! - sorted[lo]!) * (idx - lo)\n  }\n  return q(0.75) - q(0.25)\n}\n\n/**\n * Welch's t-test and 95% interval for two independent samples.\n *\n * Uses the Student-t distribution at every finite degree of freedom. Fewer\n * than two observations per side and zero observed standard error cannot\n * define a t reference distribution; those cases return an explicit status,\n * `NaN` inferential fields, and no interval rather than fabricated certainty.\n */\nexport function welchsTTest(a: number[], b: number[]): WelchTestResult {\n  assertFiniteSample('a', a)\n  assertFiniteSample('b', b)\n  const mA = mean(a)\n  const mB = mean(b)\n  const delta = mB - mA\n  if (a.length < 2 || b.length < 2) {\n    return {\n      status: 'insufficient-sample',\n      meanA: mA,\n      meanB: mB,\n      delta,\n      standardError: Number.NaN,\n      t: Number.NaN,\n      df: Number.NaN,\n      p: Number.NaN,\n      ci95: null,\n      cohensD: null,\n    }\n  }\n\n  const vA = variance(a, mA)\n  const vB = variance(b, mB)\n  const seSquared = vA / a.length + vB / b.length\n  const d = cohensD(a, b)\n  if (seSquared === 0) {\n    return {\n      status: 'zero-variance',\n      meanA: mA,\n      meanB: mB,\n      delta,\n      standardError: 0,\n      t: delta === 0 ? 0 : Math.sign(delta) * Number.POSITIVE_INFINITY,\n      df: Number.NaN,\n      p: Number.NaN,\n      ci95: null,\n      cohensD: null,\n    }\n  }\n\n  const standardError = Math.sqrt(seSquared)\n  const t = delta / standardError\n  const df =\n    (seSquared * seSquared) /\n    ((vA / a.length) ** 2 / (a.length - 1) + (vB / b.length) ** 2 / (b.length - 1))\n  const p = 2 * (1 - studentTCdf(Math.abs(t), df))\n  if (d === null) {\n    throw new Error('welchsTTest: non-zero standard error produced no pooled effect size')\n  }\n  const halfWidth = studentTQuantile(0.975, df) * standardError\n  return {\n    status: 'ok',\n    meanA: mA,\n    meanB: mB,\n    delta,\n    standardError,\n    t,\n    df,\n    p,\n    ci95: [delta - halfWidth, delta + halfWidth],\n    cohensD: d,\n  }\n}\n\nfunction variance(xs: number[], m: number): number {\n  return xs.reduce((acc, x) => acc + (x - m) ** 2, 0) / (xs.length - 1)\n}\n\nfunction assertFiniteSample(name: string, sample: readonly number[]): void {\n  const invalid = sample.find((value) => !Number.isFinite(value))\n  if (invalid !== undefined) {\n    throw new RangeError(\n      `welchsTTest: sample ${name} must contain only finite values, got ${invalid}`,\n    )\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAwGA,SAAgB,kBACd,SACA,UAA2B,CAAC,GACZ;CAChB,MAAM,kBAAkB,QAAQ,mBAAmB;CACnD,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,cAAc,QAAQ,uBAAuB;CAEnD,MAAM,UAA2B,QAAQ,KAAK,MAAM;EAClD,MAAM,aAAa,YAAY,EAAE,UAAU,EAAE,SAAS;EACtD,IAAI,WAAW,WAAW,uBACxB,MAAM,IAAI,MAAM,oDAAoD,EAAE,OAAO,EAAE;EAEjF,MAAM,EAAE,OAAO,OAAO,OAAO,OAAO,OAAO,SAAS,GAAG,GAAG,IAAI,MAAM;EAKpE,MAAM,cAAc,IAAI,EAAE,QAAQ;EAClC,MAAM,eAAe,IAAI,EAAE,SAAS;EACpC,MAAM,iBAAiB,cAAc,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,KAAK;EACxE,MAAM,kBAAkB,eAAe,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,KAAK;EAC1E,MAAM,SAAS,kBAAkB;EACjC,MAAM,cAAc,KAAK,IAAI,aAAa,YAAY;EAKtD,MAAM,eAAe,MAAM,QAAQ,KAAK,IAAI,CAAC,KAAK;EAElD,IAAI;EACJ,IAAI,CAAC,QACH,UAAU;OACL,IAAI,WAAW,WAAW,QAAQ,IAAI,SAAS,cAEpD,WAD0B,EAAE,iBAAiB,QAAQ,IAAI,QAAQ,KACnC,aAAa;OAE3C,UAAU;EAGZ,OAAO;GACL,QAAQ,EAAE;GACV,cAAc;GACd,eAAe;GACf;GACA,SAAS;GACT,QAAQ;GACR,SAAS;GACT,QAAQ;GACR;GACA,KAAK;GACL;EACF;CACF,CAAC;CAED,OAAO;EACL;EACA,eAAe,QAAQ,MAAM,MAAM,EAAE,YAAY,WAAW;EAC5D,aAAa,QAAQ,MAAM,MAAM,EAAE,YAAY,UAAU;CAC3D;AACF;AAEA,SAAS,KAAK,IAAsB;CAClC,OAAO,GAAG,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;;AAGA,SAAgB,IAAI,IAAsB;CACxC,IAAI,GAAG,WAAW,GAAG,OAAO;CAC5B,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;CAC3C,MAAM,KAAK,MAAc;EACvB,MAAM,MAAM,KAAK,OAAO,SAAS;EACjC,MAAM,KAAK,KAAK,MAAM,GAAG;EACzB,MAAM,KAAK,KAAK,KAAK,GAAG;EACxB,OAAO,OAAO,OAAQ,OAAO,MAAO,OAAO,QAAS,MAAM;CAC5D;CACA,OAAO,EAAE,GAAI,IAAI,EAAE,GAAI;AACzB;;;;;;;;;AAUA,SAAgB,YAAY,GAAa,GAA8B;CACrE,mBAAmB,KAAK,CAAC;CACzB,mBAAmB,KAAK,CAAC;CACzB,MAAM,KAAK,KAAK,CAAC;CACjB,MAAM,KAAK,KAAK,CAAC;CACjB,MAAM,QAAQ,KAAK;CACnB,IAAI,EAAE,SAAS,KAAK,EAAE,SAAS,GAC7B,OAAO;EACL,QAAQ;EACR,OAAO;EACP,OAAO;EACP;EACA,eAAe;EACf,GAAG;EACH,IAAI;EACJ,GAAG;EACH,MAAM;EACN,SAAS;CACX;CAGF,MAAM,KAAK,SAAS,GAAG,EAAE;CACzB,MAAM,KAAK,SAAS,GAAG,EAAE;CACzB,MAAM,YAAY,KAAK,EAAE,SAAS,KAAK,EAAE;CACzC,MAAM,IAAI,QAAQ,GAAG,CAAC;CACtB,IAAI,cAAc,GAChB,OAAO;EACL,QAAQ;EACR,OAAO;EACP,OAAO;EACP;EACA,eAAe;EACf,GAAG,UAAU,IAAI,IAAI,KAAK,KAAK,KAAK,IAAI,OAAO;EAC/C,IAAI;EACJ,GAAG;EACH,MAAM;EACN,SAAS;CACX;CAGF,MAAM,gBAAgB,KAAK,KAAK,SAAS;CACzC,MAAM,IAAI,QAAQ;CAClB,MAAM,KACH,YAAY,cACX,KAAK,EAAE,WAAW,KAAK,EAAE,SAAS,MAAM,KAAK,EAAE,WAAW,KAAK,EAAE,SAAS;CAC9E,MAAM,IAAI,KAAK,IAAI,YAAY,KAAK,IAAI,CAAC,GAAG,EAAE;CAC9C,IAAI,MAAM,MACR,MAAM,IAAI,MAAM,qEAAqE;CAEvF,MAAM,YAAY,iBAAiB,MAAO,EAAE,IAAI;CAChD,OAAO;EACL,QAAQ;EACR,OAAO;EACP,OAAO;EACP;EACA;EACA;EACA;EACA;EACA,MAAM,CAAC,QAAQ,WAAW,QAAQ,SAAS;EAC3C,SAAS;CACX;AACF;AAEA,SAAS,SAAS,IAAc,GAAmB;CACjD,OAAO,GAAG,QAAQ,KAAK,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,KAAK,GAAG,SAAS;AACrE;AAEA,SAAS,mBAAmB,MAAc,QAAiC;CACzE,MAAM,UAAU,OAAO,MAAM,UAAU,CAAC,OAAO,SAAS,KAAK,CAAC;CAC9D,IAAI,YAAY,KAAA,GACd,MAAM,IAAI,WACR,uBAAuB,KAAK,wCAAwC,SACtE;AAEJ"}

package/dist/{benchmark-CWeqGl7x.js → benchmark-BhT16ep9.js}

@@ -1,4 +1,4 @@
-import { t as assertValidAnalystUsageReceipt } from "./usage-receipt-t7vAzCRQ.js";
+import { i as assertValidAnalystUsageReceipt } from "./types-BI4fT3HN.js";
 import { performance } from "node:perf_hooks";
 import { linearSumAssignment } from "linear-sum-assignment";
 //#region src/analyst/benchmark-scoring.ts
@@ -551,4 +551,4 @@ function mergeRegistryUsage(result) {
 //#endregion
 export { scoreAnalystFindings as a, summarizeAnalystBenchmarkRunner as i, runAnalystBenchmark as n, traceStoreEvidenceResolver as r, registryBenchmarkRunner as t };
 
-//# sourceMappingURL=benchmark-CWeqGl7x.js.map
+//# sourceMappingURL=benchmark-BhT16ep9.js.map