@tangle-network/agent-eval 0.128.2 → 0.130.0

package/dist/baseline-hG3K85h4.d.ts

@@ -0,0 +1,125 @@
+import { C as TraceEvent, _ as Span } from "./schema-BtVldJ3T.js";
+import { s as TraceStore } from "./store-CT9YIIve.js";
+//#region src/tool-use-metrics.d.ts
+interface ToolUseMetrics {
+  runId: string;
+  totalCalls: number;
+  /** Calls whose arguments were captured and can be compared for duplication. */
+  callsWithCapturedArgs: number;
+  byTool: Record<string, ToolStats>;
+  errorRate: number;
+  /** Ratio of captured-argument calls already seen with the same tool name and arguments. */
+  duplicateRate: number;
+  /** Ratio of error calls followed by ≥1 retry on same tool. */
+  retryRate: number;
+  /** Optional: of the calls agent made, fraction the evaluator marked as "correct selection". */
+  selectionAccuracy?: number;
+}
+interface ToolStats {
+  calls: number;
+  callsWithCapturedArgs: number;
+  errors: number;
+  avgLatencyMs: number;
+  duplicates: number;
+}
+interface ToolUseOptions {
+  /** Map of spanId → whether the evaluator judged the tool selection correct. Optional. */
+  selectionLabels?: Record<string, boolean>;
+}
+declare function computeToolUseMetrics(store: TraceStore, runId: string, options?: ToolUseOptions): Promise<ToolUseMetrics>;
+//#endregion
+//#region src/trajectory.d.ts
+interface TrajectoryStep {
+  index: number;
+  span: Span;
+  /** Depth in the span tree from the root. 0 = top-level. */
+  depth: number;
+  /** Events attached to this span. */
+  events: TraceEvent[];
+}
+interface Trajectory {
+  runId: string;
+  steps: TrajectoryStep[];
+  llmTurns: number;
+  toolCalls: number;
+  judgeVerdicts: number;
+  retrievals: number;
+  totalDurationMs: number;
+}
+declare function buildTrajectory(store: TraceStore, runId: string): Promise<Trajectory>;
+//#endregion
+//#region src/baseline.d.ts
+/**
+ * Baseline regression detection.
+ *
+ * Lifted from ADC baseline.ts. Every promotion-blocking signal boils down
+ * to: "is this run measurably worse than baseline?" — with enough
+ * statistical rigor to distinguish noise from drift.
+ *
+ * Uses:
+ *   - Welch's t-test (unequal variance) for per-metric mean comparison
+ *   - Cohen's d for effect size magnitude
+ *   - IQR for stability flag (unstable samples can't be trusted for comparisons)
+ *
+ * Returns a structured verdict: improved | regressed | stable | unstable.
+ */
+interface MetricSamples {
+  /** Stable metric key (e.g. "overallScore", "firstTokenMs"). */
+  metric: string;
+  /** Whether higher values are better. */
+  higherIsBetter: boolean;
+  baseline: number[];
+  candidate: number[];
+}
+interface MetricVerdict {
+  metric: string;
+  baselineMean: number;
+  candidateMean: number;
+  delta: number;
+  cohensD: number;
+  welchT: number;
+  welchDf: number;
+  welchP: number;
+  stable: boolean;
+  /** IQR of the combined samples — used as a rough stability indicator. */
+  iqr: number;
+  verdict: 'improved' | 'regressed' | 'stable' | 'unstable';
+}
+interface BaselineReport {
+  metrics: MetricVerdict[];
+  /** True if any critical metric regressed. */
+  hasRegression: boolean;
+  /** True if any metric is unstable (too noisy to judge). */
+  hasUnstable: boolean;
+}
+interface BaselineOptions {
+  /** Effect size threshold for meaningful delta (default 0.5 — medium effect). */
+  effectThreshold?: number;
+  /** p-value threshold for statistical significance (default 0.05). */
+  alpha?: number;
+  /** IQR/mean ratio above which samples are flagged unstable (default 0.30). */
+  unstableCvThreshold?: number;
+}
+/**
+ * Compare candidate samples against baseline per metric. Verdict logic:
+ *   - unstable: IQR/|mean| > threshold on either set — not enough signal
+ *   - improved: meaningful effect in the "better" direction AND p < alpha
+ *   - regressed: meaningful effect in the "worse" direction AND p < alpha
+ *   - stable: otherwise (no significant change)
+ */
+declare function compareToBaseline(samples: MetricSamples[], options?: BaselineOptions): BaselineReport;
+/** Inter-quartile range; 0 when the sample has no spread. */
+declare function iqr(xs: number[]): number;
+/**
+ * Welch's t-test — unequal-variance two-sample t. Uses the same Student-t
+ * CDF as `pairedTTest` (via incomplete beta); falls back to normal tail
+ * when df is large.
+ */
+declare function welchsTTest(a: number[], b: number[]): {
+  t: number;
+  df: number;
+  p: number;
+};
+//#endregion
+export { compareToBaseline as a, Trajectory as c, ToolStats as d, ToolUseMetrics as f, MetricVerdict as i, TrajectoryStep as l, computeToolUseMetrics as m, BaselineReport as n, iqr as o, ToolUseOptions as p, MetricSamples as r, welchsTTest as s, BaselineOptions as t, buildTrajectory as u };
+//# sourceMappingURL=baseline-hG3K85h4.d.ts.map

package/dist/baseline-hG3K85h4.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"baseline-hG3K85h4.d.ts","names":[],"sources":["../src/tool-use-metrics.ts","../src/trajectory.ts","../src/baseline.ts"],"mappings":";;;UAaiB;EACf;EACA;;EAEA;EACA,QAAQ,eAAe;EACvB;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;;EAEf,kBAAkB;;iBAGE,sBACpB,OAAO,YACP,eACA,UAAS,iBACR,QAAQ;;;UChCM;EACf;EACA,MAAM;;EAEN;;EAEA,QAAQ;;UAGO;EACf;EACA,OAAO;EACP;EACA;EACA;EACA;EACA;;iBAGoB,gBAAgB,OAAO,YAAY,gBAAgB,QAAQ;;;;;;;;;;;;;;;;;UCfhE;;EAEf;;EAEA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;;UAGe;EACf,SAAS;;EAET;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;;;;;;;;;iBAUc,kBACd,SAAS,iBACT,UAAS,kBACR;;iBA8Da,IAAI;;;;;;iBAiBJ,YAAY,aAAa;EAAgB;EAAW;EAAY"}