@tangle-network/agent-eval 0.95.0 → 0.95.1

package/dist/authenticity/index.d.ts

@@ -24,11 +24,11 @@
  * `extractProducedState(...).artifacts` yields, so any consumer can feed a run's
  * produced state straight in.
  */
-export interface ProducedFile {
+interface ProducedFile {
     path: string;
     content?: string;
 }
-export interface AuthenticitySignals {
+interface AuthenticitySignals {
     /** Human label for the domain (e.g. 'fhenix-fhe'). */
     label: string;
     /** A file the task REQUIRES (e.g. /\.sol$/ for an on-chain task). */
@@ -55,7 +55,7 @@ export interface AuthenticitySignals {
         wiring?: number;
     };
 }
-export interface AuthenticityResult {
+interface AuthenticityResult {
     /** Deterministic realness, 0 (BS) … 100 (real on real infra). */
     realness: number;
     requiredArtifactPresent: boolean;
@@ -78,19 +78,19 @@ export interface AuthenticityResult {
 }
 /** Deterministic authenticity scan of produced files. Pure — same files in,
  *  same score out. No LLM, no IO. */
-export declare function scoreAuthenticity(files: readonly ProducedFile[], signals: AuthenticitySignals): AuthenticityResult;
-export interface RealnessGate {
+declare function scoreAuthenticity(files: readonly ProducedFile[], signals: AuthenticitySignals): AuthenticityResult;
+interface RealnessGate {
     gated: boolean;
     reason?: string;
 }
 /** Anti-Goodhart gate: a required-artifact-missing or faked submission is
  *  capped and cannot rank high regardless of buildability. */
-export declare function gateRealness(r: AuthenticityResult, opts?: {
+declare function gateRealness(r: AuthenticityResult, opts?: {
     floor?: number;
     requireArtifact?: boolean;
     requireArtifactWired?: boolean;
 }): RealnessGate;
-export interface AuthenticityNuance {
+interface AuthenticityNuance {
     /** 0 (nothing mocked) … 100 (entirely mocked). */
     mockedPct: number;
     /** 0 (genuine) … 100 (a hollow facade / cargo-culted). */
@@ -101,18 +101,18 @@ export interface AuthenticityNuance {
 }
 /** A minimal completion fn — inject your model caller (router/tcloud). Keeps
  *  this module free of any specific LLM client. */
-export type CompleteFn = (system: string, user: string) => Promise<string>;
+type CompleteFn = (system: string, user: string) => Promise<string>;
 /**
  * LLM nuance scoring — judges the "looks real but is hollow" axis structure
  * misses. Inject a `complete` caller; returns mocked/fake/unique % + a verdict.
  * Fail-soft: a bad/unparseable response yields a worst-case (fully-fake) read,
  * never a false pass.
  */
-export declare function scoreAuthenticityNuance(files: readonly ProducedFile[], complete: CompleteFn, opts?: {
+declare function scoreAuthenticityNuance(files: readonly ProducedFile[], complete: CompleteFn, opts?: {
     intent?: string;
     prioritize?: RegExp;
 }): Promise<AuthenticityNuance>;
-export interface RealnessJudgment {
+interface RealnessJudgment {
     /** 0 (facade/simulator) … 100 (real implementation on the intended infra). */
     isReal: number;
     rationale: string;
@@ -125,13 +125,13 @@ export interface RealnessJudgment {
  * consumer supplies `intent` (what the deliverable should be) and `rubric` (domain
  * specifics of real-vs-fake). Fail-closed: a bad response reads as fully fake.
  */
-export declare function judgeRealnessLlm(files: readonly ProducedFile[], complete: CompleteFn, opts?: {
+declare function judgeRealnessLlm(files: readonly ProducedFile[], complete: CompleteFn, opts?: {
     intent?: string;
     rubric?: string;
     prioritize?: RegExp;
 }): Promise<RealnessJudgment>;
-export type RealnessBand = 'clean-real' | 'clean-fake' | 'gray';
-export interface BlendedRealness extends AuthenticityResult {
+type RealnessBand = 'clean-real' | 'clean-fake' | 'gray';
+interface BlendedRealness extends AuthenticityResult {
     /** Final realness after (only-when-needed) LLM adjudication, 0…100. */
     blendedRealness: number;
     band: RealnessBand;
@@ -151,10 +151,11 @@ export interface BlendedRealness extends AuthenticityResult {
  * Domain-agnostic: the gray-band TRIGGER is structural; the LLM judges via the
  * consumer-supplied `intent`. Fail-closed (a bad LLM response reads as fully fake).
  */
-export declare function scoreRealnessBlended(files: readonly ProducedFile[], signals: AuthenticitySignals, complete: CompleteFn, opts?: {
+declare function scoreRealnessBlended(files: readonly ProducedFile[], signals: AuthenticitySignals, complete: CompleteFn, opts?: {
     intent?: string;
     rubric?: string;
     grayBand?: [number, number];
     mockGrayThreshold?: number;
 }): Promise<BlendedRealness>;
-//# sourceMappingURL=index.d.ts.map
+
+export { type AuthenticityNuance, type AuthenticityResult, type AuthenticitySignals, type BlendedRealness, type CompleteFn, type ProducedFile, type RealnessBand, type RealnessGate, type RealnessJudgment, gateRealness, judgeRealnessLlm, scoreAuthenticity, scoreAuthenticityNuance, scoreRealnessBlended };

package/dist/{baseline.d.ts → baseline-Bbid3WoO.d.ts}

@@ -1,3 +1,38 @@
+import { T as TraceStore } from './store-BcFXE6LG.js';
+
+/**
+ * Tool-use metrics — derived purely from trace data.
+ *
+ * No scoring assumptions: consumers supply optional ground-truth tool
+ * selections per turn + optional "information used downstream" signals.
+ * Without those, we still compute descriptive metrics (error rate,
+ * retry rate, duplicate-call rate) that are useful on their own.
+ */
+
+interface ToolUseMetrics {
+    runId: string;
+    totalCalls: number;
+    byTool: Record<string, ToolStats>;
+    errorRate: number;
+    /** Ratio of calls with identical (toolName, argHash) already seen earlier in the same run. */
+    duplicateRate: number;
+    /** Ratio of error calls followed by ≥1 retry on same tool. */
+    retryRate: number;
+    /** Optional: of the calls agent made, fraction the evaluator marked as "correct selection". */
+    selectionAccuracy?: number;
+}
+interface ToolStats {
+    calls: number;
+    errors: number;
+    avgLatencyMs: number;
+    duplicates: number;
+}
+interface ToolUseOptions {
+    /** Map of spanId → whether the evaluator judged the tool selection correct. Optional. */
+    selectionLabels?: Record<string, boolean>;
+}
+declare function computeToolUseMetrics(store: TraceStore, runId: string, options?: ToolUseOptions): Promise<ToolUseMetrics>;
+
 /**
  * Baseline regression detection.
  *
@@ -12,7 +47,7 @@
  *
  * Returns a structured verdict: improved | regressed | stable | unstable.
  */
-export interface MetricSamples {
+interface MetricSamples {
     /** Stable metric key (e.g. "overallScore", "firstTokenMs"). */
     metric: string;
     /** Whether higher values are better. */
@@ -20,7 +55,7 @@ export interface MetricSamples {
     baseline: number[];
     candidate: number[];
 }
-export interface MetricVerdict {
+interface MetricVerdict {
     metric: string;
     baselineMean: number;
     candidateMean: number;
@@ -34,14 +69,14 @@ export interface MetricVerdict {
     iqr: number;
     verdict: 'improved' | 'regressed' | 'stable' | 'unstable';
 }
-export interface BaselineReport {
+interface BaselineReport {
     metrics: MetricVerdict[];
     /** True if any critical metric regressed. */
     hasRegression: boolean;
     /** True if any metric is unstable (too noisy to judge). */
     hasUnstable: boolean;
 }
-export interface BaselineOptions {
+interface BaselineOptions {
     /** Effect size threshold for meaningful delta (default 0.5 — medium effect). */
     effectThreshold?: number;
     /** p-value threshold for statistical significance (default 0.05). */
@@ -56,17 +91,18 @@ export interface BaselineOptions {
  *   - regressed: meaningful effect in the "worse" direction AND p < alpha
  *   - stable: otherwise (no significant change)
  */
-export declare function compareToBaseline(samples: MetricSamples[], options?: BaselineOptions): BaselineReport;
+declare function compareToBaseline(samples: MetricSamples[], options?: BaselineOptions): BaselineReport;
 /** Inter-quartile range; 0 when the sample has no spread. */
-export declare function iqr(xs: number[]): number;
+declare function iqr(xs: number[]): number;
 /**
  * Welch's t-test — unequal-variance two-sample t. Uses the same Student-t
  * CDF as `pairedTTest` (via incomplete beta); falls back to normal tail
  * when df is large.
  */
-export declare function welchsTTest(a: number[], b: number[]): {
+declare function welchsTTest(a: number[], b: number[]): {
     t: number;
     df: number;
     p: number;
 };
-//# sourceMappingURL=baseline.d.ts.map
+
+export { type BaselineOptions as B, type MetricSamples as M, type ToolStats as T, type BaselineReport as a, type MetricVerdict as b, computeToolUseMetrics as c, type ToolUseMetrics as d, type ToolUseOptions as e, compareToBaseline as f, iqr as i, welchsTTest as w };