@tangle-network/agent-eval 0.33.0 → 0.34.0

package/dist/reporting.d.ts

@@ -1,10 +1,12 @@
-export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from './rubric-predictive-validity-CMHypZ_M.js';
-export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, P as PairedBootstrapOptions, b as PairedBootstrapResult, R as ReleaseConfidenceAxis, c as ReleaseConfidenceAxisName, d as ReleaseConfidenceInput, e as ReleaseConfidenceIssue, f as ReleaseConfidenceMetrics, g as ReleaseConfidenceScorecard, h as ReleaseConfidenceStatus, i as ReleaseConfidenceThresholds, j as ReleaseTraceEvidence, k as RenderReleaseReportOptions, V as Verdict, l as assertReleaseConfidence, m as bhAdjust, n as bootstrapCi, o as evaluateReleaseConfidence, p as judgeReplayGate, q as pairedBootstrap, r as pairedWilcoxon, s as releaseTraceEvidenceFromMultiShotTrials, t as renderReleaseReport } from './release-report-DLWbBPtH.js';
+export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from './rubric-predictive-validity-BvaNwfBE.js';
+export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, P as PairedBootstrapOptions, b as PairedBootstrapResult, R as ReleaseConfidenceAxis, c as ReleaseConfidenceAxisName, d as ReleaseConfidenceInput, e as ReleaseConfidenceIssue, f as ReleaseConfidenceMetrics, g as ReleaseConfidenceScorecard, h as ReleaseConfidenceStatus, i as ReleaseConfidenceThresholds, j as ReleaseTraceEvidence, k as RenderReleaseReportOptions, V as Verdict, l as assertReleaseConfidence, m as benjaminiHochberg, n as bootstrapCi, o as evaluateReleaseConfidence, p as judgeReplayGate, q as pairedBootstrap, r as releaseTraceEvidenceFromMultiShotTrials, s as renderReleaseReport, w as wilcoxonSignedRank } from './release-report-ChfmCmLi.js';
 export { I as InterimReleaseConfidence, a as InterimReleaseConfidenceInput, P as PairedEvalueOptions, b as PairedEvalueSequence, c as PairedEvalueStep, S as SequentialDecision, e as evaluateInterimReleaseConfidence, p as pairedEvalueSequence } from './sequential-5iSVfzl2.js';
-export { C as GainDistributionBin, F as GainDistributionFigureSpec, H as GainDistributionOptions, J as ParetoFigureSpec, K as ParetoPoint, L as RESEARCH_REPORT_HARD_PAIR_FLOOR, N as ResearchReport, O as ResearchReportCandidate, Q as ResearchReportDecision, U as ResearchReportMethodology, W as ResearchReportOptions, X as ResearchReportRecommendation, Y as SummaryTable, Z as SummaryTableOptions, _ as SummaryTableRow, $ as gainHistogram, a0 as paretoChart, a1 as researchReport, a2 as summaryTable } from './summary-report-D7AQS7eB.js';
-import './run-record-BfX5y68A.js';
+export { C as GainDistributionBin, F as GainDistributionFigureSpec, H as GainDistributionOptions, J as ParetoFigureSpec, K as ParetoPoint, L as RESEARCH_REPORT_HARD_PAIR_FLOOR, N as ResearchReport, O as ResearchReportCandidate, Q as ResearchReportDecision, U as ResearchReportMethodology, W as ResearchReportOptions, X as ResearchReportRecommendation, Y as SummaryTable, Z as SummaryTableOptions, _ as SummaryTableRow, $ as gainHistogram, a0 as paretoChart, a1 as researchReport, a2 as summaryTable } from './summary-report-BPJVzIeW.js';
+import './run-record-YinVdFwu.js';
 import './errors-mje_cKOs.js';
 import './outcome-store-D6KWmYvj.js';
+import './judge-calibration-DilmB3Ml.js';
+import '@tangle-network/tcloud';
 import './dataset-ueRVTUoY.js';
 import './failure-cluster-Cw65_5FY.js';
 import './store-Db2Bv8Cf.js';

package/dist/reporting.js

@@ -5,7 +5,7 @@ import {
   judgeReplayGate,
   releaseTraceEvidenceFromMultiShotTrials,
   renderReleaseReport
-} from "./chunk-WGXZAQLR.js";
+} from "./chunk-LGAPK7NA.js";
 import {
   rubricPredictiveValidity
 } from "./chunk-YRZ4M5GS.js";
@@ -15,22 +15,23 @@ import {
 } from "./chunk-MAZ26DC7.js";
 import {
   RESEARCH_REPORT_HARD_PAIR_FLOOR,
-  bhAdjust,
   gainHistogram,
-  pairedBootstrap,
-  pairedWilcoxon,
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-CXJOVDJR.js";
-import "./chunk-4L3WJXQJ.js";
+} from "./chunk-TQL7BAOY.js";
+import {
+  benjaminiHochberg,
+  pairedBootstrap,
+  wilcoxonSignedRank
+} from "./chunk-KHZRNY3F.js";
 import "./chunk-VSMTAMNK.js";
 import "./chunk-QYJT52YW.js";
 import "./chunk-PZ5AY32C.js";
 export {
   RESEARCH_REPORT_HARD_PAIR_FLOOR,
   assertReleaseConfidence,
-  bhAdjust,
+  benjaminiHochberg,
   bootstrapCi,
   evaluateInterimReleaseConfidence,
   evaluateReleaseConfidence,
@@ -38,12 +39,12 @@ export {
   judgeReplayGate,
   pairedBootstrap,
   pairedEvalueSequence,
-  pairedWilcoxon,
   paretoChart,
   releaseTraceEvidenceFromMultiShotTrials,
   renderReleaseReport,
   researchReport,
   rubricPredictiveValidity,
-  summaryTable
+  summaryTable,
+  wilcoxonSignedRank
 };
 //# sourceMappingURL=reporting.js.map

package/dist/{researcher-BRHa5Jxo.d.ts → researcher-CfnL3HEb.d.ts}

@@ -1,7 +1,7 @@
+import { a as RunSplitTag, b as RunTokenUsage, c as RunJudgeMetadata, J as JudgeScoresRecord, A as AgentProfileCell, d as AgentProfileCellInput, R as RunRecord } from './run-record-YinVdFwu.js';
 import { A as AgentEvalError, C as CaptureIntegrityError } from './errors-mje_cKOs.js';
 import { R as RawProviderSink, P as ProviderRedactor, a as RunIntegrityExpectations, b as RunIntegrityReport } from './integrity-DYR5gWlb.js';
-import { a as RunSplitTag, b as RunTokenUsage, c as RunJudgeMetadata, J as JudgeScoresRecord, R as RunRecord } from './run-record-BfX5y68A.js';
-import { W as ResearchReportOptions, N as ResearchReport, a3 as GateDecision } from './summary-report-D7AQS7eB.js';
+import { W as ResearchReportOptions, N as ResearchReport, a3 as GateDecision } from './summary-report-BPJVzIeW.js';
 import { T as TraceEmitter, R as RunCompleteHook } from './emitter-DP_cSSiw.js';
 import { T as TraceStore } from './store-Db2Bv8Cf.js';
 
@@ -126,6 +126,21 @@ interface LlmClientOptions {
     /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */
     redactor?: ProviderRedactor;
 }
+/**
+ * True when an error is a transient transport/network fault worth retrying,
+ * as opposed to a deterministic failure (4xx schema reject, JSON parse) that
+ * a retry cannot fix. Inspects `LlmCallError.status`, then the error's
+ * name/message/code, then recurses into `error.cause` — undici nests the
+ * real socket fault one or more levels under `.cause`.
+ *
+ * This is THE retry classifier for the package: `callLlm` and
+ * `withJudgeRetry` both route through it, so a connection-class error is
+ * treated identically whether it surfaces in the HTTP client or a
+ * TCloud-backed judge.
+ */
+declare function isTransientLlmError(err: unknown): boolean;
+/** Exponential backoff: 500ms, 1s, 2s, 4s, ... capped at 16s. Attempt is 0-indexed. */
+declare function backoffMs(attempt: number): number;
 /**
  * Strip a ```json / ``` code fence if the model emitted one.
  * Idempotent for naked JSON. Some models (claude-code via router, certain
@@ -322,6 +337,12 @@ interface CampaignRunOutcome {
      * Single-judge or scalar-only runs leave this unset.
      */
     judgeScores?: JudgeScoresRecord;
+    /**
+     * Agent profile cell observed by the runner. When supplied, it overrides
+     * `EvalCampaignOptions.agentProfile` for this run and must match the
+     * outcome's `model` and `promptHash`.
+     */
+    agentProfile?: AgentProfileCell | AgentProfileCellInput;
 }
 type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>;
 type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log';
@@ -407,6 +428,16 @@ interface EvalCampaignOptions<V> {
     now?: () => number;
     /** Override the runId generator. Tests pin this. */
     runId?: (params: CampaignFactoryParams) => string;
+    /**
+     * Agent profile cell for campaign runs. Static profiles can pass an object;
+     * routers or variant-specific harnesses can pass a factory. The campaign
+     * stamps the built cell onto every `RunRecord` and rejects profile/model or
+     * profile/prompt contradictions.
+     */
+    agentProfile?: AgentProfileCell | AgentProfileCellInput | ((params: CampaignFactoryParams & {
+        variant: V;
+        scenarioTags: Record<string, string>;
+    }) => AgentProfileCell | AgentProfileCellInput | Promise<AgentProfileCell | AgentProfileCellInput>);
 }
 interface CampaignFactoryParams {
     campaignId: string;
@@ -577,4 +608,4 @@ declare class NoopResearcher implements Researcher {
     evaluateChange(_plan: ExperimentPlan): Promise<ExperimentResult>;
 }
 
-export { CallbackResearcher as C, type EvalCampaignOptions as E, type FailedRun as F, type LlmClientOptions as L, NoopResearcher as N, type Researcher as R, type SteeringChange as S, type CallbackResearcherOptions as a, type CampaignFactoryParams as b, type CampaignIntegrityPolicy as c, type CampaignRunContext as d, type CampaignRunOutcome as e, type CampaignRunner as f, type CampaignScenario as g, type CampaignVariant as h, type EvalCampaignResult as i, type ExperimentPlan as j, type ExperimentResult as k, type FailureMode as l, type LlmCallRequest as m, type LlmCallResult as n, LlmCallError as o, LlmClient as p, type LlmMessage as q, runEvalCampaign as r, LlmRouteAssertionError as s, type LlmRouteRequirements as t, type LlmUsage as u, assertLlmRoute as v, callLlm as w, callLlmJson as x, probeLlm as y, stripFencedJson as z };
+export { probeLlm as A, stripFencedJson as B, CallbackResearcher as C, type EvalCampaignOptions as E, type FailedRun as F, type LlmClientOptions as L, NoopResearcher as N, type Researcher as R, type SteeringChange as S, type CallbackResearcherOptions as a, type CampaignFactoryParams as b, type CampaignIntegrityPolicy as c, type CampaignRunContext as d, type CampaignRunOutcome as e, type CampaignRunner as f, type CampaignScenario as g, type CampaignVariant as h, type EvalCampaignResult as i, type ExperimentPlan as j, type ExperimentResult as k, type FailureMode as l, type LlmCallRequest as m, type LlmCallResult as n, LlmCallError as o, LlmClient as p, type LlmMessage as q, runEvalCampaign as r, LlmRouteAssertionError as s, type LlmRouteRequirements as t, type LlmUsage as u, assertLlmRoute as v, backoffMs as w, callLlm as x, callLlmJson as y, isTransientLlmError as z };

package/dist/rl.d.ts

@@ -1,12 +1,12 @@
-import { R as RunRecord, a as RunSplitTag } from './run-record-BfX5y68A.js';
+import { R as RunRecord, a as RunSplitTag } from './run-record-YinVdFwu.js';
 import { V as VerificationReport } from './multi-layer-verifier-BNi4-8lR.js';
-import { t as TrialResult, V as VariantAggregate, q as PromptEvolutionResult, e as MultiShotOptimizationResult } from './summary-report-D7AQS7eB.js';
+import { t as TrialResult, V as VariantAggregate, q as PromptEvolutionResult, e as MultiShotOptimizationResult } from './summary-report-BPJVzIeW.js';
 import { O as OutcomeStore } from './outcome-store-D6KWmYvj.js';
-import { b as RubricPredictiveValidityReport } from './rubric-predictive-validity-CMHypZ_M.js';
+import { b as RubricPredictiveValidityReport } from './rubric-predictive-validity-BvaNwfBE.js';
 import { I as InterimReleaseConfidence } from './sequential-5iSVfzl2.js';
 import { S as Span, T as TraceStore } from './store-Db2Bv8Cf.js';
-import { R as Researcher, l as FailureMode, S as SteeringChange, j as ExperimentPlan, k as ExperimentResult, i as EvalCampaignResult, E as EvalCampaignOptions } from './researcher-BRHa5Jxo.js';
-export { r as runEvalCampaign } from './researcher-BRHa5Jxo.js';
+import { R as Researcher, l as FailureMode, S as SteeringChange, j as ExperimentPlan, k as ExperimentResult, i as EvalCampaignResult, E as EvalCampaignOptions } from './researcher-CfnL3HEb.js';
+export { r as runEvalCampaign } from './researcher-CfnL3HEb.js';
 import './errors-mje_cKOs.js';
 import './failure-cluster-Cw65_5FY.js';
 import './integrity-DYR5gWlb.js';

package/dist/rl.js

@@ -1,19 +1,19 @@
 import {
   runEvalCampaign
-} from "./chunk-DTEJNZYK.js";
-import "./chunk-M6RZ5LJN.js";
+} from "./chunk-SQYRO3BT.js";
+import "./chunk-VXNVVBZO.js";
+import "./chunk-DCZXFOQN.js";
 import {
   rubricPredictiveValidity
 } from "./chunk-YRZ4M5GS.js";
 import {
   evaluateInterimReleaseConfidence
 } from "./chunk-MAZ26DC7.js";
+import "./chunk-TQL7BAOY.js";
 import {
-  benjaminiHochberg
-} from "./chunk-CXJOVDJR.js";
-import {
+  benjaminiHochberg,
   wilcoxonSignedRank
-} from "./chunk-4L3WJXQJ.js";
+} from "./chunk-KHZRNY3F.js";
 import "./chunk-UBPIXOC4.js";
 import "./chunk-PC4UYEBM.js";
 import "./chunk-TVVP3ZZQ.js";