@tangle-network/agent-eval 0.115.0 → 0.115.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/dist/analyst/index.d.ts +7 -7
- package/dist/{analyze-runs-Dmz6LA9e.d.ts → analyze-runs-BYHg6Irm.d.ts} +3 -3
- package/dist/belief-state/index.d.ts +3 -3
- package/dist/belief-state/index.js +1 -1
- package/dist/benchmarks/index.d.ts +3 -3
- package/dist/benchmarks/index.js +2 -2
- package/dist/campaign/index.d.ts +13 -12
- package/dist/campaign/index.js +2 -2
- package/dist/{chunk-MNR6ZW4P.js → chunk-5NVBGKPH.js} +3 -3
- package/dist/chunk-5NVBGKPH.js.map +1 -0
- package/dist/{chunk-VK6HBGAE.js → chunk-5UF54T55.js} +53 -1
- package/dist/chunk-5UF54T55.js.map +1 -0
- package/dist/{chunk-IMWDSFUM.js → chunk-DXZRATT5.js} +2 -2
- package/dist/{chunk-S42AWHMP.js → chunk-E4BUPP7Z.js} +135 -17
- package/dist/chunk-E4BUPP7Z.js.map +1 -0
- package/dist/{chunk-WBOGKYM4.js → chunk-J6P6PK2R.js} +48 -7
- package/dist/chunk-J6P6PK2R.js.map +1 -0
- package/dist/{chunk-LOBMT6SB.js → chunk-QG5F6463.js} +2 -2
- package/dist/{chunk-RSVSSZKF.js → chunk-TLDB7WRY.js} +2 -2
- package/dist/{code-agent-session-yitf9I-F.d.ts → code-agent-session-D-g04tcy.d.ts} +8 -1
- package/dist/contract/index.d.ts +15 -15
- package/dist/contract/index.js +3 -3
- package/dist/{control-U8LBKUES.d.ts → control-CcBiAEnn.d.ts} +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/control.js +2 -2
- package/dist/{default-registry-Bcf1uKVI.d.ts → default-registry-DltpYR5u.d.ts} +1 -1
- package/dist/{gepa-DolL_Fko.d.ts → gepa-dne9JDPL.d.ts} +1 -1
- package/dist/hosted/index.d.ts +4 -4
- package/dist/{index-CWr5SIG-.d.ts → index-BTEpx9He.d.ts} +2 -2
- package/dist/index.d.ts +22 -22
- package/dist/index.js +8 -6
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-D4cXFsLt.d.ts → insight-report-IwwvqZZv.d.ts} +20 -2
- package/dist/{kind-factory-20hcaYpf.d.ts → kind-factory-DcNg13sZ.d.ts} +1 -1
- package/dist/meta-eval/index.d.ts +2 -2
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{policy-edit-az2qRmvN.d.ts → policy-edit-RLn8GWof.d.ts} +2 -2
- package/dist/{pre-registration-oNItiRBb.d.ts → pre-registration-D8h7ZxNL.d.ts} +3 -3
- package/dist/{provenance-BZmpWmn4.d.ts → provenance-Bibyg1U9.d.ts} +3 -3
- package/dist/{release-report-oBfOz8ku.d.ts → release-report-CCtzajxP.d.ts} +2 -2
- package/dist/reporting.d.ts +4 -4
- package/dist/{researcher-CaH0CwFC.d.ts → researcher-Dq-EtpbE.d.ts} +2 -2
- package/dist/rl.d.ts +6 -6
- package/dist/rl.js +2 -2
- package/dist/{rubric-predictive-validity-C-fMteAW.d.ts → rubric-predictive-validity-DYTLjGWu.d.ts} +1 -1
- package/dist/{run-record-DksGsfgv.d.ts → run-record-B7RTi_ix.d.ts} +34 -2
- package/dist/{runtime-trajectory-h5i0SZUj.d.ts → runtime-trajectory-Dws7Kpgi.d.ts} +1 -1
- package/dist/{semantic-concept-judge-CpzbtwD0.d.ts → semantic-concept-judge-DxJmRkyJ.d.ts} +1 -1
- package/dist/{summary-report-Bz-0-t8v.d.ts → summary-report-BJ5aNwZ1.d.ts} +1 -1
- package/dist/traces.d.ts +1 -1
- package/dist/traces.js +2 -2
- package/dist/{types-CgSlO6wT.d.ts → types-C5gJrOVT.d.ts} +1 -1
- package/package.json +1 -1
- package/dist/chunk-MNR6ZW4P.js.map +0 -1
- package/dist/chunk-S42AWHMP.js.map +0 -1
- package/dist/chunk-VK6HBGAE.js.map +0 -1
- package/dist/chunk-WBOGKYM4.js.map +0 -1
- /package/dist/{chunk-IMWDSFUM.js.map → chunk-DXZRATT5.js.map} +0 -0
- /package/dist/{chunk-LOBMT6SB.js.map → chunk-QG5F6463.js.map} +0 -0
- /package/dist/{chunk-RSVSSZKF.js.map → chunk-TLDB7WRY.js.map} +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { G as GainDistributionBin, P as ParetoFigureSpec } from './summary-report-
|
|
1
|
+
import { G as GainDistributionBin, P as ParetoFigureSpec } from './summary-report-BJ5aNwZ1.js';
|
|
2
2
|
import { C as ContinuousAgreement } from './judge-calibration-7C-IDmKr.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
@@ -43,6 +43,10 @@ interface InsightReport {
|
|
|
43
43
|
costQuality: {
|
|
44
44
|
cost: ScalarDistribution;
|
|
45
45
|
pareto: ParetoFigureSpec;
|
|
46
|
+
/** Cost source coverage. `uncaptured` rows are excluded from the USD
|
|
47
|
+
* distribution and Pareto chart; observed and estimated totals remain
|
|
48
|
+
* separate so reports never present estimates as billed spend. */
|
|
49
|
+
provenance?: CostProvenanceSummary;
|
|
46
50
|
/** Set when the cost/quality view is degraded because the input data
|
|
47
51
|
* doesn't fully support it — e.g. all `costUsd` were zero, or only a
|
|
48
52
|
* single candidate appears (so the Pareto is a single point). The
|
|
@@ -95,6 +99,20 @@ interface InsightReport {
|
|
|
95
99
|
* human-readable layer; the numeric sections are the evidence. */
|
|
96
100
|
recommendations: Recommendation[];
|
|
97
101
|
}
|
|
102
|
+
interface CostProvenanceSummary {
|
|
103
|
+
observed: {
|
|
104
|
+
n: number;
|
|
105
|
+
totalUsd: number;
|
|
106
|
+
};
|
|
107
|
+
estimated: {
|
|
108
|
+
n: number;
|
|
109
|
+
totalUsd: number;
|
|
110
|
+
};
|
|
111
|
+
uncaptured: {
|
|
112
|
+
n: number;
|
|
113
|
+
};
|
|
114
|
+
knownFraction: number;
|
|
115
|
+
}
|
|
98
116
|
/** Distributional summary of a scalar-valued metric. */
|
|
99
117
|
interface ScalarDistribution {
|
|
100
118
|
/** Sample count after dropping non-finite values. */
|
|
@@ -289,4 +307,4 @@ interface Recommendation {
|
|
|
289
307
|
evidencePath?: string;
|
|
290
308
|
}
|
|
291
309
|
|
|
292
|
-
export type { FailureClusterInsight as F, InsightReport as I, JudgeInsight as J, LiftInsight as L, OutcomeCorrelationInsight as O, Recommendation as R, ScalarDistribution as S, InterRaterInsight as a, ReleaseSummary as b };
|
|
310
|
+
export type { CostProvenanceSummary as C, FailureClusterInsight as F, InsightReport as I, JudgeInsight as J, LiftInsight as L, OutcomeCorrelationInsight as O, Recommendation as R, ScalarDistribution as S, InterRaterInsight as a, ReleaseSummary as b };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AxAIService, AxFunction } from '@ax-llm/ax';
|
|
2
2
|
import { T as TraceAnalysisStore } from './store-C1YxJDEK.js';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
-
import { R as RunRecord } from './run-record-
|
|
4
|
+
import { R as RunRecord } from './run-record-B7RTi_ix.js';
|
|
5
5
|
import { a as JudgeInput } from './types-C7DGg5ex.js';
|
|
6
6
|
import { b as LlmCallRequest, c as LlmCallResult } from './llm-client-DyqEH4jH.js';
|
|
7
7
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
export { C as CalibrationBin, a as CalibrationOptions, b as CalibrationPair, c as CalibrationReport, d as CorrelationResult, e as CorrelationStudyOptions, f as CorrelationStudyResult, E as EvalMetricSpec, O as OutcomePair, g as calibrationCurve, h as calibrationFromPairs, i as correlationStudy } from '../calibration-Dz8TQV4y.js';
|
|
2
2
|
export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore, O as OutcomeFilter, b as OutcomeStore } from '../outcome-store-rnXLEqSn.js';
|
|
3
|
-
export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from '../rubric-predictive-validity-
|
|
3
|
+
export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from '../rubric-predictive-validity-DYTLjGWu.js';
|
|
4
4
|
import { C as ContinuousAgreement, a as CalibrationResult, b as ContinuousCalibrationResult, c as CandidateScore, G as GoldenItem } from '../judge-calibration-7C-IDmKr.js';
|
|
5
5
|
import { S as SeriesConvergenceOptions, a as SeriesConvergenceResult } from '../series-convergence-D5OWMBg6.js';
|
|
6
6
|
import { C as CorpusAgreementReport } from '../statistics-oUbOJe-S.js';
|
|
7
7
|
import '../store-BsVi7ncX.js';
|
|
8
8
|
import '../schema-SGWcK9wa.js';
|
|
9
|
-
import '../run-record-
|
|
9
|
+
import '../run-record-B7RTi_ix.js';
|
|
10
10
|
import '@tangle-network/agent-interface';
|
|
11
11
|
import '../errors-oeQrLqXC.js';
|
|
12
12
|
import '../types-C7DGg5ex.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { J as JudgeScore } from '../types-
|
|
1
|
+
import { J as JudgeScore } from '../types-C5gJrOVT.js';
|
|
2
2
|
import { AgentProfile } from '@tangle-network/agent-interface';
|
|
3
3
|
import { M as MatrixResult } from '../types-BUxNaJ8c.js';
|
|
4
|
-
import '../run-record-
|
|
4
|
+
import '../run-record-B7RTi_ix.js';
|
|
5
5
|
import '../errors-oeQrLqXC.js';
|
|
6
6
|
import '../schema-SGWcK9wa.js';
|
|
7
7
|
import '../verdict-C9MlYujm.js';
|
package/dist/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.1.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "@tangle-network/agent-eval — wire protocol",
|
|
5
|
-
"version": "0.115.
|
|
5
|
+
"version": "0.115.2",
|
|
6
6
|
"description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
|
|
7
7
|
"contact": {
|
|
8
8
|
"name": "Tangle Network",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { A as AgentProfileCell, a as AgentProfileJson } from './run-record-
|
|
1
|
+
import { A as AgentProfileCell, a as AgentProfileJson } from './run-record-B7RTi_ix.js';
|
|
2
2
|
import { V as ValidationError } from './errors-oeQrLqXC.js';
|
|
3
|
-
import { A as AnalystFinding, E as EvidenceRef } from './kind-factory-
|
|
3
|
+
import { A as AnalystFinding, E as EvidenceRef } from './kind-factory-DcNg13sZ.js';
|
|
4
4
|
|
|
5
5
|
type PolicyEditSchemaVersion = 'policy-edit/v1';
|
|
6
6
|
declare const POLICY_EDIT_AXES: readonly ["carrier", "representation", "budget", "sampling", "output_contract", "tool_contract", "routing", "memory", "agent_profile", "deployment_target"];
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { A as AgentEvalError } from './errors-oeQrLqXC.js';
|
|
2
|
-
import { R as RunRecord } from './run-record-
|
|
2
|
+
import { R as RunRecord } from './run-record-B7RTi_ix.js';
|
|
3
3
|
import { P as PairedBootstrapOptions, M as McNemarResult, R as RiskDifferenceResult, a as PairedBootstrapResult } from './statistics-oUbOJe-S.js';
|
|
4
|
-
import { C as ChatClient } from './kind-factory-
|
|
5
|
-
import { a as JudgeDimension, S as Scenario, b as JudgeConfig } from './types-
|
|
4
|
+
import { C as ChatClient } from './kind-factory-DcNg13sZ.js';
|
|
5
|
+
import { a as JudgeDimension, S as Scenario, b as JudgeConfig } from './types-C5gJrOVT.js';
|
|
6
6
|
import { TCloud } from '@tangle-network/tcloud';
|
|
7
7
|
import { R as RawProviderSink } from './raw-provider-sink-C46HDghv.js';
|
|
8
8
|
import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { S as Scenario, g as Gate, G as GateResult, n as GateContext, C as CampaignResult, p as Mutator, f as SurfaceProposer, M as MutableSurface, h as GateDecision } from './types-
|
|
2
|
-
import { e as RedTeamCase, D as Direction, b as RunCampaignOptions } from './gepa-
|
|
3
|
-
import { R as RunRecord } from './run-record-
|
|
1
|
+
import { S as Scenario, g as Gate, G as GateResult, n as GateContext, C as CampaignResult, p as Mutator, f as SurfaceProposer, M as MutableSurface, h as GateDecision } from './types-C5gJrOVT.js';
|
|
2
|
+
import { e as RedTeamCase, D as Direction, b as RunCampaignOptions } from './gepa-dne9JDPL.js';
|
|
3
|
+
import { R as RunRecord } from './run-record-B7RTi_ix.js';
|
|
4
4
|
import { a as PairedBootstrapResult } from './statistics-oUbOJe-S.js';
|
|
5
5
|
import { HostedClient, TraceSpanEvent } from './hosted/index.js';
|
|
6
6
|
import { C as CampaignStorage } from './storage-Dw_f7WMt.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { a as DatasetSplit, b as DatasetManifest, D as DatasetScenario } from './dataset-NENEzRgk.js';
|
|
2
|
-
import { m as GateDecision } from './summary-report-
|
|
3
|
-
import { R as RunRecord, b as RunSplitTag } from './run-record-
|
|
2
|
+
import { m as GateDecision } from './summary-report-BJ5aNwZ1.js';
|
|
3
|
+
import { R as RunRecord, b as RunSplitTag } from './run-record-B7RTi_ix.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Release confidence gate.
|
package/dist/reporting.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from './rubric-predictive-validity-
|
|
2
|
-
export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, f as ReleaseConfidenceScorecard, g as ReleaseConfidenceStatus, h as ReleaseConfidenceThresholds, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-
|
|
1
|
+
export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from './rubric-predictive-validity-DYTLjGWu.js';
|
|
2
|
+
export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, f as ReleaseConfidenceScorecard, g as ReleaseConfidenceStatus, h as ReleaseConfidenceThresholds, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-CCtzajxP.js';
|
|
3
3
|
export { I as InterimReleaseConfidence, a as InterimReleaseConfidenceInput, P as PairedEvalueOptions, b as PairedEvalueSequence, c as PairedEvalueStep, S as SequentialDecision, e as evaluateInterimReleaseConfidence, p as pairedEvalueSequence } from './sequential-5iSVfzl2.js';
|
|
4
4
|
export { P as PairedBootstrapOptions, a as PairedBootstrapResult, b as benjaminiHochberg, p as pairedBootstrap, w as wilcoxonSignedRank } from './statistics-oUbOJe-S.js';
|
|
5
|
-
export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-
|
|
6
|
-
import './run-record-
|
|
5
|
+
export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-BJ5aNwZ1.js';
|
|
6
|
+
import './run-record-B7RTi_ix.js';
|
|
7
7
|
import '@tangle-network/agent-interface';
|
|
8
8
|
import './errors-oeQrLqXC.js';
|
|
9
9
|
import './schema-SGWcK9wa.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { b as RunSplitTag,
|
|
1
|
+
import { b as RunSplitTag, d as RunTokenUsage, e as RunJudgeMetadata, J as JudgeScoresRecord, A as AgentProfileCell, f as AgentProfileCellInput, R as RunRecord } from './run-record-B7RTi_ix.js';
|
|
2
2
|
import { L as LlmClientOptions, a as LlmRouteRequirements } from './llm-client-DyqEH4jH.js';
|
|
3
|
-
import { h as ResearchReportOptions, d as ResearchReport, m as GateDecision } from './summary-report-
|
|
3
|
+
import { h as ResearchReportOptions, d as ResearchReport, m as GateDecision } from './summary-report-BJ5aNwZ1.js';
|
|
4
4
|
import { T as TraceEmitter, R as RunCompleteHook } from './emitter-BRchAAAx.js';
|
|
5
5
|
import { R as RunIntegrityExpectations, a as RunIntegrityReport } from './integrity-qemeBAyx.js';
|
|
6
6
|
import { R as RawProviderSink } from './raw-provider-sink-C46HDghv.js';
|
package/dist/rl.d.ts
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import { R as RunRecord, b as RunSplitTag } from './run-record-
|
|
1
|
+
import { R as RunRecord, b as RunSplitTag } from './run-record-B7RTi_ix.js';
|
|
2
2
|
export { A as AdversarialMutation } from './adversarial-B7loGVVX.js';
|
|
3
3
|
import { S as Span } from './schema-SGWcK9wa.js';
|
|
4
4
|
import { T as TraceStore } from './store-BsVi7ncX.js';
|
|
5
5
|
export { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory, d as doublyRobust, i as inverseProbabilityWeighting, o as offPolicyEstimateAll, s as selfNormalizedImportanceWeighting } from './off-policy-DiwuKKg7.js';
|
|
6
6
|
import { b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
|
|
7
7
|
export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore } from './outcome-store-rnXLEqSn.js';
|
|
8
|
-
import { b as RubricPredictiveValidityReport } from './rubric-predictive-validity-
|
|
9
|
-
import { R as Researcher, F as FailureMode, S as SteeringChange, E as ExperimentPlan, a as ExperimentResult, b as EvalCampaignResult, c as EvalCampaignOptions } from './researcher-
|
|
10
|
-
export { r as runEvalCampaign } from './researcher-
|
|
8
|
+
import { b as RubricPredictiveValidityReport } from './rubric-predictive-validity-DYTLjGWu.js';
|
|
9
|
+
import { R as Researcher, F as FailureMode, S as SteeringChange, E as ExperimentPlan, a as ExperimentResult, b as EvalCampaignResult, c as EvalCampaignOptions } from './researcher-Dq-EtpbE.js';
|
|
10
|
+
export { r as runEvalCampaign } from './researcher-Dq-EtpbE.js';
|
|
11
11
|
import { a as VerificationReport } from './multi-layer-verifier-BsqKuLyN.js';
|
|
12
12
|
import { I as InterimReleaseConfidence } from './sequential-5iSVfzl2.js';
|
|
13
|
-
import { C as CampaignResult } from './types-
|
|
13
|
+
import { C as CampaignResult } from './types-C5gJrOVT.js';
|
|
14
14
|
import '@tangle-network/agent-interface';
|
|
15
15
|
import './errors-oeQrLqXC.js';
|
|
16
16
|
import './llm-client-DyqEH4jH.js';
|
|
17
17
|
import './raw-provider-sink-C46HDghv.js';
|
|
18
|
-
import './summary-report-
|
|
18
|
+
import './summary-report-BJ5aNwZ1.js';
|
|
19
19
|
import './failure-cluster-C48PiReX.js';
|
|
20
20
|
import './emitter-BRchAAAx.js';
|
|
21
21
|
import './integrity-qemeBAyx.js';
|
package/dist/rl.js
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
} from "./chunk-3RF76KTD.js";
|
|
11
11
|
import {
|
|
12
12
|
runEvalCampaign
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-QG5F6463.js";
|
|
14
14
|
import {
|
|
15
15
|
detectRewardHacking,
|
|
16
16
|
extractVerifiableReward,
|
|
@@ -35,7 +35,7 @@ import {
|
|
|
35
35
|
} from "./chunk-VZSRQ272.js";
|
|
36
36
|
import "./chunk-TT4KNT67.js";
|
|
37
37
|
import "./chunk-TVVP3ZZQ.js";
|
|
38
|
-
import "./chunk-
|
|
38
|
+
import "./chunk-5UF54T55.js";
|
|
39
39
|
import "./chunk-XJYR7XFV.js";
|
|
40
40
|
import "./chunk-VSMTAMNK.js";
|
|
41
41
|
import "./chunk-FUCQVFMU.js";
|
|
@@ -137,6 +137,24 @@ interface RunTokenUsage {
|
|
|
137
137
|
output: number;
|
|
138
138
|
cached?: number;
|
|
139
139
|
}
|
|
140
|
+
/**
|
|
141
|
+
* How a run's USD amount was obtained.
|
|
142
|
+
*
|
|
143
|
+
* `costUsd` remains mandatory for wire compatibility. New producers should
|
|
144
|
+
* always populate this discriminated union so a missing bill is never
|
|
145
|
+
* mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
|
|
146
|
+
* the legacy `0` sentinel while this field carries the truthful null.
|
|
147
|
+
*/
|
|
148
|
+
type RunCostProvenance = {
|
|
149
|
+
kind: 'observed';
|
|
150
|
+
usd: number;
|
|
151
|
+
} | {
|
|
152
|
+
kind: 'estimated';
|
|
153
|
+
usd: number;
|
|
154
|
+
} | {
|
|
155
|
+
kind: 'uncaptured';
|
|
156
|
+
usd: null;
|
|
157
|
+
};
|
|
140
158
|
interface RunJudgeMetadata {
|
|
141
159
|
model: string;
|
|
142
160
|
promptVersion: string;
|
|
@@ -252,8 +270,13 @@ interface RunRecord {
|
|
|
252
270
|
/** Time spent queued before execution started, if known. */
|
|
253
271
|
queueMs?: number;
|
|
254
272
|
/** Total USD cost. Mandatory — runs without a cost number are
|
|
255
|
-
* unbounded by definition and must not be admitted into the gate.
|
|
273
|
+
* unbounded by definition and must not be admitted into the gate.
|
|
274
|
+
* `0` is retained as the compatibility sentinel for an uncaptured amount;
|
|
275
|
+
* inspect `costProvenance` before treating it as observed. */
|
|
256
276
|
costUsd: number;
|
|
277
|
+
/** Observed, model-priced estimate, or genuinely uncaptured USD amount.
|
|
278
|
+
* Optional only so existing serialized RunRecords remain valid. */
|
|
279
|
+
costProvenance?: RunCostProvenance;
|
|
257
280
|
/** Token usage breakdown. */
|
|
258
281
|
tokenUsage: RunTokenUsage;
|
|
259
282
|
/** Judge-side metadata, if a judge was used. */
|
|
@@ -301,6 +324,15 @@ declare class RunRecordValidationError extends ValidationError {
|
|
|
301
324
|
* `RunRecord` on success — the validator does not coerce.
|
|
302
325
|
*/
|
|
303
326
|
declare function validateRunRecord(input: unknown): RunRecord;
|
|
327
|
+
/**
|
|
328
|
+
* Resolve provenance for both new and legacy records.
|
|
329
|
+
*
|
|
330
|
+
* Legacy producers sometimes set `outcome.raw.cost_estimated = 1`. A positive
|
|
331
|
+
* unlabeled amount is treated as observed, matching the historical contract.
|
|
332
|
+
* Zero without an explicit label is conservatively uncaptured: claiming an
|
|
333
|
+
* observed $0 would be stronger than the serialized evidence supports.
|
|
334
|
+
*/
|
|
335
|
+
declare function resolveRunCostProvenance(run: Pick<RunRecord, 'costUsd' | 'costProvenance' | 'outcome'>): RunCostProvenance;
|
|
304
336
|
/** Boolean validator — convenience for filtering arrays. */
|
|
305
337
|
declare function isRunRecord(input: unknown): input is RunRecord;
|
|
306
338
|
/** Non-throwing validator — returns a discriminated union. */
|
|
@@ -325,4 +357,4 @@ declare function roundTripRunRecord(record: RunRecord): RunRecord;
|
|
|
325
357
|
*/
|
|
326
358
|
declare function modelHasSnapshot(model: string): boolean;
|
|
327
359
|
|
|
328
|
-
export { type AgentProfileCell as A,
|
|
360
|
+
export { type AgentProfileCell as A, requireAgentProfileCell as B, resolveRunCostProvenance as C, roundTripRunRecord as D, toAgentProfileJson as E, validateAgentProfileCell as F, validateRunRecord as G, verifyAgentProfileCell as H, type JudgeScoresRecord as J, type RunRecord as R, type AgentProfileJson as a, type RunSplitTag as b, type RunCostProvenance as c, type RunTokenUsage as d, type RunJudgeMetadata as e, type AgentProfileCellInput as f, AGENT_PROFILE_KINDS as g, type AgentInterfaceProfileLike as h, type AgentProfileCellSchemaVersion as i, AgentProfileCellValidationError as j, type AgentProfileDimensionValue as k, type AgentProfileHarness as l, type AgentProfileKind as m, type AgentProfileSource as n, type AgentProfileSourceInput as o, type RunOutcome as p, RunRecordValidationError as q, agentProfileCellHashMaterial as r, agentProfileCellKey as s, assertRunAgentProfileCell as t, buildAgentInterfaceProfileCell as u, buildAgentProfileCell as v, groupRunsByAgentProfileCell as w, isRunRecord as x, modelHasSnapshot as y, parseRunRecordSafe as z };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { AxAIService } from '@ax-llm/ax';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
-
import { A as AnalystFinding, T as TraceAnalystKindSpec, a as Analyst, b as AnalystContext } from './kind-factory-
|
|
3
|
+
import { A as AnalystFinding, T as TraceAnalystKindSpec, a as Analyst, b as AnalystContext } from './kind-factory-DcNg13sZ.js';
|
|
4
4
|
import { a as TraceAnalystSpan } from './store-C1YxJDEK.js';
|
|
5
5
|
import { R as Run, S as Span, e as TraceEvent, A as Artifact, B as BudgetLedgerEntry } from './schema-SGWcK9wa.js';
|
|
6
6
|
import { T as TraceStore } from './store-BsVi7ncX.js';
|
package/dist/traces.d.ts
CHANGED
|
@@ -13,7 +13,7 @@ import { A as AnalyzeTracesOptions, b as AnalyzeTracesResult } from './analyst-C
|
|
|
13
13
|
export { a as AnalyzeTracesInput, c as AnalyzeTracesTurnSnapshot, d as analyzeTraces } from './analyst-C8HHvfJp.js';
|
|
14
14
|
import { h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, T as TraceAnalysisStore, g as TraceAnalystFilters, b as DatasetOverview, Q as QueryTracesPage, l as ViewTraceResult, V as ViewSpansResult, c as SearchTraceResult, S as SearchSpanResult } from './store-C1YxJDEK.js';
|
|
15
15
|
export { D as DEFAULT_TRACE_ANALYST_BUDGETS, E as ErrorCluster, d as SpanMatchRecord, e as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, f as TraceAnalystByteBudgets, a as TraceAnalystSpan, j as TraceAnalystTraceSummary, k as ViewTraceOversized } from './store-C1YxJDEK.js';
|
|
16
|
-
import { b as RunSplitTag,
|
|
16
|
+
import { b as RunSplitTag, d as RunTokenUsage, R as RunRecord } from './run-record-B7RTi_ix.js';
|
|
17
17
|
import { AxFunction } from '@ax-llm/ax';
|
|
18
18
|
import '@tangle-network/agent-interface';
|
|
19
19
|
|
package/dist/traces.js
CHANGED
|
@@ -28,7 +28,7 @@ import {
|
|
|
28
28
|
scoreTraceInsightReadiness,
|
|
29
29
|
tokenizeDomainWords,
|
|
30
30
|
traceAnalystOnRunComplete
|
|
31
|
-
} from "./chunk-
|
|
31
|
+
} from "./chunk-TLDB7WRY.js";
|
|
32
32
|
import {
|
|
33
33
|
FAILURE_CLASSES,
|
|
34
34
|
TRACE_SCHEMA_VERSION,
|
|
@@ -101,7 +101,7 @@ import {
|
|
|
101
101
|
TraceEmitter,
|
|
102
102
|
llmSpanFromProvider
|
|
103
103
|
} from "./chunk-TVVP3ZZQ.js";
|
|
104
|
-
import "./chunk-
|
|
104
|
+
import "./chunk-5UF54T55.js";
|
|
105
105
|
import "./chunk-XJYR7XFV.js";
|
|
106
106
|
import "./chunk-VSMTAMNK.js";
|
|
107
107
|
import {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-eval",
|
|
3
|
-
"version": "0.115.
|
|
3
|
+
"version": "0.115.2",
|
|
4
4
|
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-eval#readme",
|
|
6
6
|
"repository": {
|