@tangle-network/agent-eval 0.102.0 → 0.102.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/http.d.ts +2 -2
- package/dist/adapters/langchain.d.ts +2 -2
- package/dist/adapters/otel.d.ts +4 -4
- package/dist/analyst/index.d.ts +8 -8
- package/dist/{analyze-runs-BlJRBniC.d.ts → analyze-runs-Cd-A_K4l.d.ts} +3 -3
- package/dist/belief-state/index.d.ts +3 -3
- package/dist/benchmarks/index.d.ts +2 -2
- package/dist/campaign/index.d.ts +13 -13
- package/dist/campaign/index.js +33 -10
- package/dist/campaign/index.js.map +1 -1
- package/dist/{chunk-G6S73VA7.js → chunk-2NSLDY4B.js} +3 -2
- package/dist/{chunk-G6S73VA7.js.map → chunk-2NSLDY4B.js.map} +1 -1
- package/dist/{chunk-4LWD6GC7.js → chunk-6FIAJHCU.js} +2 -2
- package/dist/{chunk-JCUREYF5.js → chunk-6Q2DYRWV.js} +2 -2
- package/dist/{chunk-PMF5WIBX.js → chunk-7RBJANJD.js} +2 -2
- package/dist/{chunk-BOETF6BU.js → chunk-B2TMQM62.js} +2 -2
- package/dist/{chunk-LSCBODPQ.js → chunk-QIT2XZ4E.js} +10 -1
- package/dist/chunk-QIT2XZ4E.js.map +1 -0
- package/dist/{chunk-QUCGGMYM.js → chunk-YLKDN7JV.js} +3 -3
- package/dist/{chunk-52CCCXU3.js → chunk-Z3FLN24V.js} +3 -3
- package/dist/chunk-Z3FLN24V.js.map +1 -0
- package/dist/{code-agent-session-B6ZcDwyA.d.ts → code-agent-session-Ce-9u7YM.d.ts} +1 -1
- package/dist/contract/index.d.ts +16 -16
- package/dist/contract/index.js +5 -5
- package/dist/{control-DC8TELh0.d.ts → control-C8RmK9H4.d.ts} +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/control.js +2 -2
- package/dist/{corpus-ONOzGFmG.d.ts → corpus-CiSzzLa5.d.ts} +1 -1
- package/dist/{default-registry-Dhrc__SE.d.ts → default-registry-ZhqsTr4K.d.ts} +2 -2
- package/dist/diagnose.d.ts +3 -3
- package/dist/diagnose.js +1 -1
- package/dist/{gepa-bxuDoaO9.d.ts → gepa-DeyPTlvx.d.ts} +1 -1
- package/dist/hosted/index.d.ts +4 -4
- package/dist/{index-W96macmS.d.ts → index-B-bFgiAF.d.ts} +1 -1
- package/dist/index.d.ts +22 -22
- package/dist/index.js +9 -7
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-C02J3q4T.d.ts → insight-report-k0sRTzKg.d.ts} +1 -1
- package/dist/{kind-factory-OgqQSvLi.d.ts → kind-factory-D0nk7AKV.d.ts} +1 -1
- package/dist/meta-eval/index.d.ts +2 -2
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{policy-edit-Dccm9tyA.d.ts → policy-edit-BDQzzsBU.d.ts} +2 -2
- package/dist/{pre-registration-BjGZf9YA.d.ts → pre-registration-Dzg61IQA.d.ts} +3 -3
- package/dist/{provenance-BEITkFII.d.ts → provenance-BhJm32vN.d.ts} +3 -3
- package/dist/{release-report-B1tA6pKu.d.ts → release-report-BQ1Ziyu-.d.ts} +2 -2
- package/dist/reporting.d.ts +4 -4
- package/dist/{researcher-Ba2y1Foi.d.ts → researcher-B_ODTAJs.d.ts} +2 -2
- package/dist/rl.d.ts +8 -8
- package/dist/rl.js +2 -2
- package/dist/{rubric-predictive-validity-w7tun-q3.d.ts → rubric-predictive-validity-0MdjTt8R.d.ts} +1 -1
- package/dist/{run-campaign-RF3H6D4U.js → run-campaign-3NWW5PLF.js} +2 -2
- package/dist/{run-record-DEwidcqn.d.ts → run-record-MRdJ-Kq2.d.ts} +12 -1
- package/dist/{runtime-trajectory-OJDaTYHN.d.ts → runtime-trajectory-8w0_jmtR.d.ts} +1 -1
- package/dist/{semantic-concept-judge-J8xvjdc3.d.ts → semantic-concept-judge-D-IlH5v1.d.ts} +2 -2
- package/dist/{summary-report-C4uzRWh8.d.ts → summary-report-C0nnxOD8.d.ts} +1 -1
- package/dist/traces.d.ts +1 -1
- package/dist/traces.js +2 -2
- package/dist/{types-fWqEJm7h.d.ts → types-DFI_Z-ZL.d.ts} +18 -1
- package/dist/{types-BEzCBMQD.d.ts → types-Dz9cKF0g.d.ts} +1 -1
- package/dist/workflow/index.d.ts +4 -4
- package/dist/workflow/index.js +1 -1
- package/package.json +1 -1
- package/dist/chunk-52CCCXU3.js.map +0 -1
- package/dist/chunk-LSCBODPQ.js.map +0 -1
- /package/dist/{chunk-4LWD6GC7.js.map → chunk-6FIAJHCU.js.map} +0 -0
- /package/dist/{chunk-JCUREYF5.js.map → chunk-6Q2DYRWV.js.map} +0 -0
- /package/dist/{chunk-PMF5WIBX.js.map → chunk-7RBJANJD.js.map} +0 -0
- /package/dist/{chunk-BOETF6BU.js.map → chunk-B2TMQM62.js.map} +0 -0
- /package/dist/{chunk-QUCGGMYM.js.map → chunk-YLKDN7JV.js.map} +0 -0
- /package/dist/{run-campaign-RF3H6D4U.js.map → run-campaign-3NWW5PLF.js.map} +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { G as GainDistributionBin, P as ParetoFigureSpec } from './summary-report-
|
|
1
|
+
import { G as GainDistributionBin, P as ParetoFigureSpec } from './summary-report-C0nnxOD8.js';
|
|
2
2
|
import { C as ContinuousAgreement } from './judge-calibration-0p2QcWNE.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AxAIService, AxFunction } from '@ax-llm/ax';
|
|
2
2
|
import { T as TraceAnalysisStore } from './store-C1YxJDEK.js';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
-
import { g as AnalystCost, b as AnalystContext, a as Analyst } from './types-
|
|
4
|
+
import { g as AnalystCost, b as AnalystContext, a as Analyst } from './types-Dz9cKF0g.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Typed Ax output for analyst findings.
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
export { C as CalibrationBin, a as CalibrationOptions, b as CalibrationPair, c as CalibrationReport, d as CorrelationResult, e as CorrelationStudyOptions, f as CorrelationStudyResult, E as EvalMetricSpec, O as OutcomePair, g as calibrationCurve, h as calibrationFromPairs, i as correlationStudy } from '../calibration-BPmzuVPk.js';
|
|
2
2
|
export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore, O as OutcomeFilter, b as OutcomeStore } from '../outcome-store-rnXLEqSn.js';
|
|
3
|
-
export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from '../rubric-predictive-validity-
|
|
3
|
+
export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from '../rubric-predictive-validity-0MdjTt8R.js';
|
|
4
4
|
import { C as ContinuousAgreement, a as CalibrationResult, b as ContinuousCalibrationResult, c as CandidateScore, G as GoldenItem } from '../judge-calibration-0p2QcWNE.js';
|
|
5
5
|
import { S as SeriesConvergenceOptions, a as SeriesConvergenceResult } from '../series-convergence-D5OWMBg6.js';
|
|
6
6
|
import { C as CorpusAgreementReport } from '../statistics-xP-cWc5k.js';
|
|
7
7
|
import '../store-BcFXE6LG.js';
|
|
8
8
|
import '../schema-m0gsnbt3.js';
|
|
9
|
-
import '../run-record-
|
|
9
|
+
import '../run-record-MRdJ-Kq2.js';
|
|
10
10
|
import '@tangle-network/agent-interface';
|
|
11
11
|
import '../errors-CzMUYo7b.js';
|
|
12
12
|
import '../types-C7DGg5ex.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { J as JudgeScore } from '../types-
|
|
1
|
+
import { J as JudgeScore } from '../types-DFI_Z-ZL.js';
|
|
2
2
|
import { AgentProfile } from '@tangle-network/agent-interface';
|
|
3
3
|
import { M as MatrixResult } from '../types-BUxNaJ8c.js';
|
|
4
|
-
import '../run-record-
|
|
4
|
+
import '../run-record-MRdJ-Kq2.js';
|
|
5
5
|
import '../errors-CzMUYo7b.js';
|
|
6
6
|
import '../schema-m0gsnbt3.js';
|
|
7
7
|
import '../verdict-C9MlYujm.js';
|
package/dist/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.1.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "@tangle-network/agent-eval — wire protocol",
|
|
5
|
-
"version": "0.102.
|
|
5
|
+
"version": "0.102.1",
|
|
6
6
|
"description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
|
|
7
7
|
"contact": {
|
|
8
8
|
"name": "Tangle Network",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { A as AgentProfileCell, a as AgentProfileJson } from './run-record-
|
|
1
|
+
import { A as AgentProfileCell, a as AgentProfileJson } from './run-record-MRdJ-Kq2.js';
|
|
2
2
|
import { V as ValidationError } from './errors-CzMUYo7b.js';
|
|
3
|
-
import { A as AnalystFinding, E as EvidenceRef } from './types-
|
|
3
|
+
import { A as AnalystFinding, E as EvidenceRef } from './types-Dz9cKF0g.js';
|
|
4
4
|
|
|
5
5
|
type PolicyEditSchemaVersion = 'policy-edit/v1';
|
|
6
6
|
declare const POLICY_EDIT_AXES: readonly ["carrier", "representation", "budget", "sampling", "output_contract", "tool_contract", "routing", "memory", "agent_profile", "deployment_target"];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { A as AgentEvalError } from './errors-CzMUYo7b.js';
|
|
2
|
-
import { R as RunRecord } from './run-record-
|
|
3
|
-
import { C as ChatClient } from './types-
|
|
4
|
-
import { c as JudgeDimension, S as Scenario, a as JudgeConfig } from './types-
|
|
2
|
+
import { R as RunRecord } from './run-record-MRdJ-Kq2.js';
|
|
3
|
+
import { C as ChatClient } from './types-Dz9cKF0g.js';
|
|
4
|
+
import { c as JudgeDimension, S as Scenario, a as JudgeConfig } from './types-DFI_Z-ZL.js';
|
|
5
5
|
import { TCloud } from '@tangle-network/tcloud';
|
|
6
6
|
import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
|
|
7
7
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { S as Scenario, g as Gate, G as GateResult, h as GateContext, C as CampaignResult, i as Mutator, f as SurfaceProposer, M as MutableSurface, j as GateDecision } from './types-
|
|
1
|
+
import { S as Scenario, g as Gate, G as GateResult, h as GateContext, C as CampaignResult, i as Mutator, f as SurfaceProposer, M as MutableSurface, j as GateDecision } from './types-DFI_Z-ZL.js';
|
|
2
2
|
import { R as RedTeamCase } from './red-team-BWdoyleI.js';
|
|
3
|
-
import { R as RunRecord } from './run-record-
|
|
3
|
+
import { R as RunRecord } from './run-record-MRdJ-Kq2.js';
|
|
4
4
|
import { D as Direction } from './pareto-E-pembql.js';
|
|
5
5
|
import { a as PairedBootstrapResult } from './statistics-xP-cWc5k.js';
|
|
6
|
-
import { R as RunCampaignOptions, C as CampaignStorage } from './gepa-
|
|
6
|
+
import { R as RunCampaignOptions, C as CampaignStorage } from './gepa-DeyPTlvx.js';
|
|
7
7
|
import { HostedClient, TraceSpanEvent } from './hosted/index.js';
|
|
8
8
|
|
|
9
9
|
/**
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { D as DatasetSplit, c as DatasetManifest, a as DatasetScenario } from './dataset-BbGkaN2I.js';
|
|
2
|
-
import { m as GateDecision } from './summary-report-
|
|
3
|
-
import { R as RunRecord, b as RunSplitTag } from './run-record-
|
|
2
|
+
import { m as GateDecision } from './summary-report-C0nnxOD8.js';
|
|
3
|
+
import { R as RunRecord, b as RunSplitTag } from './run-record-MRdJ-Kq2.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Release confidence gate.
|
package/dist/reporting.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from './rubric-predictive-validity-
|
|
2
|
-
export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, f as ReleaseConfidenceScorecard, g as ReleaseConfidenceStatus, h as ReleaseConfidenceThresholds, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-
|
|
1
|
+
export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from './rubric-predictive-validity-0MdjTt8R.js';
|
|
2
|
+
export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, f as ReleaseConfidenceScorecard, g as ReleaseConfidenceStatus, h as ReleaseConfidenceThresholds, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-BQ1Ziyu-.js';
|
|
3
3
|
export { I as InterimReleaseConfidence, a as InterimReleaseConfidenceInput, P as PairedEvalueOptions, b as PairedEvalueSequence, c as PairedEvalueStep, S as SequentialDecision, e as evaluateInterimReleaseConfidence, p as pairedEvalueSequence } from './sequential-5iSVfzl2.js';
|
|
4
4
|
export { P as PairedBootstrapOptions, a as PairedBootstrapResult, b as benjaminiHochberg, p as pairedBootstrap, w as wilcoxonSignedRank } from './statistics-xP-cWc5k.js';
|
|
5
|
-
export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-
|
|
6
|
-
import './run-record-
|
|
5
|
+
export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-C0nnxOD8.js';
|
|
6
|
+
import './run-record-MRdJ-Kq2.js';
|
|
7
7
|
import '@tangle-network/agent-interface';
|
|
8
8
|
import './errors-CzMUYo7b.js';
|
|
9
9
|
import './schema-m0gsnbt3.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { b as RunSplitTag, c as RunTokenUsage, d as RunJudgeMetadata, J as JudgeScoresRecord, A as AgentProfileCell, e as AgentProfileCellInput, R as RunRecord } from './run-record-
|
|
1
|
+
import { b as RunSplitTag, c as RunTokenUsage, d as RunJudgeMetadata, J as JudgeScoresRecord, A as AgentProfileCell, e as AgentProfileCellInput, R as RunRecord } from './run-record-MRdJ-Kq2.js';
|
|
2
2
|
import { L as LlmClientOptions, a as LlmRouteRequirements } from './llm-client-Bj7g0rqu.js';
|
|
3
|
-
import { h as ResearchReportOptions, d as ResearchReport, m as GateDecision } from './summary-report-
|
|
3
|
+
import { h as ResearchReportOptions, d as ResearchReport, m as GateDecision } from './summary-report-C0nnxOD8.js';
|
|
4
4
|
import { T as TraceEmitter, R as RunCompleteHook } from './emitter-C2rqGH_l.js';
|
|
5
5
|
import { R as RunIntegrityExpectations, a as RunIntegrityReport } from './integrity-D2t12mMw.js';
|
|
6
6
|
import { R as RawProviderSink } from './raw-provider-sink-C46HDghv.js';
|
package/dist/rl.d.ts
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
import { R as RunRecord, b as RunSplitTag } from './run-record-
|
|
1
|
+
import { R as RunRecord, b as RunSplitTag } from './run-record-MRdJ-Kq2.js';
|
|
2
2
|
export { A as AdversarialMutation } from './adversarial-B7loGVVX.js';
|
|
3
|
-
import { P as PreferenceExtractionReport, D as DpoExportRow, G as GrpoExportRow, S as SftExportRow, E as ExtractPreferencesOptions, a as DpoLookups, b as GrpoLookups, c as SftLookups } from './corpus-
|
|
4
|
-
export { d as CorpusAppendResult, C as CorpusRecord, e as DatasetFormat, f as ExtractStepRewardsOptions, H as HarvestOptions, g as PreferenceStrategy, h as PreferenceTriple, i as PrmExportRow, j as PrmLookups, k as PrmTrainingTriple, R as RewardKind, l as RewardStats, m as RlDatasetBundle, n as RlDatasetConfig, o as RlDatasetManifest, p as RlDatasetStats, q as RunwiseStepSummary, r as StepReward, s as StepRewardJsonlRow, t as StepScorer, u as appendToCorpus, v as buildDatasetFromCorpus, w as buildRlDataset, x as datasheetToMarkdown, y as extractPreferences, z as extractStepRewards, A as prmTrainingPairs, B as readCorpus, F as runwiseStepRewardSummary, I as stepRewardsToJsonl, J as toAnthropicFormat, K as toDpoJsonl, L as toDpoRows, M as toGrpoJsonl, N as toGrpoRows, O as toPrmJsonl, Q as toPrmRows, T as toSftJsonl, U as toSftRows, V as toTRLFormat } from './corpus-
|
|
3
|
+
import { P as PreferenceExtractionReport, D as DpoExportRow, G as GrpoExportRow, S as SftExportRow, E as ExtractPreferencesOptions, a as DpoLookups, b as GrpoLookups, c as SftLookups } from './corpus-CiSzzLa5.js';
|
|
4
|
+
export { d as CorpusAppendResult, C as CorpusRecord, e as DatasetFormat, f as ExtractStepRewardsOptions, H as HarvestOptions, g as PreferenceStrategy, h as PreferenceTriple, i as PrmExportRow, j as PrmLookups, k as PrmTrainingTriple, R as RewardKind, l as RewardStats, m as RlDatasetBundle, n as RlDatasetConfig, o as RlDatasetManifest, p as RlDatasetStats, q as RunwiseStepSummary, r as StepReward, s as StepRewardJsonlRow, t as StepScorer, u as appendToCorpus, v as buildDatasetFromCorpus, w as buildRlDataset, x as datasheetToMarkdown, y as extractPreferences, z as extractStepRewards, A as prmTrainingPairs, B as readCorpus, F as runwiseStepRewardSummary, I as stepRewardsToJsonl, J as toAnthropicFormat, K as toDpoJsonl, L as toDpoRows, M as toGrpoJsonl, N as toGrpoRows, O as toPrmJsonl, Q as toPrmRows, T as toSftJsonl, U as toSftRows, V as toTRLFormat } from './corpus-CiSzzLa5.js';
|
|
5
5
|
export { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory, d as doublyRobust, i as inverseProbabilityWeighting, o as offPolicyEstimateAll, s as selfNormalizedImportanceWeighting } from './off-policy-DiwuKKg7.js';
|
|
6
6
|
import { b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
|
|
7
7
|
export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore } from './outcome-store-rnXLEqSn.js';
|
|
8
|
-
import { b as RubricPredictiveValidityReport } from './rubric-predictive-validity-
|
|
9
|
-
import { R as Researcher, F as FailureMode, S as SteeringChange, E as ExperimentPlan, a as ExperimentResult, b as EvalCampaignResult, c as EvalCampaignOptions } from './researcher-
|
|
10
|
-
export { r as runEvalCampaign } from './researcher-
|
|
8
|
+
import { b as RubricPredictiveValidityReport } from './rubric-predictive-validity-0MdjTt8R.js';
|
|
9
|
+
import { R as Researcher, F as FailureMode, S as SteeringChange, E as ExperimentPlan, a as ExperimentResult, b as EvalCampaignResult, c as EvalCampaignOptions } from './researcher-B_ODTAJs.js';
|
|
10
|
+
export { r as runEvalCampaign } from './researcher-B_ODTAJs.js';
|
|
11
11
|
import { a as VerificationReport } from './multi-layer-verifier-DUZXrPDA.js';
|
|
12
12
|
import { I as InterimReleaseConfidence } from './sequential-5iSVfzl2.js';
|
|
13
|
-
import { C as CampaignResult } from './types-
|
|
13
|
+
import { C as CampaignResult } from './types-DFI_Z-ZL.js';
|
|
14
14
|
import '@tangle-network/agent-interface';
|
|
15
15
|
import './errors-CzMUYo7b.js';
|
|
16
16
|
import './schema-m0gsnbt3.js';
|
|
17
17
|
import './store-BcFXE6LG.js';
|
|
18
18
|
import './llm-client-Bj7g0rqu.js';
|
|
19
19
|
import './raw-provider-sink-C46HDghv.js';
|
|
20
|
-
import './summary-report-
|
|
20
|
+
import './summary-report-C0nnxOD8.js';
|
|
21
21
|
import './failure-cluster-DH9Flgcf.js';
|
|
22
22
|
import './emitter-C2rqGH_l.js';
|
|
23
23
|
import './integrity-D2t12mMw.js';
|
package/dist/rl.js
CHANGED
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
} from "./chunk-3RF76KTD.js";
|
|
17
17
|
import {
|
|
18
18
|
runEvalCampaign
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-6FIAJHCU.js";
|
|
20
20
|
import "./chunk-CWNP4DV4.js";
|
|
21
21
|
import {
|
|
22
22
|
rubricPredictiveValidity
|
|
@@ -36,7 +36,7 @@ import {
|
|
|
36
36
|
} from "./chunk-VZSRQ272.js";
|
|
37
37
|
import "./chunk-SBCB6VZY.js";
|
|
38
38
|
import "./chunk-PC4UYEBM.js";
|
|
39
|
-
import "./chunk-
|
|
39
|
+
import "./chunk-2NSLDY4B.js";
|
|
40
40
|
import "./chunk-TVVP3ZZQ.js";
|
|
41
41
|
import "./chunk-ABOIVNXL.js";
|
|
42
42
|
import "./chunk-VSMTAMNK.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
planCampaignRun,
|
|
3
3
|
runCampaign
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-QIT2XZ4E.js";
|
|
5
5
|
import "./chunk-HRGUJTER.js";
|
|
6
6
|
import "./chunk-3BFEG2F6.js";
|
|
7
7
|
import "./chunk-PZ5AY32C.js";
|
|
@@ -9,4 +9,4 @@ export {
|
|
|
9
9
|
planCampaignRun,
|
|
10
10
|
runCampaign
|
|
11
11
|
};
|
|
12
|
-
//# sourceMappingURL=run-campaign-
|
|
12
|
+
//# sourceMappingURL=run-campaign-3NWW5PLF.js.map
|
|
@@ -310,5 +310,16 @@ declare function parseRunRecordSafe(input: unknown): {
|
|
|
310
310
|
};
|
|
311
311
|
/** Round-trip helper — `JSON.parse(JSON.stringify(record))` then validate. */
|
|
312
312
|
declare function roundTripRunRecord(record: RunRecord): RunRecord;
|
|
313
|
+
/**
|
|
314
|
+
* Heuristic snapshot check. Accepts:
|
|
315
|
+
* - `name@YYYY-MM-DD` (Anthropic style: `claude-sonnet-4-6@2025-04-15`)
|
|
316
|
+
* - `name-YYYYMMDD` (OpenAI style: `gpt-4o-2024-11-20`)
|
|
317
|
+
* - `name@<arbitrary-token>` (allow opaque snapshots like `@v3`)
|
|
318
|
+
* - explicit `:date-...` Vertex-style tags
|
|
319
|
+
*
|
|
320
|
+
* Rejects bare aliases like `claude-sonnet-4` or `gpt-4o` that remap
|
|
321
|
+
* silently as providers ship new snapshots.
|
|
322
|
+
*/
|
|
323
|
+
declare function modelHasSnapshot(model: string): boolean;
|
|
313
324
|
|
|
314
|
-
export { type AgentProfileCell as A,
|
|
325
|
+
export { type AgentProfileCell as A, roundTripRunRecord as B, toAgentProfileJson as C, validateAgentProfileCell as D, validateRunRecord as E, verifyAgentProfileCell as F, type JudgeScoresRecord as J, type RunRecord as R, type AgentProfileJson as a, type RunSplitTag as b, type RunTokenUsage as c, type RunJudgeMetadata as d, type AgentProfileCellInput as e, AGENT_PROFILE_KINDS as f, type AgentInterfaceProfileLike as g, type AgentProfileCellSchemaVersion as h, AgentProfileCellValidationError as i, type AgentProfileDimensionValue as j, type AgentProfileHarness as k, type AgentProfileKind as l, type AgentProfileSource as m, type AgentProfileSourceInput as n, type RunOutcome as o, RunRecordValidationError as p, agentProfileCellHashMaterial as q, agentProfileCellKey as r, assertRunAgentProfileCell as s, buildAgentInterfaceProfileCell as t, buildAgentProfileCell as u, groupRunsByAgentProfileCell as v, isRunRecord as w, modelHasSnapshot as x, parseRunRecordSafe as y, requireAgentProfileCell as z };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AxAIService } from '@ax-llm/ax';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
-
import { A as AnalystFinding, a as Analyst, b as AnalystContext } from './types-
|
|
4
|
-
import { T as TraceAnalystKindSpec } from './kind-factory-
|
|
3
|
+
import { A as AnalystFinding, a as Analyst, b as AnalystContext } from './types-Dz9cKF0g.js';
|
|
4
|
+
import { T as TraceAnalystKindSpec } from './kind-factory-D0nk7AKV.js';
|
|
5
5
|
import { a as TraceAnalystSpan } from './store-C1YxJDEK.js';
|
|
6
6
|
import { L as LlmClientOptions } from './llm-client-Bj7g0rqu.js';
|
|
7
7
|
import { S as Severity } from './multi-layer-verifier-DUZXrPDA.js';
|
package/dist/traces.d.ts
CHANGED
|
@@ -14,7 +14,7 @@ import { A as AnalyzeTracesOptions, b as AnalyzeTracesResult } from './analyst-C
|
|
|
14
14
|
export { a as AnalyzeTracesInput, c as AnalyzeTracesTurnSnapshot, d as analyzeTraces } from './analyst-C8HHvfJp.js';
|
|
15
15
|
import { h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, T as TraceAnalysisStore, g as TraceAnalystFilters, b as DatasetOverview, Q as QueryTracesPage, l as ViewTraceResult, V as ViewSpansResult, c as SearchTraceResult, S as SearchSpanResult } from './store-C1YxJDEK.js';
|
|
16
16
|
export { D as DEFAULT_TRACE_ANALYST_BUDGETS, E as ErrorCluster, d as SpanMatchRecord, e as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, f as TraceAnalystByteBudgets, a as TraceAnalystSpan, j as TraceAnalystTraceSummary, k as ViewTraceOversized } from './store-C1YxJDEK.js';
|
|
17
|
-
import { b as RunSplitTag, c as RunTokenUsage, R as RunRecord } from './run-record-
|
|
17
|
+
import { b as RunSplitTag, c as RunTokenUsage, R as RunRecord } from './run-record-MRdJ-Kq2.js';
|
|
18
18
|
import { AxFunction } from '@ax-llm/ax';
|
|
19
19
|
import '@tangle-network/agent-interface';
|
|
20
20
|
|
package/dist/traces.js
CHANGED
|
@@ -28,7 +28,7 @@ import {
|
|
|
28
28
|
scoreTraceInsightReadiness,
|
|
29
29
|
tokenizeDomainWords,
|
|
30
30
|
traceAnalystOnRunComplete
|
|
31
|
-
} from "./chunk-
|
|
31
|
+
} from "./chunk-7RBJANJD.js";
|
|
32
32
|
import {
|
|
33
33
|
TRACE_ANALYST_ACTOR_DESCRIPTION,
|
|
34
34
|
TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
|
|
@@ -106,7 +106,7 @@ import {
|
|
|
106
106
|
defaultProviderRedactor,
|
|
107
107
|
providerFromBaseUrl
|
|
108
108
|
} from "./chunk-PC4UYEBM.js";
|
|
109
|
-
import "./chunk-
|
|
109
|
+
import "./chunk-2NSLDY4B.js";
|
|
110
110
|
import {
|
|
111
111
|
TraceEmitter,
|
|
112
112
|
llmSpanFromProvider
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { c as RunTokenUsage } from './run-record-
|
|
1
|
+
import { c as RunTokenUsage } from './run-record-MRdJ-Kq2.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Pass A substrate types — `runCampaign` is the one primitive every
|
|
@@ -347,9 +347,20 @@ interface CampaignCostMeter {
|
|
|
347
347
|
* `tokenUsage`, so a cell that never reports tokens reads as a stub. Any
|
|
348
348
|
* dispatch that calls an LLM MUST report its usage. */
|
|
349
349
|
observeTokens(usage: CampaignTokenUsage): void;
|
|
350
|
+
/** Record the concrete model the backend RESOLVED this cell to at runtime.
|
|
351
|
+
* The substrate cannot see the LLM call, so it cannot know which model a
|
|
352
|
+
* vendor-locked harness actually served — only the dispatch, reading the
|
|
353
|
+
* backend's usage/terminal events, can. A dispatch whose profile declares a
|
|
354
|
+
* runtime-resolved model (the `HARNESS_NATIVE_MODEL` sentinel) MUST report
|
|
355
|
+
* the resolved, snapshot-bearing id here so the RunRecord pins a real model
|
|
356
|
+
* instead of the sentinel. Last write wins (a cell issues one logical run);
|
|
357
|
+
* optional because most dispatches declare a concrete model up front. */
|
|
358
|
+
observeModel?(model: string): void;
|
|
350
359
|
current(): number;
|
|
351
360
|
/** Accumulated token usage for this cell (zeros if never observed). */
|
|
352
361
|
tokens(): CampaignTokenUsage;
|
|
362
|
+
/** The runtime-resolved model reported via `observeModel`, if any. */
|
|
363
|
+
resolvedModel?(): string | undefined;
|
|
353
364
|
}
|
|
354
365
|
/** Source tag — required on every store write. Used by the
|
|
355
366
|
* default training-source filter (production-trace samples NOT used as
|
|
@@ -446,6 +457,12 @@ interface CampaignCellResult<TArtifact> {
|
|
|
446
457
|
* `{ input: 0, output: 0 }` when the dispatch reported none — which the
|
|
447
458
|
* backend-integrity guard reads as a stub. */
|
|
448
459
|
tokenUsage: CampaignTokenUsage;
|
|
460
|
+
/** The concrete model the backend resolved this cell to at runtime, reported
|
|
461
|
+
* by the dispatch via `ctx.cost.observeModel`. Set only when the dispatch
|
|
462
|
+
* reported it — a profile that declares a concrete model up front has no
|
|
463
|
+
* need to. Consumed by `buildRunRecord` to pin the real model when the
|
|
464
|
+
* declared model is the `HARNESS_NATIVE_MODEL` sentinel. */
|
|
465
|
+
resolvedModel?: string;
|
|
449
466
|
durationMs: number;
|
|
450
467
|
seed: number;
|
|
451
468
|
cached: boolean;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { R as RunRecord } from './run-record-
|
|
1
|
+
import { R as RunRecord } from './run-record-MRdJ-Kq2.js';
|
|
2
2
|
import { T as TraceAnalysisStore } from './store-C1YxJDEK.js';
|
|
3
3
|
import { a as JudgeInput } from './types-C7DGg5ex.js';
|
|
4
4
|
import { b as LlmCallRequest, c as LlmCallResult } from './llm-client-Bj7g0rqu.js';
|
package/dist/workflow/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { W as WorkflowTopology } from '../harness-optimizer-mOl9XX_O.js';
|
|
2
|
-
import { b as RunSplitTag, c as RunTokenUsage, R as RunRecord } from '../run-record-
|
|
3
|
-
import { A as AnalystFinding, h as AnalystSeverity, E as EvidenceRef } from '../types-
|
|
4
|
-
import { F as FailureClusterInsight } from '../insight-report-
|
|
2
|
+
import { b as RunSplitTag, c as RunTokenUsage, R as RunRecord } from '../run-record-MRdJ-Kq2.js';
|
|
3
|
+
import { A as AnalystFinding, h as AnalystSeverity, E as EvidenceRef } from '../types-Dz9cKF0g.js';
|
|
4
|
+
import { F as FailureClusterInsight } from '../insight-report-k0sRTzKg.js';
|
|
5
5
|
import { a as VerificationReport, L as LayerResult } from '../multi-layer-verifier-DUZXrPDA.js';
|
|
6
6
|
import { F as FailureClusterReport } from '../failure-cluster-DH9Flgcf.js';
|
|
7
7
|
import { R as RedactionRule, a as RedactionReport } from '../redact-B40YG2M_.js';
|
|
@@ -19,7 +19,7 @@ import '../types-C7DGg5ex.js';
|
|
|
19
19
|
import '@tangle-network/tcloud';
|
|
20
20
|
import '../llm-client-Bj7g0rqu.js';
|
|
21
21
|
import '../raw-provider-sink-C46HDghv.js';
|
|
22
|
-
import '../summary-report-
|
|
22
|
+
import '../summary-report-C0nnxOD8.js';
|
|
23
23
|
import '../judge-calibration-0p2QcWNE.js';
|
|
24
24
|
import '../verdict-C9MlYujm.js';
|
|
25
25
|
import '../control-runtime-Acf9CGhw.js';
|
package/dist/workflow/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-eval",
|
|
3
|
-
"version": "0.102.
|
|
3
|
+
"version": "0.102.1",
|
|
4
4
|
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-eval#readme",
|
|
6
6
|
"repository": {
|