@tangle-network/agent-eval 0.193.2 → 0.194.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +5 -5
- package/dist/analyst/index.js +1 -1
- package/dist/{benchmark-command-N5r3GXMP.js → benchmark-command-Boqtfx4Z.js} +2 -2
- package/dist/{benchmark-command-N5r3GXMP.js.map → benchmark-command-Boqtfx4Z.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +3 -3
- package/dist/benchmarks/index.js +2 -2
- package/dist/campaign/index.d.ts +4 -4
- package/dist/campaign/index.js +6 -6
- package/dist/{campaign-DnL0S4XL.js → campaign-Cnx0Duzs.js} +6 -6
- package/dist/{campaign-DnL0S4XL.js.map → campaign-Cnx0Duzs.js.map} +1 -1
- package/dist/cli.js +4 -4
- package/dist/contract/index.d.ts +6 -6
- package/dist/contract/index.js +3 -3
- package/dist/{default-registry-C3HTE08O.d.ts → default-registry-D44M_E4Q.d.ts} +3 -3
- package/dist/{default-registry-C3HTE08O.d.ts.map → default-registry-D44M_E4Q.d.ts.map} +1 -1
- package/dist/{define-agent-eval-DQJvVa1P.d.ts → define-agent-eval-CKTbbeyT.d.ts} +4 -4
- package/dist/{define-agent-eval-DQJvVa1P.d.ts.map → define-agent-eval-CKTbbeyT.d.ts.map} +1 -1
- package/dist/{define-agent-eval-qxAPjMSL.js → define-agent-eval-Dq0MgLnw.js} +3 -3
- package/dist/{define-agent-eval-qxAPjMSL.js.map → define-agent-eval-Dq0MgLnw.js.map} +1 -1
- package/dist/{engine-DjRV55F9.d.ts → engine-LkP-rfp1.d.ts} +2 -2
- package/dist/{engine-DjRV55F9.d.ts.map → engine-LkP-rfp1.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/hosted/index.js +1 -1
- package/dist/{index-DWQNDqPd.d.ts → index-DKUIMOst.d.ts} +6 -6
- package/dist/{index-DWQNDqPd.d.ts.map → index-DKUIMOst.d.ts.map} +1 -1
- package/dist/{index-DMPLG226.d.ts → index-Dvc9zaWD.d.ts} +41 -4
- package/dist/{index-DMPLG226.d.ts.map → index-Dvc9zaWD.d.ts.map} +1 -1
- package/dist/{index-0jEtpSM7.d.ts → index-cWnRlQLW.d.ts} +3 -3
- package/dist/{index-0jEtpSM7.d.ts.map → index-cWnRlQLW.d.ts.map} +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.js +3 -3
- package/dist/jev.d.ts +1 -1
- package/dist/{llm-judge-Cififl_V.js → llm-judge-D6NIvyrM.js} +91 -7
- package/dist/llm-judge-D6NIvyrM.js.map +1 -0
- package/dist/{matrix-CNu13gg8.d.ts → matrix-CUzY_7Ml.d.ts} +2 -2
- package/dist/{matrix-CNu13gg8.d.ts.map → matrix-CUzY_7Ml.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-B2AhDRnq.js → produced-state-BjkNRWAe.js} +2 -2
- package/dist/{produced-state-B2AhDRnq.js.map → produced-state-BjkNRWAe.js.map} +1 -1
- package/dist/rl.d.ts +1 -1
- package/dist/{search-ledger-CJiAEU3j.js → search-ledger-BbKtG_WF.js} +12 -4
- package/dist/search-ledger-BbKtG_WF.js.map +1 -0
- package/dist/{search-shipper-DUlwlCsK.js → search-shipper-Cm4Wt9SC.js} +2 -2
- package/dist/{search-shipper-DUlwlCsK.js.map → search-shipper-Cm4Wt9SC.js.map} +1 -1
- package/dist/{search-summary-BhY8p0Av.js → search-summary-DnDGR702.js} +2 -2
- package/dist/{search-summary-BhY8p0Av.js.map → search-summary-DnDGR702.js.map} +1 -1
- package/dist/{skillopt-optimization-method-C6DbFKnw.js → skillopt-optimization-method-C4GiaBzp.js} +3 -3
- package/dist/{skillopt-optimization-method-C6DbFKnw.js.map → skillopt-optimization-method-C4GiaBzp.js.map} +1 -1
- package/dist/{statistical-heldout-D-xBpbrd.d.ts → statistical-heldout-CLfkCc_B.d.ts} +2 -2
- package/dist/{statistical-heldout-D-xBpbrd.d.ts.map → statistical-heldout-CLfkCc_B.d.ts.map} +1 -1
- package/dist/{store-tool-spans-BxrfuU2t.d.ts → store-tool-spans-E-mdWPAz.d.ts} +3 -3
- package/dist/{store-tool-spans-BxrfuU2t.d.ts.map → store-tool-spans-E-mdWPAz.d.ts.map} +1 -1
- package/dist/{tool-groups-BC9O7Epn.d.ts → tool-groups-CiFSpk6C.d.ts} +2 -2
- package/dist/tool-groups-CiFSpk6C.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/{types-BWwRXofp.d.ts → types-DLtLe21j.d.ts} +9 -2
- package/dist/types-DLtLe21j.d.ts.map +1 -0
- package/dist/{types-qoxUfKQg.d.ts → types-M1Et9e6Z.d.ts} +2 -2
- package/dist/{types-qoxUfKQg.d.ts.map → types-M1Et9e6Z.d.ts.map} +1 -1
- package/docs/search-ledger.md +10 -4
- package/docs/trace-contracts.md +21 -2
- package/package.json +1 -1
- package/dist/llm-judge-Cififl_V.js.map +0 -1
- package/dist/search-ledger-CJiAEU3j.js.map +0 -1
- package/dist/tool-groups-BC9O7Epn.d.ts.map +0 -1
- package/dist/types-BWwRXofp.d.ts.map +0 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { a as RunRecord } from "./run-record-DF-fuu31.js";
|
|
2
2
|
import { a as PairedPromotionDecision, d as PairedBootstrapResult, i as PairedMcNemarEvidence, r as PairedDecisionStatistic, t as PairedDecisionMethod, u as PairedBootstrapOptions } from "./paired-promotion-decision-DPKz8ZnF.js";
|
|
3
3
|
import { r as LedgerHash } from "./canonical-CFpojCN5.js";
|
|
4
|
-
import { R as Scenario, a as CampaignResult, b as GenerationRecord, h as GateContext, j as MutableSurface, p as Gate, v as GateResult, w as JudgeScore } from "./types-
|
|
4
|
+
import { R as Scenario, a as CampaignResult, b as GenerationRecord, h as GateContext, j as MutableSurface, p as Gate, v as GateResult, w as JudgeScore } from "./types-DLtLe21j.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
//#region src/statistics/paired-binary.d.ts
|
|
7
7
|
/** A binomial proportion estimate with a confidence interval. */
|
|
@@ -1306,4 +1306,4 @@ declare function dimensionRegressions(candidate: Map<string, Record<string, Judg
|
|
|
1306
1306
|
}): DimensionRegression[];
|
|
1307
1307
|
//#endregion
|
|
1308
1308
|
export { paretoSignificanceGate as $, PairArmsOptions as A, FinalEvidenceReservation as At, PowerPreflight as B, RiskDifferenceResult as Bt, hashJson as C, verifyAttestation as Ct, ComparePairedArmsOptions as D, FinalEvidenceMeasurement as Dt, verifyManifest as E, FinalEvidenceLedger as Et, PairedCorrectness as F, EProcessStep as Ft, BuildEvidenceVectorOptions as G, pairedRiskDifference as Gt, powerPreflight as H, isBinaryOutcomeVector as Ht, PairedMetricDelta as I, eProcess as It, ParetoSignificanceGateOptions as J, passAtK as Jt, EvidenceVector as K, pairedRiskDifferenceExact as Kt, comparePairedArms as L, ExactRiskDifferenceResult as Lt, PairRunRecordsResult as M, EProcess as Mt, PairedArmRow as N, EProcessOptions as Nt, MatchedPair as O, FinalEvidenceOutcome as Ot, PairedArmsComparison as P, EProcessState as Pt, paretoPolicy as Q, pairArms as R, McNemarResult as Rt, evaluateHypothesis as S, attest as St, signManifest as T, FinalEvidenceError as Tt, AxisEvidence as U, mcnemar as Ut, PowerPreflightOptions as V, ScoreRiskDifferenceResult as Vt, AxisVerdict as W, pairedBinaryScale as Wt, PromotionPolicy as X, PromotionObjective as Y, wilson as Yt, buildEvidenceVector as Z, sequentialPairedGate as _, verifyEvidenceReceipt as _t, detectScale as a, createCampaignEvidenceReceipt as at, SignedManifest as b, AttestationVerification as bt, pairHoldout as c, EVIDENCE_RECEIPT_VERSION as ct, SequentialDecision as d, EvidenceBinding as dt, Objective as et, SequentialObservation as f, EvidenceReceipt as ft, sequentialDecide as g, isIndependentEvidence as gt, SequentialStreamState as h, createEvidenceReceipt as ht, PairedHoldout as i, CampaignEvidenceContext as it, PairArmsResult as j, openFinalEvidenceLedger as jt, MatchedRunRecordPair as k, FinalEvidenceRecord as kt, SequentialDecideFn as l, EvidenceAuthority as lt, SequentialPairedGateOptions as m, INDEPENDENT_EVIDENCE_AUTHORITY_KINDS as mt, HeldoutSignificance as n, dominates as nt, dimensionRegressions as o, CreateEvidenceReceiptInput as ot, SequentialPairedGate as p, EvidenceReceiptVerification as pt, ObjectiveSource as q, pairedRiskDifferenceScore as qt, HeldoutSignificanceOptions as r, paretoFrontier as rt, heldoutSignificance as s, EVIDENCE_AUTHORITY_KINDS as st, DimensionRegression as t, ParetoResult as tt, SequentialDecideOptions as u, EvidenceAuthorityKind as ut, HypothesisManifest as v, ATTESTATION_ALGORITHM as vt, manifestContentDigest as w, FinalEvidenceConflictError as wt, SignedManifestAlgo as x, AttestedReport as xt, HypothesisResult as y, AttestationProvenance as yt, pairRunRecords as z, ProportionInterval as zt };
|
|
1309
|
-
//# sourceMappingURL=statistical-heldout-
|
|
1309
|
+
//# sourceMappingURL=statistical-heldout-CLfkCc_B.d.ts.map
|
package/dist/{statistical-heldout-D-xBpbrd.d.ts.map → statistical-heldout-CLfkCc_B.d.ts.map}
RENAMED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistical-heldout-
|
|
1
|
+
{"version":3,"file":"statistical-heldout-CLfkCc_B.d.ts","names":[],"sources":["../src/statistics/paired-binary.ts","../src/statistics/sequential-eprocess.ts","../src/experiment/final-evidence.ts","../src/attestation.ts","../src/experiment/evidence-receipt.ts","../src/experiment/campaign-evidence.ts","../src/pareto.ts","../src/campaign/gates/promotion-policy.ts","../src/campaign/gates/power-preflight.ts","../src/paired-arms.ts","../src/pre-registration.ts","../src/campaign/gates/sequential.ts","../src/campaign/gates/statistical-heldout.ts"],"mappings":";;;;;;;UAgBiB;;EAEf;;EAEA;;EAEA;;;;;;;;;iBAUc,OAAO,mBAAmB,WAAW,sBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;iBA2CzD,sBAAsB,QAAQ;;UAU7B;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;iBAec,QACd,SAAS,6BACT,WAAW,8BACV;;UAmBc;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;;;iBAkBc,qBACd,SAAS,6BACT,WAAW,6BACX,sBACC;;;;UAkCc;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA+Bc,0BACd,SAAS,6BACT,WAAW,6BACX,sBACC;;;UAyEc;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA8Dc,0BACd,SAAS,6BACT,WAAW,6BACX,sBACC;;;;;;;;;;;;;;;;;;;;iBAiFa,kBACd,QAAQ,mBACR,OAAO;;;;;;;;;;;iBA0BO,QAAQ,WAAW,WAAW;;;UCrgB7B;;;EAGf;;;;EAIA;;;;EAIA;;;;;;;;EAQA,SAAS;;UAGM;;EAEf;;EAEA;;EAEA;;;;;;;;;;UAWe,sBAAsB;EACrC;EACA;EACA;;EAEA;;EAEA;;EAEA;;;EAGA;;UAGe;;;EAGf,OAAO,YAAY;EACnB,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA8BK,SAAS,OAAM,kBAAuB;;;cCnFhD,mBAAiB,EAAA;;;;;;GAQZ,EAAA,KAAA;;KAGC,2BAA2B,EAAE,aAAa;cAEhD,mBAAiB,EAAA;;;GAKZ,EAAA,KAAA;KAEC,2BAA2B,EAAE,aAAa;UAgCrC;EACf,aAAa;EACb,iBAAiB;EACjB;IAAY,aAAa;IAA0B,WAAW;;;KAGpD,qBAAqB;EAC3B;EAAiB,OAAO;;EACxB;EAAkB;IAAS;IAA8C;;;UAE9D;EACf,QACE,OAAO,2BACN,QAAQ;IAAuB,QAAQ;IAAqB;;EAC/D,OACE,mBACA,aAAa,2BACZ,QAAQ;IAAuB,QAAQ;IAAqB;;EAC/D,QAAQ,QAAQ,qBAAqB;;;cAI1B,2BAA2B;WAE3B;EADX,YACW,8CACT;;;cAQS,mCAAmC;EAC9C,YAAY;;;iBA+HE,wBAAwB;EAAW;IAAiB;;;;;;;;;;;;;;;;;;;;;;;cCvMvD;UAEI;;EAEf,eAAe;;EAEf;;;EAGA;;EAEA;;EAEA;;;EAGA;;UAGe;;EAEf;EACA,YAAY;EACZ,kBAAkB;;EAElB;;UAGe;EACf;;EAEA;;;;;;;;iBAiBc,OAAO,iBAAiB,YAAY,wBAAwB;;;;;;;;;;iBAoB5D,kBACd,iBACA,UAAU,iBACT;;;cC1EU;;cAGA;KAQD,gCAAgC;cAE/B;UAOI;WACN,MAAM;;WAEN;;UAGM;WACN,sBAAsB;;WAEtB;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;WACA,WAAW;;WAEX;;WAEA;;UAGM;WACN,SAAS;WACT,aAAa;;UAGP;WACN;WACA;;UAGM,mCAAmC,KAAK;;;;;;;iBAQzC,sBACd,OAAO,4BACP,YAAY,wBACX;;;;;iBA4Ba,sBAAsB,SAAS,kBAAkB;;;;;;iBAkCjD,sBAAsB,SAAS;;;;KCjJnC,0BAA0B,KACpC;EAOI,YAAY,KAAK;;;iBAGP,8BAA8B,UAAU,UAAU,GAAG;EACnE,UAAU,eAAe,GAAG;EAC5B,SAAS;EACT,SAAS;IAAA;;;;;;;;;;;;;;;;;;KCPC;UAEK,UAAU;;EAEzB;EACA,WAAW;EACX,QAAQ,WAAW;;UAGJ,aAAa;EAC5B,UAAU;EACV,WAAW;;EAEX,cAAc;IAAQ,WAAW;IAAG,WAAW;;;;iBAIjC,UAAU,GAAG,GAAG,GAAG,GAAG,GAAG,YAAY,UAAU;;;;;;iBAmB/C,eAAe,GAAG,YAAY,KAAK,YAAY,UAAU,OAAO,aAAa;;;;;KCxBjF;EAAoB;;EAAwB;EAAmB;;UAE1D;;EAEf;EACA,QAAQ;;;EAGR,WAAW;;;;EAIX;;;;EAIA;;;;;EAKA;;;;KAKU;UAEK;EACf;EACA,QAAQ;EACR,WAAW;;;;;;;;EAQX,WAAW;;;;;EAKX;;;EAGA;IAAM;IAAa;;;EAEnB,mBAAmB;;EAEnB,SAAS;;;EAGT;;EAEA;EACA;EACA,gBAAgB;EAChB;EACA;EACA,SAAS;;UAGM;;EAEf,MAAM;;;EAGN;;;EAGA;IAAQ;IAAmB;;;;;;KAMjB,mBAAmB,IAAI,mBAAmB;UAErC;;;;EAIf;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;;iBASc,oBAAoB,WAAW,kBAAkB,UAC/D,KAAK,YAAY,WAAW,YAC5B,YAAY,sBACZ,OAAM,6BACL;;;;;;cAkIU,cAAc;UA+EV,sCAAsC;;EAErD,YAAY;;;EAGZ,SAAS;;EAET;;;;;;;iBAQc,uBAAuB,qBAAqB,kBAAkB,WAAW,UACvF,SAAS,gCACR,KAAK,WAAW;;;;;;;;;;;;;;;;;UCzVF;;EAEf;;;EAGA;;EAEA;;EAEA;;;;EAIA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;EAGA;EACA;EACA;;EAEA;;EAEA;;;iBAYc,eAAe,MAAM,wBAAwB;;;;;UCzB5C;;;EAGf;;;;;EAKA;;EAEA;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;;EAEA;;;UAIe;EACf;;;;EAIA;EACA,UAAU;EACV,WAAW;;UAGI;;EAEf,OAAO;;;EAGP,kBAAkB;;EAElB,mBAAmB;;;;;;;;;;;;;;;;;;;;iBAqBL,SAAS,eAAe,gBAAgB,MAAM,kBAAkB;;UA8G/D;;EAEf;;EAEA;;EAEA,SAAS;;EAET,gBAAgB;;;UAID;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,aAAa;;EAEb;IAAY;IAAW;;;UAGR,iCAAiC;;;;EAIhD;;EAEA,YAAY;;UAGG;EACf;EACA;EACA;;;EAGA,aAAa;EACb,cAAc;;;;;;;;;;;;;;;iBAgBA,kBACd,eAAe,gBACf,MAAM,2BACL;UAmEc;EACf;EACA;EACA,UAAU;EACV,WAAW;;UAGI;EACf,OAAO;EACP,kBAAkB;EAClB,mBAAmB;;;;;;;;;iBAeL,eACd,uBAAuB,aACvB,wBAAwB,cACvB;;;;;;;;;;;;;;;;UCrWc;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;;;KAIU;UAEK,uBAAuB;;EAEtC;;EAEA,MAAM;;UAGS;EACf,UAAU;EACV;EACA;EACA;;;EAGA;;EAEA,kBAAkB;EAGlB;;;;;;;;;;;;;;;;;iBAkBoB,SAAS,GAAG,KAAK,IAAI;;;;;iBAQ3B,sBAAsB,UAAU;;;;;;iBAc1B,aAAa,GAAG,qBAAqB,QAAQ;;;;;iBAS7C,eAAe,GAAG,iBAAiB;;;;;iBAYnC,mBACpB,UAAU,gBACV;EAAY;EAAW;EAAgB;IACtC,QAAQ;;;KCrFC;UAEK;EACf,UAAU;;EAEV;;EAEA;;;EAGA;;UAGe;;;EAGf;;;;EAIA;;;EAGA;;EAEA;;;;EAIA;;;EAGA;;;;EAIA,8BAA8B;;;EAG9B,kBAAkB;;EAElB;;;;;;;EAOA,SAAS;;;;;;KAOC,wBAAwB;EAAkB,UAAU;;UAE/C,qBAAqB,qBAAqB,kBAAkB,WAAW,kBAC9E,KAAK,WAAW;;;;;;;;;;EAUxB,QAAQ,gBAAgB;;;EAGxB,SAAS;;;;;;;;;;;;;;;;;iBAuPK,qBAAqB,qBAAqB,kBAAkB,WAAW,UACrF,SAAS,8BACR,qBAAqB,WAAW;UA6GlB;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;GACd;IAAQ,SAAS;;IAAyB;IAAe;;;EAE1D,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;iBA0BK,iBAAiB,UAAS,0BAA+B;;;UC9dxD;;EAEf;;EAEA;;EAEA;;;;;;;;;;;;;iBA4Ec,YACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,SAAS,GAAG,oCACX;UAkEc;EACf,QAAQ;;;;;;;;;;;;EAYR,WAAW;;;;EAIX,iBAAiB;;;;;;;EAOjB,UAAU;;EAEV,mBAAmB;;EAEnB,SAAS;;;;EAIT;;EAEA;;EAEA;EACA;;EAEA;;EAEA;;EAEA,gBAAgB;;EAEhB;;;;EAIA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;;EAEA;EACA;;EAEA,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA6BhB,oBACd,QAAQ,eACR,OAAM,6BACL;UA2Ec;EACf;;;EAGA,WAAW;;;;EAIX;;EAEA;IAAM;IAAa;;;EAEnB,mBAAmB;;EAEnB,SAAS;;EAET;;;;EAIA;EACA;EACA;EACA;EACA;;EAEA;EACA;;EAEA;;EAEA;;;;;iBAMc,YAAY;;;;;;;;;iBAYZ,qBACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,8BACA;EACE;EACA;EACA;EACA;;;EAGA;EACA,8BAA8B;EAC9B;IAED"}
|
|
@@ -6,8 +6,8 @@ import { E as SpanSourceReader, F as SearchTraceResult, K as ViewTraceResult, N
|
|
|
6
6
|
import { et as ProviderRedactor, rt as RawProviderSink } from "./types-C0bWoSKC.js";
|
|
7
7
|
import { i as SearchArtifactRef } from "./search-ledger-types-B3El5xgS.js";
|
|
8
8
|
import { s as TraceStore } from "./store-C4avFjIi.js";
|
|
9
|
-
import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, n as TraceAnalysisEngine } from "./engine-
|
|
10
|
-
import { t as TraceToolGroupName } from "./tool-groups-
|
|
9
|
+
import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, n as TraceAnalysisEngine } from "./engine-LkP-rfp1.js";
|
|
10
|
+
import { t as TraceToolGroupName } from "./tool-groups-CiFSpk6C.js";
|
|
11
11
|
//#region src/trace/extract-usage.d.ts
|
|
12
12
|
type ExtractedUsage = RunTokenUsage;
|
|
13
13
|
type SseUsageMode = 'cumulative' | 'delta';
|
|
@@ -407,4 +407,4 @@ declare class ToolTraceMissingError extends CaptureIntegrityError {
|
|
|
407
407
|
declare function toolSpansToTraceAnalysisStore(spans: readonly ToolSpan[] | null | undefined, options?: ToolSpansToTraceAnalysisStoreOptions): TraceAnalysisStore;
|
|
408
408
|
//#endregion
|
|
409
409
|
export { AnalyzeTracesInput as A, CaptureFetchContext as B, inferDomainKeywords as C, planTraceInsightQuestions as D, isTraceInsightEvidence as E, OTEL_AGENT_EVAL_SCOPE as F, SseUsageMode as G, captureFetchToRawSink as H, OtlpExport as I, extractUsageFromSse as J, extractUsage as K, OtlpResourceSpans as L, AnalyzeTracesResult as M, analyzeTraces as N, scoreTraceInsightReadiness as O, OtlpFlatLine as P, OtlpSpan as R, domainEvidencePattern as S, isTraceInsightAttribution as T, ExtractUsageFromSseOptions as U, CaptureFetchOptions as V, ExtractedUsage as W, TraceInsightTask as _, ToolSpansToTraceAnalysisStoreOptions as a, defaultTraceInsightPanel as b, TraceInsightContext as c, TraceInsightPanelRole as d, TraceInsightPromptInput as f, TraceInsightSuite as g, TraceInsightReadiness as h, OtlpFileTraceStoreOptions as i, AnalyzeTracesOptions as j, tokenizeDomainWords as k, TraceInsightEvidence as l, TraceInsightQuestion as m, toolSpansToTraceAnalysisStore as n, otlpTextToTraceAnalysisStore as o, TraceInsightQualityGate as p, extractUsageFromResponse as q, OtlpFileTraceStore as r, TraceInsightAttribution as s, ToolTraceMissingError as t, TraceInsightFinding as u, buildTraceInsightContext as v, isTraceInsightArtifact as w, describeTraceInsightScope as x, buildTraceInsightPrompt as y, exportRunAsOtlp as z };
|
|
410
|
-
//# sourceMappingURL=store-tool-spans-
|
|
410
|
+
//# sourceMappingURL=store-tool-spans-E-mdWPAz.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store-tool-spans-
|
|
1
|
+
{"version":3,"file":"store-tool-spans-E-mdWPAz.d.ts","names":[],"sources":["../src/trace/extract-usage.ts","../src/trace/capture-fetch.ts","../src/trace/otel.ts","../src/trace/otlp-flat.ts","../src/trace-analyst/analyst.ts","../src/trace-analyst/insights.ts","../src/trace-analyst/store-otlp.ts","../src/trace-analyst/store-tool-spans.ts"],"mappings":";;;;;;;;;;;KAgBY,iBAAiB;KACjB;UAEK;;EAEf,OAAO;;;;;;;iBAuDO,aAAa,gBAAgB;;;;;;iBA6C7B,oBACd,cACA,UAAS,6BACR;;;;;;;iBA4DmB,yBACpB,UAAU,UACV,aAAa,6BACZ,QAAQ;;;UCrKM;;;EAGf;;EAEA;;;EAGA;;EAEA;;EAEA;;UAGe;;EAEf,WAAW;;;EAGX;;;EAGA;;;;;;;;EAQA,WAAW,OAAO,gBAAgB,KAAK;;EAEvC,eAAe;;iBAkDD,sBACd,cAAc,WAAW,OACzB,MAAM,iBACN,KAAK,qBACL,OAAM,6BACE,WAAW;;;cCxFR;EAA0B;EAAoC;;UAE1D;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,YAAY;IACV;IACA;MAAS;MAAsB;MAAmB;MAAsB;;;EAE1E,SAAS;IAAQ;IAAsB;IAAc,aAAa;;EAClE;IAAW;IAAc;;;UAGV;EACf;IAAY,YAAY;;EACxB,YAAY;IAAQ,cAAc;IAAuB,OAAO;;;UAGjD;EACf,eAAe;;;iBAIK,gBACpB,OAAO,YACP,eACA,gBAAe,4CACd,QAAQ;;;KClDC;UAEK;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE,MAAM;IACN;;EAEF;IAAY,YAAY;;EACxB,YAAY;EACZ,SAAS;IAAQ;IAAc;IAAuB,aAAa;;;;;UCNpD;EACf;EACA;EACA;EACA;;KAGU,sBAAsB;UAEjB;;EAEf,iBAAiB;;EAEjB,QAAQ;;EAER;;EAEA,YAAY;EACZ,SAAS,QAAQ;EACjB;EACA;EACA,aAAa;EACb;EACA,yBAAyB;EACzB,4BAA4B;EAC5B,eAAe,SAAS;EACxB,OAAO;EACP,OAAO,iBAAiB,SAAS;EACjC,SAAS;;;;;;;;iBASW,cACpB,OAAO,oBACP,SAAS,uBACR,QAAQ;;;UCnDM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;EAEA,cAAc;;;UAIC;EACf;IACE;IACA;IACA;IACA;IACA;IACA;IACA;;EAEF,WAAW,gGAET;EAEF,OAAO;IAAQ;IAAiB;IAAgB,UAAU;;;KAGhD,0BAA0B;EAE9B;EAAoB;EAA0B;;EAE9C;EACA;EACA,OAAO;EACP,WAAW;;EAGX;EACA;EACA;EACA,OAAO;EACP,WAAW;;EAGX;EACA;EACA,OAAO;EACP,MAAM;EACN;EACA,mBAAmB;;;iBAaX,uBAAuB,iBAAiB,SAAS;iBAoBjD,uBAAuB,iBAAiB,SAAS;iBAyCjD,0BAA0B,iBAAiB,SAAS;UA8CnD;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf,OAAO;EACP,WAAW;EACX,QAAQ;EACR,SAAS;EACT;;UAGe;EACf,OAAO;EACP;EACA;EACA,WAAW;EACX,OAAO;EACP,UAAU;EACV,OAAO;EACP,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;iBA0BO,oBAAoB;iBAMpB,oBAAoB,OAAO;iBAuB3B,sBAAsB,qBAAqB;iBAS3C,0BAA0B,OAAO;iBAsBjC,0BAA0B,OAAO,0BAA0B;iBAsD3D,yBAAyB,OAAO,0BAA0B;iBAa1D,2BAA2B,SAAS,sBAAsB;iBA+E1D,4BAA4B;iBA6B5B,wBAAwB,OAAO;;;UCjW9B;;EAEf,eAAe;;EAEf;;EAEA;;EAEA;;EAEA;;KAGG,gCAAgC;UAEpB,kCAAkC;;EAEjD;;;;;;;;EAQA;;uBAMa,kCAAkC;WACtC,iBAAiB;mBACT;mBACA;mBACA;mBACA;UACT;UACA;UACA;UACA;EAER,YAAY,MAAM;EA+BZ,SAAS,kBAAkB,UAAU,4BAA4B;EAOjE,SACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT;EAeG,YACJ,UAAU,qBACV,UAAU,4BACT,QAAQ;EAoDL,YACJ;IAAQ,UAAU;IAAqB;IAAe;KACtD,UAAU,4BACT,QAAQ;EAmBL,YACJ,UAAU,qBACV,UAAU,4BACT;EAMG,UACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT,QAAQ;EAuCL,UACJ;IAAQ;IAAkB;IAA6B;KACvD,UAAU,4BACT,QAAQ;EAsFL,YACJ;IAAQ;IAAkB;IAAuB;KACjD,UAAU,4BACT,QAAQ;EA4CL,WACJ;IAAQ;IAAkB;IAAiB;IAAuB;KAClE,UAAU,4BACT,QAAQ;;;EA6CL,cAAc,UAAU,4BAA4B;UAI5C;qBAmBK,WAAW,SAAS,cAAc,QAAQ;qBAE1C;UAEL;UAmBA;UAsHA;UA+DN;UAkBA;UAwDA;UAmBM;;cA+CH,2BAA2B;mBACrB;mBACA;EAEjB,YAAY,MAAM;;;YAYF,WAAW,SAAS,cAAc,QAAQ;YAkBhD;;;iBAwBI,6BACd,cACA,OAAM,uCACL;;;;cC18BU,8BAA8B;EACzC;;;;;;iBASc,8BACd,gBAAgB,+BAChB,UAAS,uCACR"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { O as TraceAnalysisStore } from "./types-CeDfCRXp.js";
|
|
2
|
-
import { h as TraceAnalysisToolDescriptor } from "./engine-
|
|
2
|
+
import { h as TraceAnalysisToolDescriptor } from "./engine-LkP-rfp1.js";
|
|
3
3
|
//#region src/analyst/tool-groups.d.ts
|
|
4
4
|
/** Named tool sets. Kinds pass `tools: TRACE_TOOL_GROUPS.failureForensics` etc. */
|
|
5
5
|
type TraceToolGroupName =
|
|
@@ -25,4 +25,4 @@ type TraceToolGroupName =
|
|
|
25
25
|
declare function buildTraceToolsForGroup(group: TraceToolGroupName, store: TraceAnalysisStore): TraceAnalysisToolDescriptor[];
|
|
26
26
|
//#endregion
|
|
27
27
|
export { buildTraceToolsForGroup as n, TraceToolGroupName as t };
|
|
28
|
-
//# sourceMappingURL=tool-groups-
|
|
28
|
+
//# sourceMappingURL=tool-groups-CiFSpk6C.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-groups-CiFSpk6C.d.ts","names":[],"sources":["../src/analyst/tool-groups.ts"],"mappings":";;;;KAmBY;;;;;;;;;;;;;;;;;;;;iBAyDI,wBACd,OAAO,oBACP,OAAO,qBACN"}
|
|
@@ -2,7 +2,7 @@ import { r as CaptureIntegrityError } from "../errors-DEE6u6ot.js";
|
|
|
2
2
|
import { b as CustomTokenPricing, c as CostLedgerHandle, p as CostProvenance } from "../cost-ledger-DbQdN3nO.js";
|
|
3
3
|
import { d as RunTokenUsage } from "../run-record-DF-fuu31.js";
|
|
4
4
|
import { t as DefaultVerdict } from "../verdict-E4eRNf7-.js";
|
|
5
|
-
import { a as TraceAnalystLimits, n as TraceAnalysisEngine } from "../engine-
|
|
5
|
+
import { a as TraceAnalystLimits, n as TraceAnalysisEngine } from "../engine-LkP-rfp1.js";
|
|
6
6
|
import { t as PrimeBridgeTransport } from "../prime-bridge-transport-6feEglLf.js";
|
|
7
7
|
import { a as RecordedTrajectoryStep, h as isRecordedTimeout } from "../steps-Bg5-KVx1.js";
|
|
8
8
|
//#region src/trace-repair/mini-swe-scaffold.d.ts
|
package/dist/traces.d.ts
CHANGED
|
@@ -4,11 +4,11 @@ import { a as RunRecord, c as RunSplitTag, d as RunTokenUsage, u as RunTerminalO
|
|
|
4
4
|
import { A as DEFAULT_TRACE_ANALYST_BUDGETS, B as TraceAnalystSpan, C as BoundedTraceAnalysisStoreOptions, D as TRACE_ANALYSIS_LIMITS, E as SpanSourceReader, F as SearchTraceResult, G as ViewTraceOversized, H as TraceAnalystSpanStatus, I as SpanMatchRecord, K as ViewTraceResult, L as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, M as ErrorCluster, N as QueryTracesPage, O as TraceAnalysisStore, P as SearchSpanResult, R as TraceAnalystByteBudgets, S as createBoundedTraceAnalysisStore, T as ReadSpanSourceResult, U as TraceAnalystTraceSummary, V as TraceAnalystSpanKind, W as ViewSpansResult, j as DatasetOverview, k as TraceAnalysisStoreContext, w as ReadSpanSourceInput, z as TraceAnalystFilters } from "./types-CeDfCRXp.js";
|
|
5
5
|
import { $ as NoopRawProviderSink, Q as InMemoryRawProviderSinkOptions, X as FileSystemRawProviderSinkOptions, Y as FileSystemRawProviderSink, Z as InMemoryRawProviderSink, at as defaultProviderRedactor, et as ProviderRedactor, it as RawProviderSinkFilter, nt as RawProviderEvent, ot as providerFromBaseUrl, rt as RawProviderSink, tt as RawProviderDirection } from "./types-C0bWoSKC.js";
|
|
6
6
|
import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, r as FileSystemTraceStoreOptions, s as TraceStore, t as EventFilter } from "./store-C4avFjIi.js";
|
|
7
|
-
import { _ as buildTraceAnalysisToolDescriptors, g as UNTRUSTED_TRACE_TEXT, h as TraceAnalysisToolDescriptor, m as TRACE_ANALYST_TOOL_NAMESPACE, p as BuildTraceAnalysisToolsOptions, v as traceAnalystFunctionGroup } from "./engine-
|
|
7
|
+
import { _ as buildTraceAnalysisToolDescriptors, g as UNTRUSTED_TRACE_TEXT, h as TraceAnalysisToolDescriptor, m as TRACE_ANALYST_TOOL_NAMESPACE, p as BuildTraceAnalysisToolsOptions, v as traceAnalystFunctionGroup } from "./engine-LkP-rfp1.js";
|
|
8
8
|
import { a as SpanHandle, i as RunCompleteHookContext, n as CaptureWrite, o as TraceEmitter, r as RunCompleteHook, s as TraceEmitterOptions, t as CaptureErrorHandler } from "./emitter-HcCmFQl_.js";
|
|
9
9
|
import { S as shareAllowed, _ as emptyRedactionReport, a as RedactionFinding, b as redactForShare, c as SafetyCategory, d as ShareSafetyVerdict, f as assessShareSafety, g as detectPersonalData, h as detectCredential, i as RedactOptions, l as ShareSafetyFinding, m as combineVerdicts, n as REDACTION_PROFILES, o as RedactionProfile, p as classifyKey, r as REDACTION_VERSION, s as RedactionReport, t as AssessOptions, u as ShareSafetyStatus, v as normalizeKey, x as redactText, y as redact } from "./redact-Cveoq6fJ.js";
|
|
10
10
|
import { C as TOOL_LATENCY_MS, D as asNumber, E as applyLlmSpanOtlpAttributes, O as contextInputTokens, S as TOOL_ARGS_CAPTURED, T as TOOL_NAME_ATTR_KEYS, _ as LlmSpanOtlpInput, a as LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, b as RUN_COST_ATTR_KEYS, c as LLM_COST_USD, d as LLM_MODEL_ATTR_KEYS, f as LLM_MODEL_NAME, g as LLM_REASONING_TOKEN_ATTR_KEYS, h as LLM_REASONING_TOKENS, i as LLM_CACHE_WRITE_TOKENS, k as firstNumberAttr, l as LLM_INPUT_TOKENS, m as LLM_OUTPUT_TOKEN_ATTR_KEYS, n as LLM_CACHED_TOKENS, o as LLM_CONTEXT_TOKENS, p as LLM_OUTPUT_TOKENS, r as LLM_CACHED_TOKEN_ATTR_KEYS, s as LLM_COST_ATTR_KEYS, t as INPUT_VALUE, u as LLM_INPUT_TOKEN_ATTR_KEYS, v as OPENINFERENCE_SPAN_KIND, w as TOOL_NAME, x as SPAN_KIND_ATTR_KEYS, y as OUTPUT_VALUE } from "./attribute-vocabulary-CcXqHODT.js";
|
|
11
|
-
import { A as AnalyzeTracesInput, B as CaptureFetchContext, C as inferDomainKeywords, D as planTraceInsightQuestions, E as isTraceInsightEvidence, F as OTEL_AGENT_EVAL_SCOPE, G as SseUsageMode, H as captureFetchToRawSink, I as OtlpExport, J as extractUsageFromSse, K as extractUsage, L as OtlpResourceSpans, M as AnalyzeTracesResult, N as analyzeTraces, O as scoreTraceInsightReadiness, P as OtlpFlatLine, R as OtlpSpan, S as domainEvidencePattern, T as isTraceInsightAttribution, U as ExtractUsageFromSseOptions, V as CaptureFetchOptions, W as ExtractedUsage, _ as TraceInsightTask, a as ToolSpansToTraceAnalysisStoreOptions, b as defaultTraceInsightPanel, c as TraceInsightContext, d as TraceInsightPanelRole, f as TraceInsightPromptInput, g as TraceInsightSuite, h as TraceInsightReadiness, i as OtlpFileTraceStoreOptions, j as AnalyzeTracesOptions, k as tokenizeDomainWords, l as TraceInsightEvidence, m as TraceInsightQuestion, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, p as TraceInsightQualityGate, q as extractUsageFromResponse, r as OtlpFileTraceStore, s as TraceInsightAttribution, t as ToolTraceMissingError, u as TraceInsightFinding, v as buildTraceInsightContext, w as isTraceInsightArtifact, x as describeTraceInsightScope, y as buildTraceInsightPrompt, z as exportRunAsOtlp } from "./store-tool-spans-
|
|
11
|
+
import { A as AnalyzeTracesInput, B as CaptureFetchContext, C as inferDomainKeywords, D as planTraceInsightQuestions, E as isTraceInsightEvidence, F as OTEL_AGENT_EVAL_SCOPE, G as SseUsageMode, H as captureFetchToRawSink, I as OtlpExport, J as extractUsageFromSse, K as extractUsage, L as OtlpResourceSpans, M as AnalyzeTracesResult, N as analyzeTraces, O as scoreTraceInsightReadiness, P as OtlpFlatLine, R as OtlpSpan, S as domainEvidencePattern, T as isTraceInsightAttribution, U as ExtractUsageFromSseOptions, V as CaptureFetchOptions, W as ExtractedUsage, _ as TraceInsightTask, a as ToolSpansToTraceAnalysisStoreOptions, b as defaultTraceInsightPanel, c as TraceInsightContext, d as TraceInsightPanelRole, f as TraceInsightPromptInput, g as TraceInsightSuite, h as TraceInsightReadiness, i as OtlpFileTraceStoreOptions, j as AnalyzeTracesOptions, k as tokenizeDomainWords, l as TraceInsightEvidence, m as TraceInsightQuestion, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, p as TraceInsightQualityGate, q as extractUsageFromResponse, r as OtlpFileTraceStore, s as TraceInsightAttribution, t as ToolTraceMissingError, u as TraceInsightFinding, v as buildTraceInsightContext, w as isTraceInsightArtifact, x as describeTraceInsightScope, y as buildTraceInsightPrompt, z as exportRunAsOtlp } from "./store-tool-spans-E-mdWPAz.js";
|
|
12
12
|
import { a as RunIntegrityReport, i as RunIntegrityIssueCode, n as RunIntegrityExpectations, o as assertRunCaptured, r as RunIntegrityIssue, s as throwIfRunIncomplete, t as RunIntegrityError } from "./integrity-CaeNeusS.js";
|
|
13
13
|
import { a as groupBy, c as judgeSpans, d as runMetricExtractor, f as runsForScenario, i as argHash, l as llmSpans, n as RunMetric, o as hasCapturedToolArgs, p as toolSpans, r as aggregateLlm, s as isRunMetric, t as RUN_METRICS, u as runFailureClass } from "./query-DBUQhehH.js";
|
|
14
14
|
import { SpanKind as SpanKind$1 } from "@tangle-network/agent-trace-contract";
|
|
@@ -175,7 +175,14 @@ declare class SearchState implements LedgerProjector<SearchLedgerEntry, SearchSt
|
|
|
175
175
|
private reservationParts;
|
|
176
176
|
/** The admission rule: committed spend, open holds, the unspent claim
|
|
177
177
|
* reserve and the new hold stay within the cap. A claim cell draws on the
|
|
178
|
-
* claim reserve instead of adding to it.
|
|
178
|
+
* claim reserve instead of adding to it.
|
|
179
|
+
*
|
|
180
|
+
* The rule admits holds, not records. Spend above a hold is legal
|
|
181
|
+
* overspend, so committed spend can pass the cap; an event that holds
|
|
182
|
+
* nothing (the claim operation, a cell an external optimizer already ran)
|
|
183
|
+
* and a claim cell the unspent reserve covers are then still admitted. The
|
|
184
|
+
* reserve was set aside at the start and overspend elsewhere cannot take it
|
|
185
|
+
* back; refusing them would leave a search that can never close. */
|
|
179
186
|
private admit;
|
|
180
187
|
/** @internal The largest hold a new non-claim cell or operation may take,
|
|
181
188
|
* and the claim reserve still unspent. */
|
|
@@ -993,4 +1000,4 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
|
|
|
993
1000
|
}
|
|
994
1001
|
//#endregion
|
|
995
1002
|
export { SearchNode as $, LabeledScenarioWrite as A, ScoredSurfaceOutcome as B, JudgeDimension as C, LabeledScenarioSampleArgs as D, LabeledScenarioRecord as E, ProposeContext as F, labelTrustRank as G, SurfaceProposer as H, ProposedCandidate as I, renderSearchSummary as J, SearchProposerView as K, RedactionStatus as L, OptimizerConfig as M, ParetoParent as N, LabeledScenarioSource as O, ProposalTrackContext as P, SearchDecisionRecord as Q, Scenario as R, JudgeConfig as S, LabelTrust as T, TraceSpan as U, SessionScript as V, isProposedCandidate as W, SearchCell as X, searchProposerView as Y, SearchCompletion as Z, GateDecision as _, CampaignResult as a, SearchUnitScore as at, GenerationRecord as b, CampaignTraceWriter as c, searchTaskSetDigest as ct, DispatchContext as d, SearchOperation as et, DispatchFn as f, GateContribution as g, GateContext as h, CampaignCostMeter as i, SearchStateView as it, MutableSurface as j, LabeledScenarioStore as k, CodeSurface as l, searchUnitScores as lt, GateCheckStatus as m, CampaignArtifactWriter as n, SearchSpend as nt, CampaignScenarioIdentity as o, searchCellId as ot, Gate as p, SearchSummaryOptions as q, CampaignCellResult as r, SearchState as rt, CampaignTokenUsage as s, searchNodeId as st, CampaignAggregates as t, SearchScoredCell as tt, ComponentSurface as u, GateResult as v, JudgeScore as w, JudgeAggregate as x, GenerationCandidate as y, ScenarioAggregate as z };
|
|
996
|
-
//# sourceMappingURL=types-
|
|
1003
|
+
//# sourceMappingURL=types-DLtLe21j.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types-DLtLe21j.d.ts","names":[],"sources":["../src/campaign/search-state.ts","../src/campaign/search-summary.ts","../src/campaign/types.ts"],"mappings":";;;;;;;;;;iBAwDgB,aAAa,kBAAkB;;iBAK/B,aACd,kBACA,gBACA,gBACA,OAAO,aACP;;iBAMc,oBAAoB,gBAAgB,eAAe;UAkBlD;EACf;EACA,UAAU;EACV,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;UAKe;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;;EAEA;EACA;;;;;;;;iBASc,iBAAiB,gBAAgB,qBAAqB;UAoBrD;EACf;;EAEA;EACA,gBAAgB;EAChB,UAAU;EACV,mBAAmB;;;EAGnB;;EAEA;;EAEA,kBAAkB;EAClB;EACA;EACA,QAAQ;EACR;;EAEA;EACA,oBAAoB;EACpB;EACA,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA,OAAO;EACP;EACA,OAAO;EACP;EACA,aAAa;EACb;EACA;EACA,SAAS;;EAET;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA;EACA;;UAGe;EACf;EACA,eAAe;EACf,aAAa;EACb;;EAEA,SAAS;;;EAGT,oBAAoB;EACpB;;UAGe;EACf;EACA;;;;;;;cA6BW,uBAAuB,gBAAgB,mBAAmB;WAC5D;UACD;UACA;UACA;UACA;UACA;UACA;mBACS;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;UACT;UACA;;mBAES;;UAET;mBACS;EAEjB,YAAY;EAgCZ,MAAM,OAAO,mBAAmB;EA6BhC,YAAY;;EAKZ,KAAK;UAIG;UAgCA;UA6CA;UAoBA;UA4BA;UA8CA;UA6GA;UAmEA;UA4EA;UAoBA;UAmDA;UAaA;;;UA2EA;;EASR;;;UAmBQ;;;UAkBA;;;;;;;;;;;UAoBA;;;EAsBR;IAAgB;IAA4B;;;EAa5C,cAAc;;EAKd,cAAc;;EAKd;IAAc;IAAkB,WAAW;;;EAK3C;;EAKA,aAAa;;EAcb,SAAS,iBAAiB;;EAM1B;;EAKA,YAAY;;EAKZ,oBAAoB;;EAKpB,SAAS,iBAAiB;;EAK1B;;EAKA,SAAS,iBAAiB;;EAM1B,YAAY;;EAMZ,cAAc,sBAAsB;;EAMpC,gBAAgB,gBAAgB,OAAO,cAAc;;;;;;;cAiB1C;WACF;WACA,QAAQ;WACR,QAAQ;WACR;IAAQ;IAAkB,WAAW;;;WAErC;WACA,OAAO;WACP,YAAY;;;;;;;WAOZ;IAAU;IAA4B;;mBAC9B;mBACA;EAEjB,YAAY,OAAO,aAAa;;MAgB5B;EAIJ,KAAK,iBAAiB;EAItB,QAAQ;;EAKR;;EAKA,SAAS;EAKT,gBAAgB;EAIhB,KAAK,iBAAiB;;EAKtB,SAAS;EAKT,KAAK,iBAAiB;;EAKtB,MAAM;IAAU;MAAyB;EAKzC,UAAU,sBAAsB;;;EAMhC,YAAY,gBAAgB,OAAO,cAAc;;EAKjD,WAAW,gBAAgB,OAAO,cAAc;;EAKhD,QAAQ,aAAa,uBAAuB;UAmCpC;;;;;;UC7sCO;WACN;WACA;;;EAGT,YAAY,0BAA0B;;EAEtC,WAAW,0BAA0B;;;iBAIvB,mBAAmB,OAAO,kBAAkB;UAW3C;;;;;;;;EAQf,OAAO;;EAEP;;;;;;;;;;;iBAYc,oBAAoB,OAAO,iBAAiB,SAAS;;;;;UCtCpD;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;;EAEA;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;;;;EAOV;;;;EAIA,eAAe,eAAe;IAAQ;IAAe;;;;;EAIrD;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;;;EAMA,cAAc,SAAS;;;;iBAKT,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;;;;WAIjC,gBAAgB;;;;WAIhB;;;;WAIA,UAAU;IAAgB;IAAgB,UAAU;;;;WAGpD,QAAQ;;;;WAIR;;;WAGA,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB,gBAAgB;;;;WAIhB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAG/D;EACf;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;UAKX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;iBASI,eAAe,OAAO;;;;UAOrB,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;;EAGlC;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;;;EAG5B;;EAEA,gBAAgB;;EAEhB;;;EAGA,YAAY;;;EAGZ;;;EAGA;EACA;EACA;EACA;;;;EAIA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;;;EAMA,cAAc;;UAGC;EACf;EACA;EACA;;;EAGA,cAAc;;UAGC;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;;;EAIA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;;;EAIA,cAAc,SAAS;;UAGR;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { c as RunSplitTag } from "./run-record-DF-fuu31.js";
|
|
2
|
-
import { R as Scenario, d as DispatchContext } from "./types-
|
|
2
|
+
import { R as Scenario, d as DispatchContext } from "./types-DLtLe21j.js";
|
|
3
3
|
//#region src/benchmarks/types.d.ts
|
|
4
4
|
type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
|
|
5
5
|
type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
|
|
@@ -99,4 +99,4 @@ declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
|
|
|
99
99
|
declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
|
|
100
100
|
//#endregion
|
|
101
101
|
export { BenchmarkFamily as a, BenchmarkSource as c, BenchmarkEvaluation as i, BenchmarkTaskKind as l, BenchmarkAdapter as n, BenchmarkResponder as o, BenchmarkDatasetItem as r, BenchmarkScenario as s, BENCHMARK_SPLIT_SEED as t, deterministicSplit as u };
|
|
102
|
-
//# sourceMappingURL=types-
|
|
102
|
+
//# sourceMappingURL=types-M1Et9e6Z.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-M1Et9e6Z.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;;;;EAKA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
|
package/docs/search-ledger.md
CHANGED
|
@@ -58,7 +58,7 @@ The same code runs in the producer's journal, the kernel, and the Intelligence v
|
|
|
58
58
|
- **Claim start:** once a node is decided `finalist`, the search only claims: no operation, node, edge or non-claim cell follows.
|
|
59
59
|
- **Parents:** a node decided `invalid` (by admission or by the divergence rule) never appears as a parent on a later edge.
|
|
60
60
|
- **Attempts:** attempts count from 1 without gaps. A `passed` or `failed` outcome is final; only a retryable `errored` outcome admits another attempt.
|
|
61
|
-
- **Budget:** at every `cell-allocated` and `operation-started
|
|
61
|
+
- **Budget:** at every `cell-allocated` and `operation-started` that holds a reservation, committed spend plus open reservations plus the unspent claim reserve plus the new reservation stays within `maxUsd`. A claim cell draws on the unspent claim reserve first, and a cell the reserve covers is admitted even when overspend elsewhere took committed spend past the cap. An event without a reservation is always admitted: the rule admits holds, and refusing to record work would leave a search that can never close. Spend above a reservation is recorded as overspend, never refused. An unknown cost counts as its proven floor.
|
|
62
62
|
- **Completion:** `search-closed` needs every allocated cell settled or cancelled, every started operation recorded, and every node with an edge and a terminal decision.
|
|
63
63
|
- **Claim:** the claim names every node decided `finalist`, estimates each against the root on test, and tests each at `1 - (1 - confidence) / k` for its k finalists. A `ship` claim needs a promoted finalist, held-out test units, a pinned judge, and every test unit scored by the root and the shipped finalist; any other claim keeps the root.
|
|
64
64
|
|
|
@@ -204,10 +204,11 @@ The ports:
|
|
|
204
204
|
|
|
205
205
|
**Budget.**
|
|
206
206
|
Every reservation passes the ledger's admission rule: committed spend plus open reservations plus the unspent claim reserve plus the new hold stays within `maxUsd`.
|
|
207
|
+
A claim cell draws on the reserve held since the start, so overspend on earlier cells cannot stop the claim.
|
|
207
208
|
The kernel checks `state.budget.headroomUsd` first and prices an expansion as one proposal plus `childrenPerProposal` screens.
|
|
208
209
|
A hard lane holds its maximum; an estimate lane holds 1.5 times the p99 of its settled cells once 20 settled, else its prior.
|
|
209
210
|
Spend above a hold is recorded as overspend, never refused.
|
|
210
|
-
Expansion stops at `maxNodes`, `maxExpansions`, the deadline, after `patience` expansions without a new leader, when the cap cannot admit one more expansion, or when the proposer stops.
|
|
211
|
+
Expansion stops at `maxNodes`, `maxExpansions`, the deadline, after `patience` expansions without a new leader, when the cap cannot admit one more expansion or overspend took the search past it, or when the proposer stops.
|
|
211
212
|
Allocated cells still run; at the deadline the ones not yet started are cancelled.
|
|
212
213
|
A search without a test split closes with no claim: the leader is decided `selected` when it has a scored cell, and every other undecided node `rejected` with its estimate against the leader.
|
|
213
214
|
A search with a test split closes with its claim (below).
|
|
@@ -255,7 +256,12 @@ When expansion stops, the kernel:
|
|
|
255
256
|
|
|
256
257
|
When the power check fails (a continuous claim needs at least 20 test units to be decided at all) or the reserve cannot cover one finalist, the claim closes as `test-cannot-resolve` and spends nothing on test cells.
|
|
257
258
|
The claim records its rule (`SEARCH_CLAIM_RULE`, whose revision digests every parameter), the family-wise confidence, the power check, each finalist's test estimate and deciding interval, and its reason.
|
|
258
|
-
|
|
259
|
+
`verifySearchClaim(state)` makes the claim again from the closed ledger alone and compares it byte for byte: the power check, the finalists, each finalist's test estimate and interval, the selection, the decision and the reason.
|
|
260
|
+
It reads no blob, so a store that keeps digests only can run it.
|
|
261
|
+
The finalists are the nodes decided `finalist`, in decision order; whether the budget covered k finalists' test cells is read from the claim cells that ran.
|
|
262
|
+
It returns `verified`, `mismatch` with the differences, or `unknown` for a claim another rule revision made.
|
|
263
|
+
The projector checks the claim's structure (every finalist named, the Bonferroni confidence of each test, a ship's held-out units and pinned judge) but not its numbers, so a store keeps what `verifySearchClaim` derives, never the producer's claim as written.
|
|
264
|
+
`runSearch` runs it on every close and on every rerun of a closed ledger, throws on `mismatch`, and returns it as `SearchRunResult.claimVerification`.
|
|
259
265
|
A judge change is a changed `search-opened` header, which `SearchRecorder.open` refuses; it starts a derived search instead of mixing verdicts.
|
|
260
266
|
|
|
261
267
|
`compareOptimizationMethods` keeps its own held-out comparison for black-box methods such as GEPA, which return one winner and never see the test split.
|
|
@@ -308,7 +314,7 @@ Those claims need the sealed test split, the claim's power check, and held-out e
|
|
|
308
314
|
- `src/campaign/search-summary.ts`: `renderSearchSummary` and `searchProposerView`.
|
|
309
315
|
- `src/campaign/search-ledger-recording.ts`: `SearchRecorder` and the surface helpers.
|
|
310
316
|
- `src/campaign/search-kernel.ts`: `runSearch`, the executor, proposer and codec ports, `searchPolicyView` and `searchDivergence`.
|
|
311
|
-
- `src/campaign/search-claim.ts`: `planSearchClaim`, `decideSearchClaim` and `searchClaimReserveUsd`.
|
|
317
|
+
- `src/campaign/search-claim.ts`: `planSearchClaim`, `decideSearchClaim`, `verifySearchClaim` and `searchClaimReserveUsd`.
|
|
312
318
|
- `src/campaign/search-policy.ts`: `SearchPolicy`, `incumbent` and `crowdedFrontierParent`.
|
|
313
319
|
- `src/campaign/allocation.ts`: `SearchAllocator` and `uniform`.
|
|
314
320
|
- `src/campaign/presets/run-optimization.ts`: `runOptimization` as a search on the kernel.
|
package/docs/trace-contracts.md
CHANGED
|
@@ -27,8 +27,8 @@ Write the contract as JSON and compile it with `compileTraceContractSpec`:
|
|
|
27
27
|
|
|
28
28
|
| Key | Checks |
|
|
29
29
|
|---|---|
|
|
30
|
-
| `run.requireCompleted` | The run status is `completed` and its end time is recorded. A failed or aborted run fails. |
|
|
31
|
-
| `run.allowedStatuses` | The run status is one of `running`, `completed`, `failed`, `aborted`. |
|
|
30
|
+
| `run.requireCompleted` | The run status is `completed` and its end time is recorded. A failed or aborted run fails. **A trace with no declared `run.status` fails too** — see the note below. |
|
|
31
|
+
| `run.allowedStatuses` | The run status is one of `running`, `completed`, `failed`, `aborted`. Same "unknown status fails" rule as above. |
|
|
32
32
|
| `run.maxDurationMs` | End minus start. An unknown duration fails. |
|
|
33
33
|
| `tools.required` | Each tool is called at least once. |
|
|
34
34
|
| `tools.forbidden` | No listed tool is called. |
|
|
@@ -47,6 +47,25 @@ Write the contract as JSON and compile it with `compileTraceContractSpec`:
|
|
|
47
47
|
Tool keys match TOOL spans only, and model keys match LLM spans only.
|
|
48
48
|
An LLM span that has a tool's name never satisfies a tool rule.
|
|
49
49
|
|
|
50
|
+
**A `run` rule needs a reader that declares `run.status`.** The checker never
|
|
51
|
+
infers `completed` from a span that merely ended without an error — only a
|
|
52
|
+
declared `run.status` attribute or an explicit error status counts, and an
|
|
53
|
+
undeclared status is `unknown`, which fails both `run.requireCompleted` and
|
|
54
|
+
`run.allowedStatuses`. A saved Claude Code **session transcript** (as opposed
|
|
55
|
+
to a `-p`/non-interactive stream, which the harness can mark `running` or
|
|
56
|
+
`completed`) carries no run-status record at all — an ordinary finished
|
|
57
|
+
interactive session and a truncated one look the same to the reader — so
|
|
58
|
+
**every `run` rule fails on a session-transcript trace**, always. Write a
|
|
59
|
+
contract for session transcripts with no `run` key, or run it against an
|
|
60
|
+
OTLP/eval-store trace that does carry a declared status.
|
|
61
|
+
|
|
62
|
+
**A `tools`/`llm` rule needs evidence its span kind was captured at all**, not
|
|
63
|
+
just a matching span. `tools.forbidden`, `tools.allowed`, `tools.maxCalls*`
|
|
64
|
+
and `retries` report `error` — not a silent pass — when the trace has zero
|
|
65
|
+
TOOL spans and no `gen_ai.tool.definitions` record either: an exporter that
|
|
66
|
+
drops a span kind must not make `forbidden` and `retries` checks pass because
|
|
67
|
+
nothing was there to check.
|
|
68
|
+
|
|
50
69
|
Parsing is strict.
|
|
51
70
|
An unknown key or an unknown status is an error that names the closest known word, so a typo never removes a check.
|
|
52
71
|
`lintTraceContractSpec` reports contradictions (a tool that is both required and forbidden) and likely mistakes (an order that uses the default `start-order` mode, which passes overlapping calls).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-eval",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.194.0",
|
|
4
4
|
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-eval#readme",
|
|
6
6
|
"repository": {
|